Merge pull request #100 from getindata/release-0.8.0

Release 0.8.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.7.0
+current_version = 0.8.0
 
 [bumpversion:file:pyproject.toml]
 

.copier-answers.yml

@@ -9,7 +9,7 @@ description: Kedro plugin with GCP Vertex AI support
 docs_url: https://kedro-vertexai.readthedocs.io/
 full_name: Kedro Vertex AI Plugin
 github_url: https://github.com/getindata/kedro-vertexai
-initial_version: 0.7.0
+initial_version: 0.8.0
 keywords:
 - kedro-plugin
 - kedro

.github/workflows/test.yml → .github/workflows/test_and_publish.yml

@@ -1,4 +1,4 @@
-name: Test
+name: Test & Publish
 
 on:
   push:
@@ -98,7 +98,7 @@ jobs:
 
   e2e_tests:
     runs-on: ubuntu-latest
-    needs: [unit_tests, sonarcloud, codeql]
+    needs: [unit_tests, sonarcloud]
     steps:
       - uses: actions/checkout@v2
 
@@ -167,4 +167,63 @@ jobs:
         run: |
           cd ./spaceflights
           export KEDRO_CONFIG_COMMIT_ID=$GITHUB_SHA
-          kedro vertexai run-once --wait-for-completion
+          kedro vertexai run-once --wait-for-completion
+  publish:
+    if: github.event.pull_request == null && github.ref == 'refs/heads/master'
+    needs: [ e2e_tests, codeql ]
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_PACKAGE: kedro_vertexai
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # necessary to enable merging, all the history is needed
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.8"
+      - name: Build package dist from source # A better way will be : https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ but pep 517 is still marked as experimental
+        run: |
+          pip install poetry
+          poetry build
+      - name:  Merge back to develop  # we have to set the config first on a fresh machine
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git checkout -b develop --track origin/develop
+          git merge master
+          git push
+      - name: Set dynamically package version as output variable # see https://github.com/actions/create-release/issues/39
+        # see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
+        id: set_package_version
+        run: |
+          echo "::set-output name=PACKAGE_VERSION::$(cat $PYTHON_PACKAGE/__init__.py | grep -Po  '\d+\.\d+\.\d+')"
+      - name: Create temporary file with the body content for the release
+        run: |
+          grep -Poz "## \[${{steps.set_package_version.outputs.PACKAGE_VERSION}}] - \d{4}-\d{2}-\d{2}[\S\s]+?(?=## \[\d+\.\d+\.\d+\]|\[.+\]:)" CHANGELOG.md > release_body.md
+      - name: Create Release # https://github.com/actions/create-release
+        id: create_release
+        uses: actions/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
+        with:
+          tag_name: ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
+          release_name: Release ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
+          body_path: ./release_body.md
+          draft: false
+          prerelease: false
+      - name: Rollback Release in case of run failure
+        if: failure() && steps.create_release.outputs.id != ''
+        uses: author/action-rollback@stable
+        with:
+          # Using a known release ID
+          release_id: ${{ steps.create_release.outputs.id }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Publish distribution to PyPI  # official action from python maintainers
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_PASSWORD }}
+          verbose: true  # trace if the upload fails

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [0.8.0] - 2022-12-09
+
+-   Added support for configuration of resources and node selectors with [Kedro node tags](https://kedro.readthedocs.io/en/stable/nodes_and_pipelines/nodes.html#how-to-tag-a-node)
+-   Added support for gpu configuration on Vertex AI (by adding `node_selectors` section and `gpu` resources entry in `vertexai.yml` configuration file)
+-   Added --auto-build option to run-once that calls 'docker build' and 'docker push' for you before running the job on VertexAI. It introduces '--yes' option to disable confirmation prompt
+
 ## [0.7.0] - 2022-09-08
 
 -   Add better MLflow authorization entrypoints (via Hooks and MLflow Request Header Provider Plugin)
@@ -42,7 +48,9 @@
 
 -   Initial version of **kedro-vertexai** plugin extracted from [kedro-kubeflow v0.6.0](https://github.com/getindata/kedro-kubeflow/tree/0.6.0)
 
-[Unreleased]: https://github.com/getindata/kedro-vertexai/compare/0.7.0...HEAD
+[Unreleased]: https://github.com/getindata/kedro-vertexai/compare/0.8.0...HEAD
+
+[0.8.0]: https://github.com/getindata/kedro-vertexai/compare/0.7.0...0.8.0
 
 [0.7.0]: https://github.com/getindata/kedro-vertexai/compare/0.6.0...0.7.0
 

docs/source/02_installation/02_configuration.md

@@ -39,23 +39,42 @@ run_config:
   resources:
 
     # For nodes that require more RAM you can increase the "memory"
-    data_import_step:
+    data-import-node:
       memory: 2Gi
 
-    # Training nodes can utilize more than one CPU if the algoritm
+    # Training nodes can utilize more than one CPU if the algorithm
     # supports it
-    model_training:
+    model-training-node:
       cpu: 8
-      memory: 1Gi
+      memory: 60Gi
 
     # GPU-capable nodes can request 1 GPU slot
-    tensorflow_step:
-      nvidia.com/gpu: 1
+    tensorflow-node:
+      gpu: 1
+
+    # Resources can be also configured via nodes tag
+    # (if there is node name and tag configuration for the same
+    # resource, tag configuration is overwritten with node one)
+    gpu_node_tag:
+      cpu: 1
+      gpu: 2
 
     # Default settings for the nodes
     __default__:
       cpu: 200m
       memory: 64Mi
+
+  # Optional section allowing to configure node selectors constraints
+  # like gpu accelerator for nodes with gpu resources.
+  # (Note that not all accelerators are available in all
+  # regions - https://cloud.google.com/compute/docs/gpus/gpu-regions-zones)
+  # and not for all machines and resources configurations - 
+  # https://cloud.google.com/vertex-ai/docs/training/configure-compute#specifying_gpus
+  node_selectors:
+    gpu_node_tag:
+      cloud.google.com/gke-accelerator: NVIDIA_TESLA_T4
+    tensorflow-step:
+      cloud.google.com/gke-accelerator: NVIDIA_TESLA_K80
 
   # Optional section allowing to generate config files at runtime,
   # useful e.g. when you need to obtain credentials dynamically and store them in credentials.yaml
@@ -140,3 +159,28 @@ def generate_config(self) -> dict:
 First one - `target_config_file` should return the name of the configuration file to be generated (e.g. `credentials.yml`) and the `generate_config` should return a dictionary, which will be then serialized into the target file as YAML. If the target file already exists during the invocation, it will be merged (see method `kedro_vertexai.dynamic_config.DynamicConfigProvider.merge_with_existing` ) with the existing one and then saved again.
 Note that the `generate_config` has access to an initialized plugin config via `self.config` property, so any values from the `vertexai.yml` configuration is accessible. 
 
+## Resources configuration
+
+Optional `resources` and `node_selectors` sections enable adjustment of the resources reservations and limits for the
+selected `Kedro` nodes. Settings for individual nodes, we can define in two ways - using the name of the node or
+its [tag](https://kedro.readthedocs.io/en/stable/nodes_and_pipelines/nodes.html#how-to-tag-a-node) (if there is node name and tag configuration for the same resource, tag configuration is overwritten with
+node one). For example, with the `vertexai.yaml` configuration file shown at the beginning of the chapter and the `Kedro`
+pipeline containing such a node:
+```python
+def create_pipeline(**kwargs):
+    return Pipeline(
+        [
+            node(
+                func=train_model,
+                inputs=["X_train", "y_train"],
+                outputs="regressor",
+                name="model_training_node",
+                tags="gpu_node_tag",
+            ),
+        ]
+    )
+```
+
+we expect this particular node to run with two `NVIDIA_TESLA_T4` GPUs, eight CPUs, and memory allocated according to
+the specified `60Gi` limit. (Note that not all accelerators are available in all [regions](https://cloud.google.com/compute/docs/gpus/gpu-regions-zones) and not for all [machines and
+resources configurations](https://cloud.google.com/vertex-ai/docs/training/configure-compute#specifying_gpus))

docs/source/03_getting_started/01_quickstart.md

@@ -102,7 +102,7 @@ kedro-vertexai
 
 ### Adjusting Data Catalog to be compatible with Vertex AI
 This change enforces raw input data existence in the image. While running locally, every intermediate dataset is stored as a `MemoryDataSet`. When running in VertexAI Pipelines, there is no shared-memory, Kedro-VertexAI plugin automatically handles intermediate dataset serialization - every intermediate dataset will be stored (as a compressed cloudpickle file) in GCS bucket specified in the `vertexai.yml` config under `run_config.root` key.
-Adjusted `catalog.yml` should look like this.
+Adjusted `catalog.yml` should look like this (note: remove the rest of the entries which comes with the spaceflights starter - you need only `companies`,`reviews`,`shuttles`.)
 
 ```yaml
 companies:
@@ -123,7 +123,7 @@ shuttles:
 
 All intermediate and output data will be stored in the location with the following pattern:
 ```
-gs://<run_config.root from vertexai.yml/kedro-vertexai-temp/<vertex ai job name>/*.bin
+gs://<run_config.root from vertexai.yml>/kedro-vertexai-temp/<vertex ai job name>/*.bin
 ```
 
 Of course if you want to use intermediate/output data and store it a location of your choice, add it to the catalog. Be aware that you cannot use local paths - use `gs://` paths instead. 

docs/spellcheck_exceptions.txt

@@ -109,3 +109,5 @@ GCS
 dataset
 IAM
 mlflow's
+CPUs
+GPUs

kedro_vertexai/__init__.py

@@ -1 +1 @@
-__version__ = "0.7.0"
+__version__ = "0.8.0"

kedro_vertexai/cli.py

@@ -4,14 +4,19 @@
 from pathlib import Path
 
 import click
-from click import ClickException, Context
+from click import ClickException, Context, confirm
 
 from .client import VertexAIPipelinesClient
 from .config import PluginConfig, RunConfig
 from .constants import KEDRO_VERTEXAI_BLOB_TEMP_DIR_NAME, VERTEXAI_RUN_ID_TAG
 from .context_helper import ContextHelper
 from .data_models import PipelineResult
-from .utils import materialize_dynamic_configuration, store_parameters_in_yaml
+from .utils import (
+    docker_build,
+    docker_push,
+    materialize_dynamic_configuration,
+    store_parameters_in_yaml,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -57,6 +62,20 @@ def list_pipelines(ctx):
 
 
 @vertexai_group.command()
+@click.option(
+    "--auto-build",
+    type=bool,
+    is_flag=True,
+    default=False,
+    help="Specify to docker build and push before scheduling a run.",
+)
+@click.option(
+    "--yes",
+    type=bool,
+    is_flag=True,
+    default=False,
+    help="Auto answer yes confirm prompts.",
+)
 @click.option(
     "-i",
     "--image",
@@ -90,6 +109,8 @@ def list_pipelines(ctx):
 @click.pass_context
 def run_once(
     ctx: Context,
+    auto_build: bool,
+    yes: bool,
     image: str,
     pipeline: str,
     params: list,
@@ -101,16 +122,36 @@ def run_once(
     context_helper = ctx.obj["context_helper"]
     config: RunConfig = context_helper.config.run_config
     client: VertexAIPipelinesClient = context_helper.vertexai_client
+    image: str = image if image else config.image
+
+    if auto_build:
+        if (splits := image.split(":"))[-1] != "latest" and len(splits) > 1:
+            logger.warning(
+                f"This operation will overwrite the target image with {splits[-1]} tag at remote location."
+            )
+
+        if not yes and not confirm("Continue?", default=True):
+            exit(1)
+
+        if (rv := docker_build(str(context_helper.context.project_path), image)) != 0:
+            exit(rv)
+        if (rv := docker_push(image)) != 0:
+            exit(rv)
+    else:
+        logger.warning(
+            "Make sure that you've built and pushed your image to run the latest version remotely.\
+ Consider using '--auto-build' parameter."
+        )
 
     run = client.run_once(
         pipeline=pipeline,
-        image=image if image else config.image,
+        image=image,
         image_pull_policy=config.image_pull_policy,
         parameters=format_params(params),
     )
 
     click.echo(
-        f"Intermediate data datasets will be stored in{os.linesep}"
+        f"Intermediate data datasets will be stored in {os.linesep}"
         f"gs://{config.root.strip('/')}/{KEDRO_VERTEXAI_BLOB_TEMP_DIR_NAME}/{run['displayName']}/*.bin"
     )