From 8a28195170d39c9546090f0a9c37d7738c281e28 Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Sun, 7 May 2023 03:53:30 +0800 Subject: [PATCH 1/2] Update pytorch runtime to fix validate images check (#3148) Signed-off-by: typhoonzero --- Makefile | 6 +++--- etc/config/metadata/runtime-images/pytorch-runtime.json | 6 +++--- pyproject.toml | 2 ++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index b8c408c42..9cfce43d9 100644 --- a/Makefile +++ b/Makefile @@ -345,20 +345,20 @@ validate-runtime-image: # Validate that runtime image meets minimum criteria fi; \ for cmd in $$required_commands ; do \ echo "=> Checking container image $$image for $$cmd..." ; \ - docker run --rm $$image which $$cmd > /dev/null 2>&1 ; \ + docker run --rm --entrypoint /bin/bash $$image -c "which $$cmd" > /dev/null 2>&1 ; \ if [ $$? -ne 0 ]; then \ echo "ERROR: Container image $$image does not meet criteria for command: $$cmd" ; \ fail=1; \ continue; \ fi; \ if [ $$cmd == "python3" ]; then \ - IMAGE_PYTHON3_MINOR_VERSION=`docker run --rm $$image $$cmd --version | cut -d' ' -f2 | cut -d'.' -f2` ; \ + IMAGE_PYTHON3_MINOR_VERSION=`docker run --rm --entrypoint /bin/bash $$image -c "$$cmd --version" | cut -d' ' -f2 | cut -d'.' -f2` ; \ if [[ $$IMAGE_PYTHON3_MINOR_VERSION -lt 8 ]]; then \ echo "WARNING: Container image $$image requires Python 3.8 or greater for latest generic component dependency installation" ; \ fail=1; \ elif [[ $$IMAGE_PYTHON3_MINOR_VERSION -ge 8 ]]; then \ echo "=> Checking notebook execution..." ; \ - docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm $$image /bin/bash -c "python3 -m pip install -r /opt/elyra/requirements-elyra.txt && \ + docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm --entrypoint /bin/bash $$image -c "python3 -m pip install -r /opt/elyra/requirements-elyra.txt && \ curl https://raw.githubusercontent.com/nteract/papermill/main/papermill/tests/notebooks/simple_execute.ipynb --output simple_execute.ipynb && \ python3 -m papermill simple_execute.ipynb output.ipynb > /dev/null" ; \ if [ $$? -ne 0 ]; then \ diff --git a/etc/config/metadata/runtime-images/pytorch-runtime.json b/etc/config/metadata/runtime-images/pytorch-runtime.json index 54cffc95e..a407766b5 100644 --- a/etc/config/metadata/runtime-images/pytorch-runtime.json +++ b/etc/config/metadata/runtime-images/pytorch-runtime.json @@ -1,8 +1,8 @@ { - "display_name": "Pytorch 1.4 with CUDA-runtime", + "display_name": "Pytorch 1.12 with CUDA-runtime", "metadata": { - "description": "PyTorch 1.4 (with GPU support)", - "image_name": "pytorch/pytorch@sha256:ee783a4c0fccc7317c150450e84579544e171dd01a3f76cf2711262aced85bf7", + "description": "PyTorch 1.12 (with GPU support)", + "image_name": "nvcr.io/nvidia/pytorch@sha256:b6080b287f79ba34c392730566a8903361453c15c3b3beff032d11a05e08f7c3", "tags": ["gpu", "pytorch"] }, "schema_name": "runtime-image" diff --git a/pyproject.toml b/pyproject.toml index 9d2cf541f..4d42f67a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,8 @@ dependencies = [ "watchdog>=2.1.3", "websocket-client", "yaspin", + # see: https://stackoverflow.com/questions/76175487/sudden-importerror-cannot-import-name-appengine-from-requests-packages-urlli + "appengine-python-standard", "kfp>=1.7.0,<2.0,!=1.7.2", # We cap the SDK to <2.0 due to possible breaking changes "pygithub", "black>=22.8.0", From 2ff61026b054bb2894c0f84e53bad81afb4dec26 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Sat, 6 May 2023 12:56:08 -0700 Subject: [PATCH 2/2] Convert --valid-only to --include-invalid on list subcommand (#3142) Signed-off-by: Kevin Bates --- docs/source/user_guide/code-snippets.md | 2 +- .../user_guide/command-line-interface.md | 2 +- docs/source/user_guide/pipeline-components.md | 6 ++--- docs/source/user_guide/runtime-conf.md | 2 +- docs/source/user_guide/runtime-image-conf.md | 2 +- elyra/metadata/metadata_app.py | 19 ++++++++-------- elyra/tests/metadata/test_metadata_app.py | 22 ++++++++++--------- 7 files changed, 28 insertions(+), 27 deletions(-) diff --git a/docs/source/user_guide/code-snippets.md b/docs/source/user_guide/code-snippets.md index 92484e982..ae78125b7 100644 --- a/docs/source/user_guide/code-snippets.md +++ b/docs/source/user_guide/code-snippets.md @@ -109,7 +109,7 @@ $ elyra-metadata list code-snippets The output lists for each code snippet the name and the name of the associated JSON formatted metadata file, which is stored in the JupyterLab data directory in the `metadata/code-snippets` subdirectory. ``` -Available metadata instances for code-snippets (includes invalid): +Available metadata instances for code-snippets: Schema Instance Resource --------- -------- -------- diff --git a/docs/source/user_guide/command-line-interface.md b/docs/source/user_guide/command-line-interface.md index 0ff5d2425..dbf8106a1 100644 --- a/docs/source/user_guide/command-line-interface.md +++ b/docs/source/user_guide/command-line-interface.md @@ -45,7 +45,7 @@ By default the `list` command displays the results in a user-friendly format. ``` $ elyra-metadata list runtime-images -Available metadata instances for runtime-images (includes invalid): +Available metadata instances for runtime-images: Schema Instance Resource ------ -------- -------- diff --git a/docs/source/user_guide/pipeline-components.md b/docs/source/user_guide/pipeline-components.md index 864b75ec1..3717ce4ee 100644 --- a/docs/source/user_guide/pipeline-components.md +++ b/docs/source/user_guide/pipeline-components.md @@ -180,7 +180,7 @@ To list component catalog entries: ```bash $ elyra-metadata list component-catalogs -Available metadata instances for component-catalogs (includes invalid): +Available metadata instances for component-catalogs: Schema Instance Resource ------ -------- -------- @@ -283,7 +283,7 @@ $ elyra-metadata list component-registries ``` In this example, there are three user-defined instances. ```bash -Available metadata instances for component-registries (includes invalid): +Available metadata instances for component-registries: Schema Instance Resource ------ -------- -------- @@ -314,7 +314,7 @@ The following component-registries instances were migrated: ['myoperators', 'air Once migrated, these entries should appear in the set of component catalogs. This can be confirmed by listing the component-catalogs instances: ```bash $ elyra-metadata list component-catalogs -Available metadata instances for component-catalogs (includes invalid): +Available metadata instances for component-catalogs: Schema Instance Resource ------ -------- -------- diff --git a/docs/source/user_guide/runtime-conf.md b/docs/source/user_guide/runtime-conf.md index 19d3ab2dc..80d3d6cdc 100644 --- a/docs/source/user_guide/runtime-conf.md +++ b/docs/source/user_guide/runtime-conf.md @@ -95,7 +95,7 @@ $ elyra-metadata list runtimes The output lists for each runtime the name and the name of the associated JSON formatted metadata file, which is stored in the JupyterLab data directory in the `metadata/runtimes` subdirectory. ``` -Available metadata instances for runtimes (includes invalid): +Available metadata instances for runtimes: Schema Instance Resource ------ -------- -------- diff --git a/docs/source/user_guide/runtime-image-conf.md b/docs/source/user_guide/runtime-image-conf.md index 13ebef006..1dd069f96 100644 --- a/docs/source/user_guide/runtime-image-conf.md +++ b/docs/source/user_guide/runtime-image-conf.md @@ -94,7 +94,7 @@ To list runtime image configurations: ```bash $ elyra-metadata list runtime-images -Available metadata instances for runtime-images (includes invalid): +Available metadata instances for runtime-images: Schema Instance Resource ------ -------- -------- diff --git a/elyra/metadata/metadata_app.py b/elyra/metadata/metadata_app.py index 71c355820..4316a169b 100644 --- a/elyra/metadata/metadata_app.py +++ b/elyra/metadata/metadata_app.py @@ -65,15 +65,15 @@ class SchemaspaceList(SchemaspaceBase): json_flag = Flag("--json", name="json", description="List complete instances as JSON", default_value=False) - valid_only_flag = Flag( - "--valid-only", - name="valid-only", - description="Only list valid instances (default includes invalid instances)", + include_invalid_flag = Flag( + "--include-invalid", + name="include-invalid", + description="Include invalid instances (default displays only valid instances)", default_value=False, ) # 'List' flags - options = [json_flag, valid_only_flag] + options = [json_flag, include_invalid_flag] def __init__(self, **kwargs): super().__init__(**kwargs) @@ -82,9 +82,8 @@ def __init__(self, **kwargs): def start(self): super().start() # process options - include_invalid = not self.valid_only_flag.value try: - metadata_instances = self.metadata_manager.get_all(include_invalid=include_invalid) + metadata_instances = self.metadata_manager.get_all(include_invalid=self.include_invalid_flag.value) except MetadataNotFoundError: metadata_instances = None @@ -97,11 +96,11 @@ def start(self): print(f"No metadata instances found for {self.schemaspace}") return - validity_clause = "includes invalid" if include_invalid else "valid only" - print(f"Available metadata instances for {self.schemaspace} ({validity_clause}):") + validity_clause = " (includes invalid)" if self.include_invalid_flag.value else "" + print(f"Available metadata instances for {self.schemaspace}{validity_clause}:") sorted_instances = sorted(metadata_instances, key=lambda inst: (inst.schema_name, inst.name)) - # pad to width of longest instance + # pad to width of the longest instance max_schema_name_len = len("Schema") max_name_len = len("Instance") max_resource_len = len("Resource") diff --git a/elyra/tests/metadata/test_metadata_app.py b/elyra/tests/metadata/test_metadata_app.py index 5ba9dd69e..276cddf0f 100644 --- a/elyra/tests/metadata/test_metadata_app.py +++ b/elyra/tests/metadata/test_metadata_app.py @@ -996,7 +996,7 @@ def test_list_instances(script_runner, mock_data_dir): resource = metadata_manager.create("another2", another) assert resource is not None - ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE) + ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--include-invalid") assert ret.success lines = ret.stdout.split("\n") assert len(lines) == 9 # always 5 more than the actual runtime count @@ -1014,13 +1014,13 @@ def test_list_instances(script_runner, mock_data_dir): # Remove the '2' runtimes and reconfirm smaller set metadata_manager.remove("valid2") metadata_manager.remove("another2") - # Include two additional invalid files as well - one for uri failure, andother missing display_name + # Include two additional invalid files as well - one for uri failure, another missing display_name metadata_dir = os.path.join(mock_data_dir, "metadata", METADATA_TEST_SCHEMASPACE) create_json_file(metadata_dir, "invalid.json", invalid_metadata_json) create_json_file(metadata_dir, "no_display_name.json", invalid_no_display_name_json) create_json_file(metadata_dir, "invalid_schema_name.json", invalid_schema_name_json) - ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE) + ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--include-invalid") assert ret.success lines = ret.stdout.split("\n") assert len(lines) == 10 # always 5 more than the actual runtime count @@ -1036,11 +1036,11 @@ def test_list_instances(script_runner, mock_data_dir): assert line_elements[4][3] == "**INVALID**" assert line_elements[4][4] == "(SchemaNotFoundError)" - ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--valid-only") + ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE) assert ret.success lines = ret.stdout.split("\n") assert len(lines) == 7 # always 5 more than the actual runtime count - assert lines[0] == f"Available metadata instances for {METADATA_TEST_SCHEMASPACE} (valid only):" + assert lines[0] == f"Available metadata instances for {METADATA_TEST_SCHEMASPACE}:" line_elements = [line.split() for line in lines[4:6]] assert line_elements[0][1] == "another" assert line_elements[1][1] == "valid" @@ -1066,7 +1066,7 @@ def test_list_json_instances(script_runner, mock_data_dir): resource = metadata_manager.create("another2", another) assert resource is not None - ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--json") + ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--json", "--include-invalid") assert ret.success # Consume results results = json.loads(ret.stdout) @@ -1076,20 +1076,22 @@ def test_list_json_instances(script_runner, mock_data_dir): metadata_manager.remove("valid2") metadata_manager.remove("another2") - # Include two additional invalid files as well - one for uri failure, andother missing display_name + # Include two additional invalid files as well - one for uri failure, another missing display_name metadata_dir = os.path.join(mock_data_dir, "metadata", METADATA_TEST_SCHEMASPACE) create_json_file(metadata_dir, "invalid.json", invalid_metadata_json) create_json_file(metadata_dir, "no_display_name.json", invalid_no_display_name_json) + # Ensure invalids are NOT listed ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--json") assert ret.success results = json.loads(ret.stdout) - assert len(results) == 4 + assert len(results) == 2 - ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--json", "--valid-only") + # Ensure invalids ARE listed + ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--json", "--include-invalid") assert ret.success results = json.loads(ret.stdout) - assert len(results) == 2 + assert len(results) == 4 def test_remove_help(script_runner):