Merge branch 'main' of https://github.com/elyra-ai/elyra into support…

…_parallel_for_operations

Makefile

@@ -345,20 +345,20 @@ validate-runtime-image: # Validate that runtime image meets minimum criteria
 	fi; \
 	for cmd in $$required_commands ; do \
         echo "=> Checking container image $$image for $$cmd..." ; \
-		docker run --rm $$image which $$cmd > /dev/null 2>&1 ; \
+		docker run --rm --entrypoint /bin/bash $$image -c "which $$cmd" > /dev/null 2>&1 ; \
 		if [ $$? -ne 0 ]; then \
 			echo "ERROR: Container image $$image does not meet criteria for command: $$cmd" ; \
 			fail=1; \
 			continue; \
 		fi; \
 		if [ $$cmd == "python3" ]; then \
-			IMAGE_PYTHON3_MINOR_VERSION=`docker run --rm $$image $$cmd --version | cut -d' ' -f2 | cut -d'.' -f2` ; \
+			IMAGE_PYTHON3_MINOR_VERSION=`docker run --rm --entrypoint /bin/bash $$image -c "$$cmd --version" | cut -d' ' -f2 | cut -d'.' -f2` ; \
 			if [[ $$IMAGE_PYTHON3_MINOR_VERSION -lt 8 ]]; then \
 				echo "WARNING: Container image $$image requires Python 3.8 or greater for latest generic component dependency installation" ; \
 				fail=1; \
 			elif [[ $$IMAGE_PYTHON3_MINOR_VERSION -ge 8 ]]; then \
 				echo "=> Checking notebook execution..." ; \
-				docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm $$image /bin/bash -c "python3 -m pip install -r /opt/elyra/requirements-elyra.txt && \
+				docker run -v $$(pwd)/etc/generic:/opt/elyra/ --rm --entrypoint /bin/bash $$image -c "python3 -m pip install -r /opt/elyra/requirements-elyra.txt && \
 							   curl https://raw.githubusercontent.com/nteract/papermill/main/papermill/tests/notebooks/simple_execute.ipynb --output simple_execute.ipynb && \
 							   python3 -m papermill simple_execute.ipynb output.ipynb > /dev/null" ; \
 				if [ $$? -ne 0 ]; then \

docs/source/user_guide/code-snippets.md

@@ -109,7 +109,7 @@ $ elyra-metadata list code-snippets
 The output lists for each code snippet the name and the name of the associated JSON formatted metadata file, which is stored in the JupyterLab data directory in the `metadata/code-snippets` subdirectory.
 
 ```
-Available metadata instances for code-snippets (includes invalid):
+Available metadata instances for code-snippets:
 
 Schema   	Instance		Resource  
 ---------   	--------		--------

docs/source/user_guide/command-line-interface.md

@@ -45,7 +45,7 @@ By default the `list` command displays the results in a user-friendly format.
 
 ```
 $ elyra-metadata list runtime-images
-Available metadata instances for runtime-images (includes invalid):
+Available metadata instances for runtime-images:
 
 Schema          Instance    Resource
 ------          --------    --------

docs/source/user_guide/pipeline-components.md

@@ -180,7 +180,7 @@ To list component catalog entries:
 ```bash
 $ elyra-metadata list component-catalogs
 
-Available metadata instances for component-catalogs (includes invalid):
+Available metadata instances for component-catalogs:
 
 Schema               Instance                            Resource
 ------               --------                            --------
@@ -283,7 +283,7 @@ $ elyra-metadata list component-registries
 ```
 In this example, there are three user-defined instances.
 ```bash
-Available metadata instances for component-registries (includes invalid):
+Available metadata instances for component-registries:
 
 Schema               Instance            Resource
 ------               --------            --------
@@ -314,7 +314,7 @@ The following component-registries instances were migrated: ['myoperators', 'air
 Once migrated, these entries should appear in the set of component catalogs.  This can be confirmed by listing the component-catalogs instances:
 ```bash
 $ elyra-metadata list component-catalogs
-Available metadata instances for component-catalogs (includes invalid):
+Available metadata instances for component-catalogs:
 
 Schema                    Instance                            Resource                                                                                                         
 ------                    --------                            --------                                                                                                             

docs/source/user_guide/runtime-conf.md

@@ -95,7 +95,7 @@ $ elyra-metadata list runtimes
 The output lists for each runtime the name and the name of the associated JSON formatted metadata file, which is stored in the JupyterLab data directory in the `metadata/runtimes` subdirectory.
 
 ```
-Available metadata instances for runtimes (includes invalid):
+Available metadata instances for runtimes:
 
 Schema   Instance  Resource  
 ------   --------  -------- 

docs/source/user_guide/runtime-image-conf.md

@@ -94,7 +94,7 @@ To list runtime image configurations:
 ```bash
 $ elyra-metadata list runtime-images
 
-Available metadata instances for runtime-images (includes invalid):
+Available metadata instances for runtime-images:
 
 Schema          Instance               Resource                                                                                                       
 ------          --------               --------                                                                                                       

elyra/metadata/metadata_app.py

@@ -65,15 +65,15 @@ class SchemaspaceList(SchemaspaceBase):
 
     json_flag = Flag("--json", name="json", description="List complete instances as JSON", default_value=False)
 
-    valid_only_flag = Flag(
-        "--valid-only",
-        name="valid-only",
-        description="Only list valid instances (default includes invalid instances)",
+    include_invalid_flag = Flag(
+        "--include-invalid",
+        name="include-invalid",
+        description="Include invalid instances (default displays only valid instances)",
         default_value=False,
     )
 
     # 'List' flags
-    options = [json_flag, valid_only_flag]
+    options = [json_flag, include_invalid_flag]
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -82,9 +82,8 @@ def __init__(self, **kwargs):
     def start(self):
         super().start()  # process options
 
-        include_invalid = not self.valid_only_flag.value
         try:
-            metadata_instances = self.metadata_manager.get_all(include_invalid=include_invalid)
+            metadata_instances = self.metadata_manager.get_all(include_invalid=self.include_invalid_flag.value)
         except MetadataNotFoundError:
             metadata_instances = None
 
@@ -97,11 +96,11 @@ def start(self):
                 print(f"No metadata instances found for {self.schemaspace}")
                 return
 
-            validity_clause = "includes invalid" if include_invalid else "valid only"
-            print(f"Available metadata instances for {self.schemaspace} ({validity_clause}):")
+            validity_clause = " (includes invalid)" if self.include_invalid_flag.value else ""
+            print(f"Available metadata instances for {self.schemaspace}{validity_clause}:")
 
             sorted_instances = sorted(metadata_instances, key=lambda inst: (inst.schema_name, inst.name))
-            # pad to width of longest instance
+            # pad to width of the longest instance
             max_schema_name_len = len("Schema")
             max_name_len = len("Instance")
             max_resource_len = len("Resource")

elyra/tests/metadata/test_metadata_app.py

@@ -996,7 +996,7 @@ def test_list_instances(script_runner, mock_data_dir):
     resource = metadata_manager.create("another2", another)
     assert resource is not None
 
-    ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE)
+    ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--include-invalid")
     assert ret.success
     lines = ret.stdout.split("\n")
     assert len(lines) == 9  # always 5 more than the actual runtime count
@@ -1014,13 +1014,13 @@ def test_list_instances(script_runner, mock_data_dir):
     # Remove the '2' runtimes and reconfirm smaller set
     metadata_manager.remove("valid2")
     metadata_manager.remove("another2")
-    # Include two additional invalid files as well - one for uri failure, andother missing display_name
+    # Include two additional invalid files as well - one for uri failure, another missing display_name
     metadata_dir = os.path.join(mock_data_dir, "metadata", METADATA_TEST_SCHEMASPACE)
     create_json_file(metadata_dir, "invalid.json", invalid_metadata_json)
     create_json_file(metadata_dir, "no_display_name.json", invalid_no_display_name_json)
     create_json_file(metadata_dir, "invalid_schema_name.json", invalid_schema_name_json)
 
-    ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE)
+    ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--include-invalid")
     assert ret.success
     lines = ret.stdout.split("\n")
     assert len(lines) == 10  # always 5 more than the actual runtime count
@@ -1036,11 +1036,11 @@ def test_list_instances(script_runner, mock_data_dir):
     assert line_elements[4][3] == "**INVALID**"
     assert line_elements[4][4] == "(SchemaNotFoundError)"
 
-    ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--valid-only")
+    ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE)
     assert ret.success
     lines = ret.stdout.split("\n")
     assert len(lines) == 7  # always 5 more than the actual runtime count
-    assert lines[0] == f"Available metadata instances for {METADATA_TEST_SCHEMASPACE} (valid only):"
+    assert lines[0] == f"Available metadata instances for {METADATA_TEST_SCHEMASPACE}:"
     line_elements = [line.split() for line in lines[4:6]]
     assert line_elements[0][1] == "another"
     assert line_elements[1][1] == "valid"
@@ -1066,7 +1066,7 @@ def test_list_json_instances(script_runner, mock_data_dir):
     resource = metadata_manager.create("another2", another)
     assert resource is not None
 
-    ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--json")
+    ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--json", "--include-invalid")
     assert ret.success
     # Consume results
     results = json.loads(ret.stdout)
@@ -1076,20 +1076,22 @@ def test_list_json_instances(script_runner, mock_data_dir):
     metadata_manager.remove("valid2")
     metadata_manager.remove("another2")
 
-    # Include two additional invalid files as well - one for uri failure, andother missing display_name
+    # Include two additional invalid files as well - one for uri failure, another missing display_name
     metadata_dir = os.path.join(mock_data_dir, "metadata", METADATA_TEST_SCHEMASPACE)
     create_json_file(metadata_dir, "invalid.json", invalid_metadata_json)
     create_json_file(metadata_dir, "no_display_name.json", invalid_no_display_name_json)
 
+    # Ensure invalids are NOT listed
     ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--json")
     assert ret.success
     results = json.loads(ret.stdout)
-    assert len(results) == 4
+    assert len(results) == 2
 
-    ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--json", "--valid-only")
+    # Ensure invalids ARE listed
+    ret = script_runner.run("elyra-metadata", "list", METADATA_TEST_SCHEMASPACE, "--json", "--include-invalid")
     assert ret.success
     results = json.loads(ret.stdout)
-    assert len(results) == 2
+    assert len(results) == 4
 
 
 def test_remove_help(script_runner):

etc/config/metadata/runtime-images/pytorch-runtime.json

@@ -1,8 +1,8 @@
 {
-  "display_name": "Pytorch 1.4 with CUDA-runtime",
+  "display_name": "Pytorch 1.12 with CUDA-runtime",
   "metadata": {
-    "description": "PyTorch 1.4 (with GPU support)",
-    "image_name": "pytorch/pytorch@sha256:ee783a4c0fccc7317c150450e84579544e171dd01a3f76cf2711262aced85bf7",
+    "description": "PyTorch 1.12 (with GPU support)",
+    "image_name": "nvcr.io/nvidia/pytorch@sha256:b6080b287f79ba34c392730566a8903361453c15c3b3beff032d11a05e08f7c3",
     "tags": ["gpu", "pytorch"]
   },
   "schema_name": "runtime-image"

pyproject.toml

@@ -48,6 +48,8 @@ dependencies = [
     "watchdog>=2.1.3",
     "websocket-client",
     "yaspin",
+    # see: https://stackoverflow.com/questions/76175487/sudden-importerror-cannot-import-name-appengine-from-requests-packages-urlli
+    "appengine-python-standard",
     "kfp>=1.7.0,<2.0,!=1.7.2",  # We cap the SDK to <2.0 due to possible breaking changes
     "pygithub",
     "black>=22.8.0",