Add support Runtime types

elyra/metadata/schemas/airflow.json

@@ -6,6 +6,8 @@
   "display_name": "Apache Airflow",
   "schemaspace": "runtimes",
   "schemaspace_id": "130b8e00-de7c-4b32-b553-b4a52824a3b5",
+  "metadata_class_name": "elyra.pipeline.runtimes_metadata.RuntimesMetadata",
+  "runtime_type": "APACHE_AIRFLOW",
   "uihints": {
     "title": "Apache Airflow runtimes",
     "icon": "elyra:runtimes",
@@ -28,6 +30,12 @@
       "description": "Additional data specific to this metadata",
       "type": "object",
       "properties": {
+        "runtime_type": {
+          "title": "Runtime Type",
+          "description": "The runtime associated with this instance",
+          "type": "string",
+          "const": "APACHE_AIRFLOW"
+        },
         "description": {
           "title": "Description",
           "description": "Description of this Apache Airflow configuration",

elyra/metadata/schemas/kfp.json

@@ -7,6 +7,7 @@
   "schemaspace": "runtimes",
   "schemaspace_id": "130b8e00-de7c-4b32-b553-b4a52824a3b5",
   "metadata_class_name": "elyra.pipeline.kfp.kfp_metadata.KfpMetadata",
+  "runtime_type": "KUBEFLOW_PIPELINES",
   "uihints": {
     "title": "Kubeflow Pipelines runtimes",
     "icon": "elyra:runtimes",
@@ -29,6 +30,12 @@
       "description": "Additional data specific to this metadata",
       "type": "object",
       "properties": {
+        "runtime_type": {
+          "title": "Runtime Type",
+          "description": "The runtime associated with this instance",
+          "type": "string",
+          "const": "KUBEFLOW_PIPELINES"
+        },
         "description": {
           "title": "Description",
           "description": "Description of this Kubeflow Pipelines configuration",

elyra/metadata/schemas/local-directory-catalog.json

@@ -39,12 +39,11 @@
           "description": "Description of this Component Catalog",
           "type": "string"
         },
-        "runtime": {
-          "title": "Runtime",
-          "description": "The runtime associated with this Component Catalog",
+        "runtime_type": {
+          "title": "Runtime Type",
+          "description": "The type of runtime associated with this Component Catalog",
           "type": "string",
-          "$comment": "This enum is dynamically generated to contain the available runtime values.",
-          "enum": ["{currently-configured-runtimes}"],
+          "enum": ["KUBEFLOW_PIPELINES", "APACHE_AIRFLOW"],
           "uihints": {
             "field_type": "dropdown",
             "category": "Runtime"
@@ -86,7 +85,7 @@
           }
         }
       },
-      "required": ["runtime", "paths"]
+      "required": ["runtime_type", "paths"]
     }
   },
   "required": ["schema_name", "display_name", "version", "metadata"]

elyra/metadata/schemas/local-file-catalog.json

@@ -39,12 +39,11 @@
           "description": "Description of this Component Catalog",
           "type": "string"
         },
-        "runtime": {
-          "title": "Runtime",
-          "description": "The runtime associated with this Component Catalog",
+        "runtime_type": {
+          "title": "Runtime Type",
+          "description": "The type of runtime associated with this Component Catalog",
           "type": "string",
-          "$comment": "This enum is dynamically generated to contain the available runtime values.",
-          "enum": ["{currently-configured-runtimes}"],
+          "enum": ["KUBEFLOW_PIPELINES", "APACHE_AIRFLOW"],
           "uihints": {
             "field_type": "dropdown",
             "category": "Runtime"
@@ -84,7 +83,7 @@
           }
         }
       },
-      "required": ["runtime", "paths"]
+      "required": ["runtime_type", "paths"]
     }
   },
   "required": ["schema_name", "display_name", "version", "metadata"]

elyra/metadata/schemas/url-catalog.json

@@ -39,12 +39,11 @@
           "description": "Description of this Component Catalog",
           "type": "string"
         },
-        "runtime": {
-          "title": "Runtime",
-          "description": "The runtime associated with this Component Catalog",
+        "runtime_type": {
+          "title": "Runtime Type",
+          "description": "The type of runtime associated with this Component Catalog",
           "type": "string",
-          "$comment": "This enum is dynamically generated to contain the available runtime values.",
-          "enum": ["{currently-configured-runtimes}"],
+          "enum": ["KUBEFLOW_PIPELINES", "APACHE_AIRFLOW"],
           "uihints": {
             "field_type": "dropdown",
             "category": "Runtime"
@@ -77,7 +76,7 @@
           }
         }
       },
-      "required": ["runtime", "paths"]
+      "required": ["runtime_type", "paths"]
     }
   },
   "required": ["schema_name", "display_name", "version", "metadata"]

elyra/metadata/schemaspaces.py

@@ -15,8 +15,6 @@
 
 from typing import Dict
 
-import entrypoints
-
 from elyra.metadata.schema import Schemaspace
 
 
@@ -67,27 +65,19 @@ def __init__(self, *args, **kwargs):
                          display_name=ComponentRegistries.COMPONENT_REGISTRIES_SCHEMASPACE_DISPLAY_NAME,
                          description="Schemaspace for instances of Elyra component registries configurations")
 
-        # get set of registered runtimes
-        self._runtime_processor_names = set()
-        for processor in entrypoints.get_group_all('elyra.pipeline.processors'):
-            # load the names of the runtime processors (skip 'local')
-            if processor.name == 'local':
-                continue
-            self._runtime_processor_names.add(processor.name)
-
     def filter_schema(self, schema: Dict) -> Dict:
         """Replace contents of Runtimes value with set of runtimes if using templated value."""
 
-        # Component-registry requires that `runtime` be a defined property so ensure its existence.
-        instance_properties = schema.get('properties', {}).get('metadata', {}).get('properties', {})
-        runtime = instance_properties.get('runtime')
-        if not runtime:
-            raise ValueError(f"{ComponentRegistries.COMPONENT_REGISTRIES_SCHEMASPACE_DISPLAY_NAME} schemas are "
-                             f"required to define a 'runtime' (string-valued) property and schema "
-                             f"\'{schema.get('name')}\' does not define 'runtime'.")
-
-        if runtime.get('enum') == ["{currently-configured-runtimes}"]:
-            runtime['enum'] = list(self._runtime_processor_names)
+        # Component-registry requires that `runtime_type` be a defined property so ensure its existence.
+        # Since schema 'component-registry' is deprecated, skip its check.
+        is_deprecated = schema.get('deprecated', False)
+        if not is_deprecated:  # Skip deprecated schemas
+            instance_properties = schema.get('properties', {}).get('metadata', {}).get('properties', {})
+            runtime_type = instance_properties.get('runtime_type')
+            if not runtime_type:
+                raise ValueError(f"{ComponentRegistries.COMPONENT_REGISTRIES_SCHEMASPACE_DISPLAY_NAME} schemas are "
+                                 f"required to define a 'runtime_type' (string-valued) property and schema "
+                                 f"\'{schema.get('name')}\' does not define 'runtime_type'.")
 
         # Component catalogs should have an associated 'metadata' class name
         # If none is provided, use the ComponentCatalogMetadata class, which implements

elyra/pipeline/airflow/component_parser_airflow.py

@@ -23,11 +23,12 @@
 from elyra.pipeline.component import Component
 from elyra.pipeline.component import ComponentParameter
 from elyra.pipeline.component import ComponentParser
+from elyra.pipeline.runtime_type import RuntimeProcessorType
 
 
 class AirflowComponentParser(ComponentParser):
-    _component_platform = "airflow"
-    _file_types = [".py"]
+    _component_platform: RuntimeProcessorType = RuntimeProcessorType.APACHE_AIRFLOW
+    _file_types: List[str] = [".py"]
 
     def parse(self, registry_entry: SimpleNamespace) -> Optional[List[Component]]:
         components: List[Component] = list()
@@ -51,7 +52,7 @@ def parse(self, registry_entry: SimpleNamespace) -> Optional[List[Component]]:
                     catalog_type=registry_entry.catalog_type,
                     source_identifier=registry_entry.component_identifier,
                     definition=self.get_class_def_as_string(component_content),
-                    runtime=self.component_platform,
+                    runtime_type=self.component_platform.name,
                     categories=registry_entry.categories,
                     properties=component_properties
                 )

elyra/pipeline/airflow/processor_airflow.py

@@ -39,12 +39,14 @@
 from elyra.pipeline.processor import PipelineProcessor
 from elyra.pipeline.processor import PipelineProcessorResponse
 from elyra.pipeline.processor import RuntimePipelineProcessor
+from elyra.pipeline.runtime_type import RuntimeProcessorType
 from elyra.util.git import GithubClient
 from elyra.util.path import get_absolute_path
 
 
 class AirflowPipelineProcessor(RuntimePipelineProcessor):
-    _type = 'airflow'
+    _type = RuntimeProcessorType.APACHE_AIRFLOW
+    _name = 'airflow'
 
     # Provide users with the ability to identify a writable directory in the
     # running container where the notebook | script is executed. The location
@@ -73,10 +75,6 @@ class AirflowPipelineProcessor(RuntimePipelineProcessor):
     # Contains mappings from class to import statement for each available Airflow operator
     class_import_map = {}
 
-    @property
-    def type(self):
-        return self._type
-
     def __init__(self, root_dir, **kwargs):
         super().__init__(root_dir, component_parser=AirflowComponentParser(), **kwargs)
         if not self.class_import_map:  # Only need to load once
@@ -432,16 +430,13 @@ def _process_list_value(self, value: str) -> Union[List, str]:
 
 class AirflowPipelineProcessorResponse(PipelineProcessorResponse):
 
-    _type = 'airflow'
+    _type = RuntimeProcessorType.APACHE_AIRFLOW
+    _name = 'airflow'
 
     def __init__(self, git_url, run_url, object_storage_url, object_storage_path):
         super().__init__(run_url, object_storage_url, object_storage_path)
         self.git_url = git_url
 
-    @property
-    def type(self):
-        return self._type
-
     def to_json(self):
         response = super().to_json()
         response['git_url'] = self.git_url

elyra/pipeline/component.py

@@ -22,6 +22,8 @@
 
 from traitlets.config import LoggingConfigurable
 
+from elyra.pipeline.runtime_type import RuntimeProcessorType
+
 
 class ComponentParameter(object):
     """
@@ -122,7 +124,7 @@ def __init__(self,
                  catalog_type: str,
                  source_identifier: Any,
                  definition: Optional[str] = None,
-                 runtime: Optional[str] = None,
+                 runtime_type: Optional[RuntimeProcessorType] = None,
                  op: Optional[str] = None,
                  categories: Optional[List[str]] = None,
                  properties: Optional[List[ComponentParameter]] = None,
@@ -136,7 +138,7 @@ def __init__(self,
                               location; one of ['url', filename', 'directory]
         :param source_identifier: Source information to help locate the component definition
         :param definition: The content of the specification file for this component
-        :param runtime: The runtime of the component (e.g. KFP or Airflow)
+        :param runtime_type: The runtime type of the component (e.g. KUBEFLOW_PIPELINES, APACHE_AIRFLOW, etc.)
         :param op: The operation name of the component; used by generic components in rendering the palette
         :param categories: A list of categories that this component belongs to; used to organize component
                            in the palette
@@ -156,7 +158,7 @@ def __init__(self,
         self._source_identifier = source_identifier
 
         self._definition = definition
-        self._runtime = runtime
+        self._runtime_type = runtime_type
         self._op = op
         self._categories = categories or []
         self._properties = properties
@@ -209,8 +211,8 @@ def definition(self) -> str:
         return self._definition
 
     @property
-    def runtime(self) -> Optional[str]:
-        return self._runtime
+    def runtime_type(self) -> Optional[RuntimeProcessorType]:
+        return self._runtime_type
 
     @property
     def op(self) -> Optional[str]:
@@ -244,10 +246,11 @@ def _log_warning(msg: str, logger: Optional[Logger] = None):
 
 
 class ComponentParser(LoggingConfigurable):  # ABC
-    _component_platform = None
+    _component_platform: RuntimeProcessorType = None
+    _file_types: List[str] = None
 
     @property
-    def component_platform(self) -> str:
+    def component_platform(self) -> RuntimeProcessorType:
         return self._component_platform
 
     @property

elyra/pipeline/component_metadata.py

@@ -56,9 +56,20 @@ class ComponentCatalogMetadata(Metadata):
     This class contains methods to trigger cache updates on modification
     and deletion of component registry metadata instances.
     """
+    def on_load(self, **kwargs: Any) -> None:
+        """Check for runtime property and update to runtime_type """
+        if 'runtime' in self.metadata:
+            if self.metadata['runtime'] == 'kfp':
+                self.metadata['runtime_type'] = 'KUBEFLOW_PIPELINES'
+            else:  # self.metadata['runtime'] == 'airflow'
+                self.metadata['runtime_type'] = 'APACHE_AIRFLOW'
+            self.metadata.pop('runtime', None)
+            getLogger('ServerApp').info(f"Upgrading component-registry {self.schema_name} instance '{self.name}' "
+                                        f"to include runtime_type '{self.metadata['runtime_type']}'...")
+            MetadataManager(schemaspace="component-registries").update(self.name, self, for_migration=True)
 
     def post_save(self, **kwargs: Any) -> None:
-        processor_type = self.metadata.get('runtime')
+        processor_type = self.metadata.get('runtime_type')
 
         # Get processor instance and update its cache
         try:
@@ -69,7 +80,7 @@ def post_save(self, **kwargs: Any) -> None:
             pass
 
     def post_delete(self, **kwargs: Any) -> None:
-        processor_type = self.metadata.get('runtime')
+        processor_type = self.metadata.get('runtime_type')
 
         # Get processor instance and update its cache
         try:

elyra/pipeline/component_registry.py

@@ -126,7 +126,7 @@ def get_component(self, component_id: str) -> Optional[Component]:
 
         if component is None:
             self.log.error(f"Component with ID '{component_id}' could not be found in any "
-                           f"{self._parser.component_platform} catalog.")
+                           f"{self._parser.component_platform.name} catalog.")
 
         return component
 
@@ -282,9 +282,10 @@ def _get_catalogs_for_runtime(self) -> List[Metadata]:
             registries = MetadataManager(schemaspace=ComponentRegistries.COMPONENT_REGISTRIES_SCHEMASPACE_ID)\
                 .get_all()
 
-            # Filter registries according to processor type
-            runtime_catalogs = [r for r in registries if r.metadata['runtime'] == self._parser.component_platform]
+            # Filter catalogs according to processor type
+            runtime_catalogs = \
+                [r for r in registries if r.metadata['runtime_type'] == self._parser.component_platform.name]
         except Exception:
-            self.log.error(f"Could not access registries for processor: {self._parser.component_platform}")
+            self.log.error(f"Could not access registries for processor type: {self._parser.component_platform.name}")
 
         return runtime_catalogs

elyra/pipeline/kfp/component_parser_kfp.py

@@ -24,11 +24,12 @@
 from elyra.pipeline.component import Component
 from elyra.pipeline.component import ComponentParameter
 from elyra.pipeline.component import ComponentParser
+from elyra.pipeline.runtime_type import RuntimeProcessorType
 
 
 class KfpComponentParser(ComponentParser):
-    _component_platform = "kfp"
-    _file_types = [".yaml"]
+    _component_platform: RuntimeProcessorType = RuntimeProcessorType.KUBEFLOW_PIPELINES
+    _file_types: List[str] = [".yaml"]
 
     def parse(self, registry_entry: SimpleNamespace) -> Optional[List[Component]]:
         # Get YAML object from component definition
@@ -51,7 +52,7 @@ def parse(self, registry_entry: SimpleNamespace) -> Optional[List[Component]]:
             catalog_type=registry_entry.catalog_type,
             source_identifier=registry_entry.component_identifier,
             definition=registry_entry.component_definition,
-            runtime=self.component_platform,
+            runtime_type=self.component_platform.name,
             categories=registry_entry.categories,
             properties=component_properties
         )

elyra/pipeline/kfp/kfp_metadata.py

@@ -27,6 +27,7 @@ class KfpMetadata(RuntimesMetadata):
     """
 
     def on_load(self, **kwargs: Any) -> None:
+        super().on_load(**kwargs)
 
         if self.metadata.get('auth_type') is None:
             # Inject auth_type property for metadata persisted using Elyra < 3.3: