diff --git a/pyatlan/model/enums.py b/pyatlan/model/enums.py index 11c1724bf..130d016a8 100644 --- a/pyatlan/model/enums.py +++ b/pyatlan/model/enums.py @@ -2216,6 +2216,25 @@ class WorkflowPackage(str, Enum): TERADATA_MINER = "atlan-teradata-miner" THOUGHTSPOT = "atlan-thoughtspot" TRINO = "atlan-trino" + ASSET_IMPORT = "csa-asset-import" + ASSET_EXPORT_BASIC = "csa-asset-export-basic" + RELATIONAL_ASSETS_BUILDER = "csa-relational-assets-builder" + + +class AssetInputHandling(str, Enum): + UPSERT = "upsert" + PARTIAL = "partial" + UPDATE = "update" + + +class AssetDeltaHandling(str, Enum): + FULL_REPLACEMENT = "full" + INCREMENTAL = "delta" + + +class AssetRemovalType(str, Enum): + ARCHIVE = "archive" + PURGE = "purge" class UTMTags(str, Enum): diff --git a/pyatlan/model/packages/__init__.py b/pyatlan/model/packages/__init__.py index ff5cbd745..fbb9a0104 100644 --- a/pyatlan/model/packages/__init__.py +++ b/pyatlan/model/packages/__init__.py @@ -1,4 +1,6 @@ # flake8: noqa +from .asset_export_basic import AssetExportBasic +from .asset_import import AssetImport from .big_query_crawler import BigQueryCrawler from .confluent_kafka_crawler import ConfluentKafkaCrawler from .connection_delete import ConnectionDelete @@ -7,8 +9,28 @@ from .glue_crawler import GlueCrawler from .postgres_crawler import PostgresCrawler from .powerbi_crawler import PowerBICrawler +from .relational_assets_builder import RelationalAssetsBuilder from .s_q_l_server_crawler import SQLServerCrawler from .sigma_crawler import SigmaCrawler from .snowflake_crawler import SnowflakeCrawler from .snowflake_miner import SnowflakeMiner from .tableau_crawler import TableauCrawler + +__all__ = [ + "BigQueryCrawler", + "ConfluentKafkaCrawler", + "ConnectionDelete", + "DbtCrawler", + "DynamoDBCrawler", + "GlueCrawler", + "PostgresCrawler", + "PowerBICrawler", + "SQLServerCrawler", + "SigmaCrawler", + "SnowflakeCrawler", + "SnowflakeMiner", + "TableauCrawler", + "AssetImport", + "AssetExportBasic", + "RelationalAssetsBuilder", +] diff --git a/pyatlan/model/packages/asset_export_basic.py b/pyatlan/model/packages/asset_export_basic.py new file mode 100644 index 000000000..b0dfcc1da --- /dev/null +++ b/pyatlan/model/packages/asset_export_basic.py @@ -0,0 +1,329 @@ +from __future__ import annotations + +from typing import List, Optional + +from pyatlan.model.enums import WorkflowPackage +from pyatlan.model.packages.base.custom_package import AbstractCustomPackage +from pyatlan.model.workflow import WorkflowMetadata + + +class AssetExportBasic(AbstractCustomPackage): + """ + Base configuration for the Asset Export package. + """ + + _NAME = "asset-export-basic" + _PACKAGE_NAME = f"@csa/{_NAME}" + _PACKAGE_PREFIX = WorkflowPackage.ASSET_EXPORT_BASIC.value + _PACKAGE_ICON = "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg" + _PACKAGE_LOGO = "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg" + + def __init__( + self, + ): + super().__init__() + self._email_addresses = None + self._delivery_type = None + self._export_scope = None + self._parameters = [] + + def glossaries_only( + self, include_archived: Optional[bool] = None + ) -> AssetExportBasic: + """ + Set up the package to export only glossaries. + + :param include_archived: Whether to include archived assets in the export (true) or only active assets (false). + + :returns: package, set up to export only glossaries + """ + self._export_scope = "GLOSSARIES_ONLY" + self._parameters.append({"name": "export_scope", "value": self._export_scope}) + self._parameters.append( + { + "name": "include_archived", + "value": include_archived, + } + ) + return self + + def enriched_only( + self, + prefix: str, + include_description: Optional[bool] = None, + include_glossaries: Optional[bool] = None, + include_data_products: Optional[bool] = None, + include_archived: Optional[bool] = None, + ) -> AssetExportBasic: + """ + Set up the package to export only enriched assets. + + :param prefix: Starting value for a qualifiedName that will determine which assets to export. + :param include_description: Whether to extract only user-entered description (false), or to also include + system-level description (true). + :param include_glossaries: Whether glossaries (and their terms and + categories) should be exported, too. + :param include_data_products: Whether data products (and their domains) + should be exported, too. + :param include_archived: Whether to include archived assets in the export (true) or + only active assets (false). + + :returns: package, set up to export only enriched assets + """ + self._export_scope = "ENRICHED_ONLY" + self._parameters.append({"name": "export_scope", "value": self._export_scope}) + params = { + "qn_prefix": prefix, + "include_description": include_description, + "include_glossaries": include_glossaries, + "include_products": include_data_products, + "include_archived": include_archived, + } + self._add_optional_params(params) + return self + + def products_only( + self, include_archived: Optional[bool] = None + ) -> AssetExportBasic: + """ + Set up the package to export only data products. + + :param include_archived: Whether to include archived assets in the export (true) or only active assets (false). + + :returns: package, set up to export only data products + """ + self._export_scope = "PRODUCTS_ONLY" + self._parameters.append({"name": "export_scope", "value": self._export_scope}) + self._parameters.append( + { + "name": "include_archived", + "value": include_archived, + } + ) + return self + + def all_assets( + self, + prefix: str, + include_description: Optional[bool] = None, + include_glossaries: Optional[bool] = None, + include_data_products: Optional[bool] = None, + include_archived: Optional[bool] = None, + ) -> AssetExportBasic: + """ + Set up the package to export all assets. + + :param prefix: Starting value for a qualifiedName that will determine which assets to export. + :param include_description: Whether to extract only user-entered description (false), or to also include + system-level description (true). + :param include_glossaries: Whether glossaries (and their terms and + categories) should be exported, too. + :param include_data_products: Whether data products (and their domains) + should be exported, too. + :param include_archived: Whether to include archived assets in the export (true) or + only active assets (false). + + :returns: package, set up to export all assets + """ + self._export_scope = "ALL" + self._parameters.append({"name": "export_scope", "value": self._export_scope}) + params = { + "qn_prefix": prefix, + "include_description": include_description, + "include_glossaries": include_glossaries, + "include_products": include_data_products, + "include_archived": include_archived, + } + self._add_optional_params(params) + + return self + + def direct(self) -> AssetExportBasic: + """ + Set up the package to deliver the export via direct download. + + :returns: package, set up to deliver the export via direct download + """ + self._delivery_type = "DIRECT" + self._add_delivery_parameters() + return self + + def email(self, email_addresses: List[str]) -> AssetExportBasic: + """ + Set up the package to deliver the export via email. + + :param email_addresses: List of email addresses to send the export to. + + :returns: package, set up to deliver the export via email + """ + self._delivery_type = "EMAIL" + self._email_addresses = email_addresses + self._add_delivery_parameters() + + return self + + def object_store(self, prefix: Optional[str] = None) -> AssetExportBasic: + """ + Set up the package to export to an object storage location. + + :param prefix: The directory (path) within the object store to upload the exported file. + + :returns: package, set up to export metadata to an object store + """ + self._delivery_type = "CLOUD" + self._add_delivery_parameters() + self._parameters.append({"name": "target_prefix", "value": prefix}) + self._parameters.append({"name": "cloud_target", "value": "{{credentialGuid}}"}) + return self + + def s3( + self, + access_key: str, + secret_key: str, + bucket: str, + region: str, + ) -> AssetExportBasic: + """ + Set up package to export to S3. + + :param access_key: AWS access key + :param secret_key: AWS secret key + :param bucket: S3 bucket to upload the export file to + :param region: AWS region + + :returns: package, set up to export metadata to S3 + """ + self._credentials_body.update( + { + "name": f"csa-{self._NAME}-{self._epoch}-0", + "auth_type": "s3", + "username": access_key, + "password": secret_key, + "extra": { + "region": region, + "s3_bucket": bucket, + }, + "connector_config_name": "csa-connectors-objectstore", + } + ) + return self + + def gcs( + self, project_id: str, service_account_json: str, bucket: str + ) -> AssetExportBasic: + """ + Set up package to export to Google Cloud Storage. + + :param project_id: ID of GCP project + :param service_account_json: service account credentials in JSON format + :param bucket: bucket to upload the export file to + + :returns: package, set up to export metadata to GCS + """ + self._credentials_body.update( + { + "name": f"csa-{self._NAME}-{self._epoch}-0", + "auth_type": "gcs", + "username": project_id, + "password": service_account_json, + "extra": { + "gcs_bucket": bucket, + }, + "connector_config_name": "csa-connectors-objectstore", + } + ) + return self + + def adls( + self, + client_id: str, + client_secret: str, + tenant_id: str, + account_name: str, + container: str, + ) -> AssetExportBasic: + """ + Set up package to export to Azure Data Lake Storage. + + :param client_id: unique application (client) ID assigned by Azure AD when the app was registered + :param client_secret: client secret for authentication + :param tenant_id: unique ID of the Azure Active Directory instance + :param account_name: name of the storage account + :param container: container to upload the export file to + + :returns: package, set up to export metadata to ADLS + """ + self._credentials_body.update( + { + "name": f"csa-{self._NAME}-{self._epoch}-0", + "auth_type": "adls", + "username": client_id, + "password": client_secret, + "extra": { + "azure_tenant_id": tenant_id, + "storage_account_name": account_name, + "adls_container": container, + }, + "connector_config_name": "csa-connectors-objectstore", + } + ) + return self + + def _add_delivery_parameters(self): + """ + Add delivery parameters to the parameters list. + """ + self._parameters.append( + { + "name": "delivery_type", + "value": self._delivery_type, + } + ) + if self._delivery_type == "EMAIL" and self._email_addresses: + self._parameters.append( + { + "name": "email_addresses", + "value": ",".join( + self._email_addresses + ), # Join the email addresses if they are in a list + } + ) + + def _get_metadata(self) -> WorkflowMetadata: + return WorkflowMetadata( + labels={ + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": self._NAME, + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": f"a-t-rcsas-l-a-s-h{self._NAME}", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true", + }, + annotations={ + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": f"https://solutions.atlan.com/{self._NAME}/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": self._PACKAGE_ICON, + "orchestration.atlan.com/logo": self._PACKAGE_LOGO, # noqa + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them " + "via the Atlan UI.", + "package.argoproj.io/homepage": f"https://packages.atlan.com/-/web/detail/{self._PACKAGE_NAME}", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", # fmt: skip + "package.argoproj.io/name": self._PACKAGE_NAME, + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": f"csa-{self._NAME}-{self._epoch}", + }, + name=f"csa-{self._NAME}-{self._epoch}", + namespace="default", + ) diff --git a/pyatlan/model/packages/asset_import.py b/pyatlan/model/packages/asset_import.py new file mode 100644 index 000000000..b312c83d4 --- /dev/null +++ b/pyatlan/model/packages/asset_import.py @@ -0,0 +1,390 @@ +from __future__ import annotations + +from json import dumps +from typing import List, Optional, Union + +from pyatlan.model.enums import AssetInputHandling, WorkflowPackage +from pyatlan.model.fields.atlan_fields import AtlanField +from pyatlan.model.packages.base.custom_package import AbstractCustomPackage +from pyatlan.model.workflow import WorkflowMetadata + + +class AssetImport(AbstractCustomPackage): + """ + Base configuration for a new Asset Import package. + """ + + _NAME = "asset-import" + _PACKAGE_NAME = f"@csa/{_NAME}" + _PACKAGE_PREFIX = WorkflowPackage.ASSET_IMPORT.value + _PACKAGE_ICON = "http://assets.atlan.com/assets/ph-cloud-arrow-up-light.svg" + _PACKAGE_LOGO = "http://assets.atlan.com/assets/ph-cloud-arrow-up-light.svg" + + def __init__( + self, + ): + self._assets_advanced = False + self._glossaries_advanced = False + self._data_product_advanced = False + super().__init__() + + def object_store(self) -> AssetImport: + """ + Set up the package to import + metadata directly from the object store. + """ + self._parameters.append({"name": "import_type", "value": "CLOUD"}) + self._parameters.append({"name": "cloud_source", "value": "{{credentialGuid}}"}) + return self + + def s3( + self, + access_key: str, + secret_key: str, + region: str, + bucket: str, + ) -> AssetImport: + """ + Set up package to import metadata from S3. + + :param access_key: AWS access key + :param secret_key: AWS secret key + :param region: AWS region + :param bucket: bucket to retrieve object store object from + + :returns: package, set up to import metadata from S3 + """ + local_creds = { + "name": f"csa-{self._NAME}-{self._epoch}-0", + "auth_type": "s3", + "username": access_key, + "password": secret_key, + "extra": { + "region": region, + "s3_bucket": bucket, + }, + "connector_config_name": "csa-connectors-objectstore", + } + self._credentials_body.update(local_creds) + return self + + def gcs( + self, project_id: str, service_account_json: str, bucket: str + ) -> AssetImport: + """ + Set up package to import metadata from GCS. + + :param project_id: ID of GCP project + :param service_account_json: service account credentials in JSON format + :param bucket: bucket to retrieve object store object from + + :returns: Package set up to import metadata from GCS + """ + local_creds = { + "name": f"csa-{self._NAME}-{self._epoch}-0", + "auth_type": "gcs", + "username": project_id, + "password": service_account_json, + "extra": { + "gcs_bucket": bucket, + }, + "connector_config_name": "csa-connectors-objectstore", + } + self._credentials_body.update(local_creds) + return self + + def adls( + self, + client_id: str, + client_secret: str, + tenant_id: str, + account_name: str, + container: str, + ) -> AssetImport: + """ + Set up package to import metadata from ADLS. + + :param client_id: unique application (client) ID assigned by Azure AD when the app was registered + :param client_secret: client secret for authentication + :param tenant_id: unique ID of the Azure Active Directory instance + :param account_name: name of the storage account + :param container: container to retrieve object store objects from + + :returns: package, set up to import metadata from ADLS + """ + local_creds = { + "name": f"csa-{self._NAME}-{self._epoch}-0", + "auth_type": "adls", + "username": client_id, + "password": client_secret, + "extra": { + "azure_tenant_id": tenant_id, + "storage_account_name": account_name, + "adls_container": container, + }, + "connector_config_name": "csa-connectors-objectstore", + } + self._credentials_body.update(local_creds) + return self + + def assets( + self, + prefix: str, + object_key: str, + input_handling: AssetInputHandling = AssetInputHandling.UPDATE, + ) -> AssetImport: + """ + Set up package to import assets. + + :param prefix: directory (path) within the object store from + which to retrieve the file containing asset metadata + :param object_key: object key (filename), + including its extension, within the object store and prefix + :param input_handling: specifies whether to allow the creation + of new assets from the input CSV (full or partial assets) + or only update existing assets in Atlan + + :returns: package, configured to import assets + """ + self._parameters.append({"name": "assets_prefix", "value": prefix}) + self._parameters.append({"name": "assets_key", "value": object_key}) + self._parameters.append( + {"name": "assets_upsert_semantic", "value": input_handling} + ) + return self + + def assets_advanced( + self, + remove_attributes: Optional[Union[List[str], List[AtlanField]]] = None, + fail_on_errors: Optional[bool] = None, + case_sensitive_match: Optional[bool] = None, + is_table_view_agnostic: Optional[bool] = None, + field_separator: Optional[str] = None, + batch_size: Optional[int] = None, + ) -> AssetImport: + """ + Set up package to import assets with advanced configuration. + + :param remove_attributes: list of attributes to clear (remove) + from assets if their value is blank in the provided file. + :param fail_on_errors: specifies whether an invalid value + in a field should cause the import to fail (`True`) or + log a warning, skip that value, and proceed (`False`). + :param case_sensitive_match: indicates whether to use + case-sensitive matching when running in update-only mode (`True`) + or to try case-insensitive matching (`False`). + :param is_table_view_agnostic: specifies whether to treat + tables, views, and materialized views as interchangeable (`True`) + or to strictly adhere to specified types in the input (`False`). + :param field_separator: character used to separate + fields in the input file (e.g., ',' or ';'). + :param batch_size: maximum number of rows + to process at a time (per API request). + + :returns: package, configured to import + assets with advanced configuration. + """ + if isinstance(remove_attributes, list) and all( + isinstance(field, AtlanField) for field in remove_attributes + ): + remove_attributes = [field.atlan_field_name for field in remove_attributes] # type: ignore + params = { + "assets_attr_to_overwrite": dumps(remove_attributes, separators=(",", ":")), + "assets_fail_on_errors": fail_on_errors, + "assets_case_sensitive": case_sensitive_match, + "assets_table_view_agnostic": is_table_view_agnostic, + "assets_field_separator": field_separator, + "assets_batch_size": batch_size, + } + self._add_optional_params(params) + self._assets_advanced = True + return self + + def glossaries( + self, + prefix: str, + object_key: str, + input_handling: AssetInputHandling = AssetInputHandling.UPDATE, + ) -> AssetImport: + """ + Set up package to import glossaries. + + :param prefix: directory (path) within the object store from + which to retrieve the file containing glossaries, categories and terms + :param object_key: object key (filename), + including its extension, within the object store and prefix + :param input_handling: specifies whether to allow the creation of new glossaries, + categories and terms from the input CSV, or ensure these are only updated + if they already exist in Atlan. + + :returns: package, configured to import glossaries, categories and terms. + """ + self._parameters.append({"name": "glossaries_prefix", "value": prefix}) + self._parameters.append({"name": "glossaries_key", "value": object_key}) + self._parameters.append( + {"name": "glossaries_upsert_semantic", "value": input_handling} + ) + return self + + def glossaries_advanced( + self, + remove_attributes: Optional[Union[List[str], List[AtlanField]]] = None, + fail_on_errors: Optional[bool] = None, + field_separator: Optional[str] = None, + batch_size: Optional[int] = None, + ) -> AssetImport: + """ + Set up package to import glossaries with advanced configuration. + + :param remove_attributes: list of attributes to clear (remove) + from assets if their value is blank in the provided file. + :param fail_on_errors: specifies whether an invalid value + in a field should cause the import to fail (`True`) or + log a warning, skip that value, and proceed (`False`). + :param field_separator: character used to separate + fields in the input file (e.g., ',' or ';'). + :param batch_size: maximum number of rows + to process at a time (per API request). + + :returns: package, configured to import + glossaries with advanced configuration. + """ + if isinstance(remove_attributes, list) and all( + isinstance(field, AtlanField) for field in remove_attributes + ): + remove_attributes = [field.atlan_field_name for field in remove_attributes] # type: ignore + params = { + "glossaries_attr_to_overwrite": dumps( + remove_attributes, separators=(",", ":") + ), + "glossaries_fail_on_errors": fail_on_errors, + "glossaries_field_separator": field_separator, + "glossaries_batch_size": batch_size, + } + self._add_optional_params(params) + self._glossaries_advanced = True + return self + + def data_products( + self, + prefix: str, + object_key: str, + input_handling: AssetInputHandling = AssetInputHandling.UPDATE, + ) -> AssetImport: + """ + Set up package to import data products. + + :param prefix: directory (path) within the object store from + which to retrieve the file containing data domains, and data products + :param object_key: object key (filename), + including its extension, within the object store and prefix + :param input_handling: specifies whether to allow the creation of new data domains, and data products + from the input CSV, or ensure these are only updated if they already exist in Atlan. + + :returns: package, configured to import data domain and data products + """ + self._parameters.append({"name": "data_products_prefix", "value": prefix}) + self._parameters.append({"name": "data_products_key", "value": object_key}) + self._parameters.append( + {"name": "data_products_upsert_semantic", "value": input_handling} + ) + return self + + def data_product_advanced( + self, + remove_attributes: Optional[Union[List[str], List[AtlanField]]] = None, + fail_on_errors: Optional[bool] = None, + field_separator: Optional[str] = None, + batch_size: Optional[int] = None, + ) -> AssetImport: + """ + Set up package to import data domain + and data products with advanced configuration. + + :param remove_attributes: list of attributes to clear (remove) + from assets if their value is blank in the provided file. + :param fail_on_errors: specifies whether an invalid value + in a field should cause the import to fail (`True`) or + log a warning, skip that value, and proceed (`False`). + :param field_separator: character used to separate + fields in the input file (e.g., ',' or ';'). + :param batch_size: maximum number of rows + to process at a time (per API request). + + :returns: package, configured to import + data domain and data products with advanced configuration. + """ + if isinstance(remove_attributes, list) and all( + isinstance(field, AtlanField) for field in remove_attributes + ): + remove_attributes = [field.atlan_field_name for field in remove_attributes] # type: ignore + params = { + "data_products_attr_to_overwrite": dumps( + remove_attributes, separators=(",", ":") + ), + "data_products_fail_on_errors": fail_on_errors, + "data_products_field_separator": field_separator, + "data_products_batch_size": batch_size, + } + self._add_optional_params(params) + self._data_product_advanced = True + return self + + def _set_required_metadata_params(self): + self._parameters.append( + dict( + name="assets_config", + value="advanced" if self._assets_advanced else "default", + ) + ) + self._parameters.append( + dict( + name="glossaries_config", + value="advanced" if self._glossaries_advanced else "default", + ) + ) + self._parameters.append( + dict( + name="data_products_config", + value="advanced" if self._data_product_advanced else "default", + ) + ) + + def _get_metadata(self) -> WorkflowMetadata: + self._set_required_metadata_params() + return WorkflowMetadata( + labels={ + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/source": self._NAME, + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": f"a-t-rcsas-l-a-s-h{self._NAME}", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true", + }, + annotations={ + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": f"https://solutions.atlan.com/{self._NAME}/", + "orchestration.atlan.com/emoji": "\U0001f680", + "orchestration.atlan.com/icon": self._PACKAGE_ICON, + "orchestration.atlan.com/logo": self._PACKAGE_LOGO, # noqa + "orchestration.atlan.com/name": "Asset Import", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Import assets from a CSV file.", + "package.argoproj.io/homepage": f"https://packages.atlan.com/-/web/detail/{self._PACKAGE_NAME}", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", # fmt: skip + "package.argoproj.io/name": self._PACKAGE_NAME, + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": f"csa-{self._NAME}-{self._epoch}", + }, + name=f"csa-{self._NAME}-{self._epoch}", + namespace="default", + ) diff --git a/pyatlan/model/packages/base/custom_package.py b/pyatlan/model/packages/base/custom_package.py new file mode 100644 index 000000000..2e2a84814 --- /dev/null +++ b/pyatlan/model/packages/base/custom_package.py @@ -0,0 +1,26 @@ +from typing import Any, Dict, Optional + +from pyatlan import utils +from pyatlan.model.packages.base.package import AbstractPackage + + +class AbstractCustomPackage(AbstractPackage): + """ + Abstract class for custom packages + """ + + def __init__( + self, + ): + super().__init__() + self._epoch = int(utils.get_epoch_timestamp()) + + def _add_optional_params(self, params: Dict[str, Optional[Any]]) -> None: + """ + Helper method to add non-None params to `self._parameters`. + + :param params: dict of param names and values. + """ + for name, value in params.items(): + if value is not None: + self._parameters.append({"name": name, "value": value}) diff --git a/pyatlan/model/packages/relational_assets_builder.py b/pyatlan/model/packages/relational_assets_builder.py new file mode 100644 index 000000000..2202774a0 --- /dev/null +++ b/pyatlan/model/packages/relational_assets_builder.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +from json import dumps +from typing import List, Optional, Union + +from pyatlan.model.enums import ( + AssetDeltaHandling, + AssetInputHandling, + AssetRemovalType, + WorkflowPackage, +) +from pyatlan.model.fields.atlan_fields import AtlanField +from pyatlan.model.packages.base.custom_package import AbstractCustomPackage +from pyatlan.model.workflow import WorkflowMetadata + + +class RelationalAssetsBuilder(AbstractCustomPackage): + """ + Base configuration for the Relational Assets Builder package. + """ + + _NAME = "relational-assets-builder" + _PACKAGE_NAME = f"@csa/{_NAME}" + _PACKAGE_PREFIX = WorkflowPackage.RELATIONAL_ASSETS_BUILDER.value + _PACKAGE_ICON = "http://assets.atlan.com/assets/ph-database-light.svg" + _PACKAGE_LOGO = "http://assets.atlan.com/assets/ph-database-light.svg" + + def __init__( + self, + ): + super().__init__() + + def direct(self) -> RelationalAssetsBuilder: + """ + Set up package to directly upload the file. + """ + self._parameters.append({"name": "import_type", "value": "DIRECT"}) + return self + + def object_store( + self, prefix: Optional[str] = None, object_key: Optional[str] = None + ) -> RelationalAssetsBuilder: + """ + Set up the package to import + metadata directly from the object store. + + :param prefix: directory (path) within the bucket/container from which to retrieve the object(s). + :param object_key: object key (filename), including its extension, within the bucket/container and + prefix. + + :returns: package, set up to import metadata from object store + """ + self._parameters.append({"name": "import_type", "value": "CLOUD"}) + self._parameters.append({"name": "assets_prefix", "value": prefix}) + self._parameters.append({"name": "assets_key", "value": object_key}) + self._parameters.append({"name": "cloud_source", "value": "{{credentialGuid}}"}) + return self + + def s3( + self, + access_key: str, + secret_key: str, + region: str, + bucket: str, + ) -> RelationalAssetsBuilder: + """ + Set up package to import metadata from S3. + + :param access_key: AWS access key + :param secret_key: AWS secret key + :param region: AWS region + :param bucket: Enter the bucket from which to retrieve the object store object(s). + + :returns: package, set up to import metadata from S3 + """ + local_creds = { + "name": f"csa-{self._NAME}-{self._epoch}-0", + "auth_type": "s3", + "username": access_key, + "password": secret_key, + "extra": { + "region": region, + "s3_bucket": bucket, + }, + "connector_config_name": "csa-connectors-objectstore", + } + self._credentials_body.update(local_creds) + return self + + def gcs( + self, project_id: str, service_account_json: str, bucket: str + ) -> RelationalAssetsBuilder: + """ + Set up package to import metadata from GCS. + + :param project_id: ID of GCP project + :param service_account_json: service account credentials in JSON format + :param bucket: the bucket from which to retrieve the object store object(s) + + :returns: Package set up to import metadata from GCS + """ + local_creds = { + "name": f"csa-{self._NAME}-{self._epoch}-0", + "auth_type": "gcs", + "username": project_id, + "password": service_account_json, + "extra": { + "gcs_bucket": bucket, + }, + "connector_config_name": "csa-connectors-objectstore", + } + self._credentials_body.update(local_creds) + return self + + def adls( + self, + client_id: str, + client_secret: str, + tenant_id: str, + account_name: str, + container: str, + ) -> RelationalAssetsBuilder: + """ + Set up package to import metadata from ADLS. + + :param client_id: unique application (client) ID assigned by Azure AD when the app was registered + :param client_secret: client secret for authentication + :param tenant_id: unique ID of the Azure Active Directory instance + :param account_name: name of the storage account + :param container: container to retrieve object store objects from + + :returns: package, set up to import metadata from ADLS + """ + local_creds = { + "name": f"csa-{self._NAME}-{self._epoch}-0", + "auth_type": "adls", + "username": client_id, + "password": client_secret, + "extra": { + "azure_tenant_id": tenant_id, + "storage_account_name": account_name, + "adls_container": container, + }, + "connector_config_name": "csa-connectors-objectstore", + } + self._credentials_body.update(local_creds) + return self + + def assets_semantics( + self, + input_handling: AssetInputHandling = AssetInputHandling.UPSERT, + delta_handling: AssetDeltaHandling = AssetDeltaHandling.INCREMENTAL, + removal_type: AssetRemovalType = AssetRemovalType.ARCHIVE, + ) -> RelationalAssetsBuilder: + """ + Set up the package to import metadata with semantics. + + :param input_handling: Whether to allow the creation of new (full or partial) assets from the input CSV, + or ensure assets are only updated if they already exist in Atlan. + :param delta_handling: Whether to treat the input file as an initial load, full replacement (deleting any + existing assets not in the file) or only incremental (no deletion of existing assets). + :param removal_type: If `delta_handling` is set to `FULL_REPLACEMENT`, this parameter specifies whether to + delete any assets not found in the latest file by archive (recoverable) or purge (non-recoverable). + If `delta_handling` is set to `INCREMENTAL`, this parameter is ignored and assets are archived. + + :returns: package, set up to import metadata with semantics + """ + self._parameters.append( + {"name": "assets_upsert_semantic", "value": input_handling} + ) + self._parameters.append({"name": "delta_semantic", "value": delta_handling}) + if delta_handling == AssetDeltaHandling.FULL_REPLACEMENT: + self._parameters.append( + {"name": "delta_removal_type", "value": removal_type} + ) + else: + self._parameters.append( + {"name": "delta_removal_type", "value": AssetRemovalType.ARCHIVE} + ) + return self + + def options( + self, + remove_attributes: Optional[Union[List[str], List[AtlanField]]] = None, + fail_on_errors: Optional[bool] = None, + field_separator: Optional[str] = None, + batch_size: Optional[int] = None, + ) -> RelationalAssetsBuilder: + """ + Set up package to import assets with advanced configuration. + + :param remove_attributes: list of attributes to clear (remove) + from assets if their value is blank in the provided file. + :param fail_on_errors: specifies whether an invalid value + in a field should cause the import to fail (`True`) or + log a warning, skip that value, and proceed (`False`). + :param field_separator: character used to separate + fields in the input file (e.g., ',' or ';'). + :param batch_size: maximum number of rows + to process at a time (per API request). + + :returns: package, set up to import assets with advanced configuration + """ + + if isinstance(remove_attributes, list) and all( + isinstance(field, AtlanField) for field in remove_attributes + ): + remove_attributes = [field.atlan_field_name for field in remove_attributes] # type: ignore + params = { + "assets_attr_to_overwrite": dumps(remove_attributes, separators=(",", ":")), + "assets_fail_on_errors": fail_on_errors, + "assets_field_separator": field_separator, + "assets_batch_size": batch_size, + } + self._add_optional_params(params) + return self + + def _get_metadata(self) -> WorkflowMetadata: + return WorkflowMetadata( + labels={ + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/source": self._NAME, + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": f"a-t-rcsas-l-a-s-h{self._NAME}", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true", + }, + annotations={ + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": f"https://solutions.atlan.com/{self._NAME}/", + "orchestration.atlan.com/emoji": "\U0001f680", + "orchestration.atlan.com/icon": self._PACKAGE_ICON, + "orchestration.atlan.com/logo": self._PACKAGE_LOGO, # noqa + "orchestration.atlan.com/name": "Relational Assets Builder", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Build (and update) relational assets managed through a CSV file.", + "package.argoproj.io/homepage": f"https://packages.atlan.com/-/web/detail/{self._PACKAGE_NAME}", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", # fmt: skip + "package.argoproj.io/name": self._PACKAGE_NAME, + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": f"csa-{self._NAME}-{self._epoch}", + }, + name=f"csa-{self._NAME}-{self._epoch}", + namespace="default", + ) diff --git a/pyatlan/model/packages/tableau_crawler.py b/pyatlan/model/packages/tableau_crawler.py index 8594536be..248168edd 100644 --- a/pyatlan/model/packages/tableau_crawler.py +++ b/pyatlan/model/packages/tableau_crawler.py @@ -52,6 +52,28 @@ def __init__( source_logo=self._PACKAGE_LOGO, ) + def s3( + self, bucket_name: str, bucket_prefix: str, bucket_region: Optional[str] = None + ) -> TableauCrawler: + """ + Set up the crawler to fetch metadata directly from the S3 bucket. + + :param bucket_name: name of the S3 bucket containing the extracted metadata files + :param bucket_prefix: prefix within the S3 bucket where the extracted metadata files are located + :param bucket_region: (Optional) region where the S3 bucket is located + :returns: crawler, configured to fetch metadata directly from the S3 bucket + """ + self._parameters.append(dict(name="extraction-method", value="s3")) + self._parameters.append(dict(name="metadata-s3-bucket", value=bucket_name)) + self._parameters.append(dict(name="metadata-s3-prefix", value=bucket_prefix)) + self._parameters.append(dict(name="metadata-s3-region", value=bucket_region)) + # Advanced configuration + self.exclude(projects=[]) + self.include(projects=[]) + self.crawl_unpublished(enabled=True) + self.crawl_hidden_fields(enabled=True) + return self + def direct( self, hostname: str, @@ -80,6 +102,9 @@ def direct( } self._credentials_body.update(local_creds) self._parameters.append({"name": "extraction-method", "value": "direct"}) + self._parameters.append( + {"name": "credential-guid", "value": "{{credentialGuid}}"} + ) return self def basic_auth(self, username: str, password: str) -> TableauCrawler: @@ -187,9 +212,6 @@ def alternate_host(self, hostname: str) -> TableauCrawler: return self def _set_required_metadata_params(self): - self._parameters.append( - {"name": "credential-guid", "value": "{{credentialGuid}}"} - ) self._parameters.append( { "name": "connection", diff --git a/tests/unit/data/package_requests/asset_export_basic_all_assets_adls.json b/tests/unit/data/package_requests/asset_export_basic_all_assets_adls.json new file mode 100644 index 000000000..2507fb18a --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_all_assets_adls.json @@ -0,0 +1,121 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "ALL" + }, + { + "name": "qn_prefix", + "value": "/test/prefix" + }, + { + "name": "include_description", + "value": true + }, + { + "name": "include_glossaries", + "value": true + }, + { + "name": "include_products", + "value": true + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "adls", + "username": "test-client-id", + "password": "test-client-secret", + "extra": { + "azure_tenant_id": "test-tenant-id", + "storage_account_name": "test-storage-account", + "adls_container": "test-adls-container" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_export_basic_all_assets_gcs.json b/tests/unit/data/package_requests/asset_export_basic_all_assets_gcs.json new file mode 100644 index 000000000..7b456e54b --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_all_assets_gcs.json @@ -0,0 +1,119 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "ALL" + }, + { + "name": "qn_prefix", + "value": "/test/prefix" + }, + { + "name": "include_description", + "value": true + }, + { + "name": "include_glossaries", + "value": true + }, + { + "name": "include_products", + "value": true + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "gcs", + "username": "test-project-id", + "password": "test-service-account-json", + "extra": { + "gcs_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_export_basic_all_assets_s3.json b/tests/unit/data/package_requests/asset_export_basic_all_assets_s3.json new file mode 100644 index 000000000..9c061f7e9 --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_all_assets_s3.json @@ -0,0 +1,120 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "ALL" + }, + { + "name": "qn_prefix", + "value": "/test/prefix" + }, + { + "name": "include_description", + "value": true + }, + { + "name": "include_glossaries", + "value": true + }, + { + "name": "include_products", + "value": true + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "s3", + "username": "test-access-key", + "password": "test-secret-key", + "extra": { + "region": "us-west-1", + "s3_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_export_basic_enriched_adls.json b/tests/unit/data/package_requests/asset_export_basic_enriched_adls.json new file mode 100644 index 000000000..c8fbd5a7b --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_enriched_adls.json @@ -0,0 +1,121 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "ENRICHED_ONLY" + }, + { + "name": "qn_prefix", + "value": "/test/prefix" + }, + { + "name": "include_description", + "value": true + }, + { + "name": "include_glossaries", + "value": true + }, + { + "name": "include_products", + "value": true + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "adls", + "username": "test-client-id", + "password": "test-client-secret", + "extra": { + "azure_tenant_id": "test-tenant-id", + "storage_account_name": "test-storage-account", + "adls_container": "test-adls-container" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_export_basic_enriched_gcs.json b/tests/unit/data/package_requests/asset_export_basic_enriched_gcs.json new file mode 100644 index 000000000..6ce350f45 --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_enriched_gcs.json @@ -0,0 +1,119 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "ENRICHED_ONLY" + }, + { + "name": "qn_prefix", + "value": "/test/prefix" + }, + { + "name": "include_description", + "value": true + }, + { + "name": "include_glossaries", + "value": true + }, + { + "name": "include_products", + "value": true + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "gcs", + "username": "test-project-id", + "password": "test-service-account-json", + "extra": { + "gcs_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_export_basic_enriched_s3.json b/tests/unit/data/package_requests/asset_export_basic_enriched_s3.json new file mode 100644 index 000000000..c61ae1617 --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_enriched_s3.json @@ -0,0 +1,120 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "ENRICHED_ONLY" + }, + { + "name": "qn_prefix", + "value": "/test/prefix" + }, + { + "name": "include_description", + "value": true + }, + { + "name": "include_glossaries", + "value": true + }, + { + "name": "include_products", + "value": true + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "s3", + "username": "test-access-key", + "password": "test-secret-key", + "extra": { + "region": "us-west-1", + "s3_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_export_basic_glossaries_adls.json b/tests/unit/data/package_requests/asset_export_basic_glossaries_adls.json new file mode 100644 index 000000000..454bd37fd --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_glossaries_adls.json @@ -0,0 +1,105 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "GLOSSARIES_ONLY" + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "adls", + "username": "test-client-id", + "password": "test-client-secret", + "extra": { + "azure_tenant_id": "test-tenant-id", + "storage_account_name": "test-storage-account", + "adls_container": "test-adls-container" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_export_basic_glossaries_gcs.json b/tests/unit/data/package_requests/asset_export_basic_glossaries_gcs.json new file mode 100644 index 000000000..3a07dd320 --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_glossaries_gcs.json @@ -0,0 +1,103 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "GLOSSARIES_ONLY" + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "gcs", + "username": "test-project-id", + "password": "test-service-account-json", + "extra": { + "gcs_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_export_basic_glossaries_s3.json b/tests/unit/data/package_requests/asset_export_basic_glossaries_s3.json new file mode 100644 index 000000000..b48711871 --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_glossaries_s3.json @@ -0,0 +1,104 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "GLOSSARIES_ONLY" + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "s3", + "username": "test-access-key", + "password": "test-secret-key", + "extra": { + "region": "us-west-1", + "s3_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_export_basic_products_adls.json b/tests/unit/data/package_requests/asset_export_basic_products_adls.json new file mode 100644 index 000000000..a31303260 --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_products_adls.json @@ -0,0 +1,105 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "PRODUCTS_ONLY" + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "adls", + "username": "test-client-id", + "password": "test-client-secret", + "extra": { + "azure_tenant_id": "test-tenant-id", + "storage_account_name": "test-storage-account", + "adls_container": "test-adls-container" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_export_basic_products_gcs.json b/tests/unit/data/package_requests/asset_export_basic_products_gcs.json new file mode 100644 index 000000000..5c326d9c0 --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_products_gcs.json @@ -0,0 +1,103 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "PRODUCTS_ONLY" + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "gcs", + "username": "test-project-id", + "password": "test-service-account-json", + "extra": { + "gcs_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_export_basic_products_s3.json b/tests/unit/data/package_requests/asset_export_basic_products_s3.json new file mode 100644 index 000000000..3f436d4c4 --- /dev/null +++ b/tests/unit/data/package_requests/asset_export_basic_products_s3.json @@ -0,0 +1,104 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-export-basic", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-export-basic", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-export-basic/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-down-light.svg", + "orchestration.atlan.com/name": "Asset Export (Basic)", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Export assets with all enrichment that could be made against them via the Atlan UI.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-export-basic", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-export-basic", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-export-basic-123456" + }, + "name": "csa-asset-export-basic-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "export_scope", + "value": "PRODUCTS_ONLY" + }, + { + "name": "include_archived", + "value": true + }, + { + "name": "delivery_type", + "value": "CLOUD" + }, + { + "name": "target_prefix", + "value": "/test/prefix" + }, + { + "name": "cloud_target", + "value": "{{credentialGuid}}" + } + ] + }, + "templateRef": { + "name": "csa-asset-export-basic", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-export-basic" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-export-basic-123456-0", + "authType": "s3", + "username": "test-access-key", + "password": "test-secret-key", + "extra": { + "region": "us-west-1", + "s3_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_import_adls.json b/tests/unit/data/package_requests/asset_import_adls.json new file mode 100644 index 000000000..2d82a21c6 --- /dev/null +++ b/tests/unit/data/package_requests/asset_import_adls.json @@ -0,0 +1,193 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-import", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-import", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-import/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-up-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-up-light.svg", + "orchestration.atlan.com/name": "Asset Import", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Import assets from a CSV file.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-import", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-import", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-import-123456" + }, + "name": "csa-asset-import-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "import_type", + "value": "CLOUD" + }, + { + "name": "cloud_source", + "value": "{{credentialGuid}}" + }, + { + "name": "assets_prefix", + "value": "/test/prefix" + }, + { + "name": "assets_key", + "value": "assets-test.csv" + }, + { + "name": "assets_upsert_semantic", + "value": "upsert" + }, + { + "name": "assets_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "assets_fail_on_errors", + "value": true + }, + { + "name": "assets_case_sensitive", + "value": false + }, + { + "name": "assets_field_separator", + "value": "," + }, + { + "name": "assets_batch_size", + "value": 20 + }, + { + "name": "glossaries_prefix", + "value": "/test/prefix" + }, + { + "name": "glossaries_key", + "value": "glossaries-test.csv" + }, + { + "name": "glossaries_upsert_semantic", + "value": "update" + }, + { + "name": "glossaries_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "glossaries_fail_on_errors", + "value": true + }, + { + "name": "glossaries_field_separator", + "value": "," + }, + { + "name": "glossaries_batch_size", + "value": 20 + }, + { + "name": "data_products_prefix", + "value": "/test/prefix" + }, + { + "name": "data_products_key", + "value": "data-products-test.csv" + }, + { + "name": "data_products_upsert_semantic", + "value": "update" + }, + { + "name": "data_products_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "data_products_fail_on_errors", + "value": true + }, + { + "name": "data_products_field_separator", + "value": "," + }, + { + "name": "data_products_batch_size", + "value": 20 + }, + { + "name": "assets_config", + "value": "advanced" + }, + { + "name": "glossaries_config", + "value": "advanced" + }, + { + "name": "data_products_config", + "value": "advanced" + } + ] + }, + "templateRef": { + "name": "csa-asset-import", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-import" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-import-123456-0", + "authType": "adls", + "username": "test-client-id", + "password": "test-client-secret", + "extra": { + "azure_tenant_id": "test-tenant-id", + "storage_account_name": "test-storage-account", + "adls_container": "test-adls-container" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_import_default.json b/tests/unit/data/package_requests/asset_import_default.json new file mode 100644 index 000000000..c72967e72 --- /dev/null +++ b/tests/unit/data/package_requests/asset_import_default.json @@ -0,0 +1,140 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-import", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-import", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-import/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-up-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-up-light.svg", + "orchestration.atlan.com/name": "Asset Import", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Import assets from a CSV file.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-import", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-import", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-import-123456" + }, + "name": "csa-asset-import-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "import_type", + "value": "CLOUD" + }, + { + "name": "cloud_source", + "value": "{{credentialGuid}}" + }, + { + "name": "assets_prefix", + "value": "/test/prefix" + }, + { + "name": "assets_key", + "value": "assets-test.csv" + }, + { + "name": "assets_upsert_semantic", + "value": "update" + }, + { + "name": "glossaries_prefix", + "value": "/test/prefix" + }, + { + "name": "glossaries_key", + "value": "glossaries-test.csv" + }, + { + "name": "glossaries_upsert_semantic", + "value": "upsert" + }, + { + "name": "data_products_prefix", + "value": "/test/prefix" + }, + { + "name": "data_products_key", + "value": "data-products-test.csv" + }, + { + "name": "data_products_upsert_semantic", + "value": "upsert" + }, + { + "name": "assets_config", + "value": "default" + }, + { + "name": "glossaries_config", + "value": "default" + }, + { + "name": "data_products_config", + "value": "default" + } + ] + }, + "templateRef": { + "name": "csa-asset-import", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-import" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-import-123456-0", + "authType": "s3", + "username": "test-access-key", + "password": "test-secret-key", + "extra": { + "region": "us-west-1", + "s3_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_import_gcs.json b/tests/unit/data/package_requests/asset_import_gcs.json new file mode 100644 index 000000000..e0883d352 --- /dev/null +++ b/tests/unit/data/package_requests/asset_import_gcs.json @@ -0,0 +1,191 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-import", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-import", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-import/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-up-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-up-light.svg", + "orchestration.atlan.com/name": "Asset Import", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Import assets from a CSV file.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-import", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-import", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-import-123456" + }, + "name": "csa-asset-import-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "import_type", + "value": "CLOUD" + }, + { + "name": "cloud_source", + "value": "{{credentialGuid}}" + }, + { + "name": "assets_prefix", + "value": "/test/prefix" + }, + { + "name": "assets_key", + "value": "assets-test.csv" + }, + { + "name": "assets_upsert_semantic", + "value": "upsert" + }, + { + "name": "assets_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "assets_fail_on_errors", + "value": true + }, + { + "name": "assets_case_sensitive", + "value": false + }, + { + "name": "assets_field_separator", + "value": "," + }, + { + "name": "assets_batch_size", + "value": 20 + }, + { + "name": "glossaries_prefix", + "value": "/test/prefix" + }, + { + "name": "glossaries_key", + "value": "glossaries-test.csv" + }, + { + "name": "glossaries_upsert_semantic", + "value": "update" + }, + { + "name": "glossaries_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "glossaries_fail_on_errors", + "value": true + }, + { + "name": "glossaries_field_separator", + "value": "," + }, + { + "name": "glossaries_batch_size", + "value": 20 + }, + { + "name": "data_products_prefix", + "value": "/test/prefix" + }, + { + "name": "data_products_key", + "value": "data-products-test.csv" + }, + { + "name": "data_products_upsert_semantic", + "value": "update" + }, + { + "name": "data_products_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "data_products_fail_on_errors", + "value": true + }, + { + "name": "data_products_field_separator", + "value": "," + }, + { + "name": "data_products_batch_size", + "value": 20 + }, + { + "name": "assets_config", + "value": "advanced" + }, + { + "name": "glossaries_config", + "value": "advanced" + }, + { + "name": "data_products_config", + "value": "advanced" + } + ] + }, + "templateRef": { + "name": "csa-asset-import", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-import" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-import-123456-0", + "authType": "gcs", + "username": "test-project-id", + "password": "test-service-account-json", + "extra": { + "gcs_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/asset_import_s3.json b/tests/unit/data/package_requests/asset_import_s3.json new file mode 100644 index 000000000..0789f11ad --- /dev/null +++ b/tests/unit/data/package_requests/asset_import_s3.json @@ -0,0 +1,192 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "asset-import", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hasset-import", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/asset-import/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-cloud-arrow-up-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-cloud-arrow-up-light.svg", + "orchestration.atlan.com/name": "Asset Import", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Import assets from a CSV file.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/asset-import", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/asset-import", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-asset-import-123456" + }, + "name": "csa-asset-import-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "import_type", + "value": "CLOUD" + }, + { + "name": "cloud_source", + "value": "{{credentialGuid}}" + }, + { + "name": "assets_prefix", + "value": "/test/prefix" + }, + { + "name": "assets_key", + "value": "assets-test.csv" + }, + { + "name": "assets_upsert_semantic", + "value": "partial" + }, + { + "name": "assets_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "assets_fail_on_errors", + "value": true + }, + { + "name": "assets_case_sensitive", + "value": false + }, + { + "name": "assets_field_separator", + "value": "," + }, + { + "name": "assets_batch_size", + "value": 20 + }, + { + "name": "glossaries_prefix", + "value": "/test/prefix" + }, + { + "name": "glossaries_key", + "value": "glossaries-test.csv" + }, + { + "name": "glossaries_upsert_semantic", + "value": "update" + }, + { + "name": "glossaries_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "glossaries_fail_on_errors", + "value": true + }, + { + "name": "glossaries_field_separator", + "value": "," + }, + { + "name": "glossaries_batch_size", + "value": 20 + }, + { + "name": "data_products_prefix", + "value": "/test/prefix" + }, + { + "name": "data_products_key", + "value": "data-products-test.csv" + }, + { + "name": "data_products_upsert_semantic", + "value": "update" + }, + { + "name": "data_products_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "data_products_fail_on_errors", + "value": true + }, + { + "name": "data_products_field_separator", + "value": "," + }, + { + "name": "data_products_batch_size", + "value": 20 + }, + { + "name": "assets_config", + "value": "advanced" + }, + { + "name": "glossaries_config", + "value": "advanced" + }, + { + "name": "data_products_config", + "value": "advanced" + } + ] + }, + "templateRef": { + "name": "csa-asset-import", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/asset-import" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-asset-import-123456-0", + "authType": "s3", + "username": "test-access-key", + "password": "test-secret-key", + "extra": { + "region": "us-west-1", + "s3_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/relational_assets_builder_adls.json b/tests/unit/data/package_requests/relational_assets_builder_adls.json new file mode 100644 index 000000000..29f75d384 --- /dev/null +++ b/tests/unit/data/package_requests/relational_assets_builder_adls.json @@ -0,0 +1,129 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "relational-assets-builder", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hrelational-assets-builder", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/relational-assets-builder/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-database-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-database-light.svg", + "orchestration.atlan.com/name": "Relational Assets Builder", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Build (and update) relational assets managed through a CSV file.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/relational-assets-builder", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/relational-assets-builder", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-relational-assets-builder-123456" + }, + "name": "csa-relational-assets-builder-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "import_type", + "value": "CLOUD" + }, + { + "name": "assets_prefix", + "value": "/test/prefix" + }, + { + "name": "assets_key", + "value": "assets-test.csv" + }, + { + "name": "cloud_source", + "value": "{{credentialGuid}}" + }, + { + "name": "assets_upsert_semantic", + "value": "partial" + }, + { + "name": "delta_semantic", + "value": "full" + }, + { + "name": "delta_removal_type", + "value": "archive" + }, + { + "name": "assets_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "assets_fail_on_errors", + "value": true + }, + { + "name": "assets_field_separator", + "value": "," + }, + { + "name": "assets_batch_size", + "value": 20 + } + ] + }, + "templateRef": { + "name": "csa-relational-assets-builder", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/relational-assets-builder" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-relational-assets-builder-123456-0", + "authType": "adls", + "username": "test-client-id", + "password": "test-client-secret", + "extra": { + "azure_tenant_id": "test-tenant-id", + "storage_account_name": "test-storage-account", + "adls_container": "test-adls-container" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/relational_assets_builder_gcs.json b/tests/unit/data/package_requests/relational_assets_builder_gcs.json new file mode 100644 index 000000000..275f529f3 --- /dev/null +++ b/tests/unit/data/package_requests/relational_assets_builder_gcs.json @@ -0,0 +1,127 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "relational-assets-builder", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hrelational-assets-builder", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/relational-assets-builder/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-database-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-database-light.svg", + "orchestration.atlan.com/name": "Relational Assets Builder", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Build (and update) relational assets managed through a CSV file.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/relational-assets-builder", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/relational-assets-builder", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-relational-assets-builder-123456" + }, + "name": "csa-relational-assets-builder-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "import_type", + "value": "CLOUD" + }, + { + "name": "assets_prefix", + "value": "/test/prefix" + }, + { + "name": "assets_key", + "value": "assets-test.csv" + }, + { + "name": "cloud_source", + "value": "{{credentialGuid}}" + }, + { + "name": "assets_upsert_semantic", + "value": "update" + }, + { + "name": "delta_semantic", + "value": "full" + }, + { + "name": "delta_removal_type", + "value": "purge" + }, + { + "name": "assets_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "assets_fail_on_errors", + "value": true + }, + { + "name": "assets_field_separator", + "value": "," + }, + { + "name": "assets_batch_size", + "value": 20 + } + ] + }, + "templateRef": { + "name": "csa-relational-assets-builder", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/relational-assets-builder" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-relational-assets-builder-123456-0", + "authType": "gcs", + "username": "test-project-id", + "password": "test-service-account-json", + "extra": { + "gcs_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/relational_assets_builder_s3.json b/tests/unit/data/package_requests/relational_assets_builder_s3.json new file mode 100644 index 000000000..44f068908 --- /dev/null +++ b/tests/unit/data/package_requests/relational_assets_builder_s3.json @@ -0,0 +1,128 @@ +{ + "metadata": { + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/preview": "true", + "orchestration.atlan.com/source": "relational-assets-builder", + "orchestration.atlan.com/sourceCategory": "utility", + "orchestration.atlan.com/type": "custom", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-rcsas-l-a-s-hrelational-assets-builder", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/atlan-ui": "true" + }, + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "kotlin,utility", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/relational-assets-builder/", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-database-light.svg", + "orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-database-light.svg", + "orchestration.atlan.com/name": "Relational Assets Builder", + "package.argoproj.io/author": "Atlan CSA", + "package.argoproj.io/description": "Build (and update) relational assets managed through a CSV file.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/relational-assets-builder", + "package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]", + "package.argoproj.io/name": "@csa/relational-assets-builder", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "csa-relational-assets-builder-123456" + }, + "name": "csa-relational-assets-builder-123456", + "namespace": "default" + }, + "spec": { + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "import_type", + "value": "CLOUD" + }, + { + "name": "assets_prefix", + "value": "/test/prefix" + }, + { + "name": "assets_key", + "value": "assets-test.csv" + }, + { + "name": "cloud_source", + "value": "{{credentialGuid}}" + }, + { + "name": "assets_upsert_semantic", + "value": "upsert" + }, + { + "name": "delta_semantic", + "value": "delta" + }, + { + "name": "delta_removal_type", + "value": "archive" + }, + { + "name": "assets_attr_to_overwrite", + "value": "[\"certificateStatus\",\"announcementType\"]" + }, + { + "name": "assets_fail_on_errors", + "value": true + }, + { + "name": "assets_field_separator", + "value": "," + }, + { + "name": "assets_batch_size", + "value": 20 + } + ] + }, + "templateRef": { + "name": "csa-relational-assets-builder", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "entrypoint": "main", + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@csa/relational-assets-builder" + } + } + }, + "payload": [ + { + "parameter": "credentialGuid", + "type": "credential", + "body": { + "name": "csa-relational-assets-builder-123456-0", + "authType": "s3", + "username": "test-access-key", + "password": "test-secret-key", + "extra": { + "region": "us-west-1", + "s3_bucket": "my-bucket" + }, + "connectorConfigName": "csa-connectors-objectstore" + } + } + ] +} diff --git a/tests/unit/data/package_requests/tableau_access_token.json b/tests/unit/data/package_requests/tableau_access_token.json index b6ab64051..01fad95a1 100644 --- a/tests/unit/data/package_requests/tableau_access_token.json +++ b/tests/unit/data/package_requests/tableau_access_token.json @@ -51,6 +51,10 @@ "name": "extraction-method", "value": "direct" }, + { + "name": "credential-guid", + "value": "{{credentialGuid}}" + }, { "name": "include-filter", "value": "{\"test-project-guid-1\": {}, \"test-project-guid-2\": {}}" @@ -67,10 +71,6 @@ "name": "crawl-hidden-datasource-fields", "value": "false" }, - { - "name": "credential-guid", - "value": "{{credentialGuid}}" - }, { "name": "connection", "value": "{\"typeName\": \"Connection\", \"attributes\": {\"qualifiedName\": \"default/tableau/123456\", \"name\": \"test-tableau-access-token-conn\", \"adminUsers\": [], \"adminGroups\": [], \"connectorName\": \"tableau\", \"isDiscoverable\": true, \"isEditable\": false, \"adminRoles\": [\"admin-guid-1234\"], \"category\": \"bi\", \"allowQuery\": false, \"allowQueryPreview\": false, \"rowLimit\": 0, \"defaultCredentialGuid\": \"{{credentialGuid}}\", \"sourceLogo\": \"https://img.icons8.com/color/480/000000/tableau-software.png\"}, \"guid\": \"-1234567890000000000000000\"}" diff --git a/tests/unit/data/package_requests/tableau_basic.json b/tests/unit/data/package_requests/tableau_basic.json index a42e0656d..481a8c732 100644 --- a/tests/unit/data/package_requests/tableau_basic.json +++ b/tests/unit/data/package_requests/tableau_basic.json @@ -51,6 +51,10 @@ "name": "extraction-method", "value": "direct" }, + { + "name": "credential-guid", + "value": "{{credentialGuid}}" + }, { "name": "include-filter", "value": "{\"test-project-guid-1\": {}, \"test-project-guid-2\": {}}" @@ -67,10 +71,6 @@ "name": "crawl-hidden-datasource-fields", "value": "false" }, - { - "name": "credential-guid", - "value": "{{credentialGuid}}" - }, { "name": "connection", "value": "{\"typeName\": \"Connection\", \"attributes\": {\"qualifiedName\": \"default/tableau/123456\", \"name\": \"test-tableau-basic-conn\", \"adminUsers\": [], \"adminGroups\": [], \"connectorName\": \"tableau\", \"isDiscoverable\": true, \"isEditable\": false, \"adminRoles\": [\"admin-guid-1234\"], \"category\": \"bi\", \"allowQuery\": false, \"allowQueryPreview\": false, \"rowLimit\": 0, \"defaultCredentialGuid\": \"{{credentialGuid}}\", \"sourceLogo\": \"https://img.icons8.com/color/480/000000/tableau-software.png\"}, \"guid\": \"-1234567890000000000000000\"}" diff --git a/tests/unit/data/package_requests/tableau_offline.json b/tests/unit/data/package_requests/tableau_offline.json new file mode 100644 index 000000000..61a853104 --- /dev/null +++ b/tests/unit/data/package_requests/tableau_offline.json @@ -0,0 +1,113 @@ +{ + "metadata": { + "annotations": { + "orchestration.atlan.com/allowSchedule": "true", + "orchestration.atlan.com/categories": "tableau,crawler", + "orchestration.atlan.com/dependentPackage": "", + "orchestration.atlan.com/docsUrl": "https://ask.atlan.com/hc/en-us/articles/6332449996689", + "orchestration.atlan.com/emoji": "🚀", + "orchestration.atlan.com/icon": "https://img.icons8.com/color/480/000000/tableau-software.png", + "orchestration.atlan.com/logo": "https://img.icons8.com/color/480/000000/tableau-software.png", + "orchestration.atlan.com/marketplaceLink": "https://packages.atlan.com/-/web/detail/@atlan/tableau", + "orchestration.atlan.com/name": "tableau Assets", + "package.argoproj.io/author": "Atlan", + "package.argoproj.io/description": "Package to crawl Tableau assets and publish to Atlan for discovery.", + "package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@atlan/tableau", + "package.argoproj.io/keywords": "[\"tableau\",\"bi\",\"connector\",\"crawler\"]", + "package.argoproj.io/name": "@atlan/tableau", + "package.argoproj.io/parent": ".", + "package.argoproj.io/registry": "https://packages.atlan.com", + "package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git", + "package.argoproj.io/support": "support@atlan.com", + "orchestration.atlan.com/atlanName": "atlan-tableau-default-tableau-123456" + }, + "labels": { + "orchestration.atlan.com/certified": "true", + "orchestration.atlan.com/source": "tableau", + "orchestration.atlan.com/sourceCategory": "bi", + "orchestration.atlan.com/type": "connector", + "orchestration.atlan.com/verified": "true", + "package.argoproj.io/installer": "argopm", + "package.argoproj.io/name": "a-t-ratlans-l-a-s-htableau", + "package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com", + "orchestration.atlan.com/default-tableau-123456": "true", + "orchestration.atlan.com/atlan-ui": "true" + }, + "name": "atlan-tableau-123456", + "namespace": "default" + }, + "spec": { + "entrypoint": "main", + "templates": [ + { + "name": "main", + "dag": { + "tasks": [ + { + "name": "run", + "arguments": { + "parameters": [ + { + "name": "extraction-method", + "value": "s3" + }, + { + "name": "metadata-s3-bucket", + "value": "test-bucket" + }, + { + "name": "metadata-s3-prefix", + "value": "test-prefix" + }, + { + "name": "metadata-s3-region", + "value": "test-region" + }, + { + "name": "exclude-filter", + "value": "{}" + }, + { + "name": "include-filter", + "value": "{}" + }, + { + "name": "crawl-unpublished-worksheets-dashboard", + "value": "true" + }, + { + "name": "crawl-hidden-datasource-fields", + "value": "true" + }, + { + "name": "connection", + "value": "{\"typeName\": \"Connection\", \"attributes\": {\"qualifiedName\": \"default/tableau/123456\", \"name\": \"test-tableau-offline-conn\", \"adminUsers\": [], \"adminGroups\": [], \"connectorName\": \"tableau\", \"isDiscoverable\": true, \"isEditable\": false, \"adminRoles\": [\"admin-guid-1234\"], \"category\": \"bi\", \"allowQuery\": false, \"allowQueryPreview\": false, \"rowLimit\": 0, \"defaultCredentialGuid\": \"{{credentialGuid}}\", \"sourceLogo\": \"https://img.icons8.com/color/480/000000/tableau-software.png\"}, \"guid\": \"-1234567890000000000000000\"}" + }, + { + "name": "atlas-auth-type", + "value": "internal" + }, + { + "name": "publish-mode", + "value": "production" + } + ] + }, + "templateRef": { + "name": "atlan-tableau", + "template": "main", + "clusterScope": true + } + } + ] + } + } + ], + "workflowMetadata": { + "annotations": { + "package.argoproj.io/name": "@atlan/tableau" + } + } + }, + "payload": [] +} diff --git a/tests/unit/test_packages.py b/tests/unit/test_packages.py index c4f8a15e3..f8d81bb84 100644 --- a/tests/unit/test_packages.py +++ b/tests/unit/test_packages.py @@ -5,7 +5,11 @@ import pytest from pyatlan.errors import InvalidRequestError +from pyatlan.model.assets.core import Asset +from pyatlan.model.enums import AssetDeltaHandling, AssetInputHandling, AssetRemovalType from pyatlan.model.packages import ( + AssetExportBasic, + AssetImport, BigQueryCrawler, ConfluentKafkaCrawler, ConnectionDelete, @@ -14,6 +18,7 @@ GlueCrawler, PostgresCrawler, PowerBICrawler, + RelationalAssetsBuilder, SigmaCrawler, SnowflakeCrawler, SnowflakeMiner, @@ -30,6 +35,7 @@ GLUE_IAM_USER = "glue_iam_user.json" TABLEAU_BASIC = "tableau_basic.json" TABLEAU_ACCESS_TOKEN = "tableau_access_token.json" +TABLEAU_OFFLINE = "tableau_offline.json" POWERBI_DELEGATED_USER = "powerbi_delegated_user.json" POWEBI_SERVICE_PRINCIPAL = "powerbi_service_principal.json" CONFLUENT_KAFKA_DIRECT = "confluent_kafka_direct.json" @@ -46,6 +52,26 @@ POSTGRES_S3_OFFLINE = "postgres_s3_offline.json" CONNECTION_DELETE_HARD = "connection_delete_hard.json" CONNECTION_DELETE_SOFT = "connection_delete_soft.json" +ASSET_IMPORT_S3 = "asset_import_s3.json" +ASSET_IMPORT_GCS = "asset_import_gcs.json" +ASSET_EXPORT_BASIC_GLOSSARIES = "asset_export_glossaries.json" +ASSET_IMPORT_ADLS = "asset_import_adls.json" +ASSET_IMPORT_DEFAULT = "asset_import_default.json" +ASSET_EXPORT_BASIC_GLOSSARIES_ONLY_S3 = "asset_export_basic_glossaries_s3.json" +ASSET_EXPORT_BASIC_PRODUCTS_ONLY_S3 = "asset_export_basic_products_s3.json" +ASSET_EXPORT_BASIC_ENRICHED_ONLY_S3 = "asset_export_basic_enriched_s3.json" +ASSET_EXPORT_BASIC_ALL_ASSETS_S3 = "asset_export_basic_all_assets_s3.json" +ASSET_EXPORT_BASIC_GLOSSARIES_ONLY_ADLS = "asset_export_basic_glossaries_adls.json" +ASSET_EXPORT_BASIC_ALL_ASSETS_ADLS = "asset_export_basic_all_assets_adls.json" +ASSET_EXPORT_BASIC_PRODUCTS_ONLY_ADLS = "asset_export_basic_products_adls.json" +ASSET_EXPORT_BASIC_ENRICHED_ONLY_ADLS = "asset_export_basic_enriched_adls.json" +ASSET_EXPORT_BASIC_ALL_ASSETS_GCS = "asset_export_basic_all_assets_gcs.json" +ASSET_EXPORT_BASIC_GLOSSARIES_ONLY_GCS = "asset_export_basic_glossaries_gcs.json" +ASSET_EXPORT_BASIC_PRODUCTS_ONLY_GCS = "asset_export_basic_products_gcs.json" +ASSET_EXPORT_BASIC_ENRICHED_ONLY_GCS = "asset_export_basic_enriched_gcs.json" +RELATIONAL_ASSETS_BUILDER_S3 = "relational_assets_builder_s3.json" +RELATIONAL_ASSETS_BUILDER_ADLS = "relational_assets_builder_adls.json" +RELATIONAL_ASSETS_BUILDER_GCS = "relational_assets_builder_gcs.json" class NonSerializable: @@ -232,6 +258,23 @@ def test_tableau_package(mock_package_env): ) assert request_json == load_json(TABLEAU_ACCESS_TOKEN) + tableau_offline = ( + TableauCrawler( + connection_name="test-tableau-offline-conn", + admin_roles=["admin-guid-1234"], + admin_groups=None, + admin_users=None, + ) + .s3( + bucket_name="test-bucket", + bucket_prefix="test-prefix", + bucket_region="test-region", + ) + .to_workflow() + ) + request_json = loads(tableau_offline.json(by_alias=True, exclude_none=True)) + assert request_json == load_json(TABLEAU_OFFLINE) + def test_powerbi_package(mock_package_env): powerbi_delegated_user = ( @@ -594,6 +637,535 @@ def test_connection_delete_package(mock_package_env): assert request_json == load_json(CONNECTION_DELETE_SOFT) +def test_asset_import(mock_package_env): + # Case 1: Importing assets, glossaries, and data products from S3 with advanced configuration + asset_import_s3 = ( + AssetImport() + .object_store() + .s3( + access_key="test-access-key", + secret_key="test-secret-key", + bucket="my-bucket", + region="us-west-1", + ) + .assets( + prefix="/test/prefix", + object_key="assets-test.csv", + input_handling=AssetInputHandling.PARTIAL, + ) + .assets_advanced( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + case_sensitive_match=False, + field_separator=",", + batch_size=20, + ) + .glossaries( + prefix="/test/prefix", + object_key="glossaries-test.csv", + input_handling=AssetInputHandling.UPDATE, + ) + .glossaries_advanced( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + field_separator=",", + batch_size=20, + ) + .data_products( + prefix="/test/prefix", + object_key="data-products-test.csv", + input_handling=AssetInputHandling.UPDATE, + ) + .data_product_advanced( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + field_separator=",", + batch_size=20, + ) + ).to_workflow() + + request_json_s3 = loads(asset_import_s3.json(by_alias=True, exclude_none=True)) + assert request_json_s3 == load_json(ASSET_IMPORT_S3) + + # Case 2: Importing assets, glossaries, and data products from GCS with advanced configuration + asset_import_gcs = ( + AssetImport() + .object_store() + .gcs( + service_account_json="test-service-account-json", + project_id="test-project-id", + bucket="my-bucket", + ) + .assets( + prefix="/test/prefix", + object_key="assets-test.csv", + input_handling=AssetInputHandling.UPSERT, + ) + .assets_advanced( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + case_sensitive_match=False, + field_separator=",", + batch_size=20, + ) + .glossaries( + prefix="/test/prefix", + object_key="glossaries-test.csv", + input_handling=AssetInputHandling.UPDATE, + ) + .glossaries_advanced( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + field_separator=",", + batch_size=20, + ) + .data_products( + prefix="/test/prefix", + object_key="data-products-test.csv", + input_handling=AssetInputHandling.UPDATE, + ) + .data_product_advanced( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + field_separator=",", + batch_size=20, + ) + ).to_workflow() + + request_json_gcs = loads(asset_import_gcs.json(by_alias=True, exclude_none=True)) + assert request_json_gcs == load_json(ASSET_IMPORT_GCS) + + # Case 3: Importing assets, glossaries, and data products from Adls with advanced configuration + asset_import_adls = ( + AssetImport() + .object_store() + .adls( + client_id="test-client-id", + client_secret="test-client-secret", + tenant_id="test-tenant-id", + account_name="test-storage-account", + container="test-adls-container", + ) + .assets( + prefix="/test/prefix", + object_key="assets-test.csv", + input_handling=AssetInputHandling.UPSERT, + ) + .assets_advanced( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + case_sensitive_match=False, + field_separator=",", + batch_size=20, + ) + .glossaries( + prefix="/test/prefix", + object_key="glossaries-test.csv", + input_handling=AssetInputHandling.UPDATE, + ) + .glossaries_advanced( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + field_separator=",", + batch_size=20, + ) + .data_products( + prefix="/test/prefix", + object_key="data-products-test.csv", + input_handling=AssetInputHandling.UPDATE, + ) + .data_product_advanced( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + field_separator=",", + batch_size=20, + ) + ).to_workflow() + + request_json_adls = loads(asset_import_adls.json(by_alias=True, exclude_none=True)) + assert request_json_adls == load_json(ASSET_IMPORT_ADLS) + + # Case 4: Importing assets, glossaries, and data products from S3 with default configuration + asset_import_default = ( + AssetImport() + .object_store() + .s3( + access_key="test-access-key", + secret_key="test-secret-key", + bucket="my-bucket", + region="us-west-1", + ) + .assets( + prefix="/test/prefix", + object_key="assets-test.csv", + input_handling=AssetInputHandling.UPDATE, + ) + .glossaries( + prefix="/test/prefix", + object_key="glossaries-test.csv", + input_handling=AssetInputHandling.UPSERT, + ) + .data_products( + prefix="/test/prefix", + object_key="data-products-test.csv", + input_handling=AssetInputHandling.UPSERT, + ) + ).to_workflow() + + request_json_default = loads( + asset_import_default.json(by_alias=True, exclude_none=True) + ) + assert request_json_default == load_json(ASSET_IMPORT_DEFAULT) + + +def test_asset_export_basic(mock_package_env): + # Case 1: Export assets with glossaries only using s3 + asset_export_basic_glossaries_only_s3 = ( + AssetExportBasic() + .glossaries_only(include_archived=True) + .object_store(prefix="/test/prefix") + .s3( + access_key="test-access-key", + secret_key="test-secret-key", + bucket="my-bucket", + region="us-west-1", + ) + ).to_workflow() + + request_json_s3 = loads( + asset_export_basic_glossaries_only_s3.json(by_alias=True, exclude_none=True) + ) + assert request_json_s3 == load_json(ASSET_EXPORT_BASIC_GLOSSARIES_ONLY_S3) + + # Case 2: Export assets with Products only using s3 + asset_export_basic_products_only_s3 = ( + AssetExportBasic() + .products_only(include_archived=True) + .object_store(prefix="/test/prefix") + .s3( + access_key="test-access-key", + secret_key="test-secret-key", + bucket="my-bucket", + region="us-west-1", + ) + ).to_workflow() + + request_json_s3 = loads( + asset_export_basic_products_only_s3.json(by_alias=True, exclude_none=True) + ) + assert request_json_s3 == load_json(ASSET_EXPORT_BASIC_PRODUCTS_ONLY_S3) + + # Case 3: Export assets with Enriched only using s3 + asset_export_basic_enriched_only_s3 = ( + AssetExportBasic() + .enriched_only( + prefix="/test/prefix", + include_description=True, + include_glossaries=True, + include_data_products=True, + include_archived=True, + ) + .object_store(prefix="/test/prefix") + .s3( + access_key="test-access-key", + secret_key="test-secret-key", + bucket="my-bucket", + region="us-west-1", + ) + ).to_workflow() + + request_json_s3 = loads( + asset_export_basic_enriched_only_s3.json(by_alias=True, exclude_none=True) + ) + assert request_json_s3 == load_json(ASSET_EXPORT_BASIC_ENRICHED_ONLY_S3) + + # Case 4: Export all assets using s3 + asset_export_basic_all_assets_s3 = ( + AssetExportBasic() + .all_assets( + prefix="/test/prefix", + include_description=True, + include_glossaries=True, + include_data_products=True, + include_archived=True, + ) + .object_store(prefix="/test/prefix") + .s3( + access_key="test-access-key", + secret_key="test-secret-key", + bucket="my-bucket", + region="us-west-1", + ) + ).to_workflow() + + request_json_s3 = loads( + asset_export_basic_all_assets_s3.json(by_alias=True, exclude_none=True) + ) + assert request_json_s3 == load_json(ASSET_EXPORT_BASIC_ALL_ASSETS_S3) + + # Case 1: Export assets with glossaries only using adls + asset_export_basic_glossaries_only_adls = ( + AssetExportBasic() + .glossaries_only(include_archived=True) + .object_store(prefix="/test/prefix") + .adls( + client_id="test-client-id", + client_secret="test-client-secret", + tenant_id="test-tenant-id", + account_name="test-storage-account", + container="test-adls-container", + ) + ).to_workflow() + + request_json_adls = loads( + asset_export_basic_glossaries_only_adls.json(by_alias=True, exclude_none=True) + ) + assert request_json_adls == load_json(ASSET_EXPORT_BASIC_GLOSSARIES_ONLY_ADLS) + + # Case 2: Export assets with Products only using adls + asset_export_basic_products_only_adls = ( + AssetExportBasic() + .products_only(include_archived=True) + .object_store(prefix="/test/prefix") + .adls( + client_id="test-client-id", + client_secret="test-client-secret", + tenant_id="test-tenant-id", + account_name="test-storage-account", + container="test-adls-container", + ) + ).to_workflow() + + request_json_adls = loads( + asset_export_basic_products_only_adls.json(by_alias=True, exclude_none=True) + ) + assert request_json_adls == load_json(ASSET_EXPORT_BASIC_PRODUCTS_ONLY_ADLS) + + # Case 3: Export assets with Enriched only using adls + asset_export_basic_enriched_only_adls = ( + AssetExportBasic() + .enriched_only( + prefix="/test/prefix", + include_description=True, + include_glossaries=True, + include_data_products=True, + include_archived=True, + ) + .object_store(prefix="/test/prefix") + .adls( + client_id="test-client-id", + client_secret="test-client-secret", + tenant_id="test-tenant-id", + account_name="test-storage-account", + container="test-adls-container", + ) + ).to_workflow() + + request_json_adls = loads( + asset_export_basic_enriched_only_adls.json(by_alias=True, exclude_none=True) + ) + assert request_json_adls == load_json(ASSET_EXPORT_BASIC_ENRICHED_ONLY_ADLS) + + # Case 4: Export all assets using adls + asset_export_basic_all_assets_adls = ( + AssetExportBasic() + .all_assets( + prefix="/test/prefix", + include_description=True, + include_glossaries=True, + include_data_products=True, + include_archived=True, + ) + .object_store(prefix="/test/prefix") + .adls( + client_id="test-client-id", + client_secret="test-client-secret", + tenant_id="test-tenant-id", + account_name="test-storage-account", + container="test-adls-container", + ) + ).to_workflow() + + request_json_adls = loads( + asset_export_basic_all_assets_adls.json(by_alias=True, exclude_none=True) + ) + assert request_json_adls == load_json(ASSET_EXPORT_BASIC_ALL_ASSETS_ADLS) + + # Case 1: Export assets with glossaries only using gcs + asset_export_basic_glossaries_only_gcs = ( + AssetExportBasic() + .glossaries_only(include_archived=True) + .object_store(prefix="/test/prefix") + .gcs( + service_account_json="test-service-account-json", + project_id="test-project-id", + bucket="my-bucket", + ) + ).to_workflow() + + request_json_gcs = loads( + asset_export_basic_glossaries_only_gcs.json(by_alias=True, exclude_none=True) + ) + assert request_json_gcs == load_json(ASSET_EXPORT_BASIC_GLOSSARIES_ONLY_GCS) + + # Case 2: Export assets with Products only using gcs + asset_export_basic_products_only_gcs = ( + AssetExportBasic() + .products_only(include_archived=True) + .object_store(prefix="/test/prefix") + .gcs( + service_account_json="test-service-account-json", + project_id="test-project-id", + bucket="my-bucket", + ) + ).to_workflow() + + request_json_gcs = loads( + asset_export_basic_products_only_gcs.json(by_alias=True, exclude_none=True) + ) + assert request_json_gcs == load_json(ASSET_EXPORT_BASIC_PRODUCTS_ONLY_GCS) + + # Case 3: Export assets with Enriched only using adls + asset_export_basic_enriched_only_gcs = ( + AssetExportBasic() + .enriched_only( + prefix="/test/prefix", + include_description=True, + include_glossaries=True, + include_data_products=True, + include_archived=True, + ) + .object_store(prefix="/test/prefix") + .gcs( + service_account_json="test-service-account-json", + project_id="test-project-id", + bucket="my-bucket", + ) + ).to_workflow() + + request_json_gcs = loads( + asset_export_basic_enriched_only_gcs.json(by_alias=True, exclude_none=True) + ) + assert request_json_gcs == load_json(ASSET_EXPORT_BASIC_ENRICHED_ONLY_GCS) + + # Case 4: Export all assets using adls + asset_export_basic_all_assets_gcs = ( + AssetExportBasic() + .all_assets( + prefix="/test/prefix", + include_description=True, + include_glossaries=True, + include_data_products=True, + include_archived=True, + ) + .object_store(prefix="/test/prefix") + .gcs( + service_account_json="test-service-account-json", + project_id="test-project-id", + bucket="my-bucket", + ) + ).to_workflow() + + request_json_gcs = loads( + asset_export_basic_all_assets_gcs.json(by_alias=True, exclude_none=True) + ) + assert request_json_gcs == load_json(ASSET_EXPORT_BASIC_ALL_ASSETS_GCS) + + +def test_relational_assets_builder(mock_package_env): + # Case 1: Build/Update relational assets from S3 with advanced configuration + relational_assets_builder_s3 = ( + RelationalAssetsBuilder() + .object_store( + prefix="/test/prefix", + object_key="assets-test.csv", + ) + .s3( + access_key="test-access-key", + secret_key="test-secret-key", + bucket="my-bucket", + region="us-west-1", + ) + .assets_semantics( + input_handling=AssetInputHandling.UPSERT, + delta_handling=AssetDeltaHandling.INCREMENTAL, + ) + .options( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + field_separator=",", + batch_size=20, + ) + ).to_workflow() + + request_json_s3 = loads( + relational_assets_builder_s3.json(by_alias=True, exclude_none=True) + ) + assert request_json_s3 == load_json(RELATIONAL_ASSETS_BUILDER_S3) + + # Case 2: Build/Update relational assets from adls with advanced configuration + relational_assets_builder_adls = ( + RelationalAssetsBuilder() + .object_store( + prefix="/test/prefix", + object_key="assets-test.csv", + ) + .adls( + client_id="test-client-id", + client_secret="test-client-secret", + tenant_id="test-tenant-id", + account_name="test-storage-account", + container="test-adls-container", + ) + .assets_semantics( + input_handling=AssetInputHandling.PARTIAL, + delta_handling=AssetDeltaHandling.FULL_REPLACEMENT, + removal_type=AssetRemovalType.ARCHIVE, + ) + .options( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + field_separator=",", + batch_size=20, + ) + ).to_workflow() + + request_json_adls = loads( + relational_assets_builder_adls.json(by_alias=True, exclude_none=True) + ) + assert request_json_adls == load_json(RELATIONAL_ASSETS_BUILDER_ADLS) + + # Case 3: Build/Update relational assets from gcs with advanced configuration + relational_assets_builder_gcs = ( + RelationalAssetsBuilder() + .object_store( + prefix="/test/prefix", + object_key="assets-test.csv", + ) + .gcs( + service_account_json="test-service-account-json", + project_id="test-project-id", + bucket="my-bucket", + ) + .assets_semantics( + input_handling=AssetInputHandling.UPDATE, + delta_handling=AssetDeltaHandling.FULL_REPLACEMENT, + removal_type=AssetRemovalType.PURGE, + ) + .options( + remove_attributes=[Asset.CERTIFICATE_STATUS, Asset.ANNOUNCEMENT_TYPE], + fail_on_errors=True, + field_separator=",", + batch_size=20, + ) + ).to_workflow() + + request_json_gcs = loads( + relational_assets_builder_gcs.json(by_alias=True, exclude_none=True) + ) + assert request_json_gcs == load_json(RELATIONAL_ASSETS_BUILDER_GCS) + + @pytest.mark.parametrize( "test_assets", [