Skip to content

Commit 816db08

Browse files
0xquarkAryamanz29
0xquark
authored and
Aryamanz29
committed
Add unit tests for relational_assets_builder package
Signed-off-by: Karanjot Singh <[email protected]>
1 parent cfcf9a5 commit 816db08

File tree

5 files changed

+583
-50
lines changed

5 files changed

+583
-50
lines changed

‎pyatlan/model/packages/relational_assets_builder.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from __future__ import annotations
22

3+
from json import dumps
34
from typing import List, Optional
45

56
from pyatlan.model.enums import (
@@ -49,9 +50,9 @@ def object_store(
4950
5051
:returns: package, set up to import metadata from object store
5152
"""
53+
self._parameters.append({"name": "import_type", "value": "CLOUD"})
5254
self._parameters.append({"name": "assets_prefix", "value": prefix})
5355
self._parameters.append({"name": "assets_key", "value": object_key})
54-
self._parameters.append({"name": "import_type", "value": "CLOUD"})
5556
self._parameters.append({"name": "cloud_source", "value": "{{credentialGuid}}"})
5657
return self
5758

@@ -206,7 +207,7 @@ def options(
206207
):
207208
remove_attributes = [field.atlan_field_name for field in remove_attributes] # type: ignore
208209
params = {
209-
"assets_attr_to_overwrite": str(remove_attributes),
210+
"assets_attr_to_overwrite": dumps(remove_attributes, separators=(",", ":")),
210211
"assets_fail_on_errors": fail_on_errors,
211212
"assets_field_separator": field_separator,
212213
"assets_batch_size": batch_size,
@@ -224,15 +225,15 @@ def _get_metadata(self) -> WorkflowMetadata:
224225
"orchestration.atlan.com/preview": "true",
225226
"orchestration.atlan.com/verified": "true",
226227
"package.argoproj.io/installer": "argopm",
227-
"package.argoproj.io/name": f"a-t-ratlans-l-a-s-h{self._NAME}",
228+
"package.argoproj.io/name": f"a-t-rcsas-l-a-s-h{self._NAME}",
228229
"package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com",
229230
"orchestration.atlan.com/atlan-ui": "true",
230231
},
231232
annotations={
232233
"orchestration.atlan.com/allowSchedule": "true",
233234
"orchestration.atlan.com/categories": "kotlin,utility",
234235
"orchestration.atlan.com/dependentPackage": "",
235-
"orchestration.atlan.com/docsUrl": f"https://solutions.atlan.com/{self._NAME}",
236+
"orchestration.atlan.com/docsUrl": f"https://solutions.atlan.com/{self._NAME}/",
236237
"orchestration.atlan.com/emoji": "\U0001f680",
237238
"orchestration.atlan.com/icon": self._PACKAGE_ICON,
238239
"orchestration.atlan.com/logo": self._PACKAGE_LOGO, # noqa

‎tests/unit/data/package_requests/relational_assets_builder_adls.json

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
{
2+
"metadata": {
3+
"labels": {
4+
"orchestration.atlan.com/certified": "true",
5+
"orchestration.atlan.com/preview": "true",
6+
"orchestration.atlan.com/source": "relational-assets-builder",
7+
"orchestration.atlan.com/sourceCategory": "utility",
8+
"orchestration.atlan.com/type": "custom",
9+
"orchestration.atlan.com/verified": "true",
10+
"package.argoproj.io/installer": "argopm",
11+
"package.argoproj.io/name": "a-t-rcsas-l-a-s-hrelational-assets-builder",
12+
"package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com",
13+
"orchestration.atlan.com/atlan-ui": "true"
14+
},
15+
"annotations": {
16+
"orchestration.atlan.com/allowSchedule": "true",
17+
"orchestration.atlan.com/categories": "kotlin,utility",
18+
"orchestration.atlan.com/dependentPackage": "",
19+
"orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/relational-assets-builder/",
20+
"orchestration.atlan.com/emoji": "🚀",
21+
"orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-database-light.svg",
22+
"orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-database-light.svg",
23+
"orchestration.atlan.com/name": "Relational Assets Builder",
24+
"package.argoproj.io/author": "Atlan CSA",
25+
"package.argoproj.io/description": "Build (and update) relational assets managed through a CSV file.",
26+
"package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/relational-assets-builder",
27+
"package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]",
28+
"package.argoproj.io/name": "@csa/relational-assets-builder",
29+
"package.argoproj.io/parent": ".",
30+
"package.argoproj.io/registry": "https://packages.atlan.com",
31+
"package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git",
32+
"package.argoproj.io/support": "[email protected]",
33+
"orchestration.atlan.com/atlanName": "csa-relational-assets-builder-123456"
34+
},
35+
"name": "csa-relational-assets-builder-123456",
36+
"namespace": "default"
37+
},
38+
"spec": {
39+
"templates": [
40+
{
41+
"name": "main",
42+
"dag": {
43+
"tasks": [
44+
{
45+
"name": "run",
46+
"arguments": {
47+
"parameters": [
48+
{
49+
"name": "import_type",
50+
"value": "CLOUD"
51+
},
52+
{
53+
"name": "assets_prefix",
54+
"value": "/test/prefix"
55+
},
56+
{
57+
"name": "assets_key",
58+
"value": "assets-test.csv"
59+
},
60+
{
61+
"name": "cloud_source",
62+
"value": "{{credentialGuid}}"
63+
},
64+
{
65+
"name": "assets_upsert_semantic",
66+
"value": "partial"
67+
},
68+
{
69+
"name": "delta_semantic",
70+
"value": "full"
71+
},
72+
{
73+
"name": "delta_removal_type",
74+
"value": "archive"
75+
},
76+
{
77+
"name": "assets_attr_to_overwrite",
78+
"value": "[\"certificateStatus\",\"announcementType\"]"
79+
},
80+
{
81+
"name": "assets_fail_on_errors",
82+
"value": true
83+
},
84+
{
85+
"name": "assets_field_separator",
86+
"value": ","
87+
},
88+
{
89+
"name": "assets_batch_size",
90+
"value": 20
91+
}
92+
]
93+
},
94+
"templateRef": {
95+
"name": "csa-relational-assets-builder",
96+
"template": "main",
97+
"clusterScope": true
98+
}
99+
}
100+
]
101+
}
102+
}
103+
],
104+
"entrypoint": "main",
105+
"workflowMetadata": {
106+
"annotations": {
107+
"package.argoproj.io/name": "@csa/relational-assets-builder"
108+
}
109+
}
110+
},
111+
"payload": [
112+
{
113+
"parameter": "credentialGuid",
114+
"type": "credential",
115+
"body": {
116+
"name": "csa-relational-assets-builder-123456-0",
117+
"authType": "adls",
118+
"username": "test-client-id",
119+
"password": "test-client-secret",
120+
"extra": {
121+
"azure_tenant_id": "test-tenant-id",
122+
"storage_account_name": "test-storage-account",
123+
"adls_container": "test-adls-container"
124+
},
125+
"connectorConfigName": "csa-connectors-objectstore"
126+
}
127+
}
128+
]
129+
}

‎tests/unit/data/package_requests/relational_assets_builder_gcs.json

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
{
2+
"metadata": {
3+
"labels": {
4+
"orchestration.atlan.com/certified": "true",
5+
"orchestration.atlan.com/preview": "true",
6+
"orchestration.atlan.com/source": "relational-assets-builder",
7+
"orchestration.atlan.com/sourceCategory": "utility",
8+
"orchestration.atlan.com/type": "custom",
9+
"orchestration.atlan.com/verified": "true",
10+
"package.argoproj.io/installer": "argopm",
11+
"package.argoproj.io/name": "a-t-rcsas-l-a-s-hrelational-assets-builder",
12+
"package.argoproj.io/registry": "httpsc-o-l-o-ns-l-a-s-hs-l-a-s-hpackages.atlan.com",
13+
"orchestration.atlan.com/atlan-ui": "true"
14+
},
15+
"annotations": {
16+
"orchestration.atlan.com/allowSchedule": "true",
17+
"orchestration.atlan.com/categories": "kotlin,utility",
18+
"orchestration.atlan.com/dependentPackage": "",
19+
"orchestration.atlan.com/docsUrl": "https://solutions.atlan.com/relational-assets-builder/",
20+
"orchestration.atlan.com/emoji": "🚀",
21+
"orchestration.atlan.com/icon": "http://assets.atlan.com/assets/ph-database-light.svg",
22+
"orchestration.atlan.com/logo": "http://assets.atlan.com/assets/ph-database-light.svg",
23+
"orchestration.atlan.com/name": "Relational Assets Builder",
24+
"package.argoproj.io/author": "Atlan CSA",
25+
"package.argoproj.io/description": "Build (and update) relational assets managed through a CSV file.",
26+
"package.argoproj.io/homepage": "https://packages.atlan.com/-/web/detail/@csa/relational-assets-builder",
27+
"package.argoproj.io/keywords": "[\"kotlin\",\"utility\"]",
28+
"package.argoproj.io/name": "@csa/relational-assets-builder",
29+
"package.argoproj.io/parent": ".",
30+
"package.argoproj.io/registry": "https://packages.atlan.com",
31+
"package.argoproj.io/repository": "git+https://github.com/atlanhq/marketplace-packages.git",
32+
"package.argoproj.io/support": "[email protected]",
33+
"orchestration.atlan.com/atlanName": "csa-relational-assets-builder-123456"
34+
},
35+
"name": "csa-relational-assets-builder-123456",
36+
"namespace": "default"
37+
},
38+
"spec": {
39+
"templates": [
40+
{
41+
"name": "main",
42+
"dag": {
43+
"tasks": [
44+
{
45+
"name": "run",
46+
"arguments": {
47+
"parameters": [
48+
{
49+
"name": "import_type",
50+
"value": "CLOUD"
51+
},
52+
{
53+
"name": "assets_prefix",
54+
"value": "/test/prefix"
55+
},
56+
{
57+
"name": "assets_key",
58+
"value": "assets-test.csv"
59+
},
60+
{
61+
"name": "cloud_source",
62+
"value": "{{credentialGuid}}"
63+
},
64+
{
65+
"name": "assets_upsert_semantic",
66+
"value": "update"
67+
},
68+
{
69+
"name": "delta_semantic",
70+
"value": "full"
71+
},
72+
{
73+
"name": "delta_removal_type",
74+
"value": "purge"
75+
},
76+
{
77+
"name": "assets_attr_to_overwrite",
78+
"value": "[\"certificateStatus\",\"announcementType\"]"
79+
},
80+
{
81+
"name": "assets_fail_on_errors",
82+
"value": true
83+
},
84+
{
85+
"name": "assets_field_separator",
86+
"value": ","
87+
},
88+
{
89+
"name": "assets_batch_size",
90+
"value": 20
91+
}
92+
]
93+
},
94+
"templateRef": {
95+
"name": "csa-relational-assets-builder",
96+
"template": "main",
97+
"clusterScope": true
98+
}
99+
}
100+
]
101+
}
102+
}
103+
],
104+
"entrypoint": "main",
105+
"workflowMetadata": {
106+
"annotations": {
107+
"package.argoproj.io/name": "@csa/relational-assets-builder"
108+
}
109+
}
110+
},
111+
"payload": [
112+
{
113+
"parameter": "credentialGuid",
114+
"type": "credential",
115+
"body": {
116+
"name": "csa-relational-assets-builder-123456-0",
117+
"authType": "gcs",
118+
"username": "test-project-id",
119+
"password": "test-service-account-json",
120+
"extra": {
121+
"gcs_bucket": "my-bucket"
122+
},
123+
"connectorConfigName": "csa-connectors-objectstore"
124+
}
125+
}
126+
]
127+
}

0 commit comments

Comments
 (0)