add fuzzy input and refactor discovery

* add fuzzy input

*

* fix tests

* fix tests

* refactor discovery

* replace discover

* refactoring: speed up working with dashboard

* get rid of click

* enforce all yes

* fix deletion

* fix-yes-no

* allow cur1 in dependency

cid/common.py

@@ -3,14 +3,14 @@
 import urllib
 import logging
 import functools
+import webbrowser
 from string import Template
 from typing import Dict
 from pkg_resources import resource_string
 from importlib.metadata import entry_points
 from functools import cached_property
 
 import yaml
-import click
 import requests
 from botocore.exceptions import ClientError, NoCredentialsError, CredentialRetrievalError
 
@@ -362,12 +362,11 @@ def get_template_parameters(self, parameters: dict, param_prefix: str='', others
             prefix = '' if value.get('global') else param_prefix
             if isinstance(value, str):
                 params[key] = value
-            elif isinstance(value, dict) and str(value.get('type')).endswith('tag_and_cost_category_fields'):
-                cur_version = '2' if str(value.get('type')).startswith('cur2.') else '1'
+            elif isinstance(value, dict) and value.get('type') == 'cur.tag_and_cost_category_fields':
                 params[key] = get_parameter(
                     param_name=prefix + key,
                     message=f"Required parameter: {key} ({value.get('description')})",
-                    choices=self.get_cur(cur_version).tag_and_cost_category_fields + ["'none'"],
+                    choices=self.cur.tag_and_cost_category_fields + ["'none'"],
                 )
             elif isinstance(value, dict) and value.get('type') == 'athena':
                 if get_parameters().get(prefix + key): # priority to user input
@@ -434,9 +433,7 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs
 
         self.ensure_subscription()
 
-        # In case if we cannot discover datasets, we need to discover dashboards
-        # TODO: check if datasets returns explicit permission denied and only then discover dashboards as a workaround
-        self.qs.dashboards
+        self.qs.pre_discover()
 
         dashboard_id = dashboard_id or get_parameters().get('dashboard-id')
         category_filter = [cat for cat in get_parameters().get('category', '').upper().split(',') if cat]
@@ -479,7 +476,7 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs
         dashboard_definition = self.get_definition("dashboard", id=dashboard_id)
         dashboard = None
         try:
-            dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
+            dashboard = self.qs.discover_dashboard(dashboard_id)
         except CidCritical:
             pass
 
@@ -528,11 +525,10 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs
 
         compatible = self.check_dashboard_version_compatibility(dashboard_id)
         if not recursive and compatible == False:
-            if get_parameter(
+            if get_yesno_parameter(
                 param_name=f'confirm-recursive',
                 message=f'This is a major update and require recursive action. This could lead to the loss of dataset customization. Continue anyway?',
-                choices=['yes', 'no'],
-                default='yes') != 'yes':
+                default='yes'):
                 return
             logger.info("Switch to recursive mode")
             recursive = True
@@ -654,19 +650,17 @@ def open(self, dashboard_id, **kwargs):
         if not dashboard_id:
             dashboard_id = self.qs.select_dashboard(force=True)
 
-        dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
-
-        click.echo('Getting dashboard status...', nl=False)
-        if dashboard is not None:
-            if dashboard.version.get('Status') not in ['CREATION_SUCCESSFUL']:
-                print(json.dumps(dashboard.version.get('Errors'),
-                      indent=4, sort_keys=True, default=str))
-                click.echo(
-                    f'\nDashboard is unhealthy, please check errors above.')
-            click.echo('healthy, opening...')
-            click.launch(self.qs_url.format(dashboard_id=dashboard_id, **self.qs_url_params))
-        else:
-            click.echo('not deployed.')
+        dashboard = self.qs.discover_dashboard(dashboard_id)
+
+        logger.info('Getting dashboard status...')
+        if not dashboard:
+            logger.error(f'{dashboard_id} is not deployed.')
+            return None
+        if dashboard.version.get('Status') not in ['CREATION_SUCCESSFUL', 'UPDATE_IN_PROGRESS', 'UPDATE_SUCCESSFUL']:
+            cid_print(json.dumps(dashboard.version.get('Errors'), indent=4, sort_keys=True, default=str))
+            cid_print(f'Dashboard {dashboard_id} is unhealthy, please check errors above.')
+        logger.info('healthy, opening...')
+        webbrowser.open(self.qs_url.format(dashboard_id=dashboard_id, **self.qs_url_params))
 
         return dashboard_id
 
@@ -683,7 +677,7 @@ def status(self, dashboard_id, **kwargs):
                 if not dashboard_id:
                     print('No dashboard selected')
                     return
-            dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
+            dashboard = self.qs.discover_dashboard(dashboard_id)
 
             if dashboard is not None:
                 dashboard.display_status()
@@ -725,11 +719,7 @@ def status(self, dashboard_id, **kwargs):
                         logger.info(f'Updating dashboard: {dashboard.id} with Recursive = {recursive}')
                         self._deploy(dashboard_id, recursive=recursive, update=True)
                         logger.info('Rediscover dashboards after update')
-
-                        refresh_overrides = [
-                            dashboard.id
-                        ]
-                        self.qs.discover_dashboards(refresh_overrides = refresh_overrides)
+                        self.qs.discover_dashboards(refresh_overrides=[dashboard.id])
                 self.qs.clear_dashboard_selection()
                 dashboard_id = None
             else:
@@ -748,7 +738,7 @@ def delete(self, dashboard_id, **kwargs):
                 return
 
         if self.qs.dashboards and dashboard_id in self.qs.dashboards:
-            datasets = self.qs.discover_dashboard(dashboardId=dashboard_id).datasets # save for later
+            datasets = self.qs.discover_dashboard(dashboard_id).datasets # save for later
         else:
             dashboard_definition = self.get_definition("dashboard", id=dashboard_id)
             datasets = {d: None for d in (dashboard_definition or {}).get('dependsOn', {}).get('datasets', [])}
@@ -793,16 +783,14 @@ def delete_dataset(self, name: str, id: str=None):
                         logger.debug(f'Picking the first of dataset databases: {dataset.schemas}')
                         self.athena.DatabaseName = schema
 
-                    if get_parameter(
+                    if get_yesno_parameter(
                         param_name=f'confirm-{dataset.name}',
                         message=f'Delete QuickSight Dataset {dataset.name}?',
-                        choices=['yes', 'no'],
-                        default='no') == 'yes':
+                        default='no'):
                         print(f'Deleting dataset {dataset.name} ({dataset.id})')
                         self.qs.delete_dataset(dataset.id)
                     else:
-                        logger.info(f'Skipping dataset {dataset.name}')
-                        print      (f'Skipping dataset {dataset.name}')
+                        cid_print(f'Skipping dataset {dataset.name}')
                         return False
                 if not dataset.datasources:
                     continue
@@ -855,7 +843,7 @@ def delete_view(self, view_name):
     def cleanup(self, **kwargs):
         """Delete unused resources (QuickSight datasets not used in Dashboards)"""
 
-        self.qs.discover_dashboards()
+        self.qs.pre_discover()
         self.qs.discover_datasets()
         references = {}
         for dashboard in self.qs.dashboards.values():
@@ -893,9 +881,9 @@ def _share(self, dashboard_id, **kwargs):
                 return
         else:
             # Describe dashboard by the ID given, no discovery
-            self.qs.discover_dashboard(dashboardId=dashboard_id)
+            self.qs.discover_dashboard(dashboard_id)
 
-        dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
+        dashboard = self.qs.discover_dashboard(dashboard_id)
 
         if dashboard is None:
             print('not deployed.')
@@ -1066,7 +1054,7 @@ def update(self, dashboard_id, recursive=False, force=False, **kwargs):
     def check_dashboard_version_compatibility(self, dashboard_id):
         """ Returns True | False | None if could not check """
         try:
-            dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
+            dashboard = self.qs.discover_dashboard(dashboard_id)
         except CidCritical:
             print(f'Dashboard "{dashboard_id}" is not deployed')
             return None
@@ -1086,7 +1074,7 @@ def check_dashboard_version_compatibility(self, dashboard_id):
 
     def update_dashboard(self, dashboard_id, dashboard_definition):
 
-        dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
+        dashboard = self.qs.discover_dashboard(dashboard_id)
         if not dashboard:
             print(f'Dashboard "{dashboard_id}" is not deployed')
             return
@@ -1363,7 +1351,7 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non
         # Read dataset definition from template
         data = self.get_data_from_definition('dataset', dataset_definition)
         template = Template(json.dumps(data))
-        cur1_required = dataset_definition.get('dependsOn', dict()).get('cur')
+        cur1_required = dataset_definition.get('dependsOn', dict()).get('cur') or dataset_definition.get('dependsOn', dict()).get('cur1')
         cur2_required = dataset_definition.get('dependsOn', dict()).get('cur2')
         athena_datasource = None
 

cid/helpers/athena.py

@@ -81,16 +81,18 @@ def DatabaseName(self) -> str:
         athena_databases = self.list_databases()
 
         # check if we have a default database
-        print(athena_databases)
+        logger.info(f'athena_databases = {athena_databases}')
         default_databases = [database for database in athena_databases if database == self.defaults.get('DatabaseName')]
+        if 'cid_cur' in athena_databases:
+            default_databases = ['cid_cur']
 
         # Ask user
         choices = list(athena_databases)
         if self.defaults.get('DatabaseName') not in choices:
             choices.append(self.defaults.get('DatabaseName') + ' (CREATE NEW)')
         self._DatabaseName = get_parameter(
             param_name='athena-database',
-            message="Select AWS Athena database to use",
+            message="Select AWS Athena database to use as default",
             choices=choices,
             default=default_databases[0] if default_databases else None,
         )
@@ -391,11 +393,10 @@ def wait_for_view(self, view_name: str, poll_interval=1, timeout=60) -> None:
 
 
     def delete_table(self, name: str, catalog: str=None, database: str=None):
-        if get_parameter(
+        if not get_yesno_parameter(
                 param_name=f'confirm-{name}',
                 message=f'Delete Athena table {name}?',
-                choices=['yes', 'no'],
-                default='no') != 'yes':
+                default='no'):
             return False
 
         try:
@@ -415,11 +416,10 @@ def delete_table(self, name: str, catalog: str=None, database: str=None):
         return True
 
     def delete_view(self, name: str, catalog: str=None, database: str=None):
-        if get_parameter(
+        if not get_yesno_parameter(
                 param_name=f'confirm-{name}',
                 message=f'Delete Athena view {name}?',
-                choices=['yes', 'no'],
-                default='no') != 'yes':
+                default='no'):
             return False
 
         try:
@@ -545,7 +545,8 @@ def create_or_update_view(self, view_name, view_query):
                         param_name='view-' + view_name + '-override',
                         message=f'The existing view is different. Override?',
                         choices=['retry diff', 'proceed and override', 'keep existing', 'exit'],
-                        default='retry diff'
+                        default='retry diff',
+                        fuzzy=False,
                     )
                     if choice == 'retry diff':
                         unset_parameter('view-' + view_name + '-override')