Voicu aws/refresh datasets command (#754)

* Refresh datasets in status command

---------

Co-authored-by: Voicu Chirtes <voicuaws@gmail.com>
Co-authored-by: Voicu <99695945+VoicuAWS@users.noreply.github.com>

cid/cli.py

@@ -1,4 +1,6 @@
+import os
 import logging
+import platform
 
 import click
 
@@ -14,7 +16,6 @@
 print(f'{prog_name} {version}\n')
 
 if __version__ != latest_version and latest_version != 'UNDEFINED':
-
     print('\033[93mUPDATE AVAILABLE\033[0m')
     print(f'\033[93mA new version {latest_version} is available, please consider update cid-cmd package via pip\033[0m\n\n')
     logger.info(f'A new version {latest_version} is available, please consider update cid-cmd package via pip')
@@ -67,6 +68,11 @@ def wrapper(ctx, **kwargs):
 @click.option('-y', '--yes', help='confirm all', is_flag=True, default=False)
 @click.pass_context
 def main(ctx, **kwargs):
+
+    # enable color for windows terminal
+    if platform.system() == "Windows":
+        os.system('color') #nosec B605, B607
+
     ctx.obj = Cid(**kwargs)
 
 

cid/common.py

@@ -23,7 +23,7 @@
 from cid import utils
 from cid.base import CidBase
 from cid.plugin import Plugin
-from cid.utils import get_parameter, get_parameters, set_parameters, unset_parameter, get_yesno_parameter, cid_print, isatty, merge_objects
+from cid.utils import get_parameter, get_parameters, set_parameters, unset_parameter, get_yesno_parameter, cid_print, isatty, merge_objects, IsolatedParameters
 from cid.helpers.account_map import AccountMap
 from cid.helpers import Athena, CUR, Glue, QuickSight, Dashboard, Dataset, Datasource, csv2view, Organizations
 from cid.helpers.quicksight.template import Template as CidQsTemplate
@@ -631,24 +631,64 @@ def open(self, dashboard_id, **kwargs):
     @command
     def status(self, dashboard_id, **kwargs):
         """Check QuickSight dashboard status"""
-
-        if not dashboard_id:
-            if not self.qs.dashboards:
-                print('No deployed dashboards found')
-                return
-            dashboard_id = self.qs.select_dashboard(force=True)
+        next_selection = None
+        while next_selection != 'exit':
             if not dashboard_id:
-                print('No dashboard selected')
-                return
-            dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
-        else:
+                if not self.qs.dashboards:
+                    print('No deployed dashboards found')
+                    return
+                dashboard_id = self.qs.select_dashboard(force=True)
+                if not dashboard_id:
+                    print('No dashboard selected')
+                    return
             dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
 
-        if dashboard is not None:
-            dashboard.display_status()
-            dashboard.display_url(self.qs_url, **self.qs_url_params)
-        else:
-            click.echo('not deployed.')
+            if dashboard is not None:
+                dashboard.display_status()
+                dashboard.display_url(self.qs_url, **self.qs_url_params)
+                with IsolatedParameters():
+                    next_selection = get_parameter(
+                        param_name=f'{dashboard.id}',
+                        message="Please make a selection",
+                        choices={
+                            '[◀] Back': 'back',
+                            '[↗] Open': 'open',
+                            '[◴] Refresh datasets': 'refresh',
+                            '[↺] Update dashboard': 'update',
+                            '[✕] Exit': 'exit',
+                        }
+                    )
+                    if next_selection == 'open':
+                        self.open(dashboard.id, **kwargs)
+
+                    elif next_selection == 'refresh':
+                        dashboard.refresh_datasets()
+
+                    elif next_selection == 'update':
+                        if dashboard.latest:
+                            if not get_yesno_parameter(
+                                    param_name=f'redeploy-{dashboard.id}',
+                                    message=f'\nThe selected dashboard {dashboard.id} is already on the latest version.\nDo you want to re-deploy it?',
+                                    default='no'):
+                                logger.info(f'Not re-deploying {dashboard.id} as it is on latest version.\n')
+                                continue
+                        recursive = get_parameter(
+                            param_name='recursive',
+                            message=f'\nRecursive update the Datasets and Views in addition to the Dashboard update?\nATTENTION: This could lead to the loss of dataset customization.\nRecursive update?',
+                            choices={
+                                '[→] Simple Update (only dashboard)': 'simple',
+                                '[⇶] Recursive Update (dashboard and all dependencies)': 'recursive',
+                            },
+                            default='simple'
+                        ) == 'recursive'
+                        logger.info(f'Updating dashboard: {dashboard.id} with Recursive = {recursive}')
+                        self._deploy(dashboard_id, recursive=recursive, update=True)
+                        logger.info('Rediscover dashboards after update')
+                        self.qs.discover_dashboards()
+                self.qs.clear_dashboard_selection()
+                dashboard_id = None
+            else:
+                cid_print('not deployed.')
 
     @command
     def delete(self, dashboard_id, **kwargs):

cid/helpers/quicksight/__init__.py

@@ -2,6 +2,7 @@
 import json
 import uuid
 import time
+import datetime
 import logging
 from string import Template
 from typing import Dict, List, Union
@@ -15,7 +16,7 @@
 from cid.helpers.quicksight.dataset import Dataset
 from cid.helpers.quicksight.datasource import Datasource
 from cid.helpers.quicksight.template import Template as CidQsTemplate
-from cid.utils import get_parameter, get_parameters, exec_env, cid_print, ago
+from cid.utils import get_parameter, get_parameters, exec_env, cid_print, ago, unset_parameter
 from cid.exceptions import CidCritical, CidError
 
 logger = logging.getLogger(__name__)
@@ -604,6 +605,10 @@ def list_data_sources(self) -> list:
             logger.debug(exc, exc_info=True)
             return list()
 
+    def clear_dashboard_selection (self):
+        """ Clears the current dashboard selection. """
+        unset_parameter('dashboard-id')
+
     def select_dashboard(self, force=False) -> str:
         """ Select from a list of discovered dashboards """
         dashboard_id = get_parameters().get('dashboard-id')
@@ -614,8 +619,9 @@ def select_dashboard(self, force=False) -> str:
             return None
         choices = {}
         for dashboard in self.dashboards.values():
-            health = 'healthy' if dashboard.health else 'unhealthy'
-            key = f'{dashboard.name} ({dashboard.arn}, {health}, {dashboard.status})'
+            health = '' if dashboard.health else ' UNHEALTHY'
+            status = '' if dashboard.status == 'up to date' else ' ' + dashboard.status.upper()
+            key = f'{dashboard.name} ({dashboard.arn.split("/")[-1]}){health}{status}'
             notice = dashboard.definition.get('deprecationNotice', '')
             if notice:
                 key = f'{key} {notice}'
@@ -950,6 +956,29 @@ def discover_datasets(self, _datasets: list=None):
             logger.debug(exc, exc_info=True)
             logger.info('No datasets found')
 
+    def refresh_dataset(self, dataset_id):
+        """ Refresh the dataset """
+
+        logger.info(f'Starting refresh for dataset: {dataset_id}')
+        status = 'FAILED'
+        try:
+            response = self.client.describe_data_set(
+                AwsAccountId=self.account_id,
+                DataSetId=dataset_id)
+            mode = response.get('DataSet').get('ImportMode')
+            if mode == 'DIRECT_QUERY':
+                return mode, 'DIRECT'
+            response = self.client.create_ingestion(
+                DataSetId=dataset_id,
+                IngestionId=datetime.datetime.now().strftime("%d%m%y-%H%M%S-%f"),
+                AwsAccountId=self.account_id)
+            status = response.get('IngestionStatus')
+        except self.client.exceptions.AccessDeniedException:
+            logger.error(f'Access denied refreshing dataset: {dataset_id}')
+        except Exception as exc:
+            logger.debug(exc, exc_info=True)
+            raise CidError(f'Unable to list refresh dataset {dataset_id}: {str(exc)}') from exc
+        return mode, status
 
     def describe_data_source(self, id: str, update: bool=False) -> Datasource:
         """ Describes an AWS QuickSight DataSource """

cid/helpers/quicksight/dashboard.py

@@ -81,7 +81,7 @@ def status(self) -> str:
             if self.version.get('Status') not in ['CREATION_SUCCESSFUL']:
                 self._status = 'broken'
                 self.status_detail = f"{self.version.get('Status')}: {self.version.get('Errors')}"
-            # Not dicovered yet
+            # Not discovered yet
             elif not self.definition:
                 self._status = 'undiscovered'
             # Missing dataset
@@ -147,13 +147,17 @@ def display_status(self) -> None:
         if self.datasets:
             cid_print(f"  <BOLD>Datasets:<END>")
             for dataset_name, dataset_id in  sorted(self.datasets.items()):
-                status = self.qs.get_dataset_last_ingestion(dataset_id) or '<BLUE>DIRECT<END>'
+                status = self.qs.get_dataset_last_ingestion(dataset_id) or '<BLUE>DIRECT<END>' #todo fix this Blue using dataset import type.
                 cid_print(f'    {dataset_name: <36} ({dataset_id: <36}) {status}')
 
-        print('\n')
-        if get_yesno_parameter('display-raw', 'Display dashboard raw data?', default='yes'):
-            print(json.dumps(self.raw, indent=4, sort_keys=True, default=str))
-
     def display_url(self, url_template: str, launch: bool = False, **kwargs) -> None:
         url = url_template.format(dashboard_id=self.id, **kwargs)
         print(f"#######\n####### {self.name} is available at: " + url + "\n#######")
+
+    def refresh_datasets(self) -> None:
+        """Refresh datasets of dashboard"""
+        if self.datasets:
+            cid_print(f"  <BOLD>Refreshing Datasets:<END>")
+            for dataset_name, dataset_id in  sorted(self.datasets.items()):
+                mode, status = self.qs.refresh_dataset(dataset_id)
+                cid_print(f'    {dataset_name: <36} ({dataset_id: <36}) Refresh Status: {status} Mode: {mode}')