Expand functionality of status command to support update and on-demand dataset refresh

cid/common.py

@@ -23,7 +23,7 @@
 from cid import utils
 from cid.base import CidBase
 from cid.plugin import Plugin
-from cid.utils import get_parameter, get_parameters, set_parameters, unset_parameter, get_yesno_parameter, cid_print, isatty, merge_objects
+from cid.utils import get_parameter, get_parameters, set_parameters, unset_parameter, get_yesno_parameter, cid_print, isatty, merge_objects, IsolatedParameters
 from cid.helpers.account_map import AccountMap
 from cid.helpers import Athena, CUR, Glue, QuickSight, Dashboard, Dataset, Datasource, csv2view, Organizations
 from cid.helpers.quicksight.template import Template as CidQsTemplate
@@ -631,24 +631,59 @@ def open(self, dashboard_id, **kwargs):
     @command
     def status(self, dashboard_id, **kwargs):
         """Check QuickSight dashboard status"""
-
-        if not dashboard_id:
-            if not self.qs.dashboards:
-                print('No deployed dashboards found')
-                return
-            dashboard_id = self.qs.select_dashboard(force=True)
+        next_selection = None
+        while next_selection != 'exit':
             if not dashboard_id:
-                print('No dashboard selected')
-                return
-            dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
-        else:
+                if not self.qs.dashboards:
+                    print('No deployed dashboards found')
+                    return
+                dashboard_id = self.qs.select_dashboard(force=True)
+                if not dashboard_id:
+                    print('No dashboard selected')
+                    return
             dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
 
-        if dashboard is not None:
-            dashboard.display_status()
-            dashboard.display_url(self.qs_url, **self.qs_url_params)
-        else:
-            click.echo('not deployed.')
+            if dashboard is not None:
+                dashboard.display_status()
+                dashboard.display_url(self.qs_url, **self.qs_url_params)
+                with IsolatedParameters():
+                    next_selections = {
+                        'Refresh all datasets of this dashboard': 'refresh',
+                        'Update dashboard': 'update',
+                        'Go back to dashboard selection': 'goback',
+                        'Exit': 'exit'
+                    }
+                    next_selection = get_parameter(
+                        param_name='next-selection',
+                        message="Please make a selection",
+                        choices=next_selections,
+                    )
+                    if next_selection == 'refresh':
+                        dashboard.refresh_datasets()
+
+                    if next_selection == 'update':
+                        if dashboard.latest:
+                            if not get_yesno_parameter(
+                                    param_name=f'redeploy-{dashboard.id}',
+                                    message=f'\nThe selected dashboard {dashboard.id} is already on the latest version.\nDo you want to re-deploy it?',
+                                    default='no'):
+                                logger.info(f'Not re-deploying {dashboard.id} as it is on latest version.\n')
+                                continue
+                        recursive = False
+                        if get_yesno_parameter(
+                                    param_name='recursive',
+                                    message=f'\nRecursive update the Datasets and Views in addition to the Dashboard update?\nATTENTION: This could lead to the loss of dataset customization.\nRecursive update?',
+                                    default='no'):
+                            logger.info("Recursive update selected")
+                            recursive = True
+                        logger.info(f'Updating dashboard: {dashboard.id} wiht Recursive = {recursive}')
+                        self._deploy(dashboard_id, recursive=recursive, update=True)
+                        logger.info('Rediscover dashboards after update')
+                        self.qs.discover_dashboards()
+                self.qs.clear_dashboard_selection()
+                dashboard_id = None
+            else:
+                click.echo('not deployed.')
 
     @command
     def delete(self, dashboard_id, **kwargs):

cid/helpers/quicksight/__init__.py

@@ -2,6 +2,7 @@
 import json
 import uuid
 import time
+import datetime
 import logging
 from string import Template
 from typing import Dict, List, Union
@@ -15,7 +16,7 @@
 from cid.helpers.quicksight.dataset import Dataset
 from cid.helpers.quicksight.datasource import Datasource
 from cid.helpers.quicksight.template import Template as CidQsTemplate
-from cid.utils import get_parameter, get_parameters, exec_env, cid_print, ago
+from cid.utils import get_parameter, get_parameters, exec_env, cid_print, ago, unset_parameter
 from cid.exceptions import CidCritical, CidError
 
 logger = logging.getLogger(__name__)
@@ -604,6 +605,10 @@ def list_data_sources(self) -> list:
             logger.debug(exc, exc_info=True)
             return list()
 
+    def clear_dashboard_selection (self):
+        """ Clears the current dashboard selection. """
+        unset_parameter('dashboard-id')
+
     def select_dashboard(self, force=False) -> str:
         """ Select from a list of discovered dashboards """
         dashboard_id = get_parameters().get('dashboard-id')
@@ -914,7 +919,7 @@ def get_dataset_last_ingestion(self, dataset_id) -> str:
         except self.client.exceptions.AccessDeniedException:
             return '<YELLOW>AccessDenied<END>'
         if not ingestions:
-            return None
+            return None #todo fix this using dataset import type. None could be if refresh never ran too, no ? 
         last_ingestion = ingestions[0] # Suppose it is the latest
         status = last_ingestion.get('IngestionStatus')
         time_ago = ago(last_ingestion.get('CreatedTime'))
@@ -950,6 +955,30 @@ def discover_datasets(self, _datasets: list=None):
             logger.debug(exc, exc_info=True)
             logger.info('No datasets found')
 
+    def refresh_dataset(self, dataset_id):
+        """ Refresh the dataset """
+
+        logger.info(f'Starting refresh for dataset: {dataset_id}')
+        status = 'FAILED'
+        try:
+            response = self.client.describe_data_set(
+                AwsAccountId=self.account_id,
+                DataSetId=dataset_id)
+            mode = response.get('DataSet').get('ImportMode')
+            if mode == 'DIRECT_QUERY':
+                return mode, 'DIRECT'
+            response = self.client.create_ingestion(
+                DataSetId=dataset_id,
+                IngestionId=datetime.datetime.now().strftime("%d%m%y-%H%M%S-%f"),
+                AwsAccountId=self.account_id)
+            status = response.get('IngestionStatus')
+        except self.client.exceptions.AccessDeniedException:
+            logger.info(f'Access denied refreshing dataset: {dataset_id}')
+        except Exception as exc:
+            logger.debug(exc, exc_info=True)
+            logger.info(f'Unable to list refresh dataset {dataset_id}: {str(exc)}')
+            raise CidError(f'Unable to list refresh dataset {dataset_id}: {str(exc)}') from exc
+        return mode, status
 
     def describe_data_source(self, id: str, update: bool=False) -> Datasource:
         """ Describes an AWS QuickSight DataSource """

cid/helpers/quicksight/dashboard.py

@@ -147,13 +147,21 @@ def display_status(self) -> None:
         if self.datasets:
             cid_print(f"  <BOLD>Datasets:<END>")
             for dataset_name, dataset_id in  sorted(self.datasets.items()):
-                status = self.qs.get_dataset_last_ingestion(dataset_id) or '<BLUE>DIRECT<END>'
+                status = self.qs.get_dataset_last_ingestion(dataset_id) or '<BLUE>DIRECT<END>' #todo fix this Blue using dataset import type.
                 cid_print(f'    {dataset_name: <36} ({dataset_id: <36}) {status}')
 
-        print('\n')
+        """print('\n')
         if get_yesno_parameter('display-raw', 'Display dashboard raw data?', default='yes'):
-            print(json.dumps(self.raw, indent=4, sort_keys=True, default=str))
+            print(json.dumps(self.raw, indent=4, sort_keys=True, default=str))"""
 
     def display_url(self, url_template: str, launch: bool = False, **kwargs) -> None:
         url = url_template.format(dashboard_id=self.id, **kwargs)
         print(f"#######\n####### {self.name} is available at: " + url + "\n#######")
+
+    def refresh_datasets(self) -> None:
+        """Refresh datasets of dashboard"""
+        if self.datasets:
+            cid_print(f"  <BOLD>Refreshing Datasets:<END>")
+            for dataset_name, dataset_id in  sorted(self.datasets.items()):
+                mode, status = self.qs.refresh_dataset(dataset_id)
+                cid_print(f'    {dataset_name: <36} ({dataset_id: <36}) Refresh Status: {status} Mode: {mode}')