make safer update (#1026)

* add incremental refresh option

cid/common.py

@@ -1090,11 +1090,10 @@ def update_dashboard(self, dashboard_id, dashboard_definition):
             print(f'Dashboard "{dashboard_id}" is not deployed')
             return
 
-        if isinstance(dashboard.deployedTemplate, CidQsTemplate):
-            print(f'Deployed template: {dashboard.deployedTemplate.arn}')
-        if isinstance(dashboard.sourceTemplate, CidQsTemplate):
-            print(f"Latest template:   {dashboard.sourceTemplate.arn}/version/{dashboard.sourceTemplate.version}")
-
+        if isinstance(dashboard.deployed_template, CidQsTemplate):
+            print(f'Deployed template: {dashboard.deployed_template.arn}')
+        if isinstance(dashboard.source_template, CidQsTemplate):
+            print(f"Latest template:   {dashboard.source_template.arn}/version/{dashboard.source_template.version}")
         try:
             cid_print(f'\nUpdating {dashboard_id} from <BOLD>{dashboard.cid_version}<END> to <BOLD>{dashboard.latest_available_cid_version}<END>')
         except:

cid/helpers/quicksight/__init__.py

@@ -255,23 +255,23 @@ def _safe_int(value, default=None):
                 try:
                     _template = self.describe_template(**params)
                     if isinstance(_template, CidQsTemplate):
-                        dashboard.deployedTemplate = _template
+                        dashboard.deployed_template = _template
                 except Exception as exc:
                     logger.debug(exc, exc_info=True)
                     logger.info(f'Unable to describe template for {dashboardId}, {exc}')
             else:
                 logger.info("Minimum template version could not be found for Dashboard {dashboardId}: {_template_arn}, deployed template could not be described")
         else: # Dashboard is not template based but definition based
             try:
-                dashboard.deployedDefinition = self.describe_dashboard_definition(dashboard_id=dashboardId, refresh=refresh)
+                dashboard.deployed_definition = self.describe_dashboard_definition(dashboard_id=dashboardId, refresh=refresh)
             except CidError as exc:
                 logger.info('Exception on reading dashboard definition {dashboardId}: {exc}. Not critical. Continue.')
 
         if 'data' in _definition:
             # Resolve source definition (the latest definition publicly available)
             data_stream = io.StringIO(_definition["data"])
             definition_data = yaml.safe_load(data_stream)
-            dashboard.sourceDefinition = CidQsDefinition(definition_data)
+            dashboard.source_definition = CidQsDefinition(definition_data)
 
         # Fetch datasets (works for both TEMPLATE and DEFINITION based dashboards)
         for dataset in dashboard.version.get('DataSetArns', []):
@@ -289,21 +289,21 @@ def _safe_int(value, default=None):
                 logger.info(f'Invalid dataset {dataset_id}')
         logger.info(f"{dashboard.name} has {len(dashboard.datasets)} datasets")
 
-        # Fetch the latest version of sourceTemplate referenced in definition
+        # Fetch the latest version of source_template referenced in definition
         source_template_account_id = _definition.get('sourceAccountId')
         template_id = _definition.get('templateId')
         region = _definition.get('region', 'us-east-1')
         if template_id:
             try:
                 logger.debug(f'Loading latest source template {template_id} from source account {source_template_account_id} in {region}')
                 template = self.describe_template(template_id, account_id=source_template_account_id, region=region)
-                dashboard.sourceTemplate = template
+                dashboard.source_template = template
             except Exception as exc:
                 logger.debug(exc, exc_info=True)
                 logger.info(f'Unable to describe template {template_id} in {source_template_account_id} ({region})')
 
         # Checking for version override in template definition
-        for dashboard_template in [dashboard.deployedTemplate, dashboard.sourceTemplate]:
+        for dashboard_template in [dashboard.deployed_template, dashboard.source_template]:
             if not isinstance(dashboard_template, CidQsTemplate)\
                 or int(dashboard_template.version) <= 0 \
                 or not version_obj:
@@ -585,21 +585,18 @@ def discover_data_sources(self) -> None:
                     self.describe_data_source(d)
         except Exception as exc:
             logger.debug(exc, exc_info=True)
-    
+
     def discover_dashboards(self, refresh_overrides: List[str]=[], refresh: bool = False) -> None:
         """ Discover deployed dashboards
-        
         :param refresh_overrides: a list of dashboard ids to refresh
         :param refresh: force refresh all dashboards
         """
-
         if refresh or self._dashboards is None:
             self._dashboards = {}
         else:
             for dashboard_id in refresh_overrides:
                 if dashboard_id in self._dashboards:
                     del self._dashboards[dashboard_id]
-
         logger.info('Discovering deployed dashboards')
         deployed_dashboards=self.list_dashboards()
         logger.info(f'Found {len(deployed_dashboards)} deployed dashboards')
@@ -611,9 +608,7 @@ def discover_dashboards(self, refresh_overrides: List[str]=[], refresh: bool = F
             dashboard_id = dashboard.get('DashboardId')
             bar.set_description(f'Discovering {dashboard_name[:10]:<10}', refresh=True)
             logger.info(f'Discovering "{dashboard_name}"')
-
             refresh = dashboard_id in refresh_overrides
-
             self.discover_dashboard(dashboard_id, refresh=refresh)
 
     def list_dashboards(self) -> list:
@@ -1259,14 +1254,31 @@ def ensure_dataset_refresh_schedule(self, dataset_id, schedules: list):
                 return
 
         for schedule in schedules:
-
             # Get the list of existing schedules with the same id
             existing_schedule = None
             for existing in existing_schedules:
                 if schedule["ScheduleId"] == existing["ScheduleId"]:
                     existing_schedule = existing
                     break
 
+            refresh_configuration = schedule.pop('RefreshConfiguration', {})
+            if refresh_configuration:
+                # schedule exists so we need to update
+                logger.debug(f'Updating refresh schedule configuration with id {schedule["ScheduleId"]} for dataset {dataset_id}.')
+                try:
+                    self.client.put_data_set_refresh_properties(
+                        DataSetId=dataset_id,
+                        AwsAccountId=self.account_id,
+                        DataSetRefreshProperties={'RefreshConfiguration': refresh_configuration}
+                    )
+                    logger.debug(f'Refresh schedule configuration with id {schedule["ScheduleId"]} for dataset {dataset_id} is updated.')
+                except self.client.exceptions.ResourceNotFoundException:
+                    logger.error(f'Unable to update refresh schedule configuration with id {schedule["ScheduleId"]}. Dataset {dataset_id} does not exist.')
+                except self.client.exceptions.AccessDeniedException:
+                    logger.error(f'Unable to update refresh schedule configuration with id {schedule["ScheduleId"]}. Please add quicksight:UpdateDataSet permission.')
+                except Exception as exc:
+                    logger.error(f'Unable to update refresh schedule configuration with id {schedule["ScheduleId"]} for dataset "{dataset_id}": {str(exc)}')
+
             # Verify that all schedule parameters are set
             schedule["ScheduleId"] = schedule.get("ScheduleId", "cid")
             if "ScheduleFrequency" not in schedule:

cid/helpers/quicksight/dashboard.py

@@ -18,13 +18,13 @@ def __init__(self, raw: dict, qs=None) -> None:
         # Initialize properties
         self.datasets: Dict[str, str] = {}
         # Deployed template
-        self._deployedTemplate: CidQsTemplate = None
-        self._deployedDefinition: CidQsDefinition = None
+        self._deployed_template: CidQsTemplate = None
+        self._deployed_definition: CidQsDefinition = None
         self._status = str()
         self.status_detail = str()
         # Source template in origin account
-        self.sourceTemplate: CidQsTemplate = None
-        self.sourceDefinition: CidQsDefinition = None
+        self.source_template: CidQsTemplate = None
+        self.source_definition: CidQsDefinition = None
         self.qs = qs
 
     @property
@@ -36,75 +36,69 @@ def version(self) -> dict:
         return self.get_property('Version')
 
     @property
-    def deployedTemplate(self) -> CidQsTemplate:
-        return self._deployedTemplate
+    def deployed_template(self) -> CidQsTemplate:
+        return self._deployed_template
 
-    @deployedTemplate.setter
-    def deployedTemplate(self, template: CidQsTemplate) -> None:
-        self._deployedTemplate = template
+    @deployed_template.setter
+    def deployed_template(self, template: CidQsTemplate) -> None:
+        self._deployed_template = template
 
     @property
-    def deployedDefinition(self) -> CidQsTemplate:
-        return self._deployedDefinition
+    def deployed_definition(self) -> CidQsTemplate:
+        return self._deployed_definition
 
-    @deployedDefinition.setter
-    def deployedDefinition(self, definition: CidQsDefinition) -> None:
-        self._deployedDefinition = definition
+    @deployed_definition.setter
+    def deployed_definition(self, definition: CidQsDefinition) -> None:
+        self._deployed_definition = definition
 
     @property
     def template_id(self) -> str:
-        if isinstance(self.deployedTemplate, CidQsTemplate):
-            return self.deployedTemplate.id
+        if isinstance(self.deployed_template, CidQsTemplate):
+            return self.deployed_template.id
         return None
 
     @property
     def template_arn(self) -> str:
-        if isinstance(self.deployedTemplate, CidQsTemplate):
-            return self.deployedTemplate.arn
+        if isinstance(self.deployed_template, CidQsTemplate):
+            return self.deployed_template.arn
         return None
 
     @property
     def deployed_cid_version(self) -> int:
-        if isinstance(self.deployedTemplate, CidQsTemplate):
-            return self.deployedTemplate.cid_version
-        elif isinstance(self.deployedDefinition, CidQsDefinition):
-            return self.deployedDefinition.cid_version
+        if isinstance(self.deployed_template, CidQsTemplate):
+            return self.deployed_template.cid_version
+        elif isinstance(self.deployed_definition, CidQsDefinition):
+            return self.deployed_definition.cid_version
         else:
             return None
 
     @property
     def latest(self) -> bool:
-        return self.latest_available_cid_version == self.deployed_cid_version
+        try:
+            return self.latest_available_cid_version == self.deployed_cid_version
+        except Exception as exc:
+            logger.debug(f'Failed to determine if latest for dashboards: {self.id}. {exc}')
+            return None
 
     @property
     def health(self) -> bool:
         return self.status not in ['broken']
-
-    @property
-    def origin_type(self) -> str:
-        if self.deployedTemplate is not None:
-            return "TEMPLATE"
-        elif self.deployedDefinition is not None:
-            return "DEFINITION"
-        else:
-            return "UNKNOWN"
-
 
     @property
     def cid_version(self) -> int:
-        if self.origin_type == "TEMPLATE":
-            return self.deployedTemplate.cid_version
-        elif self.origin_type == "DEFINITION":
-            return self.deployedDefinition.cid_version
+        if self.deployed_template:
+            return self.deployed_template.cid_version
+        elif self.deployed_definition:
+            return self.deployed_definition.cid_version
         else:
             return None
-    
+
     @property
     def latest_available_cid_version(self) -> int:
-        if self.origin_type == "TEMPLATE":
-            return self.sourceTemplate.cid_version
-        elif self.origin_type == "DEFINITION":
-            return self.sourceDefinition.cid_version
+        if self.source_template:
+            return self.source_template.cid_version
+        elif self.source_definition:
+            return self.source_definition.cid_version
         else:
             return None
 
@@ -126,7 +120,7 @@ def status(self) -> str:
                 logger.info(f"Found datasets: {self.datasets}")
                 logger.info(f"Required datasets: {self.definition.get('dependsOn').get('datasets')}")
             # Source Template has changed
-            elif self.deployedTemplate and self.sourceTemplate and self.deployedTemplate.arn and self.sourceTemplate.arn and not self.deployedTemplate.arn.startswith(self.sourceTemplate.arn):
+            elif self.deployed_template and self.source_template and self.deployed_template.arn and self.source_template.arn and not self.deployed_template.arn.startswith(self.source_template.arn):
                 self._status = 'legacy'
             elif not self.latest_available_cid_version or not self.deployed_cid_version:
                 self._status = 'undetermined'