From b8a5d62b3cedc630ac99190972bbde320af2bc34 Mon Sep 17 00:00:00 2001 From: Iakov GAN <82834333+iakov-aws@users.noreply.github.com> Date: Sat, 16 Nov 2024 17:37:16 +0100 Subject: [PATCH] make safer update (#1026) * add incremental refresh option --- cid/common.py | 9 ++-- cid/helpers/quicksight/__init__.py | 38 +++++++++----- cid/helpers/quicksight/dashboard.py | 80 +++++++++++++---------------- 3 files changed, 66 insertions(+), 61 deletions(-) diff --git a/cid/common.py b/cid/common.py index ebaad349..9efabe16 100644 --- a/cid/common.py +++ b/cid/common.py @@ -1090,11 +1090,10 @@ def update_dashboard(self, dashboard_id, dashboard_definition): print(f'Dashboard "{dashboard_id}" is not deployed') return - if isinstance(dashboard.deployedTemplate, CidQsTemplate): - print(f'Deployed template: {dashboard.deployedTemplate.arn}') - if isinstance(dashboard.sourceTemplate, CidQsTemplate): - print(f"Latest template: {dashboard.sourceTemplate.arn}/version/{dashboard.sourceTemplate.version}") - + if isinstance(dashboard.deployed_template, CidQsTemplate): + print(f'Deployed template: {dashboard.deployed_template.arn}') + if isinstance(dashboard.source_template, CidQsTemplate): + print(f"Latest template: {dashboard.source_template.arn}/version/{dashboard.source_template.version}") try: cid_print(f'\nUpdating {dashboard_id} from {dashboard.cid_version} to {dashboard.latest_available_cid_version}') except: diff --git a/cid/helpers/quicksight/__init__.py b/cid/helpers/quicksight/__init__.py index 0a99a9fd..5baff4c4 100644 --- a/cid/helpers/quicksight/__init__.py +++ b/cid/helpers/quicksight/__init__.py @@ -255,7 +255,7 @@ def _safe_int(value, default=None): try: _template = self.describe_template(**params) if isinstance(_template, CidQsTemplate): - dashboard.deployedTemplate = _template + dashboard.deployed_template = _template except Exception as exc: logger.debug(exc, exc_info=True) logger.info(f'Unable to describe template for {dashboardId}, {exc}') @@ -263,7 +263,7 @@ def _safe_int(value, default=None): logger.info("Minimum template version could not be found for Dashboard {dashboardId}: {_template_arn}, deployed template could not be described") else: # Dashboard is not template based but definition based try: - dashboard.deployedDefinition = self.describe_dashboard_definition(dashboard_id=dashboardId, refresh=refresh) + dashboard.deployed_definition = self.describe_dashboard_definition(dashboard_id=dashboardId, refresh=refresh) except CidError as exc: logger.info('Exception on reading dashboard definition {dashboardId}: {exc}. Not critical. Continue.') @@ -271,7 +271,7 @@ def _safe_int(value, default=None): # Resolve source definition (the latest definition publicly available) data_stream = io.StringIO(_definition["data"]) definition_data = yaml.safe_load(data_stream) - dashboard.sourceDefinition = CidQsDefinition(definition_data) + dashboard.source_definition = CidQsDefinition(definition_data) # Fetch datasets (works for both TEMPLATE and DEFINITION based dashboards) for dataset in dashboard.version.get('DataSetArns', []): @@ -289,7 +289,7 @@ def _safe_int(value, default=None): logger.info(f'Invalid dataset {dataset_id}') logger.info(f"{dashboard.name} has {len(dashboard.datasets)} datasets") - # Fetch the latest version of sourceTemplate referenced in definition + # Fetch the latest version of source_template referenced in definition source_template_account_id = _definition.get('sourceAccountId') template_id = _definition.get('templateId') region = _definition.get('region', 'us-east-1') @@ -297,13 +297,13 @@ def _safe_int(value, default=None): try: logger.debug(f'Loading latest source template {template_id} from source account {source_template_account_id} in {region}') template = self.describe_template(template_id, account_id=source_template_account_id, region=region) - dashboard.sourceTemplate = template + dashboard.source_template = template except Exception as exc: logger.debug(exc, exc_info=True) logger.info(f'Unable to describe template {template_id} in {source_template_account_id} ({region})') # Checking for version override in template definition - for dashboard_template in [dashboard.deployedTemplate, dashboard.sourceTemplate]: + for dashboard_template in [dashboard.deployed_template, dashboard.source_template]: if not isinstance(dashboard_template, CidQsTemplate)\ or int(dashboard_template.version) <= 0 \ or not version_obj: @@ -585,21 +585,18 @@ def discover_data_sources(self) -> None: self.describe_data_source(d) except Exception as exc: logger.debug(exc, exc_info=True) - + def discover_dashboards(self, refresh_overrides: List[str]=[], refresh: bool = False) -> None: """ Discover deployed dashboards - :param refresh_overrides: a list of dashboard ids to refresh :param refresh: force refresh all dashboards """ - if refresh or self._dashboards is None: self._dashboards = {} else: for dashboard_id in refresh_overrides: if dashboard_id in self._dashboards: del self._dashboards[dashboard_id] - logger.info('Discovering deployed dashboards') deployed_dashboards=self.list_dashboards() logger.info(f'Found {len(deployed_dashboards)} deployed dashboards') @@ -611,9 +608,7 @@ def discover_dashboards(self, refresh_overrides: List[str]=[], refresh: bool = F dashboard_id = dashboard.get('DashboardId') bar.set_description(f'Discovering {dashboard_name[:10]:<10}', refresh=True) logger.info(f'Discovering "{dashboard_name}"') - refresh = dashboard_id in refresh_overrides - self.discover_dashboard(dashboard_id, refresh=refresh) def list_dashboards(self) -> list: @@ -1259,7 +1254,6 @@ def ensure_dataset_refresh_schedule(self, dataset_id, schedules: list): return for schedule in schedules: - # Get the list of existing schedules with the same id existing_schedule = None for existing in existing_schedules: @@ -1267,6 +1261,24 @@ def ensure_dataset_refresh_schedule(self, dataset_id, schedules: list): existing_schedule = existing break + refresh_configuration = schedule.pop('RefreshConfiguration', {}) + if refresh_configuration: + # schedule exists so we need to update + logger.debug(f'Updating refresh schedule configuration with id {schedule["ScheduleId"]} for dataset {dataset_id}.') + try: + self.client.put_data_set_refresh_properties( + DataSetId=dataset_id, + AwsAccountId=self.account_id, + DataSetRefreshProperties={'RefreshConfiguration': refresh_configuration} + ) + logger.debug(f'Refresh schedule configuration with id {schedule["ScheduleId"]} for dataset {dataset_id} is updated.') + except self.client.exceptions.ResourceNotFoundException: + logger.error(f'Unable to update refresh schedule configuration with id {schedule["ScheduleId"]}. Dataset {dataset_id} does not exist.') + except self.client.exceptions.AccessDeniedException: + logger.error(f'Unable to update refresh schedule configuration with id {schedule["ScheduleId"]}. Please add quicksight:UpdateDataSet permission.') + except Exception as exc: + logger.error(f'Unable to update refresh schedule configuration with id {schedule["ScheduleId"]} for dataset "{dataset_id}": {str(exc)}') + # Verify that all schedule parameters are set schedule["ScheduleId"] = schedule.get("ScheduleId", "cid") if "ScheduleFrequency" not in schedule: diff --git a/cid/helpers/quicksight/dashboard.py b/cid/helpers/quicksight/dashboard.py index 5476fd37..eff073ab 100644 --- a/cid/helpers/quicksight/dashboard.py +++ b/cid/helpers/quicksight/dashboard.py @@ -18,13 +18,13 @@ def __init__(self, raw: dict, qs=None) -> None: # Initialize properties self.datasets: Dict[str, str] = {} # Deployed template - self._deployedTemplate: CidQsTemplate = None - self._deployedDefinition: CidQsDefinition = None + self._deployed_template: CidQsTemplate = None + self._deployed_definition: CidQsDefinition = None self._status = str() self.status_detail = str() # Source template in origin account - self.sourceTemplate: CidQsTemplate = None - self.sourceDefinition: CidQsDefinition = None + self.source_template: CidQsTemplate = None + self.source_definition: CidQsDefinition = None self.qs = qs @property @@ -36,75 +36,69 @@ def version(self) -> dict: return self.get_property('Version') @property - def deployedTemplate(self) -> CidQsTemplate: - return self._deployedTemplate + def deployed_template(self) -> CidQsTemplate: + return self._deployed_template - @deployedTemplate.setter - def deployedTemplate(self, template: CidQsTemplate) -> None: - self._deployedTemplate = template + @deployed_template.setter + def deployed_template(self, template: CidQsTemplate) -> None: + self._deployed_template = template @property - def deployedDefinition(self) -> CidQsTemplate: - return self._deployedDefinition + def deployed_definition(self) -> CidQsTemplate: + return self._deployed_definition - @deployedDefinition.setter - def deployedDefinition(self, definition: CidQsDefinition) -> None: - self._deployedDefinition = definition + @deployed_definition.setter + def deployed_definition(self, definition: CidQsDefinition) -> None: + self._deployed_definition = definition @property def template_id(self) -> str: - if isinstance(self.deployedTemplate, CidQsTemplate): - return self.deployedTemplate.id + if isinstance(self.deployed_template, CidQsTemplate): + return self.deployed_template.id return None @property def template_arn(self) -> str: - if isinstance(self.deployedTemplate, CidQsTemplate): - return self.deployedTemplate.arn + if isinstance(self.deployed_template, CidQsTemplate): + return self.deployed_template.arn return None @property def deployed_cid_version(self) -> int: - if isinstance(self.deployedTemplate, CidQsTemplate): - return self.deployedTemplate.cid_version - elif isinstance(self.deployedDefinition, CidQsDefinition): - return self.deployedDefinition.cid_version + if isinstance(self.deployed_template, CidQsTemplate): + return self.deployed_template.cid_version + elif isinstance(self.deployed_definition, CidQsDefinition): + return self.deployed_definition.cid_version else: return None @property def latest(self) -> bool: - return self.latest_available_cid_version == self.deployed_cid_version + try: + return self.latest_available_cid_version == self.deployed_cid_version + except Exception as exc: + logger.debug(f'Failed to determine if latest for dashboards: {self.id}. {exc}') + return None @property def health(self) -> bool: return self.status not in ['broken'] - - @property - def origin_type(self) -> str: - if self.deployedTemplate is not None: - return "TEMPLATE" - elif self.deployedDefinition is not None: - return "DEFINITION" - else: - return "UNKNOWN" - @property def cid_version(self) -> int: - if self.origin_type == "TEMPLATE": - return self.deployedTemplate.cid_version - elif self.origin_type == "DEFINITION": - return self.deployedDefinition.cid_version + if self.deployed_template: + return self.deployed_template.cid_version + elif self.deployed_definition: + return self.deployed_definition.cid_version else: return None - + @property def latest_available_cid_version(self) -> int: - if self.origin_type == "TEMPLATE": - return self.sourceTemplate.cid_version - elif self.origin_type == "DEFINITION": - return self.sourceDefinition.cid_version + if self.source_template: + return self.source_template.cid_version + elif self.source_definition: + return self.source_definition.cid_version else: return None @@ -126,7 +120,7 @@ def status(self) -> str: logger.info(f"Found datasets: {self.datasets}") logger.info(f"Required datasets: {self.definition.get('dependsOn').get('datasets')}") # Source Template has changed - elif self.deployedTemplate and self.sourceTemplate and self.deployedTemplate.arn and self.sourceTemplate.arn and not self.deployedTemplate.arn.startswith(self.sourceTemplate.arn): + elif self.deployed_template and self.source_template and self.deployed_template.arn and self.source_template.arn and not self.deployed_template.arn.startswith(self.source_template.arn): self._status = 'legacy' elif not self.latest_available_cid_version or not self.deployed_cid_version: self._status = 'undetermined'