From a29ea5bd0289bac980501210317218ffccaae440 Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Wed, 23 Aug 2023 22:47:37 +0200 Subject: [PATCH 1/5] fix logic --- cid/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cid/common.py b/cid/common.py index d74407a3..65183fe7 100644 --- a/cid/common.py +++ b/cid/common.py @@ -1394,7 +1394,7 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non else: raise CidCritical(f'User choice is not to update {found_dataset.name}.') elif not diff: - if not get_parameter( + if get_parameter( param_name=found_dataset.name.lower().replace(' ', '-') + '-override', message=f'Cannot get sql diff for {found_dataset.name}. Continue?', choices=['override', 'exit'], From b7ff40d8c48c1e6fac77d8ea24da161e268062c2 Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Tue, 29 Aug 2023 12:44:07 +0200 Subject: [PATCH 2/5] refactor athena error messages --- cid/helpers/athena.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/cid/helpers/athena.py b/cid/helpers/athena.py index e554a4f6..0b30ca44 100644 --- a/cid/helpers/athena.py +++ b/cid/helpers/athena.py @@ -236,15 +236,17 @@ def execute_query(self, sql_query, sleep_duration=1, database: str=None, catalog # Get Query ID query_id = response.get('QueryExecutionId') - - # Get Query Status + if not query_id: + logger.debug(f'Full query: {sql_query}') + raise CidCritical(f'Athena cannot start query. Answer is: {response}') + # Get Query Status for the first time query_status = self.client.get_query_execution(QueryExecutionId=query_id) - except self.client.exceptions.InvalidRequestException as e: + except self.client.exceptions.InvalidRequestException as exc: logger.debug(f'Full query: {sql_query}') - raise CidCritical(f'InvalidRequestException: {e}') - except Exception as e: + raise CidCritical(f'InvalidRequestException: {exc}') from exc + except Exception as exc: logger.debug(f'Full query: {sql_query}') - raise CidCritical(f'Athena query failed: {e}') + raise CidCritical(f'Athena query failed: {exc}') from exc current_status = query_status['QueryExecution']['Status']['State'] @@ -256,16 +258,14 @@ def execute_query(self, sql_query, sleep_duration=1, database: str=None, catalog # Sleep before polling again time.sleep(sleep_duration) - if (current_status == "SUCCEEDED"): + if current_status == "SUCCEEDED": return query_id - else: - failure_reason = response.get('QueryExecution', {}).get('Status', {}).get('StateChangeReason',repr(response)) - logger.info(f'Athena query failed: {failure_reason}') - logger.debug(f'Full query: {sql_query}') - if fail: - raise CidCritical(f'Athena query failed: {failure_reason}') - else: - return False + failure_reason = response.get('QueryExecution', {}).get('Status', {}).get('StateChangeReason',repr(response)) + logger.info(f'Athena query failed: {failure_reason}') + logger.debug(f'Full query: {sql_query}') + if fail: + raise CidCritical(f'Athena query failed: {failure_reason}') + return False def get_query_results(self, query_id): """ Get Query Results """ From dd7a3f041bcd4d730f17462795e268334091bf99 Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Tue, 29 Aug 2023 12:59:12 +0200 Subject: [PATCH 3/5] better warnings --- cid/helpers/quicksight/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cid/helpers/quicksight/__init__.py b/cid/helpers/quicksight/__init__.py index 7ed5b5f1..8a5042da 100644 --- a/cid/helpers/quicksight/__init__.py +++ b/cid/helpers/quicksight/__init__.py @@ -1049,9 +1049,9 @@ def get_dataset_refresh_schedules(self, dataset_id): ) return refresh_schedules.get("RefreshSchedules") except self.client.exceptions.ResourceNotFoundException as exc: - raise CidError(f'DataSource {dataset_id} does not exist') from exc + raise CidError(f'DataSet {dataset_id} does not exist') from exc except self.client.exceptions.AccessDeniedException as exc: - raise CidError(f'No quicksight:ListDataSets permission') from exc + raise CidError(f'AccessDenied when reading refresh schedule for {dataset_id}') from exc except Exception as exc: raise CidError(f'Unable to list refresh schedules for dataset {dataset_id}: {str(exc)}') from exc @@ -1080,7 +1080,7 @@ def ensure_dataset_refresh_schedule(self, dataset_id, schedules: list): default=default_timezone ) if not schedule_frequency_timezone: - logger.warning(f'Cannot get timezone. Please provide --timezone parameter. Please make sure scheduled refresh is configured manualy.') + logger.warning('Cannot get timezone. Please provide --timezone parameter. Please make sure scheduled refresh is configured manualy.') return for schedule in schedules: From 7aacd9165881b03f9946ad97f1913c8414ca8b7f Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Tue, 29 Aug 2023 13:01:09 +0200 Subject: [PATCH 4/5] reduce stacktrace noise --- cid/helpers/quicksight/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cid/helpers/quicksight/__init__.py b/cid/helpers/quicksight/__init__.py index 8a5042da..398627d5 100644 --- a/cid/helpers/quicksight/__init__.py +++ b/cid/helpers/quicksight/__init__.py @@ -1062,7 +1062,7 @@ def ensure_dataset_refresh_schedule(self, dataset_id, schedules: list): try: existing_schedules = self.get_dataset_refresh_schedules(dataset_id) except CidError as exc: - logger.debug(exc, exc_info=True) + logger.debug(f'List refresh schedule throws: {exc}') logger.warning( f'Cannot read dataset schedules for dataset = {dataset_id}. {str(exc)}. Skipping schedule management.' ' Please make sure scheduled refresh is configured manualy.' From 1235f3266d0f79578e1ffe789fbb90f2aeb5c19f Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Tue, 29 Aug 2023 13:01:57 +0200 Subject: [PATCH 5/5] reduce stacktrace noise --- cid/helpers/quicksight/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cid/helpers/quicksight/__init__.py b/cid/helpers/quicksight/__init__.py index 398627d5..848d7671 100644 --- a/cid/helpers/quicksight/__init__.py +++ b/cid/helpers/quicksight/__init__.py @@ -1051,7 +1051,7 @@ def get_dataset_refresh_schedules(self, dataset_id): except self.client.exceptions.ResourceNotFoundException as exc: raise CidError(f'DataSet {dataset_id} does not exist') from exc except self.client.exceptions.AccessDeniedException as exc: - raise CidError(f'AccessDenied when reading refresh schedule for {dataset_id}') from exc + raise CidError(f'AccessDenied') from exc except Exception as exc: raise CidError(f'Unable to list refresh schedules for dataset {dataset_id}: {str(exc)}') from exc