Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Minor fixes #610

Merged
merged 6 commits into from
Sep 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cid/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -1394,7 +1394,7 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non
else:
raise CidCritical(f'User choice is not to update {found_dataset.name}.')
elif not diff:
if not get_parameter(
if get_parameter(
param_name=found_dataset.name.lower().replace(' ', '-') + '-override',
message=f'Cannot get sql diff for {found_dataset.name}. Continue?',
choices=['override', 'exit'],
Expand Down
30 changes: 15 additions & 15 deletions cid/helpers/athena.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,15 +236,17 @@ def execute_query(self, sql_query, sleep_duration=1, database: str=None, catalog

# Get Query ID
query_id = response.get('QueryExecutionId')

# Get Query Status
if not query_id:
logger.debug(f'Full query: {sql_query}')
raise CidCritical(f'Athena cannot start query. Answer is: {response}')
# Get Query Status for the first time
query_status = self.client.get_query_execution(QueryExecutionId=query_id)
except self.client.exceptions.InvalidRequestException as e:
except self.client.exceptions.InvalidRequestException as exc:
logger.debug(f'Full query: {sql_query}')
raise CidCritical(f'InvalidRequestException: {e}')
except Exception as e:
raise CidCritical(f'InvalidRequestException: {exc}') from exc
except Exception as exc:
logger.debug(f'Full query: {sql_query}')
raise CidCritical(f'Athena query failed: {e}')
raise CidCritical(f'Athena query failed: {exc}') from exc

current_status = query_status['QueryExecution']['Status']['State']

Expand All @@ -256,16 +258,14 @@ def execute_query(self, sql_query, sleep_duration=1, database: str=None, catalog
# Sleep before polling again
time.sleep(sleep_duration)

if (current_status == "SUCCEEDED"):
if current_status == "SUCCEEDED":
return query_id
else:
failure_reason = response.get('QueryExecution', {}).get('Status', {}).get('StateChangeReason',repr(response))
logger.info(f'Athena query failed: {failure_reason}')
logger.debug(f'Full query: {sql_query}')
if fail:
raise CidCritical(f'Athena query failed: {failure_reason}')
else:
return False
failure_reason = response.get('QueryExecution', {}).get('Status', {}).get('StateChangeReason',repr(response))
logger.info(f'Athena query failed: {failure_reason}')
logger.debug(f'Full query: {sql_query}')
if fail:
raise CidCritical(f'Athena query failed: {failure_reason}')
return False

def get_query_results(self, query_id):
""" Get Query Results """
Expand Down
8 changes: 4 additions & 4 deletions cid/helpers/quicksight/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1049,9 +1049,9 @@ def get_dataset_refresh_schedules(self, dataset_id):
)
return refresh_schedules.get("RefreshSchedules")
except self.client.exceptions.ResourceNotFoundException as exc:
raise CidError(f'DataSource {dataset_id} does not exist') from exc
raise CidError(f'DataSet {dataset_id} does not exist') from exc
except self.client.exceptions.AccessDeniedException as exc:
raise CidError(f'No quicksight:ListDataSets permission') from exc
raise CidError(f'AccessDenied') from exc
except Exception as exc:
raise CidError(f'Unable to list refresh schedules for dataset {dataset_id}: {str(exc)}') from exc

Expand All @@ -1062,7 +1062,7 @@ def ensure_dataset_refresh_schedule(self, dataset_id, schedules: list):
try:
existing_schedules = self.get_dataset_refresh_schedules(dataset_id)
except CidError as exc:
logger.debug(exc, exc_info=True)
logger.debug(f'List refresh schedule throws: {exc}')
logger.warning(
f'Cannot read dataset schedules for dataset = {dataset_id}. {str(exc)}. Skipping schedule management.'
' Please make sure scheduled refresh is configured manualy.'
Expand All @@ -1080,7 +1080,7 @@ def ensure_dataset_refresh_schedule(self, dataset_id, schedules: list):
default=default_timezone
)
if not schedule_frequency_timezone:
logger.warning(f'Cannot get timezone. Please provide --timezone parameter. Please make sure scheduled refresh is configured manualy.')
logger.warning('Cannot get timezone. Please provide --timezone parameter. Please make sure scheduled refresh is configured manualy.')
return

for schedule in schedules:
Expand Down