From d37ab6a59db220e6c7495cefc06a20eb5104ff6f Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Wed, 1 Nov 2023 18:25:43 +0100 Subject: [PATCH 01/14] add catalog feature --- cid/common.py | 50 +++++++++++++++++++++++++++++------------ dashboards/catalog.yaml | 6 +++++ 2 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 dashboards/catalog.yaml diff --git a/cid/common.py b/cid/common.py index 066c46b4..7d16a0aa 100644 --- a/cid/common.py +++ b/cid/common.py @@ -1,6 +1,7 @@ import os import sys import json +import urllib import logging import functools from pathlib import Path @@ -50,6 +51,7 @@ def __init__(self, **kwargs) -> None: self.verbose = kwargs.get('verbose') set_parameters(kwargs, self.all_yes) self._logger = None + self.catalog_url = 'https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/dashboards/catalog.yaml' def aws_login(self): params = { @@ -281,28 +283,48 @@ def track(self, action, dashboard_id): logger.debug(f"Issue logging action {action} for dashboard {dashboard_id} , due to a urllib3 exception {str(e)} . This issue will be ignored") def get_page(self, source): - return requests.get(source, timeout=10) + resp = requests.get(source, timeout=10) + resp.raise_for_status() + return resp def load_resources(self): ''' load additional resources from command line parameters ''' + self.load_catalog() if get_parameters().get('resources'): source = get_parameters().get('resources') - logger.info(f'Loading resources from {source}') - resources = {} - try: - if source.startswith('https://'): - resp = self.get_page(source) - assert resp.status_code in [200, 201], f'Error {resp.status_code} while loading url. {resp.text}' - resources = yaml.safe_load(resp.text) - else: - with open(source, encoding='utf-8') as file_: - resources = yaml.safe_load(file_) - except Exception as exc: - raise CidCritical(f'Failed to load resources from {source}: {type(exc)} {exc}') - self.resources = always_merger.merge(self.resources, resources) + self.load_resource_file(source) self.resources = self.resources_with_global_parameters(self.resources) + def load_resource_file(self, source): + ''' load additional resources from resource file + ''' + logger.debug(f'Loading resources from {source}') + resources = {} + try: + if source.startswith('https://'): + resources = yaml.safe_load(self.get_page(source).text) + else: + with open(source, encoding='utf-8') as file_: + resources = yaml.safe_load(file_) + except Exception as exc: + raise CidCritical(f'Failed to load resources from {source}.') from exc + self.resources = always_merger.merge(self.resources, resources) + + def load_catalog(self, catalog_url=None): + ''' load additional resources from catalog + ''' + catalog_url = get_parameters().get('catalog') or self.catalog_url + try: + catalog = yaml.safe_load(self.get_page(catalog_url)) + except requests.exceptions.HTTPError as exc: + logger.warning(f'Failed to load catalog url: {exc}') + logger.debug(exc, exc_info=True) + return + for resource_ref in catalog.get('Resources'): + url = urllib.parse.urljoin(catalog_url, resource_ref.get("Url")) + self.load_resource_file(url) + def get_template_parameters(self, parameters: dict, param_prefix: str='', others: dict=None): """ Get template parameters. """ diff --git a/dashboards/catalog.yaml b/dashboards/catalog.yaml new file mode 100644 index 00000000..b31c3d02 --- /dev/null +++ b/dashboards/catalog.yaml @@ -0,0 +1,6 @@ +Resources: + - Url: cost-anomalies/cost-anomalies.yaml + - Url: sustainability-proxy-metrics/sustainability-proxy-metrics.yaml + - Url: cloudfront-dashboard-templates/cloudfront_standard_logs_dashboard.yaml + - Url: cloudfront-dashboard-templates/cloudfront_realtime_logs_dashboard.yaml + - Url: data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml \ No newline at end of file From 49ba72e3d79440d15cd95a0a16976b08bc0a0c03 Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Wed, 1 Nov 2023 18:29:46 +0100 Subject: [PATCH 02/14] fix --- cid/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cid/common.py b/cid/common.py index 7d16a0aa..634f6daf 100644 --- a/cid/common.py +++ b/cid/common.py @@ -316,7 +316,7 @@ def load_catalog(self, catalog_url=None): ''' catalog_url = get_parameters().get('catalog') or self.catalog_url try: - catalog = yaml.safe_load(self.get_page(catalog_url)) + catalog = yaml.safe_load(self.get_page(catalog_url).text) except requests.exceptions.HTTPError as exc: logger.warning(f'Failed to load catalog url: {exc}') logger.debug(exc, exc_info=True) From da56d7a9b2f897497f9f98b3810e66ea9f171f50 Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Wed, 1 Nov 2023 19:07:45 +0100 Subject: [PATCH 03/14] add category --- cid/builtin/core/data/resources.yaml | 6 +++ cid/common.py | 37 +++++++++++++++---- .../sustainability-proxy-metrics.yaml | 1 + 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/cid/builtin/core/data/resources.yaml b/cid/builtin/core/data/resources.yaml index 1d2ce325..0684b725 100644 --- a/cid/builtin/core/data/resources.yaml +++ b/cid/builtin/core/data/resources.yaml @@ -2,6 +2,7 @@ # QuickSight Dashboards definitions dashboards: CUDOS: + category: 'Foundational' name: CUDOS Dashboard templateId: cudos_dashboard_v3 dashboardId: cudos @@ -18,6 +19,7 @@ dashboards: minTemplateDescription: "v4.75.0" CID: + category: 'Foundational' name: Cost Intelligence Dashboard templateId: Cost_Intelligence_Dashboard dashboardId: cost_intelligence_dashboard @@ -32,6 +34,7 @@ dashboards: minTemplateDescription: "v3.1.0" KPI: + category: 'Foundational' name: KPI Dashboard templateId: kpi_dashboard dashboardId: kpi_dashboard @@ -48,6 +51,7 @@ dashboards: minTemplateDescription: "v1.2.1" TAO: + category: 'Advanced' name: Trusted Advisor Organizational View templateId: ta-organizational-view dashboardId: ta-organizational-view @@ -59,6 +63,7 @@ dashboards: minTemplateDescription: "v1.4.0" Trends: + category: 'Additional' name: Trends Dashboard templateId: cudos-trends-dashboard-template dashboardId: trends-dashboard @@ -72,6 +77,7 @@ dashboards: minTemplateDescription: "v5.0.0" Compute Optimizer: + category: 'Advanced' name: Compute Optimizer Dashboard templateId: compute_optimizer dashboardId: compute-optimizer-dashboard diff --git a/cid/common.py b/cid/common.py index 634f6daf..375461c2 100644 --- a/cid/common.py +++ b/cid/common.py @@ -385,14 +385,35 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs if dashboard_id is None: - dashboard_id = get_parameter( - param_name='dashboard-id', - message="Please select dashboard to install", - choices={ - f"[{dashboard.get('dashboardId')}] {dashboard.get('name')}" : dashboard.get('dashboardId') - for k, dashboard in self.resources.get('dashboards').items() - }, - ) + while True: + category_options = ['Foundational', 'Advanced', 'Additional'] + \ + sorted(list(set([ + f"{dashboard.get('category', 'Custom')}" + for k, dashboard in self.resources.get('dashboards').items() + if f"{dashboard.get('category', 'Custom')}" not in ('Foundational', 'Advanced', 'Additional') + ]))) + category = get_parameter( + param_name='category', + message="Please select a category of dashboard to install", + choices=category_options, + ) + print(category) + dashboard_options = { + f"[{dashboard.get('dashboardId')}] {dashboard.get('name')}" : dashboard.get('dashboardId') + for k, dashboard in self.resources.get('dashboards').items() + if dashboard.get('category', 'Custom') == category + } + dashboard_options['<<< back'] = '<<< back' + dashboard_id = get_parameter( + param_name='dashboard-id', + message="Please select dashboard to install", + choices=dashboard_options, + ) + if dashboard_id != '<<< back': + break + unset_parameter('category') + unset_parameter('dashboard-id') + if not dashboard_id: print('No dashboard selected') return diff --git a/dashboards/sustainability-proxy-metrics/sustainability-proxy-metrics.yaml b/dashboards/sustainability-proxy-metrics/sustainability-proxy-metrics.yaml index 691ec04b..9aa0f0d3 100644 --- a/dashboards/sustainability-proxy-metrics/sustainability-proxy-metrics.yaml +++ b/dashboards/sustainability-proxy-metrics/sustainability-proxy-metrics.yaml @@ -1,5 +1,6 @@ dashboards: SUS-DASH: + category: 'Additional' dependsOn: datasets: - aws_regions From 99fa4f955f8155acd74ee78d5c39a7781ee71e46 Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Wed, 1 Nov 2023 19:12:32 +0100 Subject: [PATCH 04/14] fixes --- dashboards/cost-anomalies/cost-anomalies.yaml | 1 + .../data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/dashboards/cost-anomalies/cost-anomalies.yaml b/dashboards/cost-anomalies/cost-anomalies.yaml index d9d80356..c6f169c8 100644 --- a/dashboards/cost-anomalies/cost-anomalies.yaml +++ b/dashboards/cost-anomalies/cost-anomalies.yaml @@ -1,5 +1,6 @@ dashboards: AWS COST ANOMALIES DASHBOARD: + category: 'Advanced' dependsOn: datasets: - ca_summary_view diff --git a/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml b/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml index 18522f65..d63abf9d 100644 --- a/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml +++ b/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml @@ -1,5 +1,6 @@ dashboards: DATATRANSFER COST ANALYSIS DASHBOARD ENHANCED: + category: 'Additional' dependsOn: datasets: - data_transfer_view From 8de55d3433142fed94d298a73d3794110317d60b Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Thu, 2 Nov 2023 06:58:38 +0100 Subject: [PATCH 05/14] better category choice --- cid/common.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cid/common.py b/cid/common.py index 375461c2..c7730fce 100644 --- a/cid/common.py +++ b/cid/common.py @@ -383,8 +383,8 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs # TODO: check if datasets returns explicit permission denied and only then discover dashboards as a workaround self.qs.discover_dashboards() - - if dashboard_id is None: + dashboard_id = dashboard_id or get_parameters().get('dashboard-id') + if not dashboard_id: while True: category_options = ['Foundational', 'Advanced', 'Additional'] + \ sorted(list(set([ @@ -397,10 +397,9 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs message="Please select a category of dashboard to install", choices=category_options, ) - print(category) dashboard_options = { f"[{dashboard.get('dashboardId')}] {dashboard.get('name')}" : dashboard.get('dashboardId') - for k, dashboard in self.resources.get('dashboards').items() + for k, dashboard in self.resources.get('dashboards').items() if dashboard.get('category', 'Custom') == category } dashboard_options['<<< back'] = '<<< back' From 12584c8cb3766dcab52c8eae2116f515875c16b8 Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Thu, 2 Nov 2023 07:39:38 +0100 Subject: [PATCH 06/14] cli doc enhancements and fixes --- cid/cli.py | 4 +++- cid/common.py | 35 +++++++++++++++++------------------ cid/export.py | 1 + 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/cid/cli.py b/cid/cli.py index 4572a566..961b2c40 100644 --- a/cid/cli.py +++ b/cid/cli.py @@ -111,6 +111,7 @@ def deploy(ctx, **kwargs): \b Command options: + --category TEXT The dashboards category to choose from. Not needed if dashboard-id provided directly --dashboard-id TEXT QuickSight dashboard id (cudos, cost_intelligence_dashboard, kpi_dashboard, ta-organizational-view, trends-dashboard etc) --athena-database TEXT Athena database --athena-workgroup TEXT Athena workgroup @@ -134,7 +135,7 @@ def deploy(ctx, **kwargs): @click.option('-y', '--yes', help='confirm all', is_flag=True, default=False) @cid_command def export(ctx, **kwargs): - """Expot Dashboard + """Export Dashboard \b Command options: @@ -148,6 +149,7 @@ def export(ctx, **kwargs): (definition|template) A method (definition=pull json definition of Analysis OR template=create QuickSight Template) --export-known-datasets (no|yes) If 'yes' the export will include DataSets that are already in resources file. Default = no + --category TEXT The dashboards category. Default = Custom --output A filename (.yaml) """ ctx.obj.export(**kwargs) diff --git a/cid/common.py b/cid/common.py index c7730fce..25a984cb 100644 --- a/cid/common.py +++ b/cid/common.py @@ -308,7 +308,8 @@ def load_resource_file(self, source): with open(source, encoding='utf-8') as file_: resources = yaml.safe_load(file_) except Exception as exc: - raise CidCritical(f'Failed to load resources from {source}.') from exc + logger.warning(f'Failed to load resources from {source}: {exc}') + return self.resources = always_merger.merge(self.resources, resources) def load_catalog(self, catalog_url=None): @@ -385,33 +386,31 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs dashboard_id = dashboard_id or get_parameters().get('dashboard-id') if not dashboard_id: + standard_categories = ['Foundational', 'Advanced', 'Additional'] + all_categories = set([f"{dashboard.get('category', 'Custom')}" for dashboard in self.resources.get('dashboards').values()]) + non_standard_categories = [cat for cat in all_categories if cat not in standard_categories] while True: - category_options = ['Foundational', 'Advanced', 'Additional'] + \ - sorted(list(set([ - f"{dashboard.get('category', 'Custom')}" - for k, dashboard in self.resources.get('dashboards').items() - if f"{dashboard.get('category', 'Custom')}" not in ('Foundational', 'Advanced', 'Additional') - ]))) + category_options = standard_categories + sorted(non_standard_categories) category = get_parameter( param_name='category', - message="Please select a category of dashboard to install", + message="Please select a category of dashboard to deploy", choices=category_options, ) - dashboard_options = { - f"[{dashboard.get('dashboardId')}] {dashboard.get('name')}" : dashboard.get('dashboardId') - for k, dashboard in self.resources.get('dashboards').items() - if dashboard.get('category', 'Custom') == category - } + dashboard_options = {} + for dashboard in self.resources.get('dashboards').values(): + if dashboard.get('category', 'Custom') == category: + dashboard_options[f"[{dashboard.get('dashboardId')}] {dashboard.get('name')}"] = dashboard.get('dashboardId') dashboard_options['<<< back'] = '<<< back' dashboard_id = get_parameter( param_name='dashboard-id', - message="Please select dashboard to install", + message="Please select a dashboard to deploy", choices=dashboard_options, ) - if dashboard_id != '<<< back': - break - unset_parameter('category') - unset_parameter('dashboard-id') + if dashboard_id == '<<< back': + unset_parameter('category') + unset_parameter('dashboard-id') + continue + break if not dashboard_id: print('No dashboard selected') diff --git a/cid/export.py b/cid/export.py index b158bdd6..11dab5b9 100644 --- a/cid/export.py +++ b/cid/export.py @@ -262,6 +262,7 @@ def export_analysis(qs, athena): } dashboard_resource['name'] = analysis['Name'] dashboard_resource['dashboardId'] = dashboard_id + dashboard_resource['category'] = get_parameters().get('category', 'Custom') dashboard_export_method = None if get_parameters().get('template-id'): From 4cdc9c1abbddb14eb88bb2ccdbf35d01d38941fe Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Fri, 3 Nov 2023 16:47:00 +0100 Subject: [PATCH 07/14] better choice --- cid/common.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/cid/common.py b/cid/common.py index 25a984cb..1abe9691 100644 --- a/cid/common.py +++ b/cid/common.py @@ -389,25 +389,20 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs standard_categories = ['Foundational', 'Advanced', 'Additional'] all_categories = set([f"{dashboard.get('category', 'Custom')}" for dashboard in self.resources.get('dashboards').values()]) non_standard_categories = [cat for cat in all_categories if cat not in standard_categories] - while True: - category_options = standard_categories + sorted(non_standard_categories) - category = get_parameter( - param_name='category', - message="Please select a category of dashboard to deploy", - choices=category_options, - ) - dashboard_options = {} + categories = standard_categories + sorted(non_standard_categories) + dashboard_options = {} + for category in categories: + dashboard_options[f'{category.upper()}'] = '[category]' for dashboard in self.resources.get('dashboards').values(): if dashboard.get('category', 'Custom') == category: - dashboard_options[f"[{dashboard.get('dashboardId')}] {dashboard.get('name')}"] = dashboard.get('dashboardId') - dashboard_options['<<< back'] = '<<< back' + dashboard_options[f" [{dashboard.get('dashboardId')}] {dashboard.get('name')}"] = dashboard.get('dashboardId') + while True: dashboard_id = get_parameter( param_name='dashboard-id', message="Please select a dashboard to deploy", choices=dashboard_options, ) - if dashboard_id == '<<< back': - unset_parameter('category') + if dashboard_id == '[category]': unset_parameter('dashboard-id') continue break From b5759b0c8ee20af464e6e8d82e05875dfe1d32a2 Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Fri, 3 Nov 2023 17:26:02 +0100 Subject: [PATCH 08/14] exclude some --- dashboards/catalog.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/dashboards/catalog.yaml b/dashboards/catalog.yaml index b31c3d02..2c802b07 100644 --- a/dashboards/catalog.yaml +++ b/dashboards/catalog.yaml @@ -1,6 +1,4 @@ Resources: - Url: cost-anomalies/cost-anomalies.yaml - Url: sustainability-proxy-metrics/sustainability-proxy-metrics.yaml - - Url: cloudfront-dashboard-templates/cloudfront_standard_logs_dashboard.yaml - - Url: cloudfront-dashboard-templates/cloudfront_realtime_logs_dashboard.yaml - Url: data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml \ No newline at end of file From a830b45c4b1ebc25d11d97c7522d2d2b5c34f61a Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Sun, 5 Nov 2023 16:00:15 +0100 Subject: [PATCH 09/14] allow mutiple catalogs --- cid/common.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cid/common.py b/cid/common.py index 1abe9691..7d3a73f6 100644 --- a/cid/common.py +++ b/cid/common.py @@ -51,7 +51,9 @@ def __init__(self, **kwargs) -> None: self.verbose = kwargs.get('verbose') set_parameters(kwargs, self.all_yes) self._logger = None - self.catalog_url = 'https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/dashboards/catalog.yaml' + self.catalog_urls = [ + 'https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/dashboards/catalog.yaml', + ] def aws_login(self): params = { @@ -290,7 +292,10 @@ def get_page(self, source): def load_resources(self): ''' load additional resources from command line parameters ''' - self.load_catalog() + if get_parameters().get('catalog'): + self.catalog_urls = get_parameters().get('catalog').split(',') + for catalog_url in self.catalog_urls: + self.load_catalog(catalog_url) if get_parameters().get('resources'): source = get_parameters().get('resources') self.load_resource_file(source) @@ -312,10 +317,9 @@ def load_resource_file(self, source): return self.resources = always_merger.merge(self.resources, resources) - def load_catalog(self, catalog_url=None): + def load_catalog(self, catalog_url): ''' load additional resources from catalog ''' - catalog_url = get_parameters().get('catalog') or self.catalog_url try: catalog = yaml.safe_load(self.get_page(catalog_url).text) except requests.exceptions.HTTPError as exc: From 9d7f43114be9b48bb0456963d5361af78945aea9 Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Sun, 5 Nov 2023 17:00:48 +0100 Subject: [PATCH 10/14] add a checkbox --- cid/common.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cid/common.py b/cid/common.py index 7d3a73f6..73c9be8c 100644 --- a/cid/common.py +++ b/cid/common.py @@ -112,7 +112,7 @@ def glue(self) -> Glue: 'glue': Glue(self.base.session) }) return self._clients.get('glue') - + @property def organizations(self) -> Organizations: if not self._clients.get('organizations'): @@ -399,7 +399,8 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs dashboard_options[f'{category.upper()}'] = '[category]' for dashboard in self.resources.get('dashboards').values(): if dashboard.get('category', 'Custom') == category: - dashboard_options[f" [{dashboard.get('dashboardId')}] {dashboard.get('name')}"] = dashboard.get('dashboardId') + check = '✓' if dashboard.get('dashboardId') in self.qs.dashboards else ' ' + dashboard_options[f" {check}[{dashboard.get('dashboardId')}] {dashboard.get('name')}"] = dashboard.get('dashboardId') while True: dashboard_id = get_parameter( param_name='dashboard-id', @@ -476,13 +477,13 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs choices=['yes', 'no'], default='yes') != 'yes': return - logger.info("Swich to recursive mode") + logger.info("Switch to recursive mode") recursive = True if recursive: self.create_datasets(required_datasets_names, dashboard_datasets, recursive=recursive, update=update) - # Find datasets for template or defintion + # Find datasets for template or definition if not dashboard_definition.get('datasets'): dashboard_definition['datasets'] = {} @@ -507,12 +508,12 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs if not isinstance(ds, Dataset) or ds.name != dataset_name: continue if dashboard_definition.get('templateId'): - # For templates we can additionaly verify dataset fields + # For templates we can additionally verify dataset fields dataset_fields = {col.get('Name'): col.get('Type') for col in ds.columns} src_fields = source_template.datasets.get(ds_map.get(dataset_name, dataset_name) ) - required_fileds = {col.get('Name'): col.get('DataType') for col in src_fields} + required_fields = {col.get('Name'): col.get('DataType') for col in src_fields} unmatched = {} - for k, v in required_fileds.items(): + for k, v in required_fields.items(): if k not in dataset_fields or dataset_fields[k] != v: unmatched.update({k: {'expected': v, 'found': dataset_fields.get(k)}}) logger.debug(f'unmatched_fields={unmatched}') @@ -1056,7 +1057,7 @@ def update_dashboard(self, dashboard_id, dashboard_definition): # Update dashboard print(f'\nUpdating {dashboard_id}') logger.debug(f"Updating {dashboard_id}") - + try: self.qs.update_dashboard(dashboard, dashboard_definition) print('Update completed\n') From 00cd1c7c56174160aceb2b1a01afbc6c8e5f6e1b Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Sun, 5 Nov 2023 17:04:27 +0100 Subject: [PATCH 11/14] safer code --- cid/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cid/common.py b/cid/common.py index 73c9be8c..99fb7ea5 100644 --- a/cid/common.py +++ b/cid/common.py @@ -326,7 +326,7 @@ def load_catalog(self, catalog_url): logger.warning(f'Failed to load catalog url: {exc}') logger.debug(exc, exc_info=True) return - for resource_ref in catalog.get('Resources'): + for resource_ref in catalog.get('Resources', []): url = urllib.parse.urljoin(catalog_url, resource_ref.get("Url")) self.load_resource_file(url) From 76d52add789a8bd509710f5fc35bb7f195214332 Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Sun, 5 Nov 2023 20:04:35 +0100 Subject: [PATCH 12/14] explicit error when not a yaml given --- cid/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cid/common.py b/cid/common.py index 99fb7ea5..87dad3af 100644 --- a/cid/common.py +++ b/cid/common.py @@ -309,6 +309,8 @@ def load_resource_file(self, source): try: if source.startswith('https://'): resources = yaml.safe_load(self.get_page(source).text) + if not isinstance(resources, dict): + raise CidCritical(f'Failed to load {source}. Got {type(resources)} ({repr(resources)[:150]}...)') else: with open(source, encoding='utf-8') as file_: resources = yaml.safe_load(file_) From b3d093ff109d1913c08e75efa43160306a45b8ce Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Tue, 7 Nov 2023 17:17:11 +0100 Subject: [PATCH 13/14] add deprecation feature --- cid/builtin/core/data/resources.yaml | 1 + cid/common.py | 2 ++ cid/helpers/quicksight/__init__.py | 10 ++++++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cid/builtin/core/data/resources.yaml b/cid/builtin/core/data/resources.yaml index 0684b725..81ca9202 100644 --- a/cid/builtin/core/data/resources.yaml +++ b/cid/builtin/core/data/resources.yaml @@ -4,6 +4,7 @@ dashboards: CUDOS: category: 'Foundational' name: CUDOS Dashboard + deprecationNotice: ' DEPRECATED. Replaced with CUDOS Dashboard v5.' templateId: cudos_dashboard_v3 dashboardId: cudos localConfigs: ["update-dashboard.json"] diff --git a/cid/common.py b/cid/common.py index 87dad3af..75c85524 100644 --- a/cid/common.py +++ b/cid/common.py @@ -401,6 +401,8 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs dashboard_options[f'{category.upper()}'] = '[category]' for dashboard in self.resources.get('dashboards').values(): if dashboard.get('category', 'Custom') == category: + if dashboard.get('deprecationNotice'): + continue check = '✓' if dashboard.get('dashboardId') in self.qs.dashboards else ' ' dashboard_options[f" {check}[{dashboard.get('dashboardId')}] {dashboard.get('name')}"] = dashboard.get('dashboardId') while True: diff --git a/cid/helpers/quicksight/__init__.py b/cid/helpers/quicksight/__init__.py index a303b1c2..c6cd11e2 100644 --- a/cid/helpers/quicksight/__init__.py +++ b/cid/helpers/quicksight/__init__.py @@ -615,8 +615,6 @@ def list_data_sources(self) -> list: def select_dashboard(self, force=False) -> str: """ Select from a list of discovered dashboards """ - selection = list() - dashboard_id = get_parameters().get('dashboard-id') if dashboard_id: return dashboard_id @@ -627,14 +625,18 @@ def select_dashboard(self, force=False) -> str: for dashboard in self.dashboards.values(): health = 'healthy' if dashboard.health else 'unhealthy' key = f'{dashboard.name} ({dashboard.arn}, {health}, {dashboard.status})' - if ((dashboard.latest or not dashboard.health) and not force): + notice = dashboard.definition.get('deprecationNotice', '') + if notice: + key = f'{key} {notice}' + if ((dashboard.latest or not dashboard.health or notice) and not force): choices[key] = None else: choices[key] = dashboard.id + try: dashboard_id = get_parameter( param_name='dashboard-id', - message="Please select installation(s) from the list", + message="Please select dashboard from the list", choices=choices, none_as_disabled=True, ) From 82926f818397078b3310daa9bee4810d666e39c5 Mon Sep 17 00:00:00 2001 From: Iakov Gan Date: Wed, 8 Nov 2023 13:44:08 +0100 Subject: [PATCH 14/14] reveiw fixes --- cid/common.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cid/common.py b/cid/common.py index 75c85524..27670d70 100644 --- a/cid/common.py +++ b/cid/common.py @@ -392,17 +392,17 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs dashboard_id = dashboard_id or get_parameters().get('dashboard-id') if not dashboard_id: - standard_categories = ['Foundational', 'Advanced', 'Additional'] - all_categories = set([f"{dashboard.get('category', 'Custom')}" for dashboard in self.resources.get('dashboards').values()]) + standard_categories = ['Foundational', 'Advanced', 'Additional'] # Show these categories first + all_categories = set([f"{dashboard.get('category', 'Other')}" for dashboard in self.resources.get('dashboards').values()]) non_standard_categories = [cat for cat in all_categories if cat not in standard_categories] categories = standard_categories + sorted(non_standard_categories) dashboard_options = {} for category in categories: dashboard_options[f'{category.upper()}'] = '[category]' for dashboard in self.resources.get('dashboards').values(): - if dashboard.get('category', 'Custom') == category: - if dashboard.get('deprecationNotice'): - continue + if dashboard.get('deprecationNotice'): + continue + if dashboard.get('category', 'Other') == category: check = '✓' if dashboard.get('dashboardId') in self.qs.dashboards else ' ' dashboard_options[f" {check}[{dashboard.get('dashboardId')}] {dashboard.get('name')}"] = dashboard.get('dashboardId') while True: