diff --git a/cid/builtin/core/data/resources.yaml b/cid/builtin/core/data/resources.yaml index 1d2ce325..81ca9202 100644 --- a/cid/builtin/core/data/resources.yaml +++ b/cid/builtin/core/data/resources.yaml @@ -2,7 +2,9 @@ # QuickSight Dashboards definitions dashboards: CUDOS: + category: 'Foundational' name: CUDOS Dashboard + deprecationNotice: ' DEPRECATED. Replaced with CUDOS Dashboard v5.' templateId: cudos_dashboard_v3 dashboardId: cudos localConfigs: ["update-dashboard.json"] @@ -18,6 +20,7 @@ dashboards: minTemplateDescription: "v4.75.0" CID: + category: 'Foundational' name: Cost Intelligence Dashboard templateId: Cost_Intelligence_Dashboard dashboardId: cost_intelligence_dashboard @@ -32,6 +35,7 @@ dashboards: minTemplateDescription: "v3.1.0" KPI: + category: 'Foundational' name: KPI Dashboard templateId: kpi_dashboard dashboardId: kpi_dashboard @@ -48,6 +52,7 @@ dashboards: minTemplateDescription: "v1.2.1" TAO: + category: 'Advanced' name: Trusted Advisor Organizational View templateId: ta-organizational-view dashboardId: ta-organizational-view @@ -59,6 +64,7 @@ dashboards: minTemplateDescription: "v1.4.0" Trends: + category: 'Additional' name: Trends Dashboard templateId: cudos-trends-dashboard-template dashboardId: trends-dashboard @@ -72,6 +78,7 @@ dashboards: minTemplateDescription: "v5.0.0" Compute Optimizer: + category: 'Advanced' name: Compute Optimizer Dashboard templateId: compute_optimizer dashboardId: compute-optimizer-dashboard diff --git a/cid/cli.py b/cid/cli.py index 4572a566..961b2c40 100644 --- a/cid/cli.py +++ b/cid/cli.py @@ -111,6 +111,7 @@ def deploy(ctx, **kwargs): \b Command options: + --category TEXT The dashboards category to choose from. Not needed if dashboard-id provided directly --dashboard-id TEXT QuickSight dashboard id (cudos, cost_intelligence_dashboard, kpi_dashboard, ta-organizational-view, trends-dashboard etc) --athena-database TEXT Athena database --athena-workgroup TEXT Athena workgroup @@ -134,7 +135,7 @@ def deploy(ctx, **kwargs): @click.option('-y', '--yes', help='confirm all', is_flag=True, default=False) @cid_command def export(ctx, **kwargs): - """Expot Dashboard + """Export Dashboard \b Command options: @@ -148,6 +149,7 @@ def export(ctx, **kwargs): (definition|template) A method (definition=pull json definition of Analysis OR template=create QuickSight Template) --export-known-datasets (no|yes) If 'yes' the export will include DataSets that are already in resources file. Default = no + --category TEXT The dashboards category. Default = Custom --output A filename (.yaml) """ ctx.obj.export(**kwargs) diff --git a/cid/common.py b/cid/common.py index 066c46b4..27670d70 100644 --- a/cid/common.py +++ b/cid/common.py @@ -1,6 +1,7 @@ import os import sys import json +import urllib import logging import functools from pathlib import Path @@ -50,6 +51,9 @@ def __init__(self, **kwargs) -> None: self.verbose = kwargs.get('verbose') set_parameters(kwargs, self.all_yes) self._logger = None + self.catalog_urls = [ + 'https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/dashboards/catalog.yaml', + ] def aws_login(self): params = { @@ -108,7 +112,7 @@ def glue(self) -> Glue: 'glue': Glue(self.base.session) }) return self._clients.get('glue') - + @property def organizations(self) -> Organizations: if not self._clients.get('organizations'): @@ -281,28 +285,53 @@ def track(self, action, dashboard_id): logger.debug(f"Issue logging action {action} for dashboard {dashboard_id} , due to a urllib3 exception {str(e)} . This issue will be ignored") def get_page(self, source): - return requests.get(source, timeout=10) + resp = requests.get(source, timeout=10) + resp.raise_for_status() + return resp def load_resources(self): ''' load additional resources from command line parameters ''' + if get_parameters().get('catalog'): + self.catalog_urls = get_parameters().get('catalog').split(',') + for catalog_url in self.catalog_urls: + self.load_catalog(catalog_url) if get_parameters().get('resources'): source = get_parameters().get('resources') - logger.info(f'Loading resources from {source}') - resources = {} - try: - if source.startswith('https://'): - resp = self.get_page(source) - assert resp.status_code in [200, 201], f'Error {resp.status_code} while loading url. {resp.text}' - resources = yaml.safe_load(resp.text) - else: - with open(source, encoding='utf-8') as file_: - resources = yaml.safe_load(file_) - except Exception as exc: - raise CidCritical(f'Failed to load resources from {source}: {type(exc)} {exc}') - self.resources = always_merger.merge(self.resources, resources) + self.load_resource_file(source) self.resources = self.resources_with_global_parameters(self.resources) + def load_resource_file(self, source): + ''' load additional resources from resource file + ''' + logger.debug(f'Loading resources from {source}') + resources = {} + try: + if source.startswith('https://'): + resources = yaml.safe_load(self.get_page(source).text) + if not isinstance(resources, dict): + raise CidCritical(f'Failed to load {source}. Got {type(resources)} ({repr(resources)[:150]}...)') + else: + with open(source, encoding='utf-8') as file_: + resources = yaml.safe_load(file_) + except Exception as exc: + logger.warning(f'Failed to load resources from {source}: {exc}') + return + self.resources = always_merger.merge(self.resources, resources) + + def load_catalog(self, catalog_url): + ''' load additional resources from catalog + ''' + try: + catalog = yaml.safe_load(self.get_page(catalog_url).text) + except requests.exceptions.HTTPError as exc: + logger.warning(f'Failed to load catalog url: {exc}') + logger.debug(exc, exc_info=True) + return + for resource_ref in catalog.get('Resources', []): + url = urllib.parse.urljoin(catalog_url, resource_ref.get("Url")) + self.load_resource_file(url) + def get_template_parameters(self, parameters: dict, param_prefix: str='', others: dict=None): """ Get template parameters. """ @@ -361,16 +390,32 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs # TODO: check if datasets returns explicit permission denied and only then discover dashboards as a workaround self.qs.discover_dashboards() + dashboard_id = dashboard_id or get_parameters().get('dashboard-id') + if not dashboard_id: + standard_categories = ['Foundational', 'Advanced', 'Additional'] # Show these categories first + all_categories = set([f"{dashboard.get('category', 'Other')}" for dashboard in self.resources.get('dashboards').values()]) + non_standard_categories = [cat for cat in all_categories if cat not in standard_categories] + categories = standard_categories + sorted(non_standard_categories) + dashboard_options = {} + for category in categories: + dashboard_options[f'{category.upper()}'] = '[category]' + for dashboard in self.resources.get('dashboards').values(): + if dashboard.get('deprecationNotice'): + continue + if dashboard.get('category', 'Other') == category: + check = '✓' if dashboard.get('dashboardId') in self.qs.dashboards else ' ' + dashboard_options[f" {check}[{dashboard.get('dashboardId')}] {dashboard.get('name')}"] = dashboard.get('dashboardId') + while True: + dashboard_id = get_parameter( + param_name='dashboard-id', + message="Please select a dashboard to deploy", + choices=dashboard_options, + ) + if dashboard_id == '[category]': + unset_parameter('dashboard-id') + continue + break - if dashboard_id is None: - dashboard_id = get_parameter( - param_name='dashboard-id', - message="Please select dashboard to install", - choices={ - f"[{dashboard.get('dashboardId')}] {dashboard.get('name')}" : dashboard.get('dashboardId') - for k, dashboard in self.resources.get('dashboards').items() - }, - ) if not dashboard_id: print('No dashboard selected') return @@ -436,13 +481,13 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs choices=['yes', 'no'], default='yes') != 'yes': return - logger.info("Swich to recursive mode") + logger.info("Switch to recursive mode") recursive = True if recursive: self.create_datasets(required_datasets_names, dashboard_datasets, recursive=recursive, update=update) - # Find datasets for template or defintion + # Find datasets for template or definition if not dashboard_definition.get('datasets'): dashboard_definition['datasets'] = {} @@ -467,12 +512,12 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs if not isinstance(ds, Dataset) or ds.name != dataset_name: continue if dashboard_definition.get('templateId'): - # For templates we can additionaly verify dataset fields + # For templates we can additionally verify dataset fields dataset_fields = {col.get('Name'): col.get('Type') for col in ds.columns} src_fields = source_template.datasets.get(ds_map.get(dataset_name, dataset_name) ) - required_fileds = {col.get('Name'): col.get('DataType') for col in src_fields} + required_fields = {col.get('Name'): col.get('DataType') for col in src_fields} unmatched = {} - for k, v in required_fileds.items(): + for k, v in required_fields.items(): if k not in dataset_fields or dataset_fields[k] != v: unmatched.update({k: {'expected': v, 'found': dataset_fields.get(k)}}) logger.debug(f'unmatched_fields={unmatched}') @@ -1016,7 +1061,7 @@ def update_dashboard(self, dashboard_id, dashboard_definition): # Update dashboard print(f'\nUpdating {dashboard_id}') logger.debug(f"Updating {dashboard_id}") - + try: self.qs.update_dashboard(dashboard, dashboard_definition) print('Update completed\n') diff --git a/cid/export.py b/cid/export.py index b158bdd6..11dab5b9 100644 --- a/cid/export.py +++ b/cid/export.py @@ -262,6 +262,7 @@ def export_analysis(qs, athena): } dashboard_resource['name'] = analysis['Name'] dashboard_resource['dashboardId'] = dashboard_id + dashboard_resource['category'] = get_parameters().get('category', 'Custom') dashboard_export_method = None if get_parameters().get('template-id'): diff --git a/cid/helpers/quicksight/__init__.py b/cid/helpers/quicksight/__init__.py index a303b1c2..c6cd11e2 100644 --- a/cid/helpers/quicksight/__init__.py +++ b/cid/helpers/quicksight/__init__.py @@ -615,8 +615,6 @@ def list_data_sources(self) -> list: def select_dashboard(self, force=False) -> str: """ Select from a list of discovered dashboards """ - selection = list() - dashboard_id = get_parameters().get('dashboard-id') if dashboard_id: return dashboard_id @@ -627,14 +625,18 @@ def select_dashboard(self, force=False) -> str: for dashboard in self.dashboards.values(): health = 'healthy' if dashboard.health else 'unhealthy' key = f'{dashboard.name} ({dashboard.arn}, {health}, {dashboard.status})' - if ((dashboard.latest or not dashboard.health) and not force): + notice = dashboard.definition.get('deprecationNotice', '') + if notice: + key = f'{key} {notice}' + if ((dashboard.latest or not dashboard.health or notice) and not force): choices[key] = None else: choices[key] = dashboard.id + try: dashboard_id = get_parameter( param_name='dashboard-id', - message="Please select installation(s) from the list", + message="Please select dashboard from the list", choices=choices, none_as_disabled=True, ) diff --git a/dashboards/catalog.yaml b/dashboards/catalog.yaml new file mode 100644 index 00000000..2c802b07 --- /dev/null +++ b/dashboards/catalog.yaml @@ -0,0 +1,4 @@ +Resources: + - Url: cost-anomalies/cost-anomalies.yaml + - Url: sustainability-proxy-metrics/sustainability-proxy-metrics.yaml + - Url: data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml \ No newline at end of file diff --git a/dashboards/cost-anomalies/cost-anomalies.yaml b/dashboards/cost-anomalies/cost-anomalies.yaml index d9d80356..c6f169c8 100644 --- a/dashboards/cost-anomalies/cost-anomalies.yaml +++ b/dashboards/cost-anomalies/cost-anomalies.yaml @@ -1,5 +1,6 @@ dashboards: AWS COST ANOMALIES DASHBOARD: + category: 'Advanced' dependsOn: datasets: - ca_summary_view diff --git a/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml b/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml index 18522f65..d63abf9d 100644 --- a/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml +++ b/dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml @@ -1,5 +1,6 @@ dashboards: DATATRANSFER COST ANALYSIS DASHBOARD ENHANCED: + category: 'Additional' dependsOn: datasets: - data_transfer_view diff --git a/dashboards/sustainability-proxy-metrics/sustainability-proxy-metrics.yaml b/dashboards/sustainability-proxy-metrics/sustainability-proxy-metrics.yaml index 691ec04b..9aa0f0d3 100644 --- a/dashboards/sustainability-proxy-metrics/sustainability-proxy-metrics.yaml +++ b/dashboards/sustainability-proxy-metrics/sustainability-proxy-metrics.yaml @@ -1,5 +1,6 @@ dashboards: SUS-DASH: + category: 'Additional' dependsOn: datasets: - aws_regions