Feature catalog

cid/builtin/core/data/resources.yaml

@@ -2,7 +2,9 @@
 # QuickSight Dashboards definitions
 dashboards:
   CUDOS:
+    category: 'Foundational'
     name: CUDOS Dashboard
+    deprecationNotice: ' DEPRECATED. Replaced with CUDOS Dashboard v5.'
     templateId: cudos_dashboard_v3
     dashboardId: cudos
     localConfigs: ["update-dashboard.json"]
@@ -18,6 +20,7 @@ dashboards:
       minTemplateDescription: "v4.75.0"
 
   CID:
+    category: 'Foundational'
     name: Cost Intelligence Dashboard
     templateId: Cost_Intelligence_Dashboard
     dashboardId: cost_intelligence_dashboard
@@ -32,6 +35,7 @@ dashboards:
       minTemplateDescription: "v3.1.0"
 
   KPI:
+    category: 'Foundational'
     name: KPI Dashboard
     templateId: kpi_dashboard
     dashboardId: kpi_dashboard
@@ -48,6 +52,7 @@ dashboards:
       minTemplateDescription: "v1.2.1"
 
   TAO:
+    category: 'Advanced'
     name: Trusted Advisor Organizational View
     templateId: ta-organizational-view
     dashboardId: ta-organizational-view
@@ -59,6 +64,7 @@ dashboards:
       minTemplateDescription: "v1.4.0"
 
   Trends:
+    category: 'Additional'
     name: Trends Dashboard
     templateId: cudos-trends-dashboard-template
     dashboardId: trends-dashboard
@@ -72,6 +78,7 @@ dashboards:
       minTemplateDescription: "v5.0.0"
 
   Compute Optimizer:
+    category: 'Advanced'
     name: Compute Optimizer Dashboard
     templateId: compute_optimizer
     dashboardId: compute-optimizer-dashboard

cid/cli.py

@@ -111,6 +111,7 @@ def deploy(ctx, **kwargs):
 
     \b
     Command options:
+     --category TEXT                       The dashboards category to choose from. Not needed if dashboard-id provided directly
      --dashboard-id TEXT                   QuickSight dashboard id (cudos, cost_intelligence_dashboard, kpi_dashboard, ta-organizational-view, trends-dashboard etc)
      --athena-database TEXT                Athena database
      --athena-workgroup TEXT               Athena workgroup
@@ -134,7 +135,7 @@ def deploy(ctx, **kwargs):
 @click.option('-y', '--yes', help='confirm all', is_flag=True, default=False)
 @cid_command
 def export(ctx, **kwargs):
-    """Expot Dashboard
+    """Export Dashboard
 
     \b
     Command options:
@@ -148,6 +149,7 @@ def export(ctx, **kwargs):
                (definition|template) A method (definition=pull json definition of Analysis OR template=create QuickSight Template)
         --export-known-datasets
             (no|yes)                 If 'yes' the export will include DataSets that are already in resources file. Default = no
+        --category TEXT              The dashboards category. Default = Custom
         --output                     A filename (.yaml)
     """
     ctx.obj.export(**kwargs)

cid/common.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import json
+import urllib
 import logging
 import functools
 from pathlib import Path
@@ -50,6 +51,9 @@ def __init__(self, **kwargs) -> None:
         self.verbose = kwargs.get('verbose')
         set_parameters(kwargs, self.all_yes)
         self._logger = None
+        self.catalog_urls = [
+            'https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/dashboards/catalog.yaml',
+        ]
 
     def aws_login(self):
         params = {
@@ -108,7 +112,7 @@ def glue(self) -> Glue:
                 'glue': Glue(self.base.session)
             })
         return self._clients.get('glue')
-    
+
     @property
     def organizations(self) -> Organizations:
         if not self._clients.get('organizations'):
@@ -281,28 +285,53 @@ def track(self, action, dashboard_id):
             logger.debug(f"Issue logging action {action}  for dashboard {dashboard_id} , due to a urllib3 exception {str(e)} . This issue will be ignored")
 
     def get_page(self, source):
-        return requests.get(source, timeout=10)
+        resp = requests.get(source, timeout=10)
+        resp.raise_for_status()
+        return resp
 
     def load_resources(self):
         ''' load additional resources from command line parameters
         '''
+        if get_parameters().get('catalog'):
+            self.catalog_urls = get_parameters().get('catalog').split(',')
+        for catalog_url in self.catalog_urls:
+            self.load_catalog(catalog_url)
         if get_parameters().get('resources'):
             source = get_parameters().get('resources')
-            logger.info(f'Loading resources from {source}')
-            resources = {}
-            try:
-                if source.startswith('https://'):
-                    resp = self.get_page(source)
-                    assert resp.status_code in [200, 201], f'Error {resp.status_code} while loading url. {resp.text}'
-                    resources = yaml.safe_load(resp.text)
-                else:
-                    with open(source, encoding='utf-8') as file_:
-                        resources = yaml.safe_load(file_)
-            except Exception as exc:
-                raise CidCritical(f'Failed to load resources from {source}: {type(exc)} {exc}')
-            self.resources = always_merger.merge(self.resources, resources)
+            self.load_resource_file(source)
         self.resources = self.resources_with_global_parameters(self.resources)
 
+    def load_resource_file(self, source):
+        ''' load additional resources from resource file
+        '''
+        logger.debug(f'Loading resources from {source}')
+        resources = {}
+        try:
+            if source.startswith('https://'):
+                resources = yaml.safe_load(self.get_page(source).text)
+                if not isinstance(resources, dict):
+                    raise CidCritical(f'Failed to load {source}. Got {type(resources)} ({repr(resources)[:150]}...)')
+            else:
+                with open(source, encoding='utf-8') as file_:
+                    resources = yaml.safe_load(file_)
+        except Exception as exc:
+            logger.warning(f'Failed to load resources from {source}: {exc}')
+            return
+        self.resources = always_merger.merge(self.resources, resources)
+
+    def load_catalog(self, catalog_url):
+        ''' load additional resources from catalog
+        '''
+        try:
+            catalog = yaml.safe_load(self.get_page(catalog_url).text)
+        except requests.exceptions.HTTPError as exc:
+            logger.warning(f'Failed to load catalog url: {exc}')
+            logger.debug(exc, exc_info=True)
+            return
+        for resource_ref in catalog.get('Resources', []):
+            url = urllib.parse.urljoin(catalog_url, resource_ref.get("Url"))
+            self.load_resource_file(url)
+
 
     def get_template_parameters(self, parameters: dict, param_prefix: str='', others: dict=None):
         """ Get template parameters. """
@@ -361,16 +390,32 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs
         # TODO: check if datasets returns explicit permission denied and only then discover dashboards as a workaround
         self.qs.discover_dashboards()
 
+        dashboard_id = dashboard_id or get_parameters().get('dashboard-id')
+        if not dashboard_id:
+            standard_categories = ['Foundational', 'Advanced', 'Additional'] # Show these categories first
+            all_categories = set([f"{dashboard.get('category', 'Other')}" for dashboard in self.resources.get('dashboards').values()])
+            non_standard_categories = [cat for cat in all_categories if cat not in standard_categories]
+            categories =  standard_categories + sorted(non_standard_categories)
+            dashboard_options = {}
+            for category in categories:
+                dashboard_options[f'{category.upper()}'] = '[category]'
+                for dashboard in self.resources.get('dashboards').values():
+                    if dashboard.get('deprecationNotice'):
+                        continue
+                    if dashboard.get('category', 'Other') == category:
+                        check = '✓' if dashboard.get('dashboardId') in self.qs.dashboards else ' '
+                        dashboard_options[f" {check}[{dashboard.get('dashboardId')}] {dashboard.get('name')}"] = dashboard.get('dashboardId')
+            while True:
+                dashboard_id = get_parameter(
+                    param_name='dashboard-id',
+                    message="Please select a dashboard to deploy",
+                    choices=dashboard_options,
+                )
+                if dashboard_id == '[category]':
+                    unset_parameter('dashboard-id')
+                    continue
+                break
 
-        if dashboard_id is None:
-            dashboard_id = get_parameter(
-                param_name='dashboard-id',
-                message="Please select dashboard to install",
-                choices={
-                   f"[{dashboard.get('dashboardId')}] {dashboard.get('name')}" : dashboard.get('dashboardId')
-                   for k, dashboard in self.resources.get('dashboards').items()
-                },
-            )
         if not dashboard_id:
             print('No dashboard selected')
             return
@@ -436,13 +481,13 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs
                 choices=['yes', 'no'],
                 default='yes') != 'yes':
                 return
-            logger.info("Swich to recursive mode")
+            logger.info("Switch to recursive mode")
             recursive = True
 
         if recursive:
             self.create_datasets(required_datasets_names, dashboard_datasets, recursive=recursive, update=update)
 
-        # Find datasets for template or defintion
+        # Find datasets for template or definition
         if not dashboard_definition.get('datasets'):
             dashboard_definition['datasets'] = {}
 
@@ -467,12 +512,12 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs
                     if not isinstance(ds, Dataset) or ds.name != dataset_name:
                         continue
                     if dashboard_definition.get('templateId'):
-                        # For templates we can additionaly verify dataset fields
+                        # For templates we can additionally verify dataset fields
                         dataset_fields = {col.get('Name'): col.get('Type') for col in ds.columns}
                         src_fields = source_template.datasets.get(ds_map.get(dataset_name, dataset_name) )
-                        required_fileds = {col.get('Name'): col.get('DataType') for col in src_fields}
+                        required_fields = {col.get('Name'): col.get('DataType') for col in src_fields}
                         unmatched = {}
-                        for k, v in required_fileds.items():
+                        for k, v in required_fields.items():
                             if k not in dataset_fields or dataset_fields[k] != v:
                                 unmatched.update({k: {'expected': v, 'found': dataset_fields.get(k)}})
                         logger.debug(f'unmatched_fields={unmatched}')
@@ -1016,7 +1061,7 @@ def update_dashboard(self, dashboard_id, dashboard_definition):
         # Update dashboard
         print(f'\nUpdating {dashboard_id}')
         logger.debug(f"Updating {dashboard_id}")
-        
+
         try:
             self.qs.update_dashboard(dashboard, dashboard_definition)
             print('Update completed\n')

cid/export.py

@@ -262,6 +262,7 @@ def export_analysis(qs, athena):
     }
     dashboard_resource['name'] = analysis['Name']
     dashboard_resource['dashboardId'] = dashboard_id
+    dashboard_resource['category'] = get_parameters().get('category', 'Custom')
 
     dashboard_export_method = None
     if get_parameters().get('template-id'):

cid/helpers/quicksight/__init__.py

@@ -615,8 +615,6 @@ def list_data_sources(self) -> list:
 
     def select_dashboard(self, force=False) -> str:
         """ Select from a list of discovered dashboards """
-        selection = list()
-
         dashboard_id = get_parameters().get('dashboard-id')
         if dashboard_id:
             return dashboard_id
@@ -627,14 +625,18 @@ def select_dashboard(self, force=False) -> str:
         for dashboard in self.dashboards.values():
             health = 'healthy' if dashboard.health else 'unhealthy'
             key = f'{dashboard.name} ({dashboard.arn}, {health}, {dashboard.status})'
-            if ((dashboard.latest or not dashboard.health) and not force):
+            notice = dashboard.definition.get('deprecationNotice', '')
+            if notice:
+                key = f'{key} {notice}'
+            if ((dashboard.latest or not dashboard.health or notice) and not force):
                 choices[key] = None
             else:
                 choices[key] = dashboard.id
+
         try:
             dashboard_id = get_parameter(
                 param_name='dashboard-id',
-                message="Please select installation(s) from the list",
+                message="Please select dashboard from the list",
                 choices=choices,
                 none_as_disabled=True,
             )

dashboards/catalog.yaml

@@ -0,0 +1,4 @@
+Resources:
+  - Url: cost-anomalies/cost-anomalies.yaml
+  - Url: sustainability-proxy-metrics/sustainability-proxy-metrics.yaml
+  - Url: data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml

dashboards/cost-anomalies/cost-anomalies.yaml

@@ -1,5 +1,6 @@
 dashboards:
   AWS COST ANOMALIES DASHBOARD:
+    category: 'Advanced'
     dependsOn:
       datasets:
       - ca_summary_view

dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml

@@ -1,5 +1,6 @@
 dashboards:
   DATATRANSFER COST ANALYSIS DASHBOARD ENHANCED:
+    category: 'Additional'
     dependsOn:
       datasets:
       - data_transfer_view

dashboards/sustainability-proxy-metrics/sustainability-proxy-metrics.yaml

@@ -1,5 +1,6 @@
 dashboards:
   SUS-DASH:
+    category: 'Additional'
     dependsOn:
       datasets:
       - aws_regions