Feature catalog

cid/builtin/core/data/resources.yaml

@@ -2,6 +2,7 @@
 # QuickSight Dashboards definitions
 dashboards:
   CUDOS:
+    category: 'Foundational'
     name: CUDOS Dashboard
     templateId: cudos_dashboard_v3
     dashboardId: cudos
@@ -18,6 +19,7 @@ dashboards:
       minTemplateDescription: "v4.75.0"
 
   CID:
+    category: 'Foundational'
     name: Cost Intelligence Dashboard
     templateId: Cost_Intelligence_Dashboard
     dashboardId: cost_intelligence_dashboard
@@ -32,6 +34,7 @@ dashboards:
       minTemplateDescription: "v3.1.0"
 
   KPI:
+    category: 'Foundational'
     name: KPI Dashboard
     templateId: kpi_dashboard
     dashboardId: kpi_dashboard
@@ -48,6 +51,7 @@ dashboards:
       minTemplateDescription: "v1.2.1"
 
   TAO:
+    category: 'Advanced'
     name: Trusted Advisor Organizational View
     templateId: ta-organizational-view
     dashboardId: ta-organizational-view
@@ -59,6 +63,7 @@ dashboards:
       minTemplateDescription: "v1.4.0"
 
   Trends:
+    category: 'Additional'
     name: Trends Dashboard
     templateId: cudos-trends-dashboard-template
     dashboardId: trends-dashboard
@@ -72,6 +77,7 @@ dashboards:
       minTemplateDescription: "v5.0.0"
 
   Compute Optimizer:
+    category: 'Advanced'
     name: Compute Optimizer Dashboard
     templateId: compute_optimizer
     dashboardId: compute-optimizer-dashboard

cid/common.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import json
+import urllib
 import logging
 import functools
 from pathlib import Path
@@ -50,6 +51,7 @@
         self.verbose = kwargs.get('verbose')
         set_parameters(kwargs, self.all_yes)
         self._logger = None
+        self.catalog_url = 'https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/dashboards/catalog.yaml'
 
     def aws_login(self):
         params = {
@@ -281,28 +283,48 @@
             logger.debug(f"Issue logging action {action}  for dashboard {dashboard_id} , due to a urllib3 exception {str(e)} . This issue will be ignored")
 
     def get_page(self, source):
-        return requests.get(source, timeout=10)
+        resp = requests.get(source, timeout=10)
+        resp.raise_for_status()
+        return resp
 
     def load_resources(self):
         ''' load additional resources from command line parameters
         '''
+        self.load_catalog()
         if get_parameters().get('resources'):
             source = get_parameters().get('resources')
-            logger.info(f'Loading resources from {source}')
-            resources = {}
-            try:
-                if source.startswith('https://'):
-                    resp = self.get_page(source)
-                    assert resp.status_code in [200, 201], f'Error {resp.status_code} while loading url. {resp.text}'
-                    resources = yaml.safe_load(resp.text)
-                else:
-                    with open(source, encoding='utf-8') as file_:
-                        resources = yaml.safe_load(file_)
-            except Exception as exc:
-                raise CidCritical(f'Failed to load resources from {source}: {type(exc)} {exc}')
-            self.resources = always_merger.merge(self.resources, resources)
+            self.load_resource_file(source)
         self.resources = self.resources_with_global_parameters(self.resources)
 
+    def load_resource_file(self, source):
+        ''' load additional resources from resource file
+        '''
+        logger.debug(f'Loading resources from {source}')
+        resources = {}
+        try:
+            if source.startswith('https://'):
+                resources = yaml.safe_load(self.get_page(source).text)
+            else:
+                with open(source, encoding='utf-8') as file_:
+                    resources = yaml.safe_load(file_)
+        except Exception as exc:
+            raise CidCritical(f'Failed to load resources from {source}.') from exc
+        self.resources = always_merger.merge(self.resources, resources)
+
+    def load_catalog(self, catalog_url=None):
+        ''' load additional resources from catalog
+        '''
+        catalog_url = get_parameters().get('catalog') or self.catalog_url
+        try:
+            catalog = yaml.safe_load(self.get_page(catalog_url).text)
+        except requests.exceptions.HTTPError as exc:
+            logger.warning(f'Failed to load catalog url: {exc}')
+            logger.debug(exc, exc_info=True)
+            return
+        for resource_ref in catalog.get('Resources'):
+            url = urllib.parse.urljoin(catalog_url, resource_ref.get("Url"))
+            self.load_resource_file(url)
+
 
     def get_template_parameters(self, parameters: dict, param_prefix: str='', others: dict=None):
         """ Get template parameters. """
@@ -361,16 +383,36 @@
         # TODO: check if datasets returns explicit permission denied and only then discover dashboards as a workaround
         self.qs.discover_dashboards()
 
+        dashboard_id = dashboard_id or get_parameters().get('dashboard-id')
+        if not dashboard_id:
+            while True:
+                category_options = ['Foundational', 'Advanced', 'Additional'] + \
+                    sorted(list(set([
+                        f"{dashboard.get('category', 'Custom')}"
+                        for k, dashboard in self.resources.get('dashboards').items()
+                        if f"{dashboard.get('category', 'Custom')}" not in ('Foundational', 'Advanced', 'Additional')
+                    ])))
+                category = get_parameter(
+                    param_name='category',
+                    message="Please select a category of dashboard to install",
+                    choices=category_options,
+                )
+                dashboard_options = {
+                    f"[{dashboard.get('dashboardId')}] {dashboard.get('name')}" : dashboard.get('dashboardId')
+                    for k, dashboard in self.resources.get('dashboards').items()
+                    if dashboard.get('category', 'Custom') == category
+                }
+                dashboard_options['<<< back'] = '<<< back'
+                dashboard_id = get_parameter(
+                    param_name='dashboard-id',
+                    message="Please select dashboard to install",
+                    choices=dashboard_options,
+                )
+                if dashboard_id != '<<< back':
+                    break
+                unset_parameter('category')
+                unset_parameter('dashboard-id')
 
-        if dashboard_id is None:
-            dashboard_id = get_parameter(
-                param_name='dashboard-id',
-                message="Please select dashboard to install",
-                choices={
-                   f"[{dashboard.get('dashboardId')}] {dashboard.get('name')}" : dashboard.get('dashboardId')
-                   for k, dashboard in self.resources.get('dashboards').items()
-                },
-            )
         if not dashboard_id:
             print('No dashboard selected')
             return

dashboards/catalog.yaml

@@ -0,0 +1,6 @@
+Resources:
+  - Url: cost-anomalies/cost-anomalies.yaml
+  - Url: sustainability-proxy-metrics/sustainability-proxy-metrics.yaml
+  - Url: cloudfront-dashboard-templates/cloudfront_standard_logs_dashboard.yaml
+  - Url: cloudfront-dashboard-templates/cloudfront_realtime_logs_dashboard.yaml
+  - Url: data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml

dashboards/cost-anomalies/cost-anomalies.yaml

@@ -1,5 +1,6 @@
 dashboards:
   AWS COST ANOMALIES DASHBOARD:
+    category: 'Advanced'
     dependsOn:
       datasets:
       - ca_summary_view

dashboards/data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml

@@ -1,5 +1,6 @@
 dashboards:
   DATATRANSFER COST ANALYSIS DASHBOARD ENHANCED:
+    category: 'Additional'
     dependsOn:
       datasets:
       - data_transfer_view

dashboards/sustainability-proxy-metrics/sustainability-proxy-metrics.yaml

@@ -1,5 +1,6 @@
 dashboards:
   SUS-DASH:
+    category: 'Additional'
     dependsOn:
       datasets:
       - aws_regions