Merge branch 'main' into health-events-dash-2.0.0

cfn-templates/cid-cfn.yml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
-Description: Deployment of Cloud Intelligence Dashboards v0.3.9
+Description: Deployment of Cloud Intelligence Dashboards v0.3.10
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -39,6 +39,7 @@ Metadata:
           - LambdaLayerBucketPrefix
           - DeployCUDOSDashboard
           - DataBucketsKmsKeysArns
+          - ShareDashboard
     ParameterLabels:
       PrerequisitesQuickSight:
         default: "I have enabled QuickSight Enterprise Edition AND I have a SPICE capacity in the current region."
@@ -78,6 +79,8 @@ Metadata:
         default: "IAM Role Name for QuickSight Datasource"
       QuickSightDataSetRefreshSchedule:
         default: "QuickSight DataSet Refresh Schedule [DEPRECATED]"
+      ShareDashboard:
+        default: "Share Dashboards"
       LambdaLayerBucketPrefix:
         default: "LambdaLayerBucketPrefix - Please do not change"
       GlueDataCatalog:
@@ -118,6 +121,11 @@ Parameters:
     Type: String
     Default: 'CidQuickSightDataSourceRole'
     Description: "IAM Role Name to be used on QuickSight Datasource Creation. If empty - then the Default QuickSight Role will be used; if provided other existing role, will use that Role; if name equal to 'CidQuickSightDataSourceRole', then a role will be created by this CloudFromation)."
+  ShareDashboard:
+    Type: String
+    Default: 'yes'
+    Description: "Make Dashboards visible by all users who access this QuickSight account."
+    AllowedValues: ["yes", "no"]
   CURBucketPath:
     Type: String
     MinLength: 3
@@ -1611,7 +1619,7 @@ Resources:
       Description: An AWS managed layer with a cid-cmd package installed
       Content:
         S3Bucket: !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}'
-        S3Key: 'cid-resource-lambda-layer/cid-0.3.9.zip' #replace version here if needed
+        S3Key: 'cid-resource-lambda-layer/cid-0.3.10.zip' #replace version here if needed
       CompatibleRuntimes:
         - python3.10
         - python3.11
@@ -1635,7 +1643,7 @@ Resources:
         cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ]
         quicksight-user: !Ref QuickSightUser
         account-map-source: 'dummy' #initial
-        share-with-account: 'yes'
+        share-with-account: !Ref ShareDashboard
 
   CUDOSDashboard:
     Type: Custom::CidDashboard
@@ -1655,7 +1663,7 @@ Resources:
         cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ]
         quicksight-user: !Ref QuickSightUser
         account-map-source: 'dummy' #initial
-        share-with-account: 'yes'
+        share-with-account: !Ref ShareDashboard
       Tags: # Hacky way to manage conditional dependencies
         - Key: IgnoreNeedCostIntelligenceDashboard
           Value: !If [NeedCostIntelligenceDashboard, !Ref CostIntelligenceDashboard, '']
@@ -1677,7 +1685,7 @@ Resources:
         cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ]
         quicksight-user: !Ref QuickSightUser
         account-map-source: 'dummy' #initial
-        share-with-account: 'yes'
+        share-with-account: !Ref ShareDashboard
       Tags: # Hacky way to manage conditional dependencies
         - Key: IgnoreNeedCostIntelligenceDashboard
           Value: !If [NeedCostIntelligenceDashboard, !Ref CostIntelligenceDashboard, '']
@@ -1699,7 +1707,7 @@ Resources:
         cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ]
         quicksight-user: !Ref QuickSightUser
         account-map-source: 'dummy' #initial
-        share-with-account: 'yes'
+        share-with-account: !Ref ShareDashboard
       Tags: # Hacky way to manage conditional dependencies
         - Key: IgnoreNeedCostIntelligenceDashboard
           Value: !If [NeedCostIntelligenceDashboard, !Ref CostIntelligenceDashboard, '']
@@ -1723,7 +1731,7 @@ Resources:
         glue-data-catalog: !Ref GlueDataCatalog
         cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ]
         quicksight-user: !Ref QuickSightUser
-        share-with-account: 'yes'
+        share-with-account: !Ref ShareDashboard
         view-ta-organizational-view-reports-s3FolderPath: !Sub '${OptimizationDataCollectionBucketPath}/trusted-advisor/trusted-advisor-data'
 
   ComputeOptimizerDashboard:
@@ -1743,7 +1751,7 @@ Resources:
         glue-data-catalog: !Ref GlueDataCatalog
         cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ]
         quicksight-user: !Ref QuickSightUser
-        share-with-account: 'yes'
+        share-with-account: !Ref ShareDashboard
         view-compute-optimizer-lambda-lines-s3FolderPath:       !Sub '${OptimizationDataCollectionBucketPath}/compute_optimizer/compute_optimizer_lambda'
         view-compute-optimizer-ebs-volume-lines-s3FolderPath:   !Sub '${OptimizationDataCollectionBucketPath}/compute_optimizer/compute_optimizer_ebs_volume'
         view-compute-optimizer-auto-scale-lines-s3FolderPath:   !Sub '${OptimizationDataCollectionBucketPath}/compute_optimizer/compute_optimizer_auto_scale'

cfn-templates/data-exports-aggregation.yaml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: "2010-09-09"
-Description: AWS Billing Data Export Aggregation v0.1.0
+Description: AWS Billing Data Export Aggregation v0.1.4
 Metadata:
 
   AWS::CloudFormation::Interface:
@@ -903,6 +903,7 @@ Resources:
             - {"Name": "line_item_resource_id", "Type": "string" }
             - {"Name": "line_item_unblended_cost", "Type": "double" }
             - {"Name": "line_item_usage_account_id", "Type": "string" }
+            - {"Name": "line_item_usage_account_name", "Type": "string" }
             - {"Name": "line_item_usage_amount", "Type": "double" }
             - {"Name": "line_item_usage_end_date", "Type": "timestamp" }
             - {"Name": "line_item_usage_start_date", "Type": "timestamp" }
@@ -921,20 +922,20 @@ Resources:
             - {"Name": "product_to_location", "Type": "string" }
             - {"Name": "reservation_amortized_upfront_fee_for_billing_period", "Type": "double" }
             - {"Name": "reservation_effective_cost", "Type": "double" }
-            - {"Name": "reservation_end_time", "Type": "timestamp" }
+            - {"Name": "reservation_end_time", "Type": "string" }
             - {"Name": "reservation_reservation_a_r_n", "Type": "string" }
-            - {"Name": "reservation_start_time", "Type": "timestamp" }
+            - {"Name": "reservation_start_time", "Type": "string" }
             - {"Name": "reservation_unused_amortized_upfront_fee_for_billing_period", "Type": "double" }
             - {"Name": "reservation_unused_recurring_fee", "Type": "double" }
             - {"Name": "resource_tags", "Type": "map<string,string>" }
             - {"Name": "savings_plan_amortized_upfront_commitment_for_billing_period", "Type": "double" }
-            - {"Name": "savings_plan_end_time", "Type": "timestamp" }
+            - {"Name": "savings_plan_end_time", "Type": "string" }
             - {"Name": "savings_plan_offering_type", "Type": "string" }
             - {"Name": "savings_plan_payment_option", "Type": "string" }
             - {"Name": "savings_plan_purchase_term", "Type": "string" }
             - {"Name": "savings_plan_savings_plan_a_r_n", "Type": "string" }
             - {"Name": "savings_plan_savings_plan_effective_cost", "Type": "double" }
-            - {"Name": "savings_plan_start_time", "Type": "timestamp" }
+            - {"Name": "savings_plan_start_time", "Type": "string" }
             - {"Name": "savings_plan_total_commitment_to_date", "Type": "double" }
             - {"Name": "savings_plan_used_commitment", "Type": "double" }
             - {"Name": "split_line_item_parent_resource_id", "Type": "string" }

changes/CHANGELOG-graviton-opportunities.md

@@ -1,5 +1,29 @@
 # What's new in the Graviton Opportunities Dashboard
 
+## Graviton Opportunities Dashboard v1.1.1:
+```
+cid-cmd update --dashboard-id graviton-opportunities
+Choose 'Yes' for the following prompt
+[confirm-update] No updates available, should I update it anyway? 
+```
+* Fix broken hyperlinks under Additional Resources
+
+## Graviton Opportunities Dashboard v1.1.0:
+**Important:** If attempting to update the dashboard, please update cid-cmd first. To update run these commands in your CloudShell (recommended) or other terminal:
+
+```
+python3 -m ensurepip --upgrade
+pip3 install --upgrade cid-cmd
+cid-cmd update --dashboard-id graviton-opportunities --force --recursive
+```
+**Bug fixes and improvements**
+* Mordernization mapping updated with missing instance types
+* Deleted Target Coverage and Efficiency sliders
+* Including Savings Plan covered usage for EC2
+* Updated Missing filters for RDS
+* Updates to visuals
+* New visuals for existing, potential coverage and implementation effort
+
 ## Graviton Opportunities Dashboard v1.0.3:
 * Updated modernization mapping to include r8g
 * Moved EC2 usage type filters from dashboard into SQL

cid/_version.py

@@ -1 +1 @@
-__version__ = '0.3.9'
+__version__ = '0.3.10'

cid/common.py

@@ -323,7 +323,12 @@ def load_catalog(self, catalog_url):
         ''' load additional resources from catalog
         '''
         try:
-            catalog = yaml.safe_load(self.get_page(catalog_url).text)
+            if 'https://' in catalog_url:
+                text = self.get_page(catalog_url).text
+            else:
+                with open(catalog_url, encoding='utf-8') as catalog_file:
+                    text = catalog_file.read()
+            catalog = yaml.safe_load(text)
         except (requests.exceptions.RequestException, yaml.error.MarkedYAMLError) as exc:
             logger.warning(f'Failed to load a catalog url: {exc}')
             logger.debug(exc, exc_info=True)
@@ -348,16 +353,29 @@ def get_template_parameters(self, parameters: dict, param_prefix: str='', others
                     choices=self.cur.tag_and_cost_category_fields + ["'none'"],
                 )
             elif isinstance(value, dict) and value.get('type') == 'athena':
-                if 'query' not in value:
-                    raise CidCritical(f'Failed fetching parameter {prefix}{key}: paramter with type ahena must have query value.')
-                query = value['query']
-                try:
-                    res = self.athena.query(query)[0]
-                except (self.athena.client.exceptions.ClientError, CidError, CidCritical) as exc:
-                    raise CidCritical(f'Failed fetching parameter {prefix}{key}: {exc}') from exc
-                if not res:
-                    raise CidCritical(f'Failed fetching parameter {prefix}{key}, {value}. Athena returns empty result')
-                params[key] = res[0]
+                if get_parameters().get(prefix + key): # priority to user input
+                    params[key] = get_parameters().get(prefix + key)
+                else:
+                    if 'query' not in value:
+                        raise CidCritical(f'Failed fetching parameter {prefix}{key}: parameter with type Athena must have query value.')
+                    query = value['query']
+                    try:
+                        res_list = self.athena.query(query)
+                    except (self.athena.client.exceptions.ClientError, CidError, CidCritical) as exc:
+                        raise CidCritical(f'Failed fetching parameter {prefix}{key}: {exc}') from exc
+                    if not res_list:
+                        raise CidCritical(f'Failed fetching parameter {prefix}{key}, {value}. Athena returns empty results')
+                    elif len(res_list) == 1:
+                        params[key] = '-'.join(res_list[0])
+                    else:
+                        options = ['-'.join(res) for res in res_list]
+                        default = value.get('default')
+                        params[key] = get_parameter(
+                            param_name=prefix + key,
+                            message=f"Required parameter: {key} ({value.get('description')})",
+                            choices=options,
+                            default=default if default in options else None,
+                        )
             elif isinstance(value, dict):
                 params[key] = value.get('value')
                 while params[key] is None:

dashboards/aws-feeds/aws-feeds.yaml

@@ -2939,6 +2939,7 @@ datasets:
       - aws_feeds
     schedules:
     - default
+
   aws_feeds_blog_post:
     data:
       DataSetId: 173bf994-4763-4044-a96d-1cb6cce591f4
@@ -2998,9 +2999,12 @@ datasets:
     - default
     parameters:
       data_collection_database_name:
+        type: athena
+        query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
         default: optimization_data
         description: "Enter the name of the data collection database"
         global: True
+
   aws_feeds_security_bulletin:
     data:
       DataSetId: cd5b5e17-fdde-4563-a944-1c5fc3b27684
@@ -3054,9 +3058,12 @@ datasets:
     - default
     parameters:
       data_collection_database_name:
+        type: athena
+        query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
         default: optimization_data
         description: "Enter the name of the data collection database"
         global: True
+
   aws_feeds_whats_new:
     data:
       DataSetId: 00e4f86c-96b8-49b7-b68f-8b540e4fa6b7
@@ -3116,9 +3123,12 @@ datasets:
     - default
     parameters:
       data_collection_database_name:
+        type: athena
+        query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
         default: optimization_data
         description: "Enter the name of the data collection database"
         global: True
+
   aws_feeds_youtube:
     data:
       DataSetId: 5258b9ee-48d0-48f3-9ec4-baa4acd1d006
@@ -3236,6 +3246,8 @@ views:
         "${data_collection_database_name}"."aws_feeds_youtube"
     parameters:
       data_collection_database_name:
+        type: athena
+        query: SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = 'aws_feeds_youtube'
         default: optimization_data
         description: "Enter the name of the data collection database"
         global: True

dashboards/aws-marketplace/aws-marketplace-spg.yaml

@@ -165,6 +165,6 @@ views:
       , "sum"((CASE WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "line_item_usage_amount" WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "line_item_usage_amount" WHEN ("line_item_line_item_type" = 'Usage') THEN "line_item_usage_amount" ELSE 0 END)) "usage_quantity"
       , "sum"("line_item_unblended_cost") "unblended_cost"
       FROM
-        ${cur_table_name}
+        "${cur_table_name}"
       WHERE ((("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL  '36' MONTH)) AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL  '36' MONTH))) AND ("bill_billing_entity" = 'AWS Marketplace'))
       GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21

dashboards/catalog.yaml

@@ -4,7 +4,8 @@ Resources:
   - Url: data-transfer/DataTransfer-Cost-Analysis-Dashboard.yaml
   - Url: aws-marketplace/aws-marketplace-spg.yaml
   - Url: extended-support-cost-projection/extended-support-cost-projection.yaml
-  - Url: graviton-opportunities-dashboard/graviton_dashboard.yaml
+  - Url: graviton-savings-dashboard/graviton_savings_dashboard.yaml
+  - Url: graviton-savings-dashboard/graviton_legacy.yaml # deprecated
   - Url: health-events/health-events.yaml
   - Url: scad-containers-cost-allocation/scad-containers-cost-allocation.yaml
   - Url: aws-feeds/aws-feeds.yaml