Merge branch 'main' into add-connect-dashboard

aws-samples · Nov 21, 2024 · 87087fb · 87087fb
2 parents 18bfd58 + 5d6469c
commit 87087fb
Show file tree

Hide file tree

Showing 23 changed files with 12,546 additions and 6,005 deletions.
diff --git a/cfn-templates/cid-cfn.yml b/cfn-templates/cid-cfn.yml
@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
-Description: Deployment of Cloud Intelligence Dashboards v4.0.1
+Description: Deployment of Cloud Intelligence Dashboards v4.0.5
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -137,7 +137,7 @@ Parameters:
     Type: String
     Default: '2.0'
     AllowedValues: ["1.0", "2.0"]
-    Description: "We support 2 options: CUR 2.0 from DataExports stack managed by CID or CUR 1(Legacy)."
+    Description: "We support 2 options: CUR 2.0 from DataExports stack managed by CID or CUR 1(Legacy). When you switch to CUR2 consider KeepLegacyCURTable parameter to yes for keeping Legacy CUR table for migration period."
   CURBucketPath:
     Type: String
     MinLength: 3
@@ -248,7 +248,9 @@ Conditions:
   NeedCostIntelligenceDashboard: !Equals [ !Ref DeployCostIntelligenceDashboard, "yes" ]
   NeedKPIDashboard: !Equals [ !Ref DeployKPIDashboard, "yes" ]
   NeedTAODashboard: !Equals [ !Ref DeployTAODashboard, "yes" ]
-  NeedLegacyCUR: !Equals [!Ref KeepLegacyCURTable, "yes"]
+  NeedLegacyCUR: !Or
+    - !Equals [!Ref KeepLegacyCURTable, "yes"]
+    - !Equals [!Ref CURVersion, '1.0']
   NeedComputeOptimizerDashboard: !Equals [ !Ref DeployComputeOptimizerDashboard, "yes" ]
   UseCUR2:
     Fn::And:
@@ -580,13 +582,14 @@ Resources:
       Runtime: python3.11
       Architectures: [ x86_64 ] #Compatible with arm64 but it is not supported in all regions
       MemorySize: 128
-      Timeout: 300
+      Timeout: 600
       Handler: 'index.lambda_handler'
       Code:
         ZipFile: |
           import os
           import uuid
           import json
+          import time
           import boto3
           import botocore
           import urllib3
@@ -642,12 +645,31 @@ Resources:
 
           def on_create():
               if CRAWLER:
+                  timeout_seconds = 300
+                  glue = boto3.client('glue')
                   try:
-                      boto3.client('glue').start_crawler(Name=CRAWLER)
+                      glue.start_crawler(Name=CRAWLER)
                   except Exception as exc:
-                      return (True, f'ERROR: error invoking crawler {CRAWLER} {exc}')
-                  return (True, 'INFO: crawler started. Takes 1 min to update the table.')
-              return (True, 'INFO: No actions on create')
+                      if 'CrawlerRunningException' in str(exc):
+                          print ("crawler is running already")
+                      else:
+                          return (True, f'ERROR: error invoking crawler {CRAWLER} {exc}')
+                  print('started crawler started. waiting for crawler to finish')
+                  try:
+                      start_time = time.time()
+                      while time.time() - start_time < timeout_seconds:
+                          time.sleep(1)
+                          crawler_status = glue.get_crawler(Name=CRAWLER)['Crawler']['State']
+                          print(f'status = {crawler_status}')
+                          if crawler_status in ('READY', 'STOPPING'):
+                              print("Stop waiting")
+                              break
+                      else:
+                          return (True, f"Timeout exceeded. Crawler '{CRAWLER}' did not complete. This is not a fatal error and the rest of the deployment will continue.")
+                  except Exception as exc:
+                      return (True, f'ERROR: error waiting for crawler {CRAWLER} {exc}')
+                  return (True, 'Crawler run completed.')
+              return (True, 'Ended create.')
 
           def on_delete():
               # Delete bucket (CF cannot delete if they are non-empty)
@@ -812,6 +834,7 @@ Resources:
           - Effect: Allow
             Action:
               - glue:StartCrawler
+              - glue:GetCrawler
             Resource:
               - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:crawler/${MyGlueCURCrawler}'
       Roles:
@@ -850,7 +873,7 @@ Resources:
       Path: !Ref RolePath
       PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref 'AWS::NoValue']
       Policies:
-        - PolicyName: AWSCURCrawlerComponentFunction
+        - PolicyName: CloudWatch
           PolicyDocument:
             Version: 2012-10-17
             Statement:
@@ -1758,7 +1781,7 @@ Resources:
           - LambdaLayerBucketPrefixIsManaged
           - !FindInMap [RegionMap, !Ref 'AWS::Region', BucketName]
           - !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}' # Region added for backward compatibility
-        S3Key: 'cid-resource-lambda-layer/cid-4.0.1.zip' #replace version here if needed
+        S3Key: 'cid-resource-lambda-layer/cid-4.0.5.zip' #replace version here if needed
       CompatibleRuntimes:
         - python3.10
         - python3.11

diff --git a/cfn-templates/data-exports-aggregation.yaml b/cfn-templates/data-exports-aggregation.yaml
@@ -24,6 +24,7 @@ Metadata:
         Parameters:
           - EnableSCAD
           - RolePath
+          - TimeGranularity
 
     ParameterLabels:
       ManageCOH:
@@ -40,6 +41,8 @@ Metadata:
         default: "Source Account Ids (Comma separated list)"
       EnableSCAD:
         default: "Enable Split Cost Allocation Data (SCAD) in CUR 2.0"
+      TimeGranularity:
+        default: "CUR 2.0 Granularity. Do not change."
 
 
 Parameters:
@@ -92,8 +95,13 @@ Parameters:
     Default: "yes"
   RolePath:
     Type: String
-    Default: '/'
     Description: Path for roles where PermissionBoundaries can limit location
+    Default: '/'
+  TimeGranularity:
+    Type: String
+    Description: Changing of this parameter will require redeployment of this Stack, purging of data in Destination and then additional Backfill request. HOURLY is a recommended option unless your AWS invoice is more then $50M (in this case contact your TAM when installing).
+    Default: "HOURLY"
+    AllowedValues: ["HOURLY", "DAILY", "MONTHLY"]
 
 Conditions:
   EmptySourceAccountIds: !Equals [ !Ref SourceAccountIds, '']
@@ -427,7 +435,7 @@ Resources:
           QueryStatement: !If [EnableSCAD, !FindInMap [DataExports, CUR2, SCADQuery], !FindInMap [DataExports, CUR2, DefaultQuery]]
           TableConfigurations:
             COST_AND_USAGE_REPORT:
-              TIME_GRANULARITY: "HOURLY"
+              TIME_GRANULARITY: !Ref TimeGranularity
               INCLUDE_RESOURCES: "TRUE"
               INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY: "FALSE"
               INCLUDE_SPLIT_COST_ALLOCATION_DATA: !If [EnableSCAD, "TRUE", "FALSE"]
@@ -458,7 +466,7 @@ Resources:
           QueryStatement: !If [EnableSCAD, !FindInMap [DataExports, CUR2, SCADQuery], !FindInMap [DataExports, CUR2, DefaultQuery]]
           TableConfigurations:
             COST_AND_USAGE_REPORT:
-              TIME_GRANULARITY: "HOURLY"
+              TIME_GRANULARITY: !Ref TimeGranularity
               INCLUDE_RESOURCES: "TRUE"
               INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY: "FALSE"
               INCLUDE_SPLIT_COST_ALLOCATION_DATA: !If [EnableSCAD, "TRUE", "FALSE"]

diff --git a/changes/CHANGELOG-graviton-opportunities.md b/changes/CHANGELOG-graviton-opportunities.md
@@ -1,5 +1,16 @@
 # What's new in the Graviton Opportunities Dashboard
 
+**Important:** Graviton Opportunities Dashboard has changed it's name Graviton Savings Dashboard.
+
+Please delete the legacy version of this dashboard by running 
+```
+cid-cmd delete --dashboard-id graviton-opportunities
+``` 
+Please deploy the new version of this dashboard by running 
+```
+cid-cmd deploy --dashboard-id graviton-savings
+```
+
 ## Graviton Opportunities Dashboard v1.1.1:
 ```
 cid-cmd update --dashboard-id graviton-opportunities

diff --git a/changes/CHANGELOG-graviton-savings.md b/changes/CHANGELOG-graviton-savings.md
@@ -0,0 +1,76 @@
+# What's new in the Graviton Savings Dashboard
+
+## Graviton Savings Dashboard v2.1.0:
+```
+cid-cmd update --dashboard-id graviton-savings --force --recursive
+```
+* CUR 2 Support
+* Fixed filters on Opensearch and ElastiCache Tabs
+
+
+## Graviton Savings Dashboard v2.0.0:
+```
+cid-cmd update --dashboard-id graviton-savings --force --recursive
+```
+* New Savings Implementation Effort and Reason Categorization
+* New Top and Bottom insights for Managed Services
+* New Radio buttons to toggle between Spend, Usage & Savings
+* New Potential Graviton Savings breakdown by Purchase Option and Operating System
+
+
+## Graviton Savings Dashboard Name Update ##
+**Important:** Graviton Opportunities Dashboard has changed it's name Graviton Savings Dashboard.
+
+Please delete the legacy version of this dashboard by running 
+```
+cid-cmd delete --dashboard-id graviton-opportunities
+``` 
+Please deploy the new version of this dashboard by running 
+```
+cid-cmd deploy --dashboard-id graviton-savings
+```
+
+## Graviton Opportunities Dashboard v1.1.1:
+```
+cid-cmd update --dashboard-id graviton-opportunities
+Choose 'Yes' for the following prompt
+[confirm-update] No updates available, should I update it anyway? 
+```
+* Fix broken hyperlinks under Additional Resources
+
+## Graviton Opportunities Dashboard v1.1.0:
+**Important:** If attempting to update the dashboard, please update cid-cmd first. To update run these commands in your CloudShell (recommended) or other terminal:
+
+```
+python3 -m ensurepip --upgrade
+pip3 install --upgrade cid-cmd
+cid-cmd update --dashboard-id graviton-opportunities --force --recursive
+```
+**Bug fixes and improvements**
+* Mordernization mapping updated with missing instance types
+* Deleted Target Coverage and Efficiency sliders
+* Including Savings Plan covered usage for EC2
+* Updated Missing filters for RDS
+* Updates to visuals
+* New visuals for existing, potential coverage and implementation effort
+
+## Graviton Opportunities Dashboard v1.0.3:
+* Updated modernization mapping to include r8g
+* Moved EC2 usage type filters from dashboard into SQL
+
+## Graviton Opportunities Dashboard v1.0.2:
+**Important:** If attempting to update the dashboard, please update cid-cmd first. To update run these commands in your CloudShell (recommended) or other terminal:
+
+```
+python3 -m ensurepip --upgrade
+pip3 install --upgrade cid-cmd
+cid-cmd update --dashboard-id graviton-opportunities --force --recursive
+```
+
+**Bug fixes and improvements**
+ * Updates to visuals
+ * Bug fix for duplicate resources caused by data collector storing multiple versions of upgraded resources
+
+
+## Graviton Opportunities Dashboard v1.0.0
+* Initial release
diff --git a/changes/CHANGELOG-hed.md b/changes/CHANGELOG-hed.md
@@ -1,4 +1,10 @@
 # What's new in Health Events Dashboard (HED)
+## v2.0.3
+* Fix case where reported resource name is not a full unique ARN to derive unique instance counts
+* Easier filtering by date ranges
+* Layout changes to event detail section
+* Minor cosmetic and usage guidance changes
+
 ## v2.0.2
 * Fix Event Category filter on Events Explorer tab
 * Minor cosmetic changes

diff --git a/cid/_version.py b/cid/_version.py
@@ -1,2 +1,2 @@
-__version__ = '4.0.1'
+__version__ = '4.0.5'
 
diff --git a/cid/builtin/core/data/queries/co/auto_scale.json b/cid/builtin/core/data/queries/co/auto_scale.json
@@ -1,7 +1,7 @@
 {
     "DatabaseName": "${athena_database_name}",
     "TableInput": {
-        "Name": "${athenaTableName}",
+        "Name": "${athena_table_name}",
         "StorageDescriptor": {
             "Location": "${s3FolderPath}",
             "Columns": [

diff --git a/cid/builtin/core/data/queries/co/ebs_volume.json b/cid/builtin/core/data/queries/co/ebs_volume.json
@@ -1,7 +1,7 @@
 {
     "DatabaseName": "${athena_database_name}",
     "TableInput": {
-        "Name": "${athenaTableName}",
+        "Name": "${athena_table_name}",
         "StorageDescriptor": {
             "Location": "${s3FolderPath}",
             "Columns": [

diff --git a/cid/builtin/core/data/queries/co/ec2_instance.json b/cid/builtin/core/data/queries/co/ec2_instance.json
@@ -1,7 +1,7 @@
 {
     "DatabaseName": "${athena_database_name}",
     "TableInput": {
-        "Name": "${athenaTableName}",
+        "Name": "${athena_table_name}",
         "StorageDescriptor": {
             "Location": "${s3FolderPath}",
             "Columns": [

diff --git a/cid/builtin/core/data/queries/co/lambda.json b/cid/builtin/core/data/queries/co/lambda.json
@@ -1,7 +1,7 @@
 {
     "DatabaseName": "${athena_database_name}",
     "TableInput": {
-        "Name": "${athenaTableName}",
+        "Name": "${athena_table_name}",
         "StorageDescriptor": {
             "Location": "${s3FolderPath}",
             "Columns": [

diff --git a/cid/builtin/core/data/queries/co/rds_database.json b/cid/builtin/core/data/queries/co/rds_database.json
@@ -1,7 +1,7 @@
 {
     "DatabaseName": "${athena_database_name}",
     "TableInput": {
-        "Name": "${athenaTableName}",
+        "Name": "${athena_table_name}",
         "StorageDescriptor": {
             "Location": "${s3FolderPath}",
             "Columns": [

diff --git a/cid/builtin/core/data/queries/shared/cur.yaml b/cid/builtin/core/data/queries/shared/cur.yaml
@@ -1,6 +1,6 @@
 DatabaseName: "${athena_database_name}"
 TableInput:
-  Name: "${athenaTableName}"
+  Name: "${athena_table_name}"
   Owner: owner
   Retention: 0
   TableType: EXTERNAL_TABLE

diff --git a/cid/builtin/core/data/queries/tao/glue_table.json b/cid/builtin/core/data/queries/tao/glue_table.json
@@ -1,7 +1,7 @@
 {
     "DatabaseName": "${athena_database_name}",
     "TableInput": {
-        "Name": "${athenaTableName}",
+        "Name": "${athena_table_name}",
         "StorageDescriptor": {
             "Location": "${s3FolderPath}",
             "Columns": [

diff --git a/cid/common.py b/cid/common.py
@@ -300,7 +300,7 @@ def track(self, action, dashboard_id):
             logger.debug(f"Issue logging action {action}  for dashboard {dashboard_id} , due to a urllib3 exception {str(e)} . This issue will be ignored")
 
     def get_page(self, source):
-        resp = requests.get(source, timeout=10)
+        resp = requests.get(source, timeout=10, headers={'User-Agent': 'cid'})
         resp.raise_for_status()
         return resp
 
@@ -435,7 +435,7 @@ def _deploy(self, dashboard_id: str=None, recursive=True, update=False, **kwargs
 
         # In case if we cannot discover datasets, we need to discover dashboards
         # TODO: check if datasets returns explicit permission denied and only then discover dashboards as a workaround
-        self.qs.discover_dashboards()
+        self.qs.dashboards
 
         dashboard_id = dashboard_id or get_parameters().get('dashboard-id')
         category_filter = [cat for cat in get_parameters().get('category', '').upper().split(',') if cat]
@@ -782,8 +782,7 @@ def delete_dataset(self, name: str, id: str=None):
                 # Check if dataset is used in some other dashboard
                 for dashboard in (self.qs.dashboards or {}).values():
                     if dataset.id in dashboard.datasets.values():
-                        logger.info(f'Dataset {dataset.name} ({dataset.id}) is still used by dashboard "{dashboard.id}". Skipping.')
-                        print      (f'Dataset {dataset.name} ({dataset.id}) is still used by dashboard "{dashboard.id}". Skipping.')
+                        cid_print(f'Dataset {dataset.name} ({dataset.id}) is still used by dashboard "{dashboard.id}". Skipping.')
                         return False
                 else: #not used
 
@@ -1073,13 +1072,13 @@ def check_dashboard_version_compatibility(self, dashboard_id):
 
         if dashboard.latest:
             cid_print("You are up to date!")
-            cid_print(f"  Version    {str(dashboard.cid_version)}")
+            cid_print(f"  Version    {dashboard.cid_version}")
         else:
             cid_print(f"An update is available:")
-            cid_print(f"  Version    {str(dashboard.cid_version): <9} ->  {str(dashboard.cid_version_latest): <9}")
+            cid_print(f"  Version    {dashboard.cid_version} ->  {dashboard.latest_available_cid_version}")
 
         try:
-            return dashboard.cid_version.compatible_versions(dashboard.cid_version_latest)
+            return dashboard.cid_version.compatible_versions(dashboard.latest_available_cid_version)
         except ValueError as exc:
             logger.info(exc)
         return None
@@ -1091,13 +1090,12 @@ def update_dashboard(self, dashboard_id, dashboard_definition):
             print(f'Dashboard "{dashboard_id}" is not deployed')
             return
 
-        if isinstance(dashboard.deployedTemplate, CidQsTemplate):
-            print(f'Deployed template: {dashboard.deployedTemplate.arn}')
-        if isinstance(dashboard.sourceTemplate, CidQsTemplate):
-            print(f"Latest template:   {dashboard.sourceTemplate.arn}/version/{dashboard.latest_version}")
-
+        if isinstance(dashboard.deployed_template, CidQsTemplate):
+            print(f'Deployed template: {dashboard.deployed_template.arn}')
+        if isinstance(dashboard.source_template, CidQsTemplate):
+            print(f"Latest template:   {dashboard.source_template.arn}/version/{dashboard.source_template.version}")
         try:
-            cid_print(f'\nUpdating {dashboard_id} from <BOLD>{dashboard.cid_version}<END> to <BOLD>{dashboard.cid_version_latest}<END>')
+            cid_print(f'\nUpdating {dashboard_id} from <BOLD>{dashboard.cid_version}<END> to <BOLD>{dashboard.latest_available_cid_version}<END>')
         except:
             cid_print(f'\nUpdating {dashboard_id}')
             logger.debug('Failed to define versions. Still continue.')
@@ -1741,6 +1739,7 @@ def get_view_query(self, view_name: str) -> str:
         columns_tpl = {
             #'athena_datasource_arn': athena_datasource.arn,
             'athena_database_name': self.athena.DatabaseName,
+            'athena_table_name': view_name,
             'cur_database':    self.cur1.database   if cur1_required else None, # for backward compatibly
             'cur_table_name':  self.cur1.table_name if cur1_required else None, # for backward compatibly
             'cur1_database':   self.cur1.database   if cur1_required else None,