Update fixes

assets/build_lambda_layer.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# This script builids a lambda layer. Outpits relative path of layer zip.
+export CID_VERSION=$(python3 -c "from cid import _version;print(_version.__version__)")
+rm -rf build
+
+function get_hash {
+    find ./cid -type f -exec md5sum {} + | md5sum | awk '{print $1}'
+}
+
+function build_layer {
+    echo 'Building a layer'
+    mkdir -p ./python
+    python3 -m pip install . -t ./python
+    zip -qr cid-$CID_VERSION.zip ./python
+    ls -l cid-$CID_VERSION.zip
+    rm -rf ./python
+}
+
+# Check if code has been changed
+previous_hash=$(cat cid-$CID_VERSION.hash)
+actual_hash=$(get_hash)
+if [ "$actual_hash" == "$previous_hash" ] && [ -e "cid-$CID_VERSION.zip" ]; then
+    echo "No changes in code. Reuse existing zip." 1>&2
+else
+    build_layer 1>&2
+    echo $actual_hash > cid-$CID_VERSION.hash
+fi
+
+ls cid-$CID_VERSION.zip

assets/publish_lambda_layer.sh

@@ -1,31 +1,25 @@
 #!/bin/bash
 # This script can be used for release or testing of lambda layers upload.
-export CID_VERSION=$(python3 -c "from cid import _version;print(_version.__version__)")
-rm -rf build
 
-echo 'Building a layer'
-mkdir -p ./python
-python3 -m pip install . -t ./python
-zip -qr cid-$CID_VERSION.zip ./python
-ls -l cid-$CID_VERSION.zip
-rm -rf ./python
+# First build layer
+layer=$(./assets/build_lambda_layer.sh)
 
+# Then publish on s3
 export AWS_REGION=us-east-1
 export STACK_SET_NAME=LayerBuckets
-
 aws cloudformation list-stack-instances \
   --stack-set-name $STACK_SET_NAME \
   --query 'Summaries[].[StackId,Region]' \
   --output text |
   while read stack_id region; do
-    echo "uploading cid-$CID_VERSION.zip to $region"
+    echo "uploading $layer to $region"
     bucket=$(aws cloudformation list-stack-resources --stack-name $stack_id \
       --query 'StackResourceSummaries[?LogicalResourceId == `LayerBucket`].PhysicalResourceId' \
       --region $region --output text)
     output=$(aws s3api put-object \
       --bucket "$bucket" \
-      --key cid-resource-lambda-layer/cid-$CID_VERSION.zip \
-      --body ./cid-$CID_VERSION.zip)
+      --key cid-resource-lambda-layer/$layer \
+      --body ./$layer)
     if [ $? -ne 0 ]; then
       echo "Error: $output"
     else
@@ -34,7 +28,7 @@ aws cloudformation list-stack-instances \
   done
 
 echo 'Cleanup'
-rm -vf ./cid-$CID_VERSION.zip
+rm -vf ./$layer
 
 
 # Publish cfn (only works for the release)

cfn-templates/cid-admin-policies.yaml

@@ -278,9 +278,7 @@ Resources:
           Effect: Allow
           Resource:
           - !Sub arn:aws:iam::${AWS::AccountId}:role/CidSpiceRefreshExecutionRole
-          - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-*-ProcessPathLambdaExec*
-          - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-*-InitLambdaExecutionRole*
-          - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-*-CidCURCrawlerRole*
+          - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-*-* #Roles created by CFN stack. Name is hardcoded here
           - !Sub arn:aws:iam::${AWS::AccountId}:role/CidQuickSightDataSourceRole
           - !Sub arn:aws:iam::${AWS::AccountId}:role/CidExecRole
 
@@ -289,18 +287,23 @@ Resources:
           - lambda:AddPermission
           - lambda:CreateFunction
           - lambda:DeleteFunction
-          - lambda:DeleteLayerVersion
           - lambda:GetFunction
-          - lambda:GetLayerVersion
           - lambda:InvokeFunction
-          - lambda:PublishLayerVersion
           - lambda:RemovePermission
+          - lambda:UpdateFunctionConfiguration
+          - lambda:UpdateFunctionCode
+          - lambda:PublishLayerVersion
+          - lambda:GetLayerVersion
+          - lambda:DeleteLayerVersion
           Effect: Allow
           Resource:
-          - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidProcessPath-DoNotRun
-          - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceDashboard
-          - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidInitialSetup-DoNotRun
-          - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidSpiceRefreshLambda
+          - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:Cid*
+          # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidProcessPath-DoNotRun
+          # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceProcessPath-DoNotRun
+          # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceDashboard
+          # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceFunctionInit-DoNotRun
+          # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidInitialSetup-DoNotRun #legacy
+          # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidSpiceRefreshLambda
           - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:layer:CidLambdaLayer*
 
         - Sid: QuickSightDashboard
@@ -311,7 +314,7 @@ Resources:
           - quicksight:DescribeDashboard
           Effect: Allow
           Resource:
-          - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/cudos
+          - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/cudos*
           - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/cost_intelligence_dashboard
           - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/kpi_dashboard
           - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/ta-organizational-view
@@ -337,7 +340,7 @@ Resources:
           - quicksight:CreateDataSet
           - quicksight:DeleteDataSet
           - quicksight:PassDataSet
-          - quicksight:DescribDataSet
+          - quicksight:DescribeDataSet
           - quicksight:DescribeDataSetPermissions
           - quicksight:UpdateDataSetPermissions
           Effect: Allow
@@ -357,7 +360,7 @@ Resources:
           - quicksight:DeleteDataSetRefreshProperties
           Effect: Allow
           Resource:
-          - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dataset/*/refresh-schedule/* # DataSetIDs are dynamic as well as shcedule ids
+          - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dataset/*/refresh-schedule/* # DataSetIDs are dynamic as well as schedule ids
 
         - Sid: CreateQueryResultsBucketS3
           Action:
@@ -450,6 +453,8 @@ Resources:
           - lambda:DeleteFunction
           - lambda:GetFunction
           - lambda:InvokeFunction
+          - lambda:UpdateFunctionConfiguration
+          - lambda:UpdateFunctionCode
           Effect: Allow
           Resource:
           - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-Analytics
@@ -515,7 +520,7 @@ Resources:
     Condition: CreateCURReplicationPolicy
     Properties:
       ManagedPolicyName: CidCURReplicationPolicy
-      Description: 'CloudIntelligenceDashboards Policy for CUR Creating and Stting Replication'
+      Description: 'CloudIntelligenceDashboards Policy for CUR Creating and Setting Replication'
       Roles:
         - !Ref RoleName
       PolicyDocument:
@@ -574,6 +579,8 @@ Resources:
           - lambda:GetFunctionCodeSigningConfig
           - lambda:GetRuntimeManagementConfig
           - lambda:InvokeFunction
+          - lambda:UpdateFunctionConfiguration
+          - lambda:UpdateFunctionCode
           Effect: Allow
           Resource:
           - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-Analytics

cfn-templates/cid-cfn.yml

@@ -315,7 +315,7 @@ Resources:
       FunctionName:  !Sub 'CidSpiceRefreshLambda${Suffix}'
       Role: !GetAtt SpiceRefreshExecutionRole.Arn
       Description: 'Refresh QuickSight DataSets for CID'
-      Runtime: python3.9
+      Runtime: python3.10
       Architectures: [ x86_64 ] #Compatible with arm64 but it is not supported in all regions
       MemorySize: 128
       Timeout: 60
@@ -450,13 +450,45 @@ Resources:
             EncryptionOption: SSE_S3
           OutputLocation: !If [ NeedAthenaQueryResultsBucket, !Sub 's3://${MyAthenaQueryResultsBucket}/', !Sub 's3://${AthenaQueryResultsBucket}/' ]
 
+  #Legacy version. Replaced by CustomResourceFunctionInit but we cannot remove it completely as it was removing workgroup on deletion of the custom resource.
+  CustomRessourceFunctionInit:
+    Type: AWS::Lambda::Function
+    Properties:
+      FunctionName: !Sub CidInitialSetup-DoNotRun${Suffix}
+      Role: !GetAtt 'InitLambdaExecutionRole.Arn'
+      Description: "CID legacy setup"
+      Runtime: python3.10
+      Handler: 'index.lambda_handler'
+      Code:
+        ZipFile: |
+          # This is a legacy lambda. You can delete it. This was kept to disable delete workgroup functionality.
+          import json
+          import urllib3
+
+          def lambda_handler(event, context):
+              url = event.get('ResponseURL')
+              json_body = json.dumps({
+                'Status': 'SUCCESS'
+                'Reason': 'legacy'
+                'PhysicalResourceId': 'keep_it_constant'
+                'StackId': event.get('StackId')
+                'RequestId': event.get('RequestId')
+                'LogicalResourceId': event.get('LogicalResourceId')
+              })
+              try:
+                  http = urllib3.PoolManager()
+                  response = http.request('PUT', url, body=json_body, headers={'content-type' : '', 'content-length' : str(len(json_body))}, retries=False)
+                  print(f"Status code: {response}")
+              except Exception as exc:
+                  print("Failed sending PUT to CFN: " + str(exc))
+
   CustomResourceFunctionInit:
     Type: AWS::Lambda::Function
     Properties:
       FunctionName: !Sub "CidCustomResourceFunctionInit-DoNotRun${Suffix}"
       Role: !GetAtt 'InitLambdaExecutionRole.Arn'
       Description: "Do what CFN cannot: start crawler, delete bucket with objects and delete an non empty workgroup"
-      Runtime: python3.9
+      Runtime: python3.10
       Architectures: [ x86_64 ] #Compatible with arm64 but it is not supported in all regions
       MemorySize: 128
       Timeout: 300
@@ -693,7 +725,7 @@ Resources:
       Role: !GetAtt 'ProcessPathLambdaExecutionRole.Arn'
       FunctionName: !Sub "CidCustomResourceProcessPath-DoNotRun${Suffix}"
       Description: "Do what CFN cannot: process string of path"
-      Runtime: python3.9
+      Runtime: python3.10
       Architectures: [ x86_64 ] #Compatible with arm64 but it is not supported in all regions
       MemorySize: 128
       Timeout: 60
@@ -893,6 +925,7 @@ Resources:
             - {"Name": "pricing_offering_class", "Type": "string" }
             - {"Name": "pricing_public_on_demand_cost", "Type": "double" }
             - {"Name": "pricing_purchase_option", "Type": "string" }
+            - {"Name": "pricing_term", "Type": "string" }
             - {"Name": "pricing_unit", "Type": "string" }
             - {"Name": "product_cache_engine", "Type": "string" }
             - {"Name": "product_current_generation", "Type": "string" }
@@ -910,6 +943,7 @@ Resources:
             - {"Name": "product_product_name", "Type": "string" }
             - {"Name": "product_region", "Type": "string" }
             - {"Name": "product_servicecode", "Type": "string" }
+            - {"Name": "product_storage", "Type": "string" }
             - {"Name": "product_tenancy", "Type": "string" }
             - {"Name": "product_to_location", "Type": "string" }
             - {"Name": "product_volume_api_name", "Type": "string" }
@@ -1175,6 +1209,8 @@ Resources:
               - Effect: Allow
                 Action:
                   - athena:GetWorkGroup
+                  - athena:CreateWorkGroup
+                  - athena:UpdateWorkGroup
                 Resource:
                   Fn::If:
                    - NeedAthenaWorkgroup
@@ -1192,6 +1228,7 @@ Resources:
                 Resource: "*" # This is needed to allow Autodetect in CID-CMD
               - Effect: Allow
                 Action:
+                  - s3:CreateBucket
                   - s3:ListBucket
                   - s3:ListBucketMultipartUploads
                   - s3:ListMultipartUploadParts
@@ -1364,7 +1401,7 @@ Resources:
       FunctionName: !Sub 'CidCustomResourceDashboard${Suffix}'
       Description: 'A lambda that manage create delete update of Athena views, QuickSight Datasets and dashboards using CID-CMD tool'
       Role: !GetAtt CidExecRole.Arn
-      Runtime: python3.9
+      Runtime: python3.10
       Architectures: [ x86_64 ] #Compatible with arm64 but it is not supported in all regions
       MemorySize: 2688
       Timeout: 300 # Time of discovery depend on number of dashboards
@@ -1470,7 +1507,7 @@ Resources:
         S3Bucket: !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}'
         S3Key: 'cid-resource-lambda-layer/cid-0.2.35.zip' #replace version here if needed
       CompatibleRuntimes:
-        - python3.9
+        - python3.10
 
   CostIntelligenceDashboard:
     Type: Custom::CidDashboard