Feature: Allow custom Role for QuickSight DataSet

cfn-templates/cid-admin-policies.yaml

@@ -281,6 +281,7 @@ Resources:
           - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-Dashbo-ProcessPathLambdaExecuti*
           - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-Dashboa-InitLambdaExecutionRole*
           - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-Dashboards-CidCURCrawlerRole*
+          - !Sub arn:aws:iam::${AWS::AccountId}:role/CidQuickSightDataSourceRole
           - !Sub arn:aws:iam::${AWS::AccountId}:role/CidExecRole
 
         - Sid: LambdaForCFN

cfn-templates/cid-cfn.yml

@@ -35,6 +35,7 @@ Metadata:
           - CidVersion
           - GlueDataCatalog
           - Suffix
+          - QuickSightDataSourceRoleName
           - QuickSightDataSetRefreshSchedule
           - LambdaLayerBucketPrefix
           - DataBuketsKmsKeyArns
@@ -73,6 +74,8 @@ Metadata:
         default: "Cid Version - Please do not change"
       Suffix:
         default: "Suffix - Please do not change"
+      QuickSightDataSourceRoleName:
+        default: "IAM Role Name to be used on QuickSight Datasource Creation (if not provided, the default QuickSight Role will be used)."
       QuickSightDataSetRefreshSchedule:
         default: "QuickSight DataSet Refresh Schedule. Must be a valid cron or empty. If empty refresh will be disabled."
       LambdaLayerBucketPrefix:
@@ -110,7 +113,11 @@ Parameters:
   QuickSightDataSetRefreshSchedule:
     Type: String
     Default: ''
-    Description: REQUIRED - cron expression on when to refresh spice datasets daily outside of business hours. Default is 4 AM utc, this should work for most customers in US and EU time zones.'
+    Description: 'Cron expression on when to refresh spice datasets via Lambda. Only needed if some difficulities with refresh scheduling via API.'
+  QuickSightDataSourceRoleName:
+    Type: String
+    Default: 'CidQuickSightDataSourceRole'
+    Description: "IAM Role Name to be used on QuckSight Datasource Creation. If empty - then the Default QuckSight Role will be used; if provided other existing role, will use that Role; if name equal to 'CidQuickSightDataSourceRole', then a role will be created by this CloudFromation)."
   CURBucketPath:
     Type: String
     MinLength: 3
@@ -247,6 +254,22 @@ Conditions:
     Fn::And:
       - !Equals [ !Ref LakeFormationEnabled, "yes" ]
       - !Condition NeedCURTable
+  UseQuickSightDataSourceRole:
+    Fn::And:
+      - !Condition NeedDatasource
+      - !Not [!Equals [ !Ref QuickSightDataSourceRoleName, "" ]]
+  NeedQuickSightDataSourceRole:
+    Fn::And:
+      - !Condition NeedDatasource
+      - !Equals [ !Ref QuickSightDataSourceRoleName, "CidQuickSightDataSourceRole" ]
+  NeedQuickSightDataSourceRoleAndCUR:
+    Fn::And:
+      - !Condition NeedQuickSightDataSourceRole
+      - !Condition NeedCUR
+  NeedQuickSightDataSourceRoleAndODC:
+    Fn::And:
+      - !Condition NeedQuickSightDataSourceRole
+      - !Condition NeedDataCollectionLab
 
 Resources:
   SpiceRefreshExecutionRole: #Role needed to schedule spice ingestion for the datasets
@@ -965,6 +988,147 @@ Resources:
       Roles:
         - !Ref CidCURCrawlerRole
 
+  QuickSightDataSourceRole:
+    Type: AWS::IAM::Role
+    Condition: NeedQuickSightDataSourceRole
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+                - quicksight.amazonaws.com
+            Action:
+              - 'sts:AssumeRole'
+      Path: /
+      RoleName: !Sub '${QuickSightDataSourceRoleName}${Suffix}'
+      Policies:
+        - PolicyName: AthenaAccess
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              - Effect: Allow
+                Action:
+                  - athena:ListDataCatalogs
+                Resource: '*' # Cannot restrict this. See https://docs.aws.amazon.com/athena/latest/ug/datacatalogs-example-policies.html#datacatalog-policy-listing-data-catalogs
+
+              - Effect: Allow
+                Action:
+                  - athena:ListDatabases
+                Resource:
+                  - Fn::If:
+                      - NeedDatabase
+                      - !Sub arn:${AWS::Partition}:athena:${AWS::Region}:${AWS::AccountId}:database/${CidDatabase}
+                      - !Sub arn:${AWS::Partition}:athena:${AWS::Region}:${AWS::AccountId}:database/${DatabaseName}
+              # - Effect: Allow
+              #   Action:
+              #     - athena:ListDatabases
+              #     - athena:ListTableMetadata
+              #   Resource:
+              #     - !Sub 'arn:${AWS::Partition}:athena:${AWS::Region}:${AWS::AccountId}:datacatalog/${GlueDataCatalog}'
+              - Effect: Allow
+                Action:
+                  - glue:GetDatabases
+                Resource:
+                  - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog'
+              - Effect: Allow
+                Action:
+                  - glue:GetTable
+                  - glue:GetPartitions
+                Resource:
+                   - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog'
+                   - Fn::If:
+                      - NeedDatabase
+                      - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${CidDatabase}
+                      - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${DatabaseName}
+                   - Fn::If:
+                      - NeedDatabase
+                      - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${CidDatabase}/*
+                      - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${DatabaseName}/*
+              - Effect: Allow
+                Action:
+                  - athena:GetQueryExecution
+                  - athena:StartQueryExecution
+                  - athena:GetQueryResultsStream
+                  - athena:GetQueryResults
+                Resource:
+                  Fn::If:
+                   - NeedAthenaWorkgroup
+                   - !Sub 'arn:${AWS::Partition}:athena:${AWS::Region}:${AWS::AccountId}:workgroup/${MyAthenaWorkGroup}'
+                   - !Sub 'arn:${AWS::Partition}:athena:${AWS::Region}:${AWS::AccountId}:workgroup/${AthenaWorkgroup}'
+              - Effect: Allow
+                Action:
+                  - s3:GetBucketLocation
+                  - s3:ListBucket
+                Resource:
+                  Fn::If:
+                    - NeedAthenaQueryResultsBucket
+                    - !Sub 'arn:${AWS::Partition}:s3:::${MyAthenaQueryResultsBucket}'
+                    - !Sub 'arn:${AWS::Partition}:s3:::${AthenaQueryResultsBucket}'
+              - Effect: Allow
+                Action:
+                  - s3:GetObject
+                  - s3:PutObject
+                Resource:
+                  Fn::If:
+                    - NeedAthenaQueryResultsBucket
+                    - !Sub 'arn:${AWS::Partition}:s3:::${MyAthenaQueryResultsBucket}/*'
+                    - !Sub 'arn:${AWS::Partition}:s3:::${AthenaQueryResultsBucket}/*'
+              - Effect: Allow
+                Action:
+                  - lakeformation:GetDataAccess
+                Resource: "*" # required https://docs.aws.amazon.com/lake-formation/latest/dg/access-control-underlying-data.html
+
+  QuickSightDataSourceRolePolicyForODCBucket:
+    Type: AWS::IAM::Policy
+    Condition: NeedQuickSightDataSourceRoleAndODC
+    Properties:
+      PolicyName: S3Access
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Sid: CidAllowDecryptDataBuketsKmsKeyArns,
+            Effect: Allow
+            Action: 'kms:Decrypt'
+            Resource: !Split [',', !Ref DataBuketsKmsKeyArns]
+          - Sid: CidAllowListBucket
+            Effect: Allow
+            Action: s3:ListBucket
+            Resource: !Sub arn:aws:s3:::${ProcessedODCPath.Bucket}
+          - Sid: CidAllowReadBucket
+            Effect: Allow
+            Action:
+              - s3:GetObject
+              - s3:GetObjectVersion
+            Resource: !Sub arn:aws:s3:::${ProcessedODCPath.Bucket}/*
+      Roles:
+        - !Ref QuickSightDataSourceRole
+  QuickSightDataSourceRolePolicyForCURBucket:
+    Type: AWS::IAM::Policy
+    Condition: NeedQuickSightDataSourceRoleAndCUR
+    Properties:
+      PolicyName: S3Access
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Sid: CidAllowDecryptDataBuketsKmsKeyArns
+            Effect: Allow
+            Action: 'kms:Decrypt'
+            Resource: !Split [',', !Ref DataBuketsKmsKeyArns]
+          - Sid: CidAllowListBucket
+            Effect: Allow
+            Action: s3:ListBucket
+            Resource: !Sub arn:aws:s3:::${ProcessedCURPath.Bucket}
+          - Sid: CidAllowReadBucket
+            Effect: Allow
+            Action:
+              - s3:GetObject
+              - s3:GetObjectVersion
+            Resource: !Sub arn:aws:s3:::${ProcessedCURPath.Bucket}/*
+      Roles:
+        - !Ref QuickSightDataSourceRole
+
   CidAthenaDataSource:
     Type: AWS::QuickSight::DataSource
     Condition: NeedDatasource
@@ -976,6 +1140,7 @@ Resources:
       DataSourceParameters:
         AthenaParameters:
           WorkGroup: !If [ NeedAthenaWorkgroup, !Ref MyAthenaWorkGroup, !Ref AthenaWorkgroup ]
+          RoleArn: !If [ UseQuickSightDataSourceRole, !Sub "arn:aws:iam::${AWS::AccountId}:role/${QuickSightDataSourceRoleName}", !Ref AWS::NoValue ]
       Permissions:
         - Actions:
             - 'quicksight:DescribeDataSource'
@@ -1319,6 +1484,7 @@ Resources:
         dashboard-id: cost_intelligence_dashboard
         athena-workgroup: !If [ NeedAthenaWorkgroup, !Ref MyAthenaWorkGroup, !Ref AthenaWorkgroup ]
         quicksight-datasource-id: !If [ NeedDatasource, !Select [ 1, !Split [ '/', !GetAtt CidAthenaDataSource.Arn]], 'CID-Athena-1']
+        quicksight-datasource-role-arn: !If [ NeedQuickSightDataSourceRole, !Sub "arn:aws:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ]
         athena-database: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ]
         glue-data-catalog: !Ref GlueDataCatalog
         cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ]
@@ -1338,6 +1504,7 @@ Resources:
         dashboard-id: cudos
         athena-workgroup: !If [ NeedAthenaWorkgroup, !Ref MyAthenaWorkGroup, !Ref AthenaWorkgroup ]
         quicksight-datasource-id: !If [ NeedDatasource, !Select [ 1, !Split [ '/', !GetAtt CidAthenaDataSource.Arn]], 'CID-Athena-1']
+        quicksight-datasource-role-arn: !If [ NeedQuickSightDataSourceRole, !Sub "arn:aws:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ]
         athena-database: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ]
         glue-data-catalog: !Ref GlueDataCatalog
         cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ]
@@ -1360,6 +1527,7 @@ Resources:
         dashboard-id: kpi_dashboard
         athena-workgroup: !If [ NeedAthenaWorkgroup, !Ref MyAthenaWorkGroup, !Ref AthenaWorkgroup ]
         quicksight-datasource-id: !If [ NeedDatasource, !Select [ 1, !Split [ '/', !GetAtt CidAthenaDataSource.Arn]], 'CID-Athena-1']
+        quicksight-datasource-role-arn: !If [ NeedQuickSightDataSourceRole, !Sub "arn:aws:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ]
         athena-database: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ]
         glue-data-catalog: !Ref GlueDataCatalog
         cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ]
@@ -1384,6 +1552,7 @@ Resources:
         dashboard-id: ta-organizational-view
         athena-workgroup: !If [ NeedAthenaWorkgroup, !Ref MyAthenaWorkGroup, !Ref AthenaWorkgroup ]
         quicksight-datasource-id: !If [ NeedDatasource, !Select [ 1, !Split [ '/', !GetAtt CidAthenaDataSource.Arn]], 'CID-Athena-1']
+        quicksight-datasource-role-arn: !If [ NeedQuickSightDataSourceRole, !Sub "arn:aws:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ]
         athena-database: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ]
         glue-data-catalog: !Ref GlueDataCatalog
         cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ]
@@ -1403,6 +1572,7 @@ Resources:
         dashboard-id: compute-optimizer-dashboard
         athena-workgroup: !If [ NeedAthenaWorkgroup, !Ref MyAthenaWorkGroup, !Ref AthenaWorkgroup ]
         quicksight-datasource-id: !If [ NeedDatasource, !Select [ 1, !Split [ '/', !GetAtt CidAthenaDataSource.Arn]], 'CID-Athena-1']
+        quicksight-datasource-role-arn: !If [ NeedQuickSightDataSourceRole, !Sub "arn:aws:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ]
         athena-database: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ]
         glue-data-catalog: !Ref GlueDataCatalog
         cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ]

cfn-templates/tests/test_deploy_with_permissions.py

@@ -119,7 +119,7 @@ def get_qs_user(): # move to tools
     """ get any valid qs user """
     qs_ = boto3.client('quicksight')
     users = qs_.list_users(AwsAccountId=account_id, Namespace='default')['UserList']
-    assert users, 'No QS users, pleas craete one.' # nosec B101:assert_used
+    assert users, 'No QS users, pleas create one.' # nosec B101:assert_used
     return users[0]['UserName']
 
 def timeit(method): # move to tools
@@ -211,7 +211,7 @@ def create_cid_as_finops():
     )
     logger.info('Finops Session created')
 
-    logger.info('As Fionps Creating CUR')
+    logger.info('As Finops Creating CUR')
     finops_cfn = finops_session.client('cloudformation')
     finops_cfn.create_stack(
         StackName="CID-CUR-Destination",
@@ -228,7 +228,7 @@ def create_cid_as_finops():
     logger.info('Stack created')
 
     logger.info('As Finops Creating Dashboards')
-    finops_cfn.create_stack(
+    res = finops_cfn.create_stack(
         StackName="Cloud-Intelligence-Dashboards",
         TemplateURL=upload_to_s3('cfn-templates/cid-cfn.yml'),
         Parameters=[
@@ -239,9 +239,11 @@ def create_cid_as_finops():
         ],
         Capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'],
     )
+    logger.debug(res)
     watch_stacks(admin_cfn, ["Cloud-Intelligence-Dashboards"])
     logger.info('Stack created')
 
+
 def test_dashboard_exists():
     """check that dashboard exists"""
     dash = boto3.client('quicksight').describe_dashboard(
@@ -250,12 +252,53 @@ def test_dashboard_exists():
     )['Dashboard']
     logger.info("Dashboard exists with status = %s", dash['Version']['Status'])
 
+
 def test_dataset_scheduled():
     """check that dataset and schedule exist"""
     schedules = boto3.client('quicksight').list_refresh_schedules(AwsAccountId=account_id, DataSetId='d01a936f-2b8f-49dd-8f95-d9c7130c5e46')['RefreshSchedules']
     if not schedules:
         raise Exception('Schedules not set') #pylint: disable=broad-exception-raised
 
+def test_ingestion_successful():
+    """check that first ingestion is successful"""
+    qs = boto3.client('quicksight')
+
+    timeout_seconds = 300
+    dataset_name = 'summary_view' # Please note that there can be already another dataset in with the same name
+
+    logger.info('Waiting for the first ingestion')
+    dataset_id = None
+    start_time = time.time()
+    while time.time() - start_time < timeout_seconds:
+        datasets = {}
+        for dst in  qs.list_data_sets(AwsAccountId=account_id)['DataSetSummaries']:
+            datasets[dst["Name"]]= dst["DataSetId"]
+        if dataset_name in datasets:
+            dataset_id = datasets[dataset_name]
+            break
+        time.sleep(2)
+    else:
+        raise AssertionError('Timeout while waiting for dataset')
+
+    logger.info('Waiting for the first ingestion results')
+    start_time = time.time()
+    while time.time() - start_time < timeout_seconds:
+        ingestions = qs.list_ingestions(AwsAccountId=account_id, DataSetId=dataset_id)['Ingestions']
+        if not ingestions:
+            time.sleep(2)
+            continue
+        latest = sorted(ingestions, key=lambda x: x['CreatedTime'])[-1]
+        logger.debug(latest['IngestionStatus'])
+        if latest['IngestionStatus'] == 'FAILED':
+            raise AssertionError(f"ingestion of dataset {dataset_name} FAILED: {latest['ErrorInfo']}")
+        elif latest['IngestionStatus'] == 'COMPLETED':
+            logger.info('Ingestion Successful')
+            logger.debug(latest)
+            break
+    else:
+        raise AssertionError(f'Timeout while waiting for {dataset_name} dataset ingestion.')
+
+
 def teardown():
     """Cleanup the test"""
     admin_cfn = boto3.client('cloudformation')
@@ -277,7 +320,7 @@ def teardown():
 
     logger.info("Deleting bucket")
     delete_bucket(f'cid-{account_id}-shared') # Cannot be done by CFN
-    logger.info("Deleting Dasbhoards stack")
+    logger.info("Deleting Dashboards stack")
     try:
         finops_cfn.delete_stack(StackName="Cloud-Intelligence-Dashboards")
     except Exception as exc:  # pylint: disable=broad-exception-caught
@@ -314,14 +357,21 @@ def teardown():
 def main():
     """ main """
     try:
-        teardown() #Try to remove previous attempt
+        try:
+            teardown() #Try to remove previous attempt
+        except Exception as exc: # pylint: disable=broad-exception-caught
+            logger.debug(exc)
         create_finops_role()
         create_cid_as_finops()
         test_dashboard_exists()
         test_dataset_scheduled()
+        test_ingestion_successful()
+    except Exception as exc:
+        logger.error(exc)
+        raise
     finally:
         for index in range(10):
-            print(f'Press Ctrl+C if you want to avoid teardown: {9-index}\a') # beeep
+            print(f'Press Ctrl+C if you want to avoid teardown: {9-index}\a') # beep
             time.sleep(1)
         teardown()