From 15f7b2fba6230f2d2a68e5c46ea471d1495b8f52 Mon Sep 17 00:00:00 2001 From: Iakov GAN <82834333+iakov-aws@users.noreply.github.com> Date: Mon, 13 Nov 2023 08:12:21 +0100 Subject: [PATCH] Update fixes (#684) --- assets/build_lambda_layer.sh | 29 +++ assets/publish_lambda_layer.sh | 20 +-- cfn-templates/cid-admin-policies.yaml | 35 ++-- cfn-templates/cid-cfn.yml | 47 ++++- .../tests/test_deploy_with_permissions.py | 167 +++++++++++------- cid/builtin/core/data/resources.yaml | 7 +- cid/helpers/athena.py | 4 +- cid/helpers/cur.py | 61 +++++-- 8 files changed, 256 insertions(+), 114 deletions(-) create mode 100755 assets/build_lambda_layer.sh diff --git a/assets/build_lambda_layer.sh b/assets/build_lambda_layer.sh new file mode 100755 index 00000000..a1bb292f --- /dev/null +++ b/assets/build_lambda_layer.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# This script builids a lambda layer. Outpits relative path of layer zip. +export CID_VERSION=$(python3 -c "from cid import _version;print(_version.__version__)") +rm -rf build + +function get_hash { + find ./cid -type f -exec md5sum {} + | md5sum | awk '{print $1}' +} + +function build_layer { + echo 'Building a layer' + mkdir -p ./python + python3 -m pip install . -t ./python + zip -qr cid-$CID_VERSION.zip ./python + ls -l cid-$CID_VERSION.zip + rm -rf ./python +} + +# Check if code has been changed +previous_hash=$(cat cid-$CID_VERSION.hash) +actual_hash=$(get_hash) +if [ "$actual_hash" == "$previous_hash" ] && [ -e "cid-$CID_VERSION.zip" ]; then + echo "No changes in code. Reuse existing zip." 1>&2 +else + build_layer 1>&2 + echo $actual_hash > cid-$CID_VERSION.hash +fi + +ls cid-$CID_VERSION.zip \ No newline at end of file diff --git a/assets/publish_lambda_layer.sh b/assets/publish_lambda_layer.sh index baa937c4..c0112062 100755 --- a/assets/publish_lambda_layer.sh +++ b/assets/publish_lambda_layer.sh @@ -1,31 +1,25 @@ #!/bin/bash # This script can be used for release or testing of lambda layers upload. -export CID_VERSION=$(python3 -c "from cid import _version;print(_version.__version__)") -rm -rf build -echo 'Building a layer' -mkdir -p ./python -python3 -m pip install . -t ./python -zip -qr cid-$CID_VERSION.zip ./python -ls -l cid-$CID_VERSION.zip -rm -rf ./python +# First build layer +layer=$(./assets/build_lambda_layer.sh) +# Then publish on s3 export AWS_REGION=us-east-1 export STACK_SET_NAME=LayerBuckets - aws cloudformation list-stack-instances \ --stack-set-name $STACK_SET_NAME \ --query 'Summaries[].[StackId,Region]' \ --output text | while read stack_id region; do - echo "uploading cid-$CID_VERSION.zip to $region" + echo "uploading $layer to $region" bucket=$(aws cloudformation list-stack-resources --stack-name $stack_id \ --query 'StackResourceSummaries[?LogicalResourceId == `LayerBucket`].PhysicalResourceId' \ --region $region --output text) output=$(aws s3api put-object \ --bucket "$bucket" \ - --key cid-resource-lambda-layer/cid-$CID_VERSION.zip \ - --body ./cid-$CID_VERSION.zip) + --key cid-resource-lambda-layer/$layer \ + --body ./$layer) if [ $? -ne 0 ]; then echo "Error: $output" else @@ -34,7 +28,7 @@ aws cloudformation list-stack-instances \ done echo 'Cleanup' -rm -vf ./cid-$CID_VERSION.zip +rm -vf ./$layer # Publish cfn (only works for the release) diff --git a/cfn-templates/cid-admin-policies.yaml b/cfn-templates/cid-admin-policies.yaml index c637e676..034b3fb9 100644 --- a/cfn-templates/cid-admin-policies.yaml +++ b/cfn-templates/cid-admin-policies.yaml @@ -278,9 +278,7 @@ Resources: Effect: Allow Resource: - !Sub arn:aws:iam::${AWS::AccountId}:role/CidSpiceRefreshExecutionRole - - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-*-ProcessPathLambdaExec* - - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-*-InitLambdaExecutionRole* - - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-*-CidCURCrawlerRole* + - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-*-* #Roles created by CFN stack. Name is hardcoded here - !Sub arn:aws:iam::${AWS::AccountId}:role/CidQuickSightDataSourceRole - !Sub arn:aws:iam::${AWS::AccountId}:role/CidExecRole @@ -289,18 +287,23 @@ Resources: - lambda:AddPermission - lambda:CreateFunction - lambda:DeleteFunction - - lambda:DeleteLayerVersion - lambda:GetFunction - - lambda:GetLayerVersion - lambda:InvokeFunction - - lambda:PublishLayerVersion - lambda:RemovePermission + - lambda:UpdateFunctionConfiguration + - lambda:UpdateFunctionCode + - lambda:PublishLayerVersion + - lambda:GetLayerVersion + - lambda:DeleteLayerVersion Effect: Allow Resource: - - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidProcessPath-DoNotRun - - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceDashboard - - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidInitialSetup-DoNotRun - - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidSpiceRefreshLambda + - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:Cid* + # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidProcessPath-DoNotRun + # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceProcessPath-DoNotRun + # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceDashboard + # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceFunctionInit-DoNotRun + # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidInitialSetup-DoNotRun #legacy + # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidSpiceRefreshLambda - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:layer:CidLambdaLayer* - Sid: QuickSightDashboard @@ -311,7 +314,7 @@ Resources: - quicksight:DescribeDashboard Effect: Allow Resource: - - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/cudos + - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/cudos* - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/cost_intelligence_dashboard - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/kpi_dashboard - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/ta-organizational-view @@ -337,7 +340,7 @@ Resources: - quicksight:CreateDataSet - quicksight:DeleteDataSet - quicksight:PassDataSet - - quicksight:DescribDataSet + - quicksight:DescribeDataSet - quicksight:DescribeDataSetPermissions - quicksight:UpdateDataSetPermissions Effect: Allow @@ -357,7 +360,7 @@ Resources: - quicksight:DeleteDataSetRefreshProperties Effect: Allow Resource: - - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dataset/*/refresh-schedule/* # DataSetIDs are dynamic as well as shcedule ids + - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dataset/*/refresh-schedule/* # DataSetIDs are dynamic as well as schedule ids - Sid: CreateQueryResultsBucketS3 Action: @@ -450,6 +453,8 @@ Resources: - lambda:DeleteFunction - lambda:GetFunction - lambda:InvokeFunction + - lambda:UpdateFunctionConfiguration + - lambda:UpdateFunctionCode Effect: Allow Resource: - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-Analytics @@ -515,7 +520,7 @@ Resources: Condition: CreateCURReplicationPolicy Properties: ManagedPolicyName: CidCURReplicationPolicy - Description: 'CloudIntelligenceDashboards Policy for CUR Creating and Stting Replication' + Description: 'CloudIntelligenceDashboards Policy for CUR Creating and Setting Replication' Roles: - !Ref RoleName PolicyDocument: @@ -574,6 +579,8 @@ Resources: - lambda:GetFunctionCodeSigningConfig - lambda:GetRuntimeManagementConfig - lambda:InvokeFunction + - lambda:UpdateFunctionConfiguration + - lambda:UpdateFunctionCode Effect: Allow Resource: - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-Analytics diff --git a/cfn-templates/cid-cfn.yml b/cfn-templates/cid-cfn.yml index 1391a4c4..e91dbdb9 100644 --- a/cfn-templates/cid-cfn.yml +++ b/cfn-templates/cid-cfn.yml @@ -315,7 +315,7 @@ Resources: FunctionName: !Sub 'CidSpiceRefreshLambda${Suffix}' Role: !GetAtt SpiceRefreshExecutionRole.Arn Description: 'Refresh QuickSight DataSets for CID' - Runtime: python3.9 + Runtime: python3.10 Architectures: [ x86_64 ] #Compatible with arm64 but it is not supported in all regions MemorySize: 128 Timeout: 60 @@ -450,13 +450,45 @@ Resources: EncryptionOption: SSE_S3 OutputLocation: !If [ NeedAthenaQueryResultsBucket, !Sub 's3://${MyAthenaQueryResultsBucket}/', !Sub 's3://${AthenaQueryResultsBucket}/' ] + #Legacy version. Replaced by CustomResourceFunctionInit but we cannot remove it completely as it was removing workgroup on deletion of the custom resource. + CustomRessourceFunctionInit: + Type: AWS::Lambda::Function + Properties: + FunctionName: !Sub CidInitialSetup-DoNotRun${Suffix} + Role: !GetAtt 'InitLambdaExecutionRole.Arn' + Description: "CID legacy setup" + Runtime: python3.10 + Handler: 'index.lambda_handler' + Code: + ZipFile: | + # This is a legacy lambda. You can delete it. This was kept to disable delete workgroup functionality. + import json + import urllib3 + + def lambda_handler(event, context): + url = event.get('ResponseURL') + json_body = json.dumps({ + 'Status': 'SUCCESS' + 'Reason': 'legacy' + 'PhysicalResourceId': 'keep_it_constant' + 'StackId': event.get('StackId') + 'RequestId': event.get('RequestId') + 'LogicalResourceId': event.get('LogicalResourceId') + }) + try: + http = urllib3.PoolManager() + response = http.request('PUT', url, body=json_body, headers={'content-type' : '', 'content-length' : str(len(json_body))}, retries=False) + print(f"Status code: {response}") + except Exception as exc: + print("Failed sending PUT to CFN: " + str(exc)) + CustomResourceFunctionInit: Type: AWS::Lambda::Function Properties: FunctionName: !Sub "CidCustomResourceFunctionInit-DoNotRun${Suffix}" Role: !GetAtt 'InitLambdaExecutionRole.Arn' Description: "Do what CFN cannot: start crawler, delete bucket with objects and delete an non empty workgroup" - Runtime: python3.9 + Runtime: python3.10 Architectures: [ x86_64 ] #Compatible with arm64 but it is not supported in all regions MemorySize: 128 Timeout: 300 @@ -693,7 +725,7 @@ Resources: Role: !GetAtt 'ProcessPathLambdaExecutionRole.Arn' FunctionName: !Sub "CidCustomResourceProcessPath-DoNotRun${Suffix}" Description: "Do what CFN cannot: process string of path" - Runtime: python3.9 + Runtime: python3.10 Architectures: [ x86_64 ] #Compatible with arm64 but it is not supported in all regions MemorySize: 128 Timeout: 60 @@ -893,6 +925,7 @@ Resources: - {"Name": "pricing_offering_class", "Type": "string" } - {"Name": "pricing_public_on_demand_cost", "Type": "double" } - {"Name": "pricing_purchase_option", "Type": "string" } + - {"Name": "pricing_term", "Type": "string" } - {"Name": "pricing_unit", "Type": "string" } - {"Name": "product_cache_engine", "Type": "string" } - {"Name": "product_current_generation", "Type": "string" } @@ -910,6 +943,7 @@ Resources: - {"Name": "product_product_name", "Type": "string" } - {"Name": "product_region", "Type": "string" } - {"Name": "product_servicecode", "Type": "string" } + - {"Name": "product_storage", "Type": "string" } - {"Name": "product_tenancy", "Type": "string" } - {"Name": "product_to_location", "Type": "string" } - {"Name": "product_volume_api_name", "Type": "string" } @@ -1175,6 +1209,8 @@ Resources: - Effect: Allow Action: - athena:GetWorkGroup + - athena:CreateWorkGroup + - athena:UpdateWorkGroup Resource: Fn::If: - NeedAthenaWorkgroup @@ -1192,6 +1228,7 @@ Resources: Resource: "*" # This is needed to allow Autodetect in CID-CMD - Effect: Allow Action: + - s3:CreateBucket - s3:ListBucket - s3:ListBucketMultipartUploads - s3:ListMultipartUploadParts @@ -1364,7 +1401,7 @@ Resources: FunctionName: !Sub 'CidCustomResourceDashboard${Suffix}' Description: 'A lambda that manage create delete update of Athena views, QuickSight Datasets and dashboards using CID-CMD tool' Role: !GetAtt CidExecRole.Arn - Runtime: python3.9 + Runtime: python3.10 Architectures: [ x86_64 ] #Compatible with arm64 but it is not supported in all regions MemorySize: 2688 Timeout: 300 # Time of discovery depend on number of dashboards @@ -1470,7 +1507,7 @@ Resources: S3Bucket: !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}' S3Key: 'cid-resource-lambda-layer/cid-0.2.35.zip' #replace version here if needed CompatibleRuntimes: - - python3.9 + - python3.10 CostIntelligenceDashboard: Type: Custom::CidDashboard diff --git a/cfn-templates/tests/test_deploy_with_permissions.py b/cfn-templates/tests/test_deploy_with_permissions.py index 011e4ca9..704f4e32 100644 --- a/cfn-templates/tests/test_deploy_with_permissions.py +++ b/cfn-templates/tests/test_deploy_with_permissions.py @@ -12,14 +12,16 @@ 3. Verify dashboard exists 3. Delete all in reverse order -This must be executed with admin priveleges. +This must be executed with admin privileges. """ import os import json import time import logging +import subprocess #nosec B404 import boto3 +import click logger = logging.getLogger(__name__) @@ -34,8 +36,15 @@ BOLD = '\033[1m' UNDERLINE = '\033[4m' +region = boto3.session.Session().region_name account_id = boto3.client('sts').get_caller_identity()['Account'] +TMP_BUCKET_PREFIX = f'cid-{account_id}-test' +TMP_BUCKET = f'{TMP_BUCKET_PREFIX}-{region}' +def build_layer(): + """delete all content and the bucket""" + layer_file = subprocess.check_output('./assets/build_lambda_layer.sh').decode().strip() #nosec B603 + upload_to_s3(layer_file, path=f'cid-resource-lambda-layer/{layer_file}') def delete_bucket(name): # move to tools """delete all content and the bucket""" @@ -50,11 +59,11 @@ def delete_bucket(name): # move to tools except s3c.exceptions.NoSuchBucket: pass -def upload_to_s3(filename): # move to tools +def upload_to_s3(filename, path=None): # move to tools """upload file object to a temporary bucket and return a public url""" - path = os.path.basename(filename) + path = path or os.path.basename(filename) s3c = boto3.client('s3') - bucket = f'{account_id}-cid-tests-deleteme' + bucket = TMP_BUCKET try: s3c.create_bucket(Bucket=bucket) except s3c.exceptions.BucketAlreadyExists: @@ -133,52 +142,63 @@ def timed(*args, **kwargs): return timed -def create_finops_role(): +def create_finops_role(update): """ Create a finops role and grant needed permissions. """ admin_cfn = boto3.client('cloudformation') admin_iam = boto3.client('iam') logger.info('As admin creating role for Finops') - role_arn = admin_iam.create_role( - Path='/', - RoleName='TestFinopsRole', - AssumeRolePolicyDocument=json.dumps({ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": {"AWS": f"arn:aws:iam::{account_id}:root" }, - "Action": "sts:AssumeRole", - } - ] - }), - Description='string' - )['Role']['Arn'] - admin_iam.put_role_policy( - RoleName='TestFinopsRole', - PolicyName='finops-access-to-bucket-with-cfn', - PolicyDocument=json.dumps({ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "ReadTestBucket", - "Effect": "Allow", - "Action": [ - "s3:List*", - "s3:Get*" - ], - "Resource": [ - f"arn:aws:s3:::{account_id}-cid-tests-deleteme", - f"arn:aws:s3:::{account_id}-cid-tests-deleteme/*" - ] - }, - ] - }), - ) - logger.info('Role Created %s', role_arn) + try: + role_arn = admin_iam.create_role( + Path='/', + RoleName='TestFinopsRole', + AssumeRolePolicyDocument=json.dumps({ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"AWS": f"arn:aws:iam::{account_id}:root" }, + "Action": "sts:AssumeRole", + } + ] + }), + Description='string' + )['Role']['Arn'] + logger.info('Role Created %s', role_arn) + except admin_iam.exceptions.EntityAlreadyExistsException as exc: + if update: + print ('role exists') + else: + raise + + try: + admin_iam.put_role_policy( + RoleName='TestFinopsRole', + PolicyName='finops-access-to-bucket-with-cfn', + PolicyDocument=json.dumps({ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ReadTestBucket", + "Effect": "Allow", + "Action": [ + "s3:List*", + "s3:Get*" + ], + "Resource": [ + f"arn:aws:s3:::{TMP_BUCKET}", + f"arn:aws:s3:::{TMP_BUCKET}/*" + ] + }, + ] + }), + ) + except Exception: + raise + logger.info('As admin creating permissions for Finops') - admin_cfn.create_stack( + params = dict( StackName="cid-admin", TemplateURL=upload_to_s3('cfn-templates/cid-admin-policies.yaml'), Parameters=[ @@ -191,11 +211,18 @@ def create_finops_role(): ], Capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'], ) + try: + admin_cfn.create_stack(**params) + except admin_cfn.exceptions.AlreadyExistsException: + try: + admin_cfn.update_stack(**params) + except admin_cfn.exceptions.ClientError as exc: + if not "No updates are to be performed" in str(exc): raise watch_stacks(admin_cfn, ["cid-admin"]) logger.info('Stack created') -def create_cid_as_finops(): +def create_cid_as_finops(update): """creates cid with finops role""" admin_cfn = boto3.client('cloudformation') @@ -213,7 +240,7 @@ def create_cid_as_finops(): logger.info('As Finops Creating CUR') finops_cfn = finops_session.client('cloudformation') - finops_cfn.create_stack( + params = dict( StackName="CID-CUR-Destination", TemplateURL=upload_to_s3('cfn-templates/cur-aggregation.yaml'), Parameters=[ @@ -224,11 +251,19 @@ def create_cid_as_finops(): ], Capabilities=['CAPABILITY_IAM'], ) + try: + finops_cfn.create_stack(**params) + except finops_cfn.exceptions.AlreadyExistsException: + try: + finops_cfn.update_stack(**params) + except finops_cfn.exceptions.ClientError as exc: + if not "No updates are to be performed" in str(exc): raise + watch_stacks(admin_cfn, ["CID-CUR-Destination"]) logger.info('Stack created') logger.info('As Finops Creating Dashboards') - res = finops_cfn.create_stack( + params = dict( StackName="Cloud-Intelligence-Dashboards", TemplateURL=upload_to_s3('cfn-templates/cid-cfn.yml'), Parameters=[ @@ -236,10 +271,18 @@ def create_cid_as_finops(): {"ParameterKey": 'PrerequisitesQuickSightPermissions', "ParameterValue": 'yes'}, {"ParameterKey": 'QuickSightUser', "ParameterValue": get_qs_user()}, {"ParameterKey": 'DeployCUDOSDashboard', "ParameterValue": 'yes'}, + {"ParameterKey": 'DeployCUDOSv5', "ParameterValue": 'yes'}, + {"ParameterKey": 'LambdaLayerBucketPrefix', "ParameterValue": TMP_BUCKET_PREFIX}, ], Capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'], ) - logger.debug(res) + try: + finops_cfn.create_stack(**params) + except finops_cfn.exceptions.AlreadyExistsException: + try: + finops_cfn.update_stack(**params) + except finops_cfn.exceptions.ClientError as exc: + if not "No updates are to be performed" in str(exc): raise watch_stacks(admin_cfn, ["Cloud-Intelligence-Dashboards"]) logger.info('Stack created') @@ -349,20 +392,25 @@ def teardown(): logger.info(exc) logger.info("Cleanup tmp bucket") - delete_bucket(f'{account_id}-cid-tests-deleteme') + delete_bucket(TMP_BUCKET) logger.info("Teardown done") @timeit -def main(): +@click.command() +@click.option('--update', help='Try to update first', is_flag=True) +@click.option('--keep', help='No teardown', is_flag=True) +def main(update, keep): """ main """ try: - try: - teardown() #Try to remove previous attempt - except Exception as exc: # pylint: disable=broad-exception-caught - logger.debug(exc) - create_finops_role() - create_cid_as_finops() + if not update: + try: + teardown() #Try to remove previous attempt + except Exception as exc: # pylint: disable=broad-exception-caught + logger.debug(exc) + build_layer() + create_finops_role(update) + create_cid_as_finops(update) test_dashboard_exists() test_dataset_scheduled() test_ingestion_successful() @@ -370,10 +418,11 @@ def main(): logger.error(exc) raise finally: - for index in range(10): - print(f'Press Ctrl+C if you want to avoid teardown: {9-index}\a') # beep - time.sleep(1) - teardown() + if not keep: + for index in range(10): + print(f'Press Ctrl+C if you want to avoid teardown: {9-index}\a') # beep + time.sleep(1) + teardown() if __name__ == '__main__': logging.basicConfig(level=logging.INFO) diff --git a/cid/builtin/core/data/resources.yaml b/cid/builtin/core/data/resources.yaml index 98042def..431eac72 100644 --- a/cid/builtin/core/data/resources.yaml +++ b/cid/builtin/core/data/resources.yaml @@ -113,6 +113,7 @@ datasets: File: cid/summary_view.json dependsOn: views: + - account_map - summary_view - ri_sp_mapping schedules: @@ -154,15 +155,17 @@ datasets: hourly_view: File: cudos/hourly_view.json dependsOn: - views: + views: - hourly_view + - account_map schedules: - default resource_view: File: cudos/resource_view.json dependsOn: - views: + views: - resource_view + - account_map schedules: - default # KPI DataSets diff --git a/cid/helpers/athena.py b/cid/helpers/athena.py index da457e30..9e56a6a4 100644 --- a/cid/helpers/athena.py +++ b/cid/helpers/athena.py @@ -121,13 +121,13 @@ def WorkGroup(self) -> str: """ Select AWS Athena workgroup """ if not self._WorkGroup: if get_parameters().get('athena-workgroup'): - self.WorkGroup = get_parameters().get('athena-workgroup') + self.WorkGroup = self._ensure_workgroup(name=get_parameters().get('athena-workgroup')) return self._WorkGroup logger.info('Selecting Athena workgroup...') workgroups = self.list_work_groups() logger.info(f'Found {len(workgroups)} workgroups: {", ".join([wg.get("Name") for wg in workgroups])}') if len(workgroups) == 0: - self.WorkGroup = self._ensure_workgroup(name=self.defaults.get('WorkGroup')) + self.WorkGroup = self._ensure_workgroup(name=self.defaults.get('WorkGroup')) elif len(workgroups) == 1: # Silently choose the only workgroup that is available self.WorkGroup = self._ensure_workgroup(name=workgroups.pop().get('Name')) diff --git a/cid/helpers/cur.py b/cid/helpers/cur.py index 616ea36c..7596766f 100644 --- a/cid/helpers/cur.py +++ b/cid/helpers/cur.py @@ -11,21 +11,42 @@ class CUR(CidBase): curRequiredColumns = [ - 'identity_line_item_id', - 'identity_time_interval', - 'bill_invoice_id', - 'bill_billing_entity', 'bill_bill_type', - 'bill_payer_account_id', - 'bill_billing_period_start_date', + 'bill_billing_entity', 'bill_billing_period_end_date', - 'line_item_usage_account_id', + 'bill_billing_period_start_date', + 'bill_invoice_id', + 'bill_payer_account_id', + 'identity_line_item_id', + 'identity_time_interval', + 'line_item_legal_entity', + 'line_item_line_item_description', 'line_item_line_item_type', - 'line_item_usage_start_date', - 'line_item_usage_end_date', + 'line_item_operation', 'line_item_product_code', + #'line_item_resource_id', + 'line_item_unblended_cost', + 'line_item_usage_account_id', + 'line_item_usage_amount', + 'line_item_usage_end_date', + 'line_item_usage_start_date', 'line_item_usage_type', - 'line_item_operation', + 'pricing_term', + 'pricing_unit', + 'product_database_engine', + 'product_deployment_option', + 'product_from_location', + 'product_group', + 'product_instance_type', + 'product_instance_type_family', + 'product_operating_system', + 'product_product_family', + 'product_product_name', + 'product_region', + 'product_servicecode', + 'product_storage', + 'product_to_location', + 'product_volume_api_name', ] riRequiredColumns = [ 'reservation_reservation_a_r_n', @@ -113,20 +134,21 @@ def hasSavingsPlans(self) -> bool: return self._hasSavingsPlans - def table_is_cur(self, table: dict=None, name: str=None) -> bool: + def table_is_cur(self, table: dict=None, name: str=None, return_reason: bool=False) -> bool: """ return True if table metadata fits CUR definition. """ try: table = table or self.athena.get_table_metadata(name) except Exception as exc: logger.debug(exc) - return False + return False if not return_reason else (False, f'cannot get table {name}. {exc}.') - if table.get('TableType') not in ['EXTERNAL_TABLE', 'VIRTUAL_VIEW']: - return False + table_name = table.get('Name') columns = [cols.get('Name') for cols in table.get('Columns')] - if not all(cols in columns for cols in self.curRequiredColumns): - return False - return True + missing_columns = [col for col in self.curRequiredColumns if col not in columns] + if missing_columns: + return False if not return_reason else (False, f"Table {table_name} does not contain columns: {','.join(missing_columns)}. You can try ALTER TABLE {table_name} ADD COLUMNS (missing_column string).") + + return True if not return_reason else (True, 'all good') @property def metadata(self) -> dict: @@ -136,8 +158,9 @@ def metadata(self) -> dict: if get_parameters().get('cur-table-name'): self._tableName = get_parameters().get('cur-table-name') self._metadata = self.athena.get_table_metadata(self._tableName) - if not self.table_is_cur(table=self._metadata): - raise CidCritical(f'Table {self._tableName} does not looks like CUR. Please check that the table exist and have fields: {self.curRequiredColumns}.') + res, message = self.table_is_cur(table=self._metadata, return_reason=True) + if not res: + raise CidCritical(f'Table {self._tableName} does not look like CUR. {message}') else: # Look all tables and filter ones with CUR fields all_tables = self.athena.list_table_metadata()