From a5717a21b7b36e371715c6833ee00090f95a4382 Mon Sep 17 00:00:00 2001 From: IA Automator Date: Wed, 10 Jan 2024 18:09:40 +0000 Subject: [PATCH] (automated) Updates from project type --- .copier-answers.yml | 4 +- .gitignore | 30 ++++++++++ .../functional_tests/entrypoint_scotsuite.sh | 55 ------------------- ...abi-cloudtrail-no-encryption-with-kms.json | 0 ...assume-role-lacks-external-id-and-mfa.json | 0 .../abi-scoutsuite-custom-ruleset.json | 2 +- .../create-scoutsuite-custom-rule.py | 2 +- .../lambda_s3_presignedurl/lambda_function.py | 0 .../lambda_s3_presignedurl/requirements.txt | 0 .../lambda_s3_presignedurl.yaml | 0 .../process-scoutsuite-report.py | 6 +- .../functional_tests/scoutsuite/scoutsuite.sh | 40 ++++++++++++++ 12 files changed, 78 insertions(+), 61 deletions(-) delete mode 100755 .project_automation/functional_tests/entrypoint_scotsuite.sh rename .project_automation/functional_tests/{ => scoutsuite}/abi-cloudtrail-no-encryption-with-kms.json (100%) rename .project_automation/functional_tests/{ => scoutsuite}/abi-iam-assume-role-lacks-external-id-and-mfa.json (100%) rename .project_automation/functional_tests/{ => scoutsuite}/abi-scoutsuite-custom-ruleset.json (98%) rename .project_automation/functional_tests/{ => scoutsuite}/create-scoutsuite-custom-rule.py (96%) rename .project_automation/functional_tests/{ => scoutsuite}/lambda_functions/source/lambda_s3_presignedurl/lambda_function.py (100%) rename .project_automation/functional_tests/{ => scoutsuite}/lambda_functions/source/lambda_s3_presignedurl/requirements.txt (100%) rename .project_automation/functional_tests/{ => scoutsuite}/lambda_s3_presignedurl.yaml (100%) rename .project_automation/functional_tests/{ => scoutsuite}/process-scoutsuite-report.py (97%) create mode 100755 .project_automation/functional_tests/scoutsuite/scoutsuite.sh diff --git a/.copier-answers.yml b/.copier-answers.yml index aafadf5..9537d2f 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,6 +1,6 @@ # This file is auto-generated, changes will be overwritten -_commit: v0.1.3 -_src_path: /task/db0b1217-7517-11ee-ad22-429c8fcd4fb9/projecttype +_commit: v0.1.4 +_src_path: /task/59da57c9-afe3-11ee-8091-0668063857e2/projecttype starting_version: v0.0.0 version_file: VERSION diff --git a/.gitignore b/.gitignore index dc6d3f5..8ae246f 100644 --- a/.gitignore +++ b/.gitignore @@ -161,15 +161,45 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +<<<<<<< before updating # taskcat +======= +# JetBrains IDE +.idea/ + +.DS_Store + +.taskcat/ +taskcat_outputs/ +*.zip +======= + +.DS_Store + +# Taskcat files +>>>>>>> after updating taskcat_outputs/ .taskcat/ # since the zips are automatically generated, we don't want to check them in lambda_functions/packages/ +<<<<<<< before updating # General .DS_Store # Ash aggregated_results.txt +======= +# Package files +lambda.zip + +# Visual Studio Code IDE +.vscode/ + +# Scoutsuite files +/scoutsuite-report/ +scoutsuite-report.zip +scoutsuite_sysout.txt +scoutsuite_s3_filename.txt +>>>>>>> after updating diff --git a/.project_automation/functional_tests/entrypoint_scotsuite.sh b/.project_automation/functional_tests/entrypoint_scotsuite.sh deleted file mode 100755 index c8a574f..0000000 --- a/.project_automation/functional_tests/entrypoint_scotsuite.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -ex - - -## NOTE: paths may differ when running in a managed task. To ensure behavior is consistent between -# managed and local tasks always use these variables for the project and project type path -PROJECT_PATH=${BASE_PATH}/project -PROJECT_TYPE_PATH=${BASE_PATH}/projecttype - -cd ${PROJECT_PATH} - -regions=(us-east-1 us-east-2 us-west-2 us-west-1) -for region in ${regions[@]} -do - echo "Cleanup running in region: $region" - export AWS_DEFAULT_REGION=$region - python3 scripts/cleanup_config.py -C scripts/cleanup_config.json -done - -echo $AWS_DEFAULT_REGION -unset AWS_DEFAULT_REGION - -echo $AWS_DEFAULT_REGION -# Run taskcat e2e test without deleting resources -taskcat test run -n -#Create Scoutsuite security scan custom rule -python3 .project_automation/functional_tests/create-scoutsuite-custom-rule.py -# Execute Scoutsuite security scan -scout aws -r us-east-1 --ruleset .project_automation/functional_tests/abi-scoutsuite-custom-ruleset.json --no-browser --max-rate 5 --max-workers 5 -# Upload Scoutsuite security scan results to S3 bucket named scoutsuite-results-aws-AWS-ACCOUNT-ID -python3 .project_automation/functional_tests/process-scoutsuite-report.py -# Delete taskcat e2e test resources -taskcat test clean ALL -# Check Scoutsuite security scan result for Danger level findings (Non-0 exit code) -scoutsuite_sysout_result=$(cat scoutsuite_sysout.txt) -rm scoutsuite_sysout.txt -if [ "$scoutsuite_sysout_result" -ne 0 ]; then - # The value is non-zero, indicating Scoutsuite report needs to be checked for security issues - exit 1 -fi - -#sleep 1800 -#Use the command above to allow for a sleep timer buffer between sequential execution of Taskcat so that the Cloudformation resources from the previous Taskcat execution can be fully deleted until this is fixed in https://github.com/aws-ia/taskcat/issues/809 - -## Executing ash tool - -#find ${PROJECT_PATH} -name lambda.zip -exec rm -rf {} \; - -#git clone https://github.com/aws-samples/automated-security-helper.git /tmp/ash - -# Set the repo path in your shell for easier access -#export PATH=$PATH:/tmp/ash - -#ash --source-dir . -#cat aggregated_results.txt - diff --git a/.project_automation/functional_tests/abi-cloudtrail-no-encryption-with-kms.json b/.project_automation/functional_tests/scoutsuite/abi-cloudtrail-no-encryption-with-kms.json similarity index 100% rename from .project_automation/functional_tests/abi-cloudtrail-no-encryption-with-kms.json rename to .project_automation/functional_tests/scoutsuite/abi-cloudtrail-no-encryption-with-kms.json diff --git a/.project_automation/functional_tests/abi-iam-assume-role-lacks-external-id-and-mfa.json b/.project_automation/functional_tests/scoutsuite/abi-iam-assume-role-lacks-external-id-and-mfa.json similarity index 100% rename from .project_automation/functional_tests/abi-iam-assume-role-lacks-external-id-and-mfa.json rename to .project_automation/functional_tests/scoutsuite/abi-iam-assume-role-lacks-external-id-and-mfa.json diff --git a/.project_automation/functional_tests/abi-scoutsuite-custom-ruleset.json b/.project_automation/functional_tests/scoutsuite/abi-scoutsuite-custom-ruleset.json similarity index 98% rename from .project_automation/functional_tests/abi-scoutsuite-custom-ruleset.json rename to .project_automation/functional_tests/scoutsuite/abi-scoutsuite-custom-ruleset.json index 5558b3a..8504d09 100644 --- a/.project_automation/functional_tests/abi-scoutsuite-custom-ruleset.json +++ b/.project_automation/functional_tests/scoutsuite/abi-scoutsuite-custom-ruleset.json @@ -1,5 +1,5 @@ { - "about": "This is a simplified ruleset intended for use by CDO Securtiy Certifiers and is maintained by Amazon's CloudSecurity team.", + "about": "This is a simplified ruleset intended for use with AWS ABI test environments.", "rules": { "cloudtrail-not-configured.json": [ { diff --git a/.project_automation/functional_tests/create-scoutsuite-custom-rule.py b/.project_automation/functional_tests/scoutsuite/create-scoutsuite-custom-rule.py similarity index 96% rename from .project_automation/functional_tests/create-scoutsuite-custom-rule.py rename to .project_automation/functional_tests/scoutsuite/create-scoutsuite-custom-rule.py index 26d0e4a..2941192 100644 --- a/.project_automation/functional_tests/create-scoutsuite-custom-rule.py +++ b/.project_automation/functional_tests/scoutsuite/create-scoutsuite-custom-rule.py @@ -40,7 +40,7 @@ def create_scoutsuite_custom_rule_file(file_name): if not os.path.exists(scoutsuite_cloudtrail_json_file_path): # If the file does not exist, create it # Source path - src = '.project_automation/functional_tests/'+file_name + src = '.project_automation/functional_tests/scoutsuite/'+file_name copy_file(src, scoutsuite_cloudtrail_json_file_path) else: logging.info(f'File {scoutsuite_cloudtrail_json_file_path} already exists') diff --git a/.project_automation/functional_tests/lambda_functions/source/lambda_s3_presignedurl/lambda_function.py b/.project_automation/functional_tests/scoutsuite/lambda_functions/source/lambda_s3_presignedurl/lambda_function.py similarity index 100% rename from .project_automation/functional_tests/lambda_functions/source/lambda_s3_presignedurl/lambda_function.py rename to .project_automation/functional_tests/scoutsuite/lambda_functions/source/lambda_s3_presignedurl/lambda_function.py diff --git a/.project_automation/functional_tests/lambda_functions/source/lambda_s3_presignedurl/requirements.txt b/.project_automation/functional_tests/scoutsuite/lambda_functions/source/lambda_s3_presignedurl/requirements.txt similarity index 100% rename from .project_automation/functional_tests/lambda_functions/source/lambda_s3_presignedurl/requirements.txt rename to .project_automation/functional_tests/scoutsuite/lambda_functions/source/lambda_s3_presignedurl/requirements.txt diff --git a/.project_automation/functional_tests/lambda_s3_presignedurl.yaml b/.project_automation/functional_tests/scoutsuite/lambda_s3_presignedurl.yaml similarity index 100% rename from .project_automation/functional_tests/lambda_s3_presignedurl.yaml rename to .project_automation/functional_tests/scoutsuite/lambda_s3_presignedurl.yaml diff --git a/.project_automation/functional_tests/process-scoutsuite-report.py b/.project_automation/functional_tests/scoutsuite/process-scoutsuite-report.py similarity index 97% rename from .project_automation/functional_tests/process-scoutsuite-report.py rename to .project_automation/functional_tests/scoutsuite/process-scoutsuite-report.py index aedb607..c7e15db 100755 --- a/.project_automation/functional_tests/process-scoutsuite-report.py +++ b/.project_automation/functional_tests/scoutsuite/process-scoutsuite-report.py @@ -81,7 +81,7 @@ def create_lambda_s3_presignedurl(): '''Function that creates the Lambda function that generates S3 presigned URLs''' # Create the ZIP deployment package for Lambda - lambda_zipped_code = zip_folder_to_bytesio('./.project_automation/functional_tests/lambda_functions/source/lambda_s3_presignedurl') + lambda_zipped_code = zip_folder_to_bytesio('./.project_automation/functional_tests/scoutsuite/lambda_functions/source/lambda_s3_presignedurl') lambda_zipped_code.seek(0) # Reset the cursor of the BytesIO object to the beginning s3_client = session.client('s3') @@ -95,7 +95,7 @@ def create_lambda_s3_presignedurl(): # Use the session to create a client for CloudFormation cf_client = session.client('cloudformation') - with open('./.project_automation/functional_tests/lambda_s3_presignedurl.yaml', 'r') as file: + with open('./.project_automation/functional_tests/scoutsuite/lambda_s3_presignedurl.yaml', 'r') as file: template_body = file.read() # Check if the stack already exists try: @@ -277,6 +277,8 @@ def upload_scoutsuite_results_zip_to_s3(scoutsuite_zip_file_path, zip_name): s3_file_with_key = time_key + '-' + zip_name # Upload the Scoutsuite results zip to an S3 bucket s3.upload_file(scoutsuite_zip_file_path, bucket_name, s3_file_with_key) + with open("scoutsuite_s3_filename.txt", "w") as file: + file.write(str(s3_file_with_key)) except ClientError as error: logging.exception (error) diff --git a/.project_automation/functional_tests/scoutsuite/scoutsuite.sh b/.project_automation/functional_tests/scoutsuite/scoutsuite.sh new file mode 100755 index 0000000..1cad0e2 --- /dev/null +++ b/.project_automation/functional_tests/scoutsuite/scoutsuite.sh @@ -0,0 +1,40 @@ +#!/bin/bash -ex + + +## NOTE: paths may differ when running in a managed task. To ensure behavior is consistent between +# managed and local tasks always use these variables for the project and project type path +PROJECT_PATH=${BASE_PATH}/project +PROJECT_TYPE_PATH=${BASE_PATH}/projecttype + +cd ${PROJECT_PATH} + +# Retrieve the AWS account ID and store it in a variable +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) + +run_scoutsuite() { + #Create Scoutsuite security scan custom rule + python3 .project_automation/functional_tests/scoutsuite/create-scoutsuite-custom-rule.py + # Execute Scoutsuite security scan + scout aws -r us-east-1 --ruleset .project_automation/functional_tests/scoutsuite/abi-scoutsuite-custom-ruleset.json --no-browser --max-rate 5 --max-workers 5 -f + # Upload Scoutsuite security scan results to S3 bucket named scoutsuite-results-aws-AWS-ACCOUNT-ID + python3 .project_automation/functional_tests/scoutsuite/process-scoutsuite-report.py + # Delete taskcat e2e test resources + taskcat test clean ALL + process_scoutsuite_report +} + +process_scoutsuite_report() { + # Check Scoutsuite security scan result for Danger level findings (Non-0 exit code) + scoutsuite_sysout_result=$(cat scoutsuite_sysout.txt) + scoutsuite_s3_filename=$(cat scoutsuite_s3_filename.txt) + rm scoutsuite_sysout.txt + rm scoutsuite_s3_filename.txt + if [ "$scoutsuite_sysout_result" -ne 0 ]; then + # The value is non-zero, indicating Scoutsuite report needs to be checked for security issues + echo "Scoutsuite report contains security issues. For details please check the log messages above or the file $scoutsuite_s3_filename in the S3 bucket named scoutsuite-results-aws-$AWS_ACCOUNT_ID in the AWS test account provided by the ABI team." + exit 1 + fi +} + +#Run Scoutsuite security test +run_scoutsuite \ No newline at end of file