Fix AWS S3 functional tests failures (#2006)

.github/workflows/aws-ci.yml

@@ -0,0 +1,80 @@
+name: CIS-AWS-CI
+
+on:
+ pull_request_target:
+ branches:
+ - main
+ - "[0-9]+.[0-9]+"
+ types: [opened, synchronize, reopened]
+ push:
+ branches:
+ - main
+ - "[0-9]+.[0-9]+"
+
+jobs:
+ Run-CSPM-AWS-Tests:
+ name: CIS AWS integration test
+ runs-on: ubuntu-22.04
+ timeout-minutes: 60
+ permissions:
+ contents: "read"
+ id-token: "write"
+ steps:
+ - name: Check out the repo
+ uses: actions/checkout@v4
+
+ - name: Init Hermit
+ run: ./bin/hermit env -r >> $GITHUB_ENV
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.9"
+
+ - name: Install Poetry
+ run: |
+ curl -sSL https://install.python-poetry.org | python3 -
+ poetry --version
+
+ - name: Build cloudbeat binary
+ uses: magefile/mage-action@v3
+ with:
+ version: latest
+ args: build
+
+ - name: Run Elasticsearch
+ uses: elastic/elastic-github-actions/elasticsearch@master
+ with:
+ stack-version: ${{ env.ELK_VERSION }}
+ security-enabled: false
+
+ - name: Elasticsearch disable dynamic mapping for findings
+ run: ./.ci/scripts/disable_dynamic_mapping.sh
+
+ - name: Run cloudbeat in background
+ env:
+ ES_HOST: http://localhost:9200
+ ES_USERNAME: elastic
+ ES_PASSWORD: changeme
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_TEST_ACC }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_TEST_ACC }}
+ AWS_ACCOUNT_TYPE: single-account
+ run: |
+ ./cloudbeat -c deploy/aws/cloudbeat-aws.yml -d '*' &
+
+ - name: Wait for cloudbeat to send events
+ run: sleep 264
+
+ - name: Check for findings
+ working-directory: ./tests
+ env:
+ USE_K8S: false
+ run: |
+ poetry install
+ poetry run pytest -k "aws" --alluredir=./allure/results/ --clean-alluredir
+
+ - name: Print cloudbeat logs
+ if: always()
+ continue-on-error: true
+ run: |
+ cat logs/cloudbeat*

.github/workflows/k8s-ci.yml

@@ -213,6 +213,8 @@ jobs:
  needs: [ Build ]
  runs-on: ubuntu-22.04
  timeout-minutes: 55
+ env:
+ KIND_CONFIG: kind-multi
  strategy:
  fail-fast: false
  matrix:
@@ -223,9 +225,6 @@ jobs:
  - test-target: pre_merge
  range: ""
  values_file: tests/test_environments/values/ci.yml
- - test-target: 'aws and not aws_s3'
- range: ""
- values_file: tests/test_environments/values/ci-aws-services.yml
  - test-target: file_system_rules
  range: "0..5"
  values_file: tests/test_environments/values/ci.yml
@@ -387,15 +386,6 @@ jobs:
  - name: Init Hermit
  run: ./bin/hermit env -r >> $GITHUB_ENV
 
- - name: Set Kind Configuration
- run: |
- echo "KIND_CONFIG=kind-multi" >> $GITHUB_ENV
- if [[ $VAR == *"aws"* ]]; then
- echo "KIND_CONFIG=kind-mono" >> $GITHUB_ENV
- fi
- env:
- VAR: ${{ matrix.test-target }}
-
  - name: Create k8s Kind Cluster
  run: |
  just create-kind-cluster ${{ env.KIND_CONFIG }}
@@ -416,15 +406,6 @@ jobs:
  run: |
  just deploy-tests-helm '${{ matrix.test-target }}' ${{ matrix.values_file }} ${{ matrix.range }}
 
- - name: Deploy cloudbeat aws
- if: contains(matrix.test-target, 'aws')
- env:
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_TEST_ACC }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_TEST_ACC }}
- run: |
- just deploy-cloudbeat-aws
- sleep 300
-
  - name: Run Tests
  id: run_tests
  run: |

deploy/aws/cloudbeat-aws.yml

@@ -6,6 +6,7 @@ cloudbeat:
  credentials:
  access_key_id: ${AWS_ACCESS_KEY_ID:""}
  secret_access_key: ${AWS_SECRET_ACCESS_KEY:""}
+ account_type: ${AWS_ACCOUNT_TYPE:""}
  type: cloudbeat/cis_aws
  # Defines how often an event is sent to the output
  period: 30s