From a0b72e4e7f9e243a21dd11ceeb2263c1509e934a Mon Sep 17 00:00:00 2001 From: Kostas Stamatakis Date: Thu, 14 Mar 2024 14:04:01 +0200 Subject: [PATCH] k8s --- .github/actions/docker-images/action.yml | 85 +++++++++++++++++ .github/actions/hermit/action.yml | 12 +++ .github/actions/k8s-ci/action.yml | 92 +++++++++++++++++++ .github/workflows/pr-ci.yml | 76 ++++++++++++++- .../packaging/docker/elastic-agent/Dockerfile | 27 ++---- scripts/packaging/docker/elastic-agent/env.sh | 9 ++ 6 files changed, 280 insertions(+), 21 deletions(-) create mode 100644 .github/actions/docker-images/action.yml create mode 100644 .github/actions/k8s-ci/action.yml create mode 100755 scripts/packaging/docker/elastic-agent/env.sh diff --git a/.github/actions/docker-images/action.yml b/.github/actions/docker-images/action.yml new file mode 100644 index 0000000000..a956718394 --- /dev/null +++ b/.github/actions/docker-images/action.yml @@ -0,0 +1,85 @@ +name: 'Docker Images' +description: 'Build docker images' +inputs: + elk-version: + description: 'ELK version' + required: true + container-image-suffix: + description: 'Container image suffix' + required: true + elastic-agent-docker-image: + description: 'Elastic-Agent docker image' + required: true + elastic-agent-docker-image-tag: + description: 'Elastic-Agent docker image tag' + required: true +runs: + using: composite + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: true + swap-storage: true + + - name: Build cloudbeat binary + shell: bash + run: mage -v build + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Build dependencies + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ci-buildx-${{ runner.os }}-${{ runner.arch }}-${{ github.workflow }} + restore-keys: | + ci-buildx-${{ runner.os }}-${{ runner.arch }}- + + - name: Build cloudbeat-docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./deploy/Dockerfile + push: false + tags: cloudbeat:latest + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/cloudbeat-${{ inputs.container-image-suffix }}.tar + + - name: Build elastic-agent + uses: docker/build-push-action@v5 + env: + GOOS: linux + GOARCH: amd64 + with: + context: . + file: ./scripts/packaging/docker/elastic-agent/Dockerfile + push: false + tags: ${{ inputs.elastic-agent-docker-image }}:${{ inputs.elastic-agent-docker-image-tag }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/elastic-agent-${{ inputs.container-image-suffix }}.tar + build-args: ELASTIC_AGENT_IMAGE=docker.elastic.co/beats/elastic-agent:${{ inputs.elastic-agent-docker-image-tag }} + + - name: Build pytest-docker + uses: docker/build-push-action@v5 + with: + context: ./tests/. + push: false + tags: cloudbeat-test:latest + cache-from: type=local,mode=max,src=/tmp/.buildx-cache + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/pytest-${{ inputs.container-image-suffix }}.tar + + - name: Rotate cache + shell: bash + run: | + ls -lahR /tmp/ || true + [[ -d /tmp/.buildx-cache-new ]] && rm -rf /tmp/.buildx-cache && mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.github/actions/hermit/action.yml b/.github/actions/hermit/action.yml index f2d305a3aa..9d04a9f99b 100644 --- a/.github/actions/hermit/action.yml +++ b/.github/actions/hermit/action.yml @@ -33,6 +33,16 @@ runs: restore-keys: | ci-hermit-env-${{ runner.os }} + - if: ${{ inputs.cache == 'true' }} + id: cache-go-deps + uses: actions/cache@v4 + with: + path: | + ~/go/pkg/ + key: ci-go-deps-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/go.mod', '**/go.sum') }} + restore-keys: | + ci-go-deps-${{ runner.os }}-${{ runner.arch }} + - name: Initialize hermit shell: bash run: ./bin/hermit env --raw >> "$GITHUB_ENV" @@ -59,5 +69,7 @@ runs: shell: bash run: pre-commit || true + - shell: bash + run: go mod download - shell: bash run: go env diff --git a/.github/actions/k8s-ci/action.yml b/.github/actions/k8s-ci/action.yml new file mode 100644 index 0000000000..4d124a8b62 --- /dev/null +++ b/.github/actions/k8s-ci/action.yml @@ -0,0 +1,92 @@ +name: 'K8S CI' +description: 'K8s integration tests' +inputs: + elk-version: + description: 'ELK version' + required: true + kind-config: + description: 'KIND configuration' + required: true + container-image-suffix: + description: 'Container image suffix' + required: true + elastic-agent-docker-image: + description: 'Elastic-Agent docker image' + required: true + elastic-agent-docker-image-tag: + description: 'Elastic-Agent docker image tag' + required: true + + test-target: + description: 'Test target' + required: true + test-range: + description: 'Test range' + required: false + default: '' + values-file: + description: 'Helm values file' + required: true +runs: + using: composite + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: true + swap-storage: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build docker images + uses: ./.github/actions/docker-images + with: + elk-version: ${{ inputs.elk-version }} + container-image-suffix: ${{ inputs.container-image-suffix }} + elastic-agent-docker-image: ${{ inputs.elastic-agent-docker-image }} + elastic-agent-docker-image-tag: ${{ inputs.elastic-agent-docker-image-tag }} + + - name: Prepare Kind Cluster for Process Tests + if: ${{ contains(inputs.kind-config, 'conf2') }} + shell: bash + run: | + # Workaround: Direct creation of a Kind cluster for config2 fails to start control-plane (kubelet fails) + # Creating and deleting a kind-mono cluster as a preparation for configuring Kind for process tests + just create-kind-cluster kind-mono + just delete-kind-cluster kind-mono + + - name: Create k8s Kind Cluster + shell: bash + run: just create-kind-cluster ${{ inputs.kind-config }} + + - name: Load images to kind + shell: bash + run: ./.ci/scripts/kind-images.sh '${{ inputs.container-image-suffix }}' '${{ inputs.kind-config }}' + + - name: Deploy tests Helm chart + id: deploy_helm + shell: bash + run: just deploy-tests-helm '${{ inputs.test-target }}' '${{ inputs.values-file }}' '${{ inputs.test-range }}' + + - name: Deploy K8s Objects + if: ${{ contains(inputs.test-target, 'object_psp') }} + shell: bash + run: just apply-k8s-test-objects + + - name: Run Tests + id: run_tests + shell: bash + run: just run-tests ${{ inputs.test-target }} kind-${{ inputs.kind-config }} + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: allure-results-ci-k8s-${{ inputs.test-target }}-${{ inputs.kind-config }} + path: tests/allure/results/ diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 7c720b90d6..81f0bf69b8 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -113,12 +113,86 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: 'us-east-2' + docker-images: + needs: [ init-hermit ] + name: Build docker images + runs-on: ubuntu-22.04 + timeout-minutes: 40 + env: + CONTAINER_SUFFIX: ${{ github.run_id }} + CI_ELASTIC_AGENT_DOCKER_TAG: "8.14.0-SNAPSHOT" + CI_ELASTIC_AGENT_DOCKER_IMAGE: "704479110758.dkr.ecr.eu-west-2.amazonaws.com/elastic-agent" + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Hermit Environment + uses: ./.github/actions/hermit + + - name: Build docker images + uses: ./.github/actions/docker-images + with: + elk-version: ${{ env.ELK_VERSION }} + container-image-suffix: ${{ github.run_id }} + elastic-agent-docker-image: ${{ env.CI_ELASTIC_AGENT_DOCKER_IMAGE }} + elastic-agent-docker-image-tag: ${{ env.CI_ELASTIC_AGENT_DOCKER_TAG }} + + ci-k8s: + needs: [ init-hermit, docker-images ] + name: ${{ matrix.test-target }}-${{ matrix.kind-config }} + runs-on: ubuntu-22.04 + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - test-target: pre_merge + kind-config: kind-multi + values-file: tests/test_environments/values/ci.yml + - test-target: pre_merge_agent + kind-config: kind-multi + values-file: tests/test_environments/values/ci-sa-agent.yml + - test-target: k8s_file_system_rules + kind-config: kind-test-files + values-file: tests/test_environments/values/ci-test-k8s-files.yml + - test-target: k8s_object_psp_rules + kind-config: kind-multi + values-file: tests/test_environments/values/ci-test-k8s-objects.yml + - test-target: k8s_process_rules + kind-config: kind-test-proc-conf1 + values-file: tests/test_environments/values/ci-test-k8s-proc-conf1.yml + - test-target: k8s_process_rules + kind-config: kind-test-proc-conf2 + values-file: tests/test_environments/values/ci-test-k8s-proc-conf2.yml + env: + CONTAINER_SUFFIX: ${{ github.run_id }} + CI_ELASTIC_AGENT_DOCKER_TAG: "8.14.0-SNAPSHOT" + CI_ELASTIC_AGENT_DOCKER_IMAGE: "704479110758.dkr.ecr.eu-west-2.amazonaws.com/elastic-agent" + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Hermit Environment + uses: ./.github/actions/hermit + + - name: Run k8s integration tests + uses: ./.github/actions/k8s-ci + with: + elk-version: ${{ env.ELK_VERSION }} + kind-config: ${{ matrix.kind-config }} + container-image-suffix: ${{ github.run_id }} + elastic-agent-docker-image: ${{ env.CI_ELASTIC_AGENT_DOCKER_IMAGE }} + elastic-agent-docker-image-tag: ${{ env.CI_ELASTIC_AGENT_DOCKER_TAG }} + test-target: ${{ matrix.test-target }} + values-file: ${{ matrix.values-file }} + upload-allure-results: needs: - ci-azure - ci-aws - ci-gcp - ci-cnvm + - ci-k8s name: Upload integration tests results runs-on: ubuntu-22.04 timeout-minutes: 60 @@ -155,7 +229,7 @@ jobs: - name: log if: always() shell: bash - run: ls -laR + run: ls -lahR || true - name: Publish allure report if: always() diff --git a/scripts/packaging/docker/elastic-agent/Dockerfile b/scripts/packaging/docker/elastic-agent/Dockerfile index 3ebd5ad855..700312bc38 100644 --- a/scripts/packaging/docker/elastic-agent/Dockerfile +++ b/scripts/packaging/docker/elastic-agent/Dockerfile @@ -1,23 +1,10 @@ -ARG ELASTIC_AGENT_IMAGE # e.g. docker.elastic.co/cloud-release/elastic-agent-cloud:8.5.0-7dbc10f8-SNAPSHOT +ARG ELASTIC_AGENT_IMAGE=docker.elastic.co/beats/elastic-agent:8.14.0-SNAPSHOT -FROM --platform=linux/amd64 ${ELASTIC_AGENT_IMAGE} as elastic_agent_amd64 -ARG STACK_VERSION # e.g. 8.5.0-SNAPSHOT -ARG VCS_REF_SHORT # e.g. abc123 -ONBUILD COPY --chown=elastic-agent cloudbeat \ - ./data/elastic-agent-${VCS_REF_SHORT}/components/cloudbeat -ONBUILD COPY --chown=elastic-agent bundle.tar.gz \ - ./data/elastic-agent-${VCS_REF_SHORT}/components/bundle.tar.gz -ONBUILD COPY --chown=elastic-agent cloudbeat.yml \ - ./data/elastic-agent-${VCS_REF_SHORT}/components/cloudbeat.yml +FROM ${ELASTIC_AGENT_IMAGE} as elastic_agent_cloudbeat +COPY --chown=elastic-agent:elastic-agent --chmod=755 cloudbeat /tmp/components/cloudbeat +COPY --chown=elastic-agent:elastic-agent --chmod=666 bundle.tar.gz /tmp/components/bundle.tar.gz +COPY --chown=elastic-agent:elastic-agent --chmod=644 cloudbeat.yml /tmp/components/cloudbeat.yml -FROM --platform=linux/arm64 ${ELASTIC_AGENT_IMAGE} as elastic_agent_arm64 -ARG STACK_VERSION # e.g. 8.5.0-SNAPSHOT -ARG VCS_REF_SHORT # e.g. abc123 -ONBUILD COPY --chown=elastic-agent cloudbeat \ - ./data/elastic-agent-${VCS_REF_SHORT}/components/cloudbeat -ONBUILD COPY --chown=elastic-agent bundle.tar.gz \ - ./data/elastic-agent-${VCS_REF_SHORT}/components/bundle.tar.gz -ONBUILD COPY --chown=elastic-agent cloudbeat.yml \ - ./data/elastic-agent-${VCS_REF_SHORT}/components/cloudbeat.yml +RUN mv /tmp/components/* "$(dirname "$(realpath "$(which elastic-agent)")")"/components/ -FROM elastic_agent_${TARGETARCH} +FROM elastic_agent_cloudbeat diff --git a/scripts/packaging/docker/elastic-agent/env.sh b/scripts/packaging/docker/elastic-agent/env.sh new file mode 100755 index 0000000000..9c24270373 --- /dev/null +++ b/scripts/packaging/docker/elastic-agent/env.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -eu + +VERSION=$(grep defaultBeatVersion version/version.go | cut -f2 -d "\"") +DEFAULT_IMAGE_TAG="${DEFAULT_IMAGE_TAG:-${VERSION}-SNAPSHOT}" +BASE_IMAGE="${BASE_IMAGE:-docker.elastic.co/beats/elastic-agent:$DEFAULT_IMAGE_TAG}" + +echo "BASE_IMAGE=${BASE_IMAGE}"