IDE integration tests #675
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "IDE integration tests" | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| contents: read # This is required for actions/checkout | |
| actions: write # This is required for trigger another action which is used by JetBrains integrateion tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| name: | |
| required: true | |
| description: "The name of the preview environment" | |
| default: "latest" | |
| version: | |
| required: true | |
| description: "The version of Gitpod to install" | |
| default: "latest" | |
| image_repo_base: | |
| type: choice | |
| required: false | |
| description: "The base repo of image" | |
| options: | |
| - "eu.gcr.io/gitpod-core-dev/build" | |
| - "eu.gcr.io/gitpod-dev-artifact/build" | |
| default: "eu.gcr.io/gitpod-core-dev/build" | |
| skip_deploy: | |
| required: false | |
| description: "Skip deploy preview environment (debug only)" | |
| skip_delete: | |
| required: false | |
| description: "Skip delete preview environment (debug only)" | |
| schedule: | |
| - cron: "0 3 * * *" | |
| jobs: | |
| create-runner: | |
| uses: gitpod-io/gce-github-runner/.github/workflows/create-vm.yml@main | |
| secrets: | |
| runner_token: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_TOKEN }} | |
| gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }} | |
| configuration: | |
| name: Configuration | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| needs: [create-runner] | |
| container: | |
| image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:pd-test-new-preview-gha.24525 | |
| outputs: | |
| name: ${{ steps.configuration.outputs.name }} | |
| version: ${{ steps.configuration.outputs.version }} | |
| image_repo_base: ${{ steps.configuration.outputs.image_repo_base }} | |
| steps: | |
| - name: "Set outputs" | |
| id: configuration | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [[ '${{ github.event.inputs.name }}' != '' && '${{ github.event.inputs.name }}' != 'latest' ]]; then | |
| # The workflow was triggered by workflow_dispatch | |
| { | |
| echo "version=${{ github.event.inputs.version }}" | |
| echo "name=${{ github.event.inputs.name }}" | |
| } >> $GITHUB_OUTPUT | |
| else | |
| # others | |
| RUNID=$(gh run list -R gitpod-io/gitpod -b main -w build.yml --limit 1 --json databaseId --jq .[0].databaseId) | |
| gh run watch "$RUNID" -R gitpod-io/gitpod | |
| CONCLUSION=$(gh run view "$RUNID" -R gitpod-io/gitpod --json jobs --jq '.jobs[] | select(.name == "Build Gitpod") | {name, status, conclusion} | .conclusion') | |
| if [ "$CONCLUSION" != "success" ]; then | |
| echo main branch build is failed, see https://github.com/gitpod-io/gitpod/actions/runs/"$RUNID" for detail | tee -a $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| fi | |
| { | |
| echo "version=$(gh run view "$RUNID" --log -R gitpod-io/gitpod | grep 'main-gha.[0-9]*' -o | head -n 1)" | |
| echo "name=ide-integration-test-${{ github.run_id }}-${{ github.run_attempt }}" | |
| } >> $GITHUB_OUTPUT | |
| fi | |
| if [[ '${{ inputs.image_repo_base }}' != '' ]]; then | |
| { | |
| echo "image_repo_base=${{ inputs.image_repo_base }}" | |
| } >> $GITHUB_OUTPUT | |
| else | |
| { | |
| echo "image_repo_base=eu.gcr.io/gitpod-core-dev/build" | |
| } >> $GITHUB_OUTPUT | |
| fi | |
| - name: Slack Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| if: failure() | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }} | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: main branch build failed | |
| SLACK_FOOTER: "<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Workflow logs>" | |
| infrastructure: | |
| name: Create preview environment infrastructure | |
| needs: [configuration, create-runner] | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| concurrency: | |
| group: ${{ needs.configuration.outputs.name }}-infrastructure | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }} | |
| service_account: ${{ secrets.DEV_PREVIEW_SA }} | |
| leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }} | |
| - name: Create preview environment infrastructure | |
| id: create | |
| uses: ./.github/actions/preview-create | |
| with: | |
| name: ${{ needs.configuration.outputs.name }} | |
| infrastructure_provider: gce | |
| large_vm: true | |
| preemptible: true | |
| image_repo_base: ${{ needs.configuration.outputs.image_repo_base }} | |
| - name: Deploy Gitpod to the preview environment | |
| id: deploy-gitpod | |
| if: github.event.inputs.skip_deploy != 'true' | |
| uses: ./.github/actions/deploy-gitpod | |
| with: | |
| name: ${{ needs.configuration.outputs.name }} | |
| version: ${{ needs.configuration.outputs.version}} | |
| image_repo_base: ${{ needs.configuration.outputs.image_repo_base }} | |
| check: | |
| name: Check for regressions | |
| needs: [configuration, infrastructure, create-runner] | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| container: | |
| image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:pd-test-new-preview-gha.24525 | |
| volumes: | |
| - /var/tmp:/var/tmp | |
| - /tmp:/tmp | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Environment | |
| env: | |
| HOME: /home/gitpod | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }} | |
| service_account: ${{ secrets.DEV_PREVIEW_SA }} | |
| leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }} | |
| - name: Integration Test | |
| shell: bash | |
| env: | |
| ROBOQUAT_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| USERNAME: ${{ secrets.IDE_INTEGRATION_TEST_USERNAME }} | |
| USER_TOKEN: ${{ secrets.IDE_INTEGRATION_TEST_USER_TOKEN }} | |
| PREVIEW_NAME: ${{ needs.configuration.outputs.name }} | |
| TEST_BUILD_ID: ${{ github.run_id }} | |
| TEST_BUILD_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| TEST_BUILD_REF: ${{ github.head_ref || github.ref }} | |
| HOME: /home/gitpod | |
| run: | | |
| set -euo pipefail | |
| export LEEWAY_WORKSPACE_ROOT="$(pwd)" | |
| leeway run dev/preview/previewctl:install | |
| previewctl install-context --branch "${PREVIEW_NAME}" --log-level debug --timeout 1m | |
| # start integration test | |
| args=() | |
| args+=( "-kubeconfig=$HOME/.kube/config" ) | |
| args+=( "-namespace=default" ) | |
| [[ "$USERNAME" != "" ]] && args+=( "-username=$USERNAME" ) | |
| args+=( "-timeout=60m" ) | |
| IDE_TESTS_DIR="$GITHUB_WORKSPACE/test/tests/ide" | |
| JETBRAINS_TESTS="$IDE_TESTS_DIR/jetbrains" | |
| VSCODE_TESTS="$IDE_TESTS_DIR/vscode" | |
| SSH_TESTS="$IDE_TESTS_DIR/ssh" | |
| go install github.com/jstemmer/go-junit-report/v2@latest | |
| FAILURE_COUNT=0 | |
| IDE_TEST_LIST=("$SSH_TESTS" "$VSCODE_TESTS" "$JETBRAINS_TESTS") | |
| for TEST_PATH in "${IDE_TEST_LIST[@]}" | |
| do | |
| TEST_NAME=$(basename "${TEST_PATH}") | |
| echo "running integration for ${TEST_NAME}" | |
| cd "${TEST_PATH}" | |
| set +e | |
| go test -parallel=3 -v ./... "${args[@]}" 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "TEST-${TEST_NAME}.xml" -iocopy | |
| RC=${PIPESTATUS[0]} | |
| set -e | |
| if [ "${RC}" -ne "0" ]; then | |
| FAILURE_COUNT=$((FAILURE_COUNT+1)) | |
| fi | |
| done | |
| exit $FAILURE_COUNT | |
| - name: Test Summary | |
| id: test_summary | |
| uses: test-summary/action@v2 | |
| with: | |
| paths: "test/tests/**/TEST-*.xml" | |
| if: always() | |
| - name: Slack Notification | |
| uses: rtCamp/action-slack-notify@v2 | |
| if: success() || failure() | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }} | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: ${{ steps.test_summary.outputs.passed }}/${{ steps.test_summary.outputs.total }} tests passed | |
| SLACK_FOOTER: "<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Workflow logs>" | |
| delete: | |
| name: Delete preview environment | |
| needs: [configuration, infrastructure, check, create-runner] | |
| if: github.event.inputs.skip_delete != 'true' && always() | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-environment | |
| with: | |
| identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }} | |
| service_account: ${{ secrets.DEV_PREVIEW_SA }} | |
| leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }} | |
| - name: Delete preview environment | |
| uses: ./.github/actions/delete-preview | |
| with: | |
| name: ${{ needs.configuration.outputs.name }} | |
| delete-runner: | |
| if: always() | |
| needs: | |
| - create-runner | |
| - configuration | |
| - infrastructure | |
| - check | |
| - delete | |
| uses: gitpod-io/gce-github-runner/.github/workflows/delete-vm.yml@main | |
| secrets: | |
| gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }} | |
| with: | |
| runner-label: ${{ needs.create-runner.outputs.label }} | |
| machine-zone: ${{ needs.create-runner.outputs.machine-zone }} |