Auto generated from templates by gromit

.github/workflows/release.yml

@@ -138,7 +138,7 @@ jobs:
  type=semver,pattern={{version}},prefix=v
  - name: push image to CI
  if: ${{ matrix.golang_cross == '1.21-bookworm' }}
- uses: docker/build-push-action@v5
+ uses: docker/build-push-action@v6
  with:
  context: "dist"
  platforms: linux/amd64,linux/arm64
@@ -167,7 +167,7 @@ jobs:
  labels: "org.opencontainers.image.title=tyk-pump (distroless) \norg.opencontainers.image.description=Tyk Analytics Pump to move analytics data from Redis to any supported back end (multiple back ends can be written to at once).\norg.opencontainers.image.vendor=tyk.io\norg.opencontainers.image.version=${{ github.ref_name }}\n"
  - name: build multiarch image
  if: ${{ matrix.golang_cross == '1.21-bookworm' }}
- uses: docker/build-push-action@v5
+ uses: docker/build-push-action@v6
  with:
  context: "dist"
  platforms: linux/amd64,linux/arm64
@@ -196,7 +196,8 @@ jobs:
  dist/*.rpm
  !dist/*PAYG*.rpm
  test-controller-api:
- needs: goreleaser
+ needs:
+ - goreleaser
  runs-on: ubuntu-latest
  outputs:
  envfiles: ${{ steps.params.outputs.envfiles }}
@@ -207,29 +208,23 @@ jobs:
  id: params
  shell: bash
  env:
- # Cover pull_request_target too
+ # startsWith covers pull_request_target too
  BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
+ VARIATION: esteban
  run: |
  set -eo pipefail
- endpoint="http://tui.internal.dev.tyk.technology/api/tyk-pump/$BASE_REF/${{ github.event_name}}/api"
- curl="curl -s --retry 5 --retry-delay 10 --fail-with-body --retry-all-errors"
- echo "pump<<EOF
- $($curl ${endpoint}/Pump)
- EOF
- sink<<EOF
- $($curl ${endpoint}/Sink)
- EOF
- envfiles<<EOF
- $($curl ${endpoint}/EnvFiles)
- EOF" | tee -a "$GITHUB_OUTPUT"
+ curl -s --retry 5 --retry-delay 10 --fail-with-body "http://tui.internal.dev.tyk.technology/v2/$VARIATION/tyk-pump/$BASE_REF/${{ github.event_name}}/api.gho" | tee -a "$GITHUB_OUTPUT"
+ [[ $VARIATION =~ test ]] && {
+ echo "::warning file=release.yml,line=100,col=5,endColumn=7::Using test variation"
+ echo "### :warning: You are using VARIATION with test in test-controller-api" >> $GITHUB_STEP_SUMMARY
+ }
  api-tests:
  needs:
- - goreleaser
  - test-controller-api
+ - goreleaser
  runs-on: ubuntu-latest-m-2
  env:
- METADATA_REPORT_PATH: /tmp/metadata.toml
- XUNIT_REPORT_PATH: ${{ github.workspace }}/reports/pytest-report.xml
+ XUNIT_REPORT_PATH: ${{ github.workspace}}/test-results.xml
  permissions:
  id-token: write # This is required for requesting the Github JWT
  contents: read # This is required for actions/checkout
@@ -238,9 +233,6 @@ jobs:
  matrix:
  envfiles: ${{ fromJson(needs.test-controller-api.outputs.envfiles) }}
  sink: ${{ fromJson(needs.test-controller-api.outputs.sink) }}
- include:
- - db: postgres15
- markers: "and not sql"
  steps:
  - uses: aws-actions/configure-aws-credentials@v4
  with:
@@ -251,20 +243,20 @@ jobs:
  uses: aws-actions/amazon-ecr-login@v2
  with:
  mask-password: 'true'
- # Only ${{ github.actor }} has access
- # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
  - name: Setup tmate session only in debug mode
  uses: mxschmitt/action-tmate@v3
  if: runner.debug == '1'
  with:
  detached: true
  limit-access-to-actor: true
- - name: fetch env from tyk-ci
+ # Only ${{ github.actor }} has access
+ # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
+ - name: fetch env from tyk-pro
  env:
  GH_TOKEN: ${{ github.token }}
  run: |
- gh release download --repo github.com/TykTechnologies/tyk-ci -p 'ci-env.tgz' -O env.tgz
- tar xzvf env.tgz
+ gh release download --repo github.com/tyklabs/tyk-pro --archive tar.gz -O env.tgz
+ mkdir auto && tar --strip-components=1 -C auto -xzvf env.tgz
  - name: env up
  shell: bash
  working-directory: auto
@@ -275,53 +267,28 @@ jobs:
  TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
  TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
  BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
- run: |
- match_tag=${{steps.ecr.outputs.registry}}/$REPO:$BASE_REF
- tags=(${{ needs.goreleaser.outputs.tags }})
- docker run -q --rm -v ~/.docker/config.json:/root/.docker/config.json tykio/gromit policy match ${tags[0]} ${match_tag} 2>versions.env
- echo '# alfa and beta have to come after the override
- tyk_alfa_image=$tyk_image
- tyk_beta_image=$tyk_image
- ECR=${{steps.ecr.outputs.registry}}
- tyk_sink_image=${{matrix.sink}}
- confs_dir=./pro-ha
- env_file=local-${{ matrix.envfiles.db }}.env' >> versions.env
- echo "::group::versions"
- cat versions.env
- echo "::endgroup::"
- # Add Tyk component config variations to $env_file
- cat confs/${{ matrix.envfiles.config }}.env >> local-${{ matrix.envfiles.db }}.env
- # bring up env, the project name is important
- docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d
- ./dash-bootstrap.sh http://localhost:3000
- docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d
- - name: Run tests
- working-directory: auto
+ run: "match_tag=${{steps.ecr.outputs.registry}}/tyk-pump:$BASE_REF\ntags=(${{ needs.goreleaser.outputs.tags }})\nset -eaxo pipefail\ndocker run -q --rm -v ~/.docker/config.json:/root/.docker/config.json tykio/gromit policy match ${tags[0]} ${match_tag} 2>versions.env\necho '# alfa and beta have to come after the override\ntyk_alfa_image=$tyk_image\ntyk_beta_image=$tyk_image\nECR=${{steps.ecr.outputs.registry}}\ntyk_sink_image=${{matrix.sink}}\nconfs_dir=./pro-ha\nenv_file=local.env' >> versions.env\ncat ./confs/${{ matrix.envfiles.config }}.env local-${{ matrix.envfiles.db }}.env > local.env\necho \"::group::versions\"\ncat versions.env local.env\necho \"::endgroup::\"\n# bring up env, the project name is important\ndocker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d\n./dash-bootstrap.sh http://localhost:3000\ndocker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d \n"
+ - name: Run API tests
  id: test_execution
  env:
  # Cover pull_request_target too
  BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name }}
  run: |
  # Generate report id
  echo "id=$(date +%s%N)" >> $GITHUB_OUTPUT
+ pytest_file="/app/pytest_ci.ini"
+ pytest="pytest -c $pytest_file --junitxml=${XUNIT_REPORT_PATH} --ci -m ${{ matrix.envfiles.apimarkers }}"
+ touch $XUNIT_REPORT_PATH
+ cmd="docker run --rm --network auto_default -v $XUNIT_REPORT_PATH:$XUNIT_REPORT_PATH ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:$BASE_REF $pytest"
  # Run tests
- set -o pipefail
- echo "### API tests ${{ matrix.envfiles.db }} ${{ matrix.envfiles.conf }}" >> $GITHUB_STEP_SUMMARY
- if docker run --rm --network auto_default --env-file pytest.env -v ${{ github.workspace }}/reports:/app/reports \
- ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:$BASE_REF \
- pytest -c pytest_ci.ini --junitxml=./${XUNIT_REPORT_PATH#"${{ github.workspace }}"} --ci -m "not local and not dind ${{ matrix.markers }}" | tee tests.out; then
- echo "All tests passed!" >> $GITHUB_STEP_SUMMARY
- else
- echo "::error title=API tests ${{ matrix.envfiles.db }} ${{ matrix.envfiles.conf }}::Test execution failed"
- cat tests.out >> $GITHUB_STEP_SUMMARY
- exit 1
- fi
- - name: Generate metadata
+ eval $cmd
+ - name: Generate metadata and upload test reports
  if: always() && steps.test_execution.outcome != 'skipped'
  id: metadata_report
  env:
  BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref}}
  REPORT_NAME: ${{ github.repository }}_${{ github.run_id }}_${{ github.run_attempt }}-${{steps.test_execution.outputs.id}}
+ METADATA_REPORT_PATH: /tmp/metadata.toml
  run: |
  # Generate metadata report
  echo "[metadata]
@@ -343,9 +310,7 @@ jobs:
  cat ${METADATA_REPORT_PATH}
  echo "::endgroup::"
  fi
- #Upload xunit report
  aws s3 cp ${XUNIT_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.xml
- #Upload metadata report
  aws s3 cp ${METADATA_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.metadata.toml
  - name: Docker logs for all components
  if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
@@ -357,45 +322,51 @@ jobs:
  TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
  ECR: ${{ steps.ecr.outputs.registry }}
  run: |
- docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml  --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose.log
+ docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose.log
  echo "::group::DockerLogs"
  cat ${{ github.workspace }}/docker-compose.log
  echo "::endgroup::"
- - name: Upload Artifact
+ - name: Upload compose logs
  uses: actions/upload-artifact@v4
  if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
  with:
  name: docker-compose-logs-${{ github.job }}-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }}
  path: ${{ github.workspace }}/docker-compose.log
  retention-days: 3
  overwrite: true
- - name: Archive Integration tests report
- if: always()
- uses: actions/upload-artifact@v4
- with:
- name: api-test-report-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }}
- retention-days: 3
- path: ${{ github.workspace }}/reports
- overwrite: true
+ test-controller-distros:
+ needs:
+ - goreleaser
+ runs-on: ubuntu-latest
+ outputs:
+ deb: ${{ steps.params.outputs.deb }}
+ rpm: ${{ steps.params.outputs.rpm }}
+ steps:
+ - name: set params
+ id: params
+ shell: bash
+ env:
+ # startsWith covers pull_request_target too
+ BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
+ VARIATION: test
+ run: |
+ set -eo pipefail
+ curl -s --retry 5 --retry-delay 10 --fail-with-body "http://tui.internal.dev.tyk.technology/v2/$VARIATION/tyk-pump/$BASE_REF/${{ github.event_name}}/api/Distros.gho" | tee -a "$GITHUB_OUTPUT"
+ [[ $VARIATION =~ test ]] && echo "::warning file=release.yml,line=100,col=5,endColumn=7::Using test variation"
  upgrade-deb:
  services:
  httpbin.org:
  image: kennethreitz/httpbin
  runs-on: ubuntu-latest
- needs: goreleaser
+ needs:
+ - test-controller-distros
  strategy:
- fail-fast: false
+ fail-fast: true
  matrix:
  arch:
  - amd64
  - arm64
- distro:
- - ubuntu:xenial
- - ubuntu:bionic
- - ubuntu:focal
- - ubuntu:jammy
- - debian:bullseye
- - debian:bookworm
+ distro: ${{ fromJson(needs.test-controller-distros.outputs.deb) }}
  steps:
  - uses: actions/checkout@v4
  with:
@@ -406,16 +377,9 @@ jobs:
  - uses: docker/setup-qemu-action@v3
  - uses: docker/setup-buildx-action@v3
  - name: generate dockerfile
- run: |
- echo 'FROM ${{ matrix.distro }}
- ARG TARGETARCH
- COPY tyk-pump*_${TARGETARCH}.deb /tyk-pump.deb
- RUN apt-get update && apt-get install -y curl
- RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.deb.sh | bash && apt-get install -y tyk-pump=1.6.0
- RUN dpkg -i tyk-pump.deb
- ' > Dockerfile
+ run: "echo 'FROM ${{ matrix.distro }}\nARG TARGETARCH\nCOPY tyk-pump*_${TARGETARCH}.deb /tyk-pump.deb\nRUN apt-get update && apt-get install -y curl\nRUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.deb.sh | bash && apt-get install -y tyk-pump=1.6.0\nRUN dpkg -i /tyk-pump.deb \n' > Dockerfile\n"
  - name: install on ${{ matrix.distro }}
- uses: docker/build-push-action@v5
+ uses: docker/build-push-action@v6
  with:
  context: "."
  platforms: linux/${{ matrix.arch }}
@@ -425,40 +389,32 @@ jobs:
  services:
  httpbin.org:
  image: kennethreitz/httpbin
- needs: goreleaser
  runs-on: ubuntu-latest
+ needs:
+ - test-controller-distros
  strategy:
- fail-fast: false
+ fail-fast: true
  matrix:
- distro:
- - amazonlinux:2023
- - registry.access.redhat.com/ubi8/ubi
- - registry.access.redhat.com/ubi9/ubi
- - amazonlinux:2
- - registry.access.redhat.com/ubi7/ubi
+ arch:
+ - amd64
+ - arm64
+ distro: ${{ fromJson(needs.test-controller-distros.outputs.rpm) }}
  steps:
  - uses: actions/checkout@v4
  with:
  fetch-depth: 1
  - uses: actions/download-artifact@v4
  with:
- name: rpm
+ name: deb
+ - uses: docker/setup-qemu-action@v3
  - uses: docker/setup-buildx-action@v3
  - name: generate dockerfile
- run: |
- echo 'FROM ${{ matrix.distro }}
- COPY tyk-pump*.x86_64.rpm /tyk-pump.rpm
- RUN command -v curl || yum install -y curl
- RUN command -v useradd || yum install -y shadow-utils
- RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.rpm.sh | bash && yum install -y tyk-pump-1.6.0-1
- RUN curl https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 -o tyk-pump.key && rpm --import tyk-pump.key
- RUN rpm --checksig tyk-pump.rpm
- RUN rpm -Uvh --force tyk-pump.rpm
- ' > Dockerfile
+ run: "echo 'FROM ${{ matrix.distro }}\nARG TARGETARCH\nCOPY tyk-pump*.x86_64.rpm /tyk-pump.rpm\nRUN command -v curl || yum install -y curl\nRUN command -v useradd || yum install -y shadow-utils\nRUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.rpm.sh | bash && yum install -y tyk-pump-1.6.0-1\nRUN curl https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 -o tyk-pump.key && rpm --import tyk-pump.key\nRUN rpm --checksig /tyk-pump.rpm\nRUN rpm -Uvh --force /tyk-pump.rpm \n' > Dockerfile\n"
  - name: install on ${{ matrix.distro }}
- uses: docker/build-push-action@v5
+ uses: docker/build-push-action@v6
  with:
  context: "."
+ platforms: linux/${{ matrix.arch }}
  file: Dockerfile
  push: false
  sbom:

ci/goreleaser/goreleaser.yml

@@ -4,6 +4,8 @@
 # This project needs CGO_ENABLED=1 and the cross-compiler toolchains for
 # - arm64
 # - amd64
+
+version: 2
 builds:
  - id: std
  ldflags: