upgrade_dm_via_tiup.yaml

name: Upgrade DM via TiUP

on:
  schedule:
    - cron: '0 17-23 * * *' # run at minute 0 every hour from 01:00 ~ 07:00 UTC+8

  workflow_dispatch:
    inputs:
      pr:
        description: 'Which PR do you want to trigger'
        required: true
        default: ''
      # TODO: use these inputs
      fromVer:
        dmVer: "v1.0.7"
      toVer:
        dmVer: "nightly"

# See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency.
concurrency:
  group: ${{ github.ref }}-${{ github.workflow }}
  cancel-in-progress: true

jobs:
  from_v1:
    name: From V1
    runs-on: ubuntu-20.04
    # skip it now, the ansible script fails with "Could not find the requested service prometheus-9090.service: host"
    # TODO enable it later.
    if: ${{ false }}
    env:
      working-directory: ${{ github.workspace }}/go/src/github.com/pingcap/tiflow

    steps:
      - name: Set up Go env
        uses: actions/setup-go@v3
        with:
          go-version: '1.21'

      - name: Check out code
        uses: actions/checkout@v2
        with:
          path: go/src/github.com/pingcap/tiflow

      - name: Check out code by workflow dispatch
        if: ${{ github.event.inputs.pr != '' }}
        uses: actions/checkout@v2
        with:
          path: go/src/github.com/pingcap/tiflow
          ref: refs/pull/${{ github.event.inputs.pr }}/head

      - name: Setup containers
        working-directory: ${{ env.working-directory }}
        run: |
          cd ${{ env.working-directory }}/dm/tests/tiup/docker
          GOPATH=${GITHUB_WORKSPACE}/go docker-compose up -d

      - name: Run test cases
        working-directory: ${{ env.working-directory }}
        run: |
          cd ${{ env.working-directory }}/dm/tests/tiup/docker
          docker-compose exec -T control bash -c "cd /go/src/github.com/pingcap/tiflow/dm && ./tests/tiup/upgrade-from-v1.sh"

      # send Slack notify if failed.
      # NOTE: With the exception of `GITHUB_TOKEN`, secrets are not passed to the runner when a workflow is triggered from a forked repository.
      - name: Slack notification
        if: ${{ failure() }}
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFY }}
        uses: Ilshidur/action-slack@2.1.0
        with:
          args: "upgrade job failed, see https://github.com/pingcap/tiflow/actions/runs/{{ GITHUB_RUN_ID }}"

  from_v2:
    name: From V2
    runs-on: ubuntu-20.04
    env:
      working-directory: ${{ github.workspace }}/go/src/github.com/pingcap/tiflow
    strategy:
      fail-fast: false
      matrix:
        previous_v2:
          ["v2.0.1", "v2.0.7", "v5.3.0"]

    steps:
      - name: Set up Go env
        uses: actions/setup-go@v3
        with:
          go-version: '1.21'

      - name: Check out code
        uses: actions/checkout@v2
        with:
          path: go/src/github.com/pingcap/tiflow

      - name: Check out code by workflow dispatch
        if: ${{ github.event.inputs.pr != '' }}
        uses: actions/checkout@v2
        with:
          path: go/src/github.com/pingcap/tiflow
          ref: refs/pull/${{ github.event.inputs.pr }}/head

      - name: Build
        if: ${{ github.ref != 'refs/heads/master' || github.event.inputs.pr != '' }}
        working-directory: ${{ env.working-directory }}
        run: make dm

      - name: Package files
        if: ${{ github.ref != 'refs/heads/master' || github.event.inputs.pr != '' }}
        run: |
          mkdir ${{ github.workspace }}/package
          cd ${{ github.workspace }}/package

          echo "package dm-master"
          mkdir dm-master
          cp ${{ env.working-directory }}/bin/dm-master dm-master
          cp -r ${{ env.working-directory }}/metrics/grafana/ dm-master/scripts
          cp -r ${{ env.working-directory }}/dm/metrics/alertmanager/ dm-master/conf
          tar -czvf dm-master-nightly-linux-amd64.tar.gz dm-master

          echo "package dm-worker"
          mkdir dm-worker
          cp ${{ env.working-directory }}/bin/dm-worker dm-worker
          cp -r ${{ env.working-directory }}/metrics/grafana/ dm-worker/scripts
          cp -r ${{ env.working-directory }}/dm/metrics/alertmanager/ dm-worker/conf
          tar -czvf dm-worker-nightly-linux-amd64.tar.gz dm-worker

          echo "package dmctl"
          mkdir dmctl
          cp ${{ env.working-directory }}/bin/dmctl dmctl
          cp -r ${{ env.working-directory }}/metrics/grafana/ dmctl/scripts
          cp -r ${{ env.working-directory }}/dm/metrics/alertmanager/ dmctl/conf
          tar -czvf dmctl-nightly-linux-amd64.tar.gz dmctl

      - name: Setup containers
        working-directory: ${{ env.working-directory }}
        run: |
          cd ${{ env.working-directory }}/dm/tests/tiup/docker
          GOPATH=${GITHUB_WORKSPACE}/go docker-compose up -d

      - name: Copy package files
        if: ${{ github.ref != 'refs/heads/master' || github.event.inputs.pr != '' }}
        run: |
          cd ${{ github.workspace }}/package
          docker cp dm-master-nightly-linux-amd64.tar.gz control:/tmp
          docker cp dm-worker-nightly-linux-amd64.tar.gz control:/tmp
          docker cp dmctl-nightly-linux-amd64.tar.gz control:/tmp

      # TODO: support more CUR_VER
      - name: Run test cases
        working-directory: ${{ env.working-directory }}
        run: |
          cd ${{ env.working-directory }}/dm/tests/tiup/docker
          docker-compose exec -e ref=${{ github.ref }} -e id=${{ github.event.inputs.pr }} -T control bash -c "cd /go/src/github.com/pingcap/tiflow/dm && ./tests/tiup/upgrade-from-v2.sh ${{ matrix.previous_v2 }} nightly"

      # if above step is passed, logs will be removed by tiup dm destroy
      - name: Copy logs to hack permission
        if: ${{ failure() }}
        run: |
          mkdir ./logs
          docker cp -L master1:/home/tidb/dm/deploy/dm-master-8261/log ./logs/master
          docker cp -L worker1:/home/tidb/dm/deploy/dm-worker-8262/log ./logs/worker1
          docker cp -L worker2:/home/tidb/dm/deploy/dm-worker-8262/log ./logs/worker2
          docker cp -L control:/tmp/tiup-dm-operation.log ./logs
          sudo chown -R runner ./logs

      # Update logs as artifact seems not stable, so we set `continue-on-error: true` here.
      - name: Upload logs
        continue-on-error: true
        uses: actions/upload-artifact@v2
        if: ${{ failure() }}
        with:
          name: upgrade-via-tiup-${{ matrix.previous_v2 }}
          path: |
            ./logs

      # send Slack notify if failed.
      # NOTE: With the exception of `GITHUB_TOKEN`, secrets are not passed to the runner when a workflow is triggered from a forked repository.
      - name: Slack notification
        if: ${{ failure() }}
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFY }}
        uses: Ilshidur/action-slack@2.1.0
        with:
          args: "upgrade-via-tiup job failed, see https://github.com/pingcap/tiflow/actions/runs/{{ GITHUB_RUN_ID }}"

  upgrade_tidb:
    name: Upgrade TiDB
    runs-on: ubuntu-20.04
    env:
      working-directory: ${{ github.workspace }}/go/src/github.com/pingcap/tiflow
    steps:
      - name: Set up Go env
        uses: actions/setup-go@v3
        with:
          go-version: '1.21'

      - name: Check out code
        uses: actions/checkout@v2
        with:
          path: go/src/github.com/pingcap/tiflow

      # TODO: support more versions
      - name: Setup containers
        working-directory: ${{ env.working-directory }}
        run: |
          cd ${{ env.working-directory }}/dm/tests/tiup/docker
          sed -i "s/tidb:v4.0.7/tidb:v3.0.19/g" docker-compose.yml
          GOPATH=${GITHUB_WORKSPACE}/go docker-compose up -d

      - name: Run test cases before upgrade
        working-directory: ${{ env.working-directory }}
        run: |
          cd ${{ env.working-directory }}/dm/tests/tiup/docker
          docker-compose exec -T control bash -c "cd /go/src/github.com/pingcap/tiflow && ./dm/tests/tiup/upgrade-tidb.sh before_upgrade nightly"

      - name: Upgrade TiDB
        working-directory: ${{ env.working-directory }}
        run: |
          cd ${{ env.working-directory }}/dm/tests/tiup/docker
          sed -i "s/tidb:v3.0.19/tidb:v4.0.7/g" docker-compose.yml
          GOPATH=${GITHUB_WORKSPACE}/go docker-compose up -d

      - name: Run test cases after upgrade
        working-directory: ${{ env.working-directory }}
        run: |
          cd ${{ env.working-directory }}/dm/tests/tiup/docker
          docker-compose exec -T control bash -c "source /root/.profile && cd /go/src/github.com/pingcap/tiflow && ./dm/tests/tiup/upgrade-tidb.sh after_upgrade nightly"

      # send Slack notify if failed.
      # NOTE: With the exception of `GITHUB_TOKEN`, secrets are not passed to the runner when a workflow is triggered from a forked repository.
      - name: Slack notification
        if: ${{ failure() }}
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFY }}
        uses: Ilshidur/action-slack@2.1.0
        with:
          args: "upgrade job failed, see https://github.com/pingcap/tiflow/actions/runs/{{ GITHUB_RUN_ID }}"