From 78001f59f5eed3eef1e0f415b7ae3cb5e7454696 Mon Sep 17 00:00:00 2001
From: Roshan Khatri <117414976+roshkhatri@users.noreply.github.com>
Date: Tue, 16 Apr 2024 07:01:36 -0700
Subject: [PATCH] Adds workflows to build release binaries and push to S3
 (#315)

[related to](https://github.com/valkey-io/valkey/issues/230)

Adds workflows to build Valkey binaries and push to S3 to make it
available to download from the website

The Workflows can be triggered by pushing a release to the repo and the
other option is manually by one of the Maintainers.

Once the workflow triggers, it will generate a matrix of Jobs for the
platforms we need to build from `utils/releasetools/build-config.json`
and then the respective Jobs are triggered. These jobs make Valkey with
respect to the platform binaries we want to release and would push to a
private S3 bucket.

---------

Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
---
 .../generate-package-build-matrix/action.yml  | 44 +++++++++
 .github/workflows/build-release-packages.yml  | 89 +++++++++++++++++++
 .../call-build-linux-arm-packages.yml         | 79 ++++++++++++++++
 .../call-build-linux-x86-packages.yml         | 73 +++++++++++++++
 utils/releasetools/build-config.json          | 28 ++++++
 5 files changed, 313 insertions(+)
 create mode 100644 .github/actions/generate-package-build-matrix/action.yml
 create mode 100644 .github/workflows/build-release-packages.yml
 create mode 100644 .github/workflows/call-build-linux-arm-packages.yml
 create mode 100644 .github/workflows/call-build-linux-x86-packages.yml
 create mode 100644 utils/releasetools/build-config.json

diff --git a/.github/actions/generate-package-build-matrix/action.yml b/.github/actions/generate-package-build-matrix/action.yml
new file mode 100644
index 0000000000..7e90f27be5
--- /dev/null
+++ b/.github/actions/generate-package-build-matrix/action.yml
@@ -0,0 +1,44 @@
+name: Generate target matrix.
+description: Matrix creation for building Valkey for different architectures and platforms.
+
+inputs:
+  ref:
+    description: The commit, tag or branch of Valkey to checkout to determine what version to use.
+    required: true
+outputs:
+  x86_64-build-matrix:
+    description: The x86_64 build matrix.
+    value: ${{ steps.set-matrix.outputs.x86matrix }}
+  arm64-build-matrix:
+    description: The arm64 build matrix.
+    value: ${{ steps.set-matrix.outputs.armmatrix }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout code for version check
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.ref }}
+        path: version-check
+
+    - name: Get targets
+      run: |
+        x86_arch=$(jq -c '[.linux_targets[] | select(.arch=="x86_64")]' utils/releasetools/build-config.json)
+        x86_matrix=$(echo "{ \"distro\" : $x86_arch }" | jq -c .)
+        echo "X86_MATRIX=$x86_matrix" >> $GITHUB_ENV
+
+        arm_arch=$(jq -c '[.linux_targets[] | select(.arch=="arm64")]' utils/releasetools/build-config.json)
+        arm_matrix=$(echo "{ \"distro\" : $arm_arch }" | jq -c .)
+        echo "ARM_MATRIX=$arm_matrix" >> $GITHUB_ENV
+      shell: bash
+
+    - id: set-matrix
+      run: |
+        echo $X86_MATRIX
+        echo $X86_MATRIX| jq .
+        echo "x86matrix=$X86_MATRIX" >> $GITHUB_OUTPUT
+        echo $ARM_MATRIX
+        echo $ARM_MATRIX| jq .
+        echo "armmatrix=$ARM_MATRIX" >> $GITHUB_OUTPUT
+      shell: bash
diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml
new file mode 100644
index 0000000000..8c32548f9e
--- /dev/null
+++ b/.github/workflows/build-release-packages.yml
@@ -0,0 +1,89 @@
+name: Build Release Packages
+
+on:
+  release:
+    types: [published]
+  
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Version of Valkey to build
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  # This job provides the version metadata from the tag for the other jobs to use.
+  release-build-get-meta:
+    name: Get metadata to build
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.get_version.outputs.VERSION }}
+    steps:
+
+      - run: |
+          echo "Version: ${{ inputs.version || github.ref_name }}"
+        shell: bash
+
+      # This step is to consolidate the three different triggers into a single "version"
+      # 1. If manual dispatch - use the version provided.
+      # 3. If tag trigger, use that tag.
+      - name: Get the version
+        id: get_version
+        run: |
+          VERSION="${INPUT_VERSION}"
+          if [ -z "${VERSION}" ]; then
+            exit 1
+          fi
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+        shell: bash
+        env:
+          # Use the dispatch variable in preference, if empty use the context ref_name which should
+          # only ever be a tag
+          INPUT_VERSION: ${{ inputs.version || github.ref_name }}
+
+  generate-build-matrix:
+      name: Generating build matrix
+      runs-on: ubuntu-latest
+      outputs:
+        x86_64-build-matrix: ${{ steps.set-matrix.outputs.x86_64-build-matrix }}
+        arm64-build-matrix: ${{ steps.set-matrix.outputs.arm64-build-matrix }}
+      steps:
+        - name: Checkout repository
+          uses: actions/checkout@v4
+        # Set up the list of target to build so we can pass the JSON to the reusable job
+        - uses: ./.github/actions/generate-package-build-matrix
+          id: set-matrix
+          with:
+            ref: ${{ inputs.version || github.ref_name }}
+  
+  release-build-linux-x86-packages:
+    needs:
+      - release-build-get-meta
+      - generate-build-matrix
+    uses: ./.github/workflows/call-build-linux-x86-packages.yml
+    with:
+      version: ${{ needs.release-build-get-meta.outputs.version }}
+      ref: ${{ inputs.version || github.ref_name }}
+      build_matrix: ${{ needs.generate-build-matrix.outputs.x86_64-build-matrix }}
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+      bucket: ${{ secrets.AWS_S3_BUCKET }}
+      access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
+      secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }}
+
+  release-build-linux-arm-packages:
+    needs:
+      - release-build-get-meta
+      - generate-build-matrix
+    uses: ./.github/workflows/call-build-linux-arm-packages.yml
+    with:
+      version: ${{ needs.release-build-get-meta.outputs.version }}
+      ref: ${{ inputs.version || github.ref_name }}
+      build_matrix: ${{ needs.generate-build-matrix.outputs.arm64-build-matrix }}
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+      bucket: ${{ secrets.AWS_S3_BUCKET }}
+      access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
+      secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }}
diff --git a/.github/workflows/call-build-linux-arm-packages.yml b/.github/workflows/call-build-linux-arm-packages.yml
new file mode 100644
index 0000000000..db987b3816
--- /dev/null
+++ b/.github/workflows/call-build-linux-arm-packages.yml
@@ -0,0 +1,79 @@
+name: Builds Linux arm binary packages into S3 bucket.
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: The version of Valkey to create.
+        type: string
+        required: true
+      ref:
+        description: The commit, tag or branch of Valkey to checkout for building that creates the version above.
+        type: string
+        required: true
+      build_matrix:
+        description: The build targets to produce as a JSON matrix.
+        type: string
+        required: true
+    secrets:
+      token:
+        description: The Github token or similar to authenticate with.
+        required: true
+      bucket:
+        description: The name of the S3 bucket to push packages into.
+        required: false
+      access_key_id:
+        description: The S3 access key id for the bucket.
+        required: false
+      secret_access_key:
+        description: The S3 secret access key for the bucket.
+        required: false
+
+permissions:
+  contents: read
+
+jobs:
+  build-valkey:
+    # Capture source tarball and generate checksum for it
+    name: Build package ${{ matrix.distro.target }} ${{ matrix.distro.arch }}
+    runs-on: 'ubuntu-latest'
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.build_matrix) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.version }}
+      
+      - name: Make Valkey
+        uses: uraimo/run-on-arch-action@v2
+        with:
+          arch: aarch64
+          distro: ${{matrix.distro.target}}
+          install: apt-get update && apt-get install -y build-essential libssl-dev
+          run: make -C src all BUILD_TLS=yes
+      
+      - name: Create Tarball and SHA256sums
+        run: |
+          TAR_FILE_NAME=valkey-${{inputs.version}}-${{matrix.distro.platform}}-${{ matrix.distro.arch}}
+          mkdir -p $TAR_FILE_NAME/bin $TAR_FILE_NAME/share
+          cp -rfv src/valkey-* $TAR_FILE_NAME/bin
+          cp -v /home/runner/work/valkey/valkey/COPYING $TAR_FILE_NAME/share/LICENSE
+          tar -czvf $TAR_FILE_NAME.tar.gz $TAR_FILE_NAME
+          sha256sum $TAR_FILE_NAME.tar.gz > $TAR_FILE_NAME.tar.gz.sha256
+          mkdir -p packages-files
+          cp -rfv $TAR_FILE_NAME.tar* packages-files/
+      
+      - name: Install AWS cli.
+        run: |
+          sudo apt-get install -y awscli
+
+      - name: Configure AWS credentials
+        run: |
+          aws configure set region us-west-2
+          aws configure set aws_access_key_id ${{ secrets.access_key_id }}
+          aws configure set aws_secret_access_key ${{ secrets.secret_access_key }}
+
+      - name: Sync to S3
+        run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/
diff --git a/.github/workflows/call-build-linux-x86-packages.yml b/.github/workflows/call-build-linux-x86-packages.yml
new file mode 100644
index 0000000000..59c9fac52f
--- /dev/null
+++ b/.github/workflows/call-build-linux-x86-packages.yml
@@ -0,0 +1,73 @@
+name: Builds Linux X86 binary packages into S3 bucket.
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: The version of Valkey to create.
+        type: string
+        required: true
+      ref:
+        description: The commit, tag or branch of Valkey to checkout for building that creates the version above.
+        type: string
+        required: true
+      build_matrix:
+        description: The build targets to produce as a JSON matrix.
+        type: string
+        required: true
+    secrets:
+      token:
+        description: The Github token or similar to authenticate with.
+        required: true
+      bucket:
+        description: The name of the S3 bucket to push packages into.
+        required: false
+      access_key_id:
+        description: The S3 access key id for the bucket.
+        required: false
+      secret_access_key:
+        description: The S3 secret access key for the bucket.
+        required: false
+
+permissions:
+  contents: read
+
+jobs:
+  build-valkey:
+    # Capture source tarball and generate checksum for it
+    name: Build package ${{ matrix.distro.target }} ${{ matrix.distro.arch }}
+    runs-on: 'ubuntu-latest'
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.build_matrix) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.version }}
+
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install -y build-essential libssl-dev jq wget awscli
+
+      - name: Make Valkey
+        run: make -C src all BUILD_TLS=yes
+      
+      - name: Create Tarball and SHA256sums
+        run: |
+          TAR_FILE_NAME=valkey-${{inputs.version}}-${{matrix.distro.platform}}-${{ matrix.distro.arch}}
+          mkdir -p $TAR_FILE_NAME/bin $TAR_FILE_NAME/share
+          cp -rfv src/valkey-* $TAR_FILE_NAME/bin
+          cp -v /home/runner/work/valkey/valkey/COPYING $TAR_FILE_NAME/share/LICENSE
+          tar -czvf $TAR_FILE_NAME.tar.gz $TAR_FILE_NAME
+          sha256sum $TAR_FILE_NAME.tar.gz > $TAR_FILE_NAME.tar.gz.sha256
+          mkdir -p packages-files
+          cp -rfv $TAR_FILE_NAME.tar* packages-files/
+
+      - name: Configure AWS credentials
+        run: |
+          aws configure set region us-west-2
+          aws configure set aws_access_key_id ${{ secrets.access_key_id }}
+          aws configure set aws_secret_access_key ${{ secrets.secret_access_key }}
+
+      - name: Sync to S3
+        run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/
diff --git a/utils/releasetools/build-config.json b/utils/releasetools/build-config.json
new file mode 100644
index 0000000000..5ee9308b3b
--- /dev/null
+++ b/utils/releasetools/build-config.json
@@ -0,0 +1,28 @@
+{
+    "linux_targets": [
+        {
+          "arch": "x86_64",
+          "target": "ubuntu18.04",
+          "type": "deb",
+          "platform": "bionic"
+        },
+        {
+          "arch": "x86_64",
+          "target": "ubuntu20.04",
+          "type": "deb",
+          "platform": "focal"
+        },
+        {
+          "arch": "arm64",
+          "target": "ubuntu18.04",
+          "type": "deb",
+          "platform": "bionic"
+        },
+        {
+          "arch": "arm64",
+          "target": "ubuntu20.04",
+          "type": "deb",
+          "platform": "focal"
+        }
+      ]
+}
\ No newline at end of file