wip: new release process improvements - docs, drafts, changelog

.github/workflows/release.yml

@@ -11,22 +11,24 @@ on:
  branches:
  - release/v*
  - releases
+ - ipdx/*
  paths:
  - build/version.go
  workflow_dispatch:
  inputs:
+ projects:
+ description: 'Projects to release (JSON array)'
+ required: false
+ default: '[]'
+ ref:
+ description: 'Ref to buuild the binaries from'
+ required: false
+ default: ''
  publish:
  description: 'Publish the release'
  required: false
  default: 'false'
- draft:
- description: 'Create a draft release'
- required: false
- default: 'true'
- target_commitish:
- description: 'The commitish value that determines where the Git tag is created from'
- required: false
- default: ''
+
 defaults:
  run:
  shell: bash
@@ -36,22 +38,29 @@ permissions:
 
 jobs:
  check:
- name: Check which projects need to be built
+ name: Check which projects need to be released
  runs-on: ubuntu-latest
  outputs:
  projects: ${{ steps.projects.outputs.projects }}
  steps:
- - uses: actions/checkout@v4
+ - name: Check out lotus
+ uses: actions/checkout@v4
  with:
  fetch-depth: 0
- - id: projects
+ ref: ${{ github.event.inputs.ref }}
+ - name: Find projects that need to be released
+ id: projects
  env:
- TARGET_REF: ${{ github.base_ref || github.ref }}
+ projects: ${{ github.event.inputs.projects }}
  run: |
  go run cmd/release/main.go --json list-projects | jq -r '.msg' |
- jq 'map(select(.version | endswith("-dev") | not))' |
- jq 'map(select(.released | not))' |
- jq 'map(select(.prerelease != ((env.TARGET_REF | sub("^refs/heads/"; "")) == "releases")))' |
+ # jq 'map(select(.version | endswith("-dev") | not))' |
+ # jq 'map(select(.released | not))' |
+ jq 'map(select(
+ (env.GITHUB_EVENT_NAME == "push" and ((env.GITHUB_REF == "refs/heads/releases" and .prerelease == false) or (env.GITHUB_REF != "refs/heads/releases" and .prerelease == true))) or
+ (env.GITHUB_EVENT_NAME == "pull_request" and ((env.GITHUB_BASE_REF == "ipdx/release-automation" and .prerelease == false) or (env.GITHUB_BASE_REF != "releases" and .prerelease == true))) or
+ (env.GITHUB_EVENT_NAME == "workflow_dispatch" and (.name as $name | env.projects | fromjson | index($name) != null))
+ ))' |
  jq -c '.' | xargs -I {} -0 echo "projects={}" |
  tee -a $GITHUB_OUTPUT
  build:
@@ -68,34 +77,47 @@ jobs:
  - macos-13 # MacOs X64
  - macos-14 # MacOS ARM64
  steps:
- - run: echo "Building on $RUNNER_OS/$RUNNER_ARCH"
- - id: project
+ - name: Print runner information
+ run: echo "Building on $RUNNER_OS/$RUNNER_ARCH"
+ - name: Make project config available
+ id: project
  env:
  projects: ${{ needs.check.outputs.projects }}
  name: ${{ matrix.project }}
  run: |
  jq -nc 'env.projects | fromjson | map(select(.name == env.name)) | .[0]' |
  xargs -I {} -0 echo "config={}" |
  tee -a $GITHUB_OUTPUT
- - uses: actions/checkout@v4
+ - name: Check out lotus
+ uses: actions/checkout@v4
  with:
  submodules: 'recursive'
- - uses: ./.github/actions/install-system-dependencies
- - uses: ./.github/actions/install-go
- - env:
-  GITHUB_TOKEN: ${{ github.token }}
-  run: make deps
- - if: matrix.project == 'node'
- env:
- GITHUB_TOKEN: ${{ github.token }}
- run: make lotus
- - if: matrix.project == 'miner'
+ - name: Install system dependencies
+  uses: ./.github/actions/install-system-dependencies
+ - name: Install Go
+ uses: ./.github/actions/install-go
+ - if: github.event.inputs.ref
+  uses: actions/checkout@v4
+ with:
+ submodules: 'recursive'
+  ref: ${{ github.event.inputs.ref }}
+ - name: Build binaries
  env:
  GITHUB_TOKEN: ${{ github.token }}
- run: make lotus-miner lotus-worker
- - if: runner.os == 'macOS'
+ binaries: ${{ matrix.project == 'node' && 'lotus' || 'lotus-miner lotus-worker' }}
+ run: |
+ make deps
+ while read -r binary; do
+ if [[ -z "$binary" ]]; then
+ continue
+ fi
+ make $binary
+ done <<< "$(jq -r '.[]' <<< "$binaries")"
+ - name: Run otool
+ if: runner.os == 'macOS'
  run: if [[ -f lotus ]]; then otool -hv lotus; fi
- - env:
+ - name: Verify binary versions
+ env:
  LOTUS_VERSION_IGNORE_COMMIT: 1
  expected: ${{ fromJSON(steps.project.outputs.config).version }}
  run: |
@@ -109,7 +131,8 @@ jobs:
  fi
  fi
  done
- - uses: actions/upload-artifact@v4
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
  with:
  name: lotus-${{ matrix.project }}-${{ runner.os }}-${{ runner.arch }}
  path: |
@@ -119,7 +142,7 @@ jobs:
  release:
  needs: [check, build]
  if: needs.check.outputs.projects != '[]'
- name: Release ${{ matrix.project }} [publish=${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}]
+ name: Release ${{ matrix.project }}
  permissions:
  # This enables the job to create and/or update GitHub releases
  contents: write
@@ -128,91 +151,156 @@ jobs:
  fail-fast: false
  matrix:
  project: ${{ fromJSON(needs.check.outputs.projects).*.name }}
- env:
- PUBLISH: ${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}
- IS_DRAFT: ${{ inputs.draft == 'true' || github.event_name == 'pull_request' }}
  steps:
- - id: project
+ - name: Make project config available
+ id: project
  env:
  projects: ${{ needs.check.outputs.projects }}
  name: ${{ matrix.project }}
  run: |
  jq -nc 'env.projects | fromjson | map(select(.name == env.name)) | .[0]' |
  xargs -I {} -0 echo "config={}" |
  tee -a $GITHUB_OUTPUT
- - uses: actions/checkout@v4
+ - name: Check out lotus
+ uses: actions/checkout@v4
  with:
  submodules: 'recursive'
  fetch-depth: 0
- - uses: actions/download-artifact@v4
+ - name: Create dist directory
+ run: mkdir dist
+ - name: Download Linux X64 binaries
+ uses: actions/download-artifact@v4
  with:
  name: lotus-${{ matrix.project }}-Linux-X64
- path: linux_amd64_v1
- - uses: actions/download-artifact@v4
+ path: dist/linux_amd64_v1
+ - name: Download macOS X64 binaries
+ uses: actions/download-artifact@v4
  with:
  name: lotus-${{ matrix.project }}-macOS-X64
- path: darwin_amd64_v1
- - uses: actions/download-artifact@v4
+ path: dist/darwin_amd64_v1
+ - name: Download macOS ARM64 binaries
+ uses: actions/download-artifact@v4
  with:
  name: lotus-${{ matrix.project }}-macOS-ARM64
- path: darwin_arm64
- - uses: ./.github/actions/install-go
- - uses: ipfs/download-ipfs-distribution-action@v1
+ path: dist/darwin_arm64
+ - name: Install Go (archives generation dependency)
+ uses: ./.github/actions/install-go
+ - name: Generate archives
+ env:
+ prefix: ${{ matrix.project == 'node' && 'lotus' || format('lotus-{0}', matrix.project) }}
+ version: ${{ fromJSON(steps.project.outputs.config).version }}
+ binaries: ${{ fromJSON(steps.project.outputs.config).binaries }}
+ run: |
+ mkdir darwin_all
+ while read -r binary; do
+ if [[ -z "$binary" ]]; then
+ continue
+ fi
+ go run github.com/randall77/makefat ./darwin_all/$binary ./darwin_amd64_v1/$binary ./darwin_arm64/$binary
+ done <<< "$(jq -r '.[]' <<< "$binaries")"
+ rm -rf darwin_amd64_v1 darwin_arm64
+ for directory in darwin_all linux_amd64_v1; do
+ mv $directory $prefix_v$version_$directory
+ tar -czf $prefix_v$version_$directory.tar.gz $directory
+ rm -rf $prefix_v$version_$directory
+ done
+ working-directory: dist
+ - name: Install Kubo (checksums generation dependency)
+ uses: ipfs/download-ipfs-distribution-action@v1
  with:
  name: kubo
  version: v0.16.0
  cache: false
- - name: Install yq
+ - name: Generate checksums
+ run: ./scripts/generate-checksums.sh
+ - name: Install zsh (changelog generation dependency)
+ run: sudo apt update && sudo apt install -y zsh
+ - name: Generate changelog
+ id: changelog
+ env:
+ tag: ${{ fromJSON(steps.project.outputs.config).tag }}
+ previous: ${{ fromJSON(steps.project.outputs.config).previous }}
+ current: ${{ github.event.inputs.ref || github.sha }}
+ run: |
+ echo "content<<EOF" | tee -a $GITHUB_OUTPUT
+ echo "# $tag" | tee -a $GITHUB_OUTPUT
+ echo "" | tee -a $GITHUB_OUTPUT
+ csplit --digits=4 --quiet --elide-empty-files CHANGELOG.md '/^# /' '{*}'
+ # Checking the files in reverse order to get to the UNRELEASED section last
+ for file in $(ls -r xx*); do
+ if grep -q "^# $tag " $file || grep -q "^# UNRELEASED" $file; then
+ tail -n +3 $file | tee -a $GITHUB_OUTPUT
+ break
+ fi
+ done
+ if [[ "$previous" != '' ]]; then
+ mkdir -p "$(go env GOPATH)/src/github.com/$GITHUB_REPOSITORY"
+ rm -rf "$(go env GOPATH)/src/github.com/$GITHUB_REPOSITORY"
+ ln -s "$(pwd)" "$(go env GOPATH)/src/github.com/$GITHUB_REPOSITORY"
+ ./scripts/mkreleaselog "$previous" "$current" | tee -a $GITHUB_OUTPUT
+ fi
+ echo "EOF" | tee -a $GITHUB_OUTPUT
+ - name: Find release
+ id: before
  env:
+ tag: ${{ fromJSON(steps.project.outputs.config).tag }}
  GITHUB_TOKEN: ${{ github.token }}
  run: |
- gh release --repo mikefarah/yq download v4.44.2 -p yq_linux_amd64
- sudo mv yq_linux_amd64 /usr/bin/yq
- sudo chmod +x /usr/bin/yq
- - if: env.IS_DRAFT == 'false'
- run: yq -i '.release.draft = false' '.goreleaser.yaml'
- - if: env.PUBLISH == 'true' && env.IS_DRAFT == 'false' && fromJSON(steps.project.outputs.config).latest && !fromJSON(steps.project.outputs.config).prerelease
- run: yq -i '.brews.[0].skip_upload = false' '.goreleaser.yaml'
- - if: matrix.project == 'node'
+ echo "release>>EOF" | tee -a $GITHUB_OUTPUT
+ gh api --paginate /repos/$GITHUB_REPOSITORY/releases --jq 'map(select(.tag_name == env.tag))' |
+ jq -s add | jq -r '.[0]' | tee -a $GITHUB_OUTPUT
+ echo "EOF" | tee -a $GITHUB_OUTPUT
+ - name: Create or update release
+ id: after
  env:
- BREW_INSTALL: |
- bin.install "lotus"
- BREW_TEST: |
- system "#{bin}/lotus --version"
+ method: ${{ steps.before.outputs.release == 'null' && 'POST' || 'PATCH' }}
+ endpoint: ${{ steps.before.outputs.release == 'null' && format('/repos/{0}/releases', github.repository) || format('/repos/{0}/releases/{1}', github.repository, fromJSON(steps.before.outputs.release).id) }}
+ tag_name: ${{ fromJSON(steps.project.outputs.config).tag }}
+ target_commitish: ${{ github.event.inputs.ref || github.base_ref || github.ref }}
+ name: ${{ fromJSON(steps.project.outputs.config).tag }}
+ body: ${{ steps.changelog.outputs.content }}
+ draft: ${{ steps.before.outputs.release == 'null' && true || fromJSON(steps.before.outputs.release).draft }}
+ prerelease: ${{ fromJSON(steps.project.outputs.config).prerelease }}
+ make_latest: ${{ fromJSON(steps.project.outputs.config).latest }}
+ discussion_category_name: ""
+ # GITHUB_TOKEN: ${{ github.token }}
  run: |
- yq -i '(.builds[] | select(.id == "lotus")).skip = false' '.goreleaser.yaml'
- yq -i '.brews[0].install = strenv(BREW_INSTALL)' '.goreleaser.yaml'
- yq -i '.brews[0].test = strenv(BREW_TEST)' '.goreleaser.yaml'
- - if: matrix.project == 'miner'
+ echo "release>>EOF" | tee -a $GITHUB_OUTPUT
+ gh api -X "$method" "$endpoint" \
+ --field tag_name="$tag_name" \
+ --field target_commitish="$target_commitish" \
+ --field name="$name" \
+ --field body="$body" \
+ --field draft=true \
+ --field prerelease=$prerelease \
+ --field make_latest=$make_latest \
+ --field discussion_category_name="$discussion_category_name" |
+ tee -a $GITHUB_OUTPUT
+ echo "EOF" | tee -a $GITHUB_OUTPUT
+ - name: Delete old assets
  env:
- BREW_INSTALL: |
- bin.install "lotus-miner"
- bin.install "lotus-worker"
- BREW_TEST: |
- system "#{bin}/lotus-miner --version"
- system "#{bin}/lotus-worker --version"
+ asset_ids: ${{ toJSON(fromJSON(steps.after.outputs.release).assets.*.id) }}
+ release_id: ${{ fromJSON(steps.after.outputs.release).id }}
+ # GITHUB_TOKEN: ${{ github.token }}
  run: |
- yq -i '(.builds[] | select(.id == "lotus-miner" or .id == "lotus-worker")).skip = false' '.goreleaser.yaml'
- yq -i '.brews[0].install = strenv(BREW_INSTALL)' '.goreleaser.yaml'
- yq -i '.brews[0].test = strenv(BREW_TEST)' '.goreleaser.yaml'
- - uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0
- with:
- distribution: goreleaser-pro
- version: 2.0.1
- args: release --clean --skip=validate ${{ env.PUBLISH == 'false' && '--snapshot' || '' }}
+ while read -r asset_id; do
+ gh api -X DELETE /repos/$GITHUB_REPOSITORY/releases/$release_id/assets/$asset_id
+ done <<< "$(jq '.[]' <<< "$asset_ids")"
+ - name: Upload new assets
  env:
- PROJECT_NAME: ${{ matrix.project == 'node' && 'lotus' || format('lotus-{0}', matrix.project) }}
- GITHUB_TOKEN: ${{ env.PUBLISH == 'true' && (secrets.GORELEASER_GITHUB_TOKEN || github.token) || '' }}
- GORELEASER_KEY: ${{ env.PUBLISH == 'true' && secrets.GORELEASER_KEY || '' }}
- TAG: ${{ fromJSON(steps.project.outputs.config).tag }}
- VERSION: ${{ fromJSON(steps.project.outputs.config).version }}
- IS_LATEST: ${{ fromJSON(steps.project.outputs.config).latest }}
- IS_PRERELEASE: ${{ fromJSON(steps.project.outputs.config).prerelease }}
- TARGET_COMMITISH: ${{ inputs.target_commitish || (github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.sha) }}
- HEADER: ''
- - run: ./scripts/generate-checksums.sh
- - if: env.PUBLISH == 'true' && env.IS_DRAFT == 'false'
+ release_id: ${{ fromJSON(steps.after.outputs.release).id }}
+ # GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ for asset in *.{tar.gz,cid,sha512}; do
+ gh api -X POST /repos/$GITHUB_REPOSITORY/releases/$release_id/assets?name=$asset \
+ --field "@$asset" |
+ jq -r '.id'
+ done
+ - name: Publish release
+ if: github.event.inputs.publish == 'true' || github.event_name == 'push'
  env:
- GITHUB_TOKEN: ${{ github.token }}
- TAG: ${{ fromJSON(steps.project.outputs.config).tag }}
- run: ./scripts/publish-checksums.sh
+ release_id: ${{ fromJSON(steps.after.outputs.release).id }}
+ # GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ gh api -X PATCH /repos/$GITHUB_REPOSITORY/releases/$release_id \
+ --field draft=false