From b5883718b03b99d46c67ee3a1fe2b3abf3e74385 Mon Sep 17 00:00:00 2001 From: frank zhu Date: Tue, 20 Feb 2024 13:14:58 -0800 Subject: [PATCH] fix: goreleaser develop build for local and CI (#11847) * add updated goreleaser templates * update zig 0.11.0 * update goreleaser develop * add back ldflags and rename goreleaser config file * update goreleaser build gha workflow and pin new versions * add back pre/post hooks and updated goreleaser Dockerfile to include LOOP plugins * comment out integration-tests * add back goreleaser_wrapper * fix path in goreleaser_utils * add multi-line json output support * save * revert everthing and add go mod tidy * revert zig and goreleaser config * update * zig 0.10.1 * update go.mod go version * add -shared in goreleaser * use zig 0.11.0 and update bash fail * add to cc * update name * test * test * add LD_LIBRARY_PATH in dockerfile * add _transform_path func * fix post-hook cp * remove post hooks * use zig * use older zig version * try zig 0.12.0-dev * add back posthook * use zig 0.10.1 again * update go.mod to 1.21.5 * try older zig version * another older zig version * add ldd_fix script with patchelf * uncomment * add comments and remove unnecessary * update go.mod version * update go.mod to 1.21.7 for core/scripts * update all go.mod to 1.21.7 and remove artifact outputs --- .../goreleaser-build-sign-publish/action.yml | 15 +++++++---- .../action_utils | 4 +-- .../goreleaser-build-publish-develop.yml | 5 ++-- .goreleaser.develop.yaml | 12 ++++----- .tool-versions | 4 +-- core/chainlink.Dockerfile | 3 ++- core/chainlink.goreleaser.Dockerfile | 13 ++++++--- core/scripts/go.mod | 2 +- go.mod | 2 +- integration-tests/.tool-versions | 2 +- integration-tests/go.mod | 2 +- tools/bin/ldd_fix | 27 +++++++++++++++++++ 12 files changed, 64 insertions(+), 27 deletions(-) create mode 100755 tools/bin/ldd_fix diff --git a/.github/actions/goreleaser-build-sign-publish/action.yml b/.github/actions/goreleaser-build-sign-publish/action.yml index fdfcbd3711a..b8760e34dc1 100644 --- a/.github/actions/goreleaser-build-sign-publish/action.yml +++ b/.github/actions/goreleaser-build-sign-publish/action.yml @@ -3,7 +3,10 @@ description: A composite action that allows building and publishing signed chain inputs: goreleaser-version: description: The goreleaser version - default: 1.15.2 + default: 1.23.0 + required: false + goreleaser-key: + description: The goreleaser key required: false zig-version: description: The zig version @@ -11,13 +14,13 @@ inputs: required: false cosign-version: description: The cosign version - default: v1.13.1 + default: v2.2.2 required: false macos-sdk-dir: description: The macos sdk directory default: MacOSX12.3.sdk required: false - # publising inputs + # publishing inputs enable-docker-publish: description: Enable publishing of docker images / manifests default: "true" @@ -75,16 +78,18 @@ runs: - name: Setup goreleaser uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0 with: - distribution: goreleaser + distribution: goreleaser-pro install-only: true version: ${{ inputs.goreleaser-version }} + env: + GORELEASER_KEY: ${{ inputs.goreleaser-key }} - name: Setup zig uses: goto-bus-stop/setup-zig@7ab2955eb728f5440978d5824358023be3a2802d # v2.2.0 with: version: ${{ inputs.zig-version }} - name: Setup cosign if: inputs.enable-cosign == 'true' - uses: sigstore/cosign-installer@11086d25041f77fe8fe7b9ea4e48e3b9192b8f19 # v3.1.2 + uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 # v3.3.0 with: cosign-release: ${{ inputs.cosign-version }} - name: Login to docker registry diff --git a/.github/actions/goreleaser-build-sign-publish/action_utils b/.github/actions/goreleaser-build-sign-publish/action_utils index bf33d0cb4c6..4aac78d6fcc 100755 --- a/.github/actions/goreleaser-build-sign-publish/action_utils +++ b/.github/actions/goreleaser-build-sign-publish/action_utils @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -x +set -euo pipefail ENABLE_COSIGN=${ENABLE_COSIGN:-false} ENABLE_GORELEASER_SNAPSHOT=${ENABLE_GORELEASER_SNAPSHOT:-false} @@ -70,9 +71,6 @@ goreleaser_release() { rm -rf cosign.pub rm -rf cosign.key fi - - echo "metadata=$(cat dist/metadata.json)" >> "$GITHUB_OUTPUT" - echo "artifacts=$(cat dist/artifacts.json)" >> "$GITHUB_OUTPUT" } "$@" diff --git a/.github/workflows/goreleaser-build-publish-develop.yml b/.github/workflows/goreleaser-build-publish-develop.yml index 942f7d22432..5d4041bbc47 100644 --- a/.github/workflows/goreleaser-build-publish-develop.yml +++ b/.github/workflows/goreleaser-build-publish-develop.yml @@ -34,8 +34,8 @@ jobs: enable-goreleaser-snapshot: "true" goreleaser-exec: ./tools/bin/goreleaser_wrapper goreleaser-config: .goreleaser.develop.yaml - # ISSUE: https://github.com/golang/go/issues/52690 - zig-version: 0.11.0-dev.3380+7e0a02ee2 # TODO: update action to v0.11.x once released + goreleaser-key: ${{ secrets.GORELEASER_KEY }} + zig-version: 0.11.0 - name: Collect Metrics if: always() id: collect-gha-metrics @@ -46,6 +46,7 @@ jobs: hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: push-chainlink-develop-goreleaser continue-on-error: true + mercury-e2e-tests: needs: [push-chainlink-develop-goreleaser] runs-on: diff --git a/.goreleaser.develop.yaml b/.goreleaser.develop.yaml index 20312491651..60949eee72e 100644 --- a/.goreleaser.develop.yaml +++ b/.goreleaser.develop.yaml @@ -10,6 +10,7 @@ env: before: hooks: + - go mod tidy - ./tools/bin/goreleaser_utils before_hook # See https://goreleaser.com/customization/build/ @@ -62,6 +63,7 @@ dockers: goarch: amd64 extra_files: - tmp/linux_amd64/libs + - tools/bin/ldd_fix build_flag_templates: - "--platform=linux/amd64" - "--pull" @@ -84,6 +86,7 @@ dockers: goarch: arm64 extra_files: - tmp/linux_arm64/libs + - tools/bin/ldd_fix build_flag_templates: - "--platform=linux/arm64" - "--pull" @@ -106,6 +109,7 @@ dockers: goarch: amd64 extra_files: - tmp/linux_amd64/libs + - tools/bin/ldd_fix build_flag_templates: - "--platform=linux/amd64" - "--pull" @@ -129,6 +133,7 @@ dockers: goarch: arm64 extra_files: - tmp/linux_arm64/libs + - tools/bin/ldd_fix build_flag_templates: - "--platform=linux/arm64" - "--pull" @@ -170,13 +175,6 @@ docker_signs: - artifacts: all stdin: "{{ .Env.COSIGN_PASSWORD }}" -archives: - - rlcp: true - files: - - src: tmp/{{ .Os }}_{{ .Arch }}/libs/* - dst: libs - strip_parent: true - checksum: name_template: "checksums.txt" diff --git a/.tool-versions b/.tool-versions index e52a1f25a81..b3c9c6c56da 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,8 +1,8 @@ -golang 1.21.5 +golang 1.21.7 mockery 2.38.0 nodejs 16.16.0 postgres 13.3 helm 3.10.3 -zig 0.10.1 +zig 0.11.0 golangci-lint 1.55.2 protoc 25.1 diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile index 22e65c0aa7a..f992ee76166 100644 --- a/core/chainlink.Dockerfile +++ b/core/chainlink.Dockerfile @@ -44,7 +44,8 @@ RUN apt-get update && apt-get install -y ca-certificates gnupg lsb-release curl RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |tee /etc/apt/sources.list.d/pgdg.list \ && apt-get update && apt-get install -y postgresql-client-15 \ - && apt-get clean all + && apt-get clean all \ + && rm -rf /var/lib/apt/lists/* COPY --from=buildgo /go/bin/chainlink /usr/local/bin/ diff --git a/core/chainlink.goreleaser.Dockerfile b/core/chainlink.goreleaser.Dockerfile index 26335a85bf1..7774c416f0e 100644 --- a/core/chainlink.goreleaser.Dockerfile +++ b/core/chainlink.goreleaser.Dockerfile @@ -6,18 +6,25 @@ FROM ubuntu:20.04 ARG CHAINLINK_USER=root ARG TARGETARCH ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y ca-certificates gnupg lsb-release curl +RUN apt-get update && apt-get install -y ca-certificates gnupg lsb-release curl patchelf # Install Postgres for CLI tools, needed specifically for DB backups RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |tee /etc/apt/sources.list.d/pgdg.list \ && apt-get update && apt-get install -y postgresql-client-15 \ - && apt-get clean all + && apt-get clean all \ + && rm -rf /var/lib/apt/lists/* -COPY . /usr/local/bin/ +COPY ./chainlink /usr/local/bin/ # Copy native libs if cgo is enabled COPY ./tmp/linux_${TARGETARCH}/libs /usr/local/bin/libs +# Temp fix to patch correctly link the libwasmvm.so +COPY ./tools/bin/ldd_fix /usr/local/bin/ldd_fix +RUN chmod +x /usr/local/bin/ldd_fix +RUN /usr/local/bin/ldd_fix +RUN apt-get remove -y patchelf + RUN if [ ${CHAINLINK_USER} != root ]; then \ useradd --uid 14933 --create-home ${CHAINLINK_USER}; \ fi diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 97f8fda74e8..b4d6860719d 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -1,6 +1,6 @@ module github.com/smartcontractkit/chainlink/core/scripts -go 1.21.3 +go 1.21.7 // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../../ diff --git a/go.mod b/go.mod index 1d3799dc999..be354048dcd 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/smartcontractkit/chainlink/v2 -go 1.21.3 +go 1.21.7 require ( github.com/Depado/ginprom v1.8.0 diff --git a/integration-tests/.tool-versions b/integration-tests/.tool-versions index ac6300f9797..7f84d58b918 100644 --- a/integration-tests/.tool-versions +++ b/integration-tests/.tool-versions @@ -1,4 +1,4 @@ -golang 1.21.5 +golang 1.21.7 k3d 5.4.6 kubectl 1.25.5 nodejs 18.13.0 diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 39d17c0ae79..02857f31230 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -1,6 +1,6 @@ module github.com/smartcontractkit/chainlink/integration-tests -go 1.21.4 +go 1.21.7 // Make sure we're working with the latest chainlink libs replace github.com/smartcontractkit/chainlink/v2 => ../ diff --git a/tools/bin/ldd_fix b/tools/bin/ldd_fix new file mode 100755 index 00000000000..dd48e9c3b30 --- /dev/null +++ b/tools/bin/ldd_fix @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# This script is used as a temp fix the ldd linking of cosm lib for binary +# Currently there is an issue with the go linker not working with zig +# https://github.com/ziglang/zig/issues/18922 + +chainlink_path="/usr/local/bin/chainlink" +libs_path="/usr/local/bin/libs" + +line=$(ldd ${chainlink_path} | grep "github.com/!cosm!wasm/wasmvm") + +if [ -z "$line" ]; then + echo "Error: Path containing 'github.com/!cosm!wasm/wasmvm' not found in the ldd output." + exit 1 +fi + +path=$(echo "$line" | awk '{print $1}') + +if [ -z "$path" ]; then + echo "Error: Failed to extract the path from the line." + exit 1 +fi + +trimmed_path=${path%.so*}.so +cosm_file=$(ls ${libs_path} | grep "\.so$" | head -n 1) + +patchelf --remove-needed "${trimmed_path}" "$chainlink_path" +patchelf --add-needed "$cosm_file" "$chainlink_path"