diff --git a/.github/workflows/codespace.yml b/.github/workflows/codespace.yml deleted file mode 100644 index f9a39b68..00000000 --- a/.github/workflows/codespace.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Build and push GitHub Codespace images - -on: - push: - branches: - - main - workflow_dispatch: - -env: - IMAGE_NAME: input-output-hk/devx-devcontainer - REGISTRY: ghcr.io - DEFAULT_TAG: latest - -jobs: - build_and_push: - permissions: - packages: write - runs-on: ubuntu-latest - # We want a GitHub Codespace image for each combination of devx developer shell option. - # But, since the purpose of GitHub Codespace is to serve a complete development environment, - # the user is likely to always expect HLS (I don't see the point otherwise). - # Therefore, it doesn't seem useful to build an image on the `-minimal` flavor (without HLS), - # or the `-static` one (especially since the latter currently requires `-minimal` to work). - # Likely, we consider using `-iog` as the default and do not generate other images. - # Then the user choices left would be between native, `-windows` or `-js` target platforms, - # and the GHC version (currently `ghc8107` and `ghc962`). - strategy: - matrix: - platform: ['x86_64-linux', 'aarch64-linux'] - target-platform: ['', '-windows', '-js'] - compiler-nix-name: ['ghc8107','ghc962'] - minimal: [false] - iog: [true] - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v2.1.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push Docker image - uses: docker/build-push-action@v4 - with: - context: . - push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.platform }}.${{ matrix.compiler-nix-name }}${{ matrix.target-platform }}-iog - build-args: | - PLATFORM=${{ matrix.platform }} - TARGET_PLATFORM=${{ matrix.target-platform }} - COMPILER_NIX_NAME=${{ matrix.compiler-nix-name }} - MINIMAL=${{ matrix.minimal }} - IOG=${{ matrix.iog }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 362e81fa..073532fc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,41 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build: + wait-for-hydra-eval: + env: + HYDRA_JOB: ci/eval + GH_TOKEN: ${{ github.token }} + name: "Wait for hydra status" + runs-on: ubuntu-latest + steps: + - name: Get specific check run status + run: | + # start with a random sleep to prevent hitting the api too hard. + while true; do + # For GitHub Apps + # conclusion=$(gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status --jq '.check_runs[] | select(.name == "ci/hydra-build:$DEV_SHELL") | .conclusion') + # For GitHub Statuses; we need --paginate because there are so many statuses + echo "Querying: gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status --paginate --jq '.statuses[] | select(.context == \"$HYDRA_JOB\") | .state'" + conclusion=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status" --paginate --jq ".statuses[] | select(.context == \"$HYDRA_JOB\") | .state") + case "$conclusion" in + success) + echo "$HYDRA_JOB succeeded" + exit 0;; + failure) + echo "$HYDRA_JOB failed" + exit 1;; + *) + echo "conclusion is: '$conclusion'" + gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status" --paginate --jq '.statuses[] | .state+"\t"+.context'|sort + WAIT=$((30 + RANDOM % 30)) + echo "$HYDRA_JOB pending. Waiting ${WAIT}s..." + sleep $WAIT;; + esac + done + + upload: + needs: wait-for-hydra-eval + name: Container Upload strategy: fail-fast: false matrix: @@ -66,39 +100,10 @@ jobs: - target-platform: "-js" platform: x86_64-darwin variant: "" - - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Install Nix with good defaults - uses: cachix/install-nix-action@v20 - with: - extra_nix_config: | - trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk= - substituters = https://cache.iog.io/ https://cache.zw3rk.com/ https://cache.nixos.org/ - nix_path: nixpkgs=channel:nixos-unstable - - name: Checkout repository - uses: actions/checkout@v3 - - name: Log in to the Container registry - uses: docker/login-action@v2.1.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Work around issue fetching cabal revision files - run: | - # This will cause the revised `.cabal` files used whan building alex - # for building GHC to be fetched using the `x86_64-linux` `.drv` for `fetchurl`. - # Later when we want the non `x86_64-linux` version later it will find the - # fixed output of the derivation is already in the `/nix/store` and will - # not try to fetch it using the platform we do not have a builder for. - if [[ "${{ matrix.platform }}" != "x86_64-linux" && "${{ matrix.target-platform }}" = "-js" ]]; then - nix build ".#hydraJobs.x86_64-linux.${{ matrix.compiler-nix-name }}-js-minimal" --show-trace - fi - - name: Compute and upload closure and developer environment to ghcr.io - env: - NIX_STORE_SECRET_KEY: ${{ secrets.SECRET_KEY }} - DEV_SHELL: ${{ matrix.platform }}.${{ matrix.compiler-nix-name }}${{ matrix.target-platform }}${{ matrix.variant }}${{ matrix.iog }}-env - run: ./extra/ghcr-upload.sh + uses: ./.github/workflows/wait-and-upload.yml + with: + platform: ${{ matrix.platform }} + target-platform: ${{ matrix.target-platform }} + compiler-nix-name: ${{ matrix.compiler-nix-name }} + minimal: ${{ matrix.variant == '-minimal' }} + iog: ${{ matrix.iog == '-iog' }} \ No newline at end of file diff --git a/.github/workflows/wait-and-upload.yml b/.github/workflows/wait-and-upload.yml new file mode 100644 index 00000000..3dc5c72e --- /dev/null +++ b/.github/workflows/wait-and-upload.yml @@ -0,0 +1,139 @@ +name: Wait and Upload + +on: + workflow_call: + inputs: + platform: + required: true + description: 'build platform' + type: string + target-platform: + required: true + description: 'target platform' + type: string + compiler-nix-name: + required: true + description: 'compiler name in nix format. e.g. ghc8107' + type: string + minimal: + description: 'without hls, hlint, ...' + type: boolean + default: true + iog: + description: 'without iog libs: libsodium, libsecp256k1, libblst, ...' + type: boolean + default: false + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + DEV_SHELL: ${{ inputs.platform }}.${{ inputs.compiler-nix-name }}${{ inputs.target-platform }}${{ inputs.minimal && '-minimal' || '' }}${{ inputs.iog && '-iog' || ''}}-env + DEFAULT_TAG: latest + GH_TOKEN: ${{ github.token }} + +jobs: + wait-for-hydra: + name: "Wait for hydra status" + runs-on: ubuntu-latest + steps: + - name: Get specific check run status + run: | + # start with a random sleep to prevent hitting the api too hard. + while true; do + # For GitHub Apps + # conclusion=$(gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status --jq '.check_runs[] | select(.name == "ci/hydra-build:$DEV_SHELL") | .conclusion') + # For GitHub Statuses; we need --paginate because there are so many statuses + echo "Querying: gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status --paginate --jq '.statuses[] | select(.context == \"ci/hydra-build:$DEV_SHELL\") | .state'" + conclusion=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status" --paginate --jq ".statuses[] | select(.context == \"ci/hydra-build:$DEV_SHELL\") | .state") + case "$conclusion" in + success) + echo "ci/hydra-build:$DEV_SHELL succeeded" + exit 0;; + failure) + echo "ci/hydra-build:$DEV_SHELL failed" + exit 1;; + *) + echo "conclusion is: '$conclusion'" + gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status --paginate --jq '.statuses[] | .state+"\t"+.context'|sort + WAIT=$((180 + RANDOM % 180)) + echo "ci/hydra-build:$DEV_SHELL pending. Waiting ${WAIT}s..." + sleep $WAIT;; + esac + done + + ghcr-upload: + needs: wait-for-hydra + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Install Nix with good defaults + uses: cachix/install-nix-action@v20 + with: + extra_nix_config: | + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk= + substituters = https://cache.iog.io/ https://cache.zw3rk.com/ https://cache.nixos.org/ + nix_path: nixpkgs=channel:nixos-unstable + - name: Checkout repository + uses: actions/checkout@v3 + - name: Log in to the Container registry + uses: docker/login-action@v2.1.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Work around issue fetching cabal revision files + run: | + # This will cause the revised `.cabal` files used whan building alex + # for building GHC to be fetched using the `x86_64-linux` `.drv` for `fetchurl`. + # Later when we want the non `x86_64-linux` version later it will find the + # fixed output of the derivation is already in the `/nix/store` and will + # not try to fetch it using the platform we do not have a builder for. + if [[ "${{ inputs.platform }}" != "x86_64-linux" && "${{ inputs.target-platform }}" = "-js" ]]; then + nix build ".#hydraJobs.x86_64-linux.${{ inputs.compiler-nix-name }}-js-minimal" --show-trace + fi + - name: Compute and upload closure and developer environment to ghcr.io + env: + NIX_STORE_SECRET_KEY: ${{ secrets.SECRET_KEY }} + run: ./extra/ghcr-upload.sh + + codespace-upload: + env: + IMAGE_NAME: input-output-hk/devx-devcontainer + needs: ghcr-upload + permissions: + packages: write + runs-on: ubuntu-latest + # We want a GitHub Codespace image for each combination of devx developer shell option. + # But, since the purpose of GitHub Codespace is to serve a complete development environment, + # the user is likely to always expect HLS (I don't see the point otherwise). + # Therefore, it doesn't seem useful to build an image on the `-minimal` flavor (without HLS), + # or the `-static` one (especially since the latter currently requires `-minimal` to work). + # Likely, we consider using `-iog` as the default and do not generate other images. + # Then the user choices left would be between native, `-windows` or `-js` target platforms, + # and the GHC version (currently `ghc8107` and `ghc962`). + if: ${{ contains(fromJSON('["x86_64-linux", "aarch64-linux"]'), inputs.platform) && contains(fromJson('["","-windows","-js"]'), inputs.target-platform) && contains(fromJson('["ghc8107","ghc962"]'), inputs.compiler-nix-name) && !inputs.minimal && inputs.iog }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2.1.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ inputs.platform }}.${{ inputs.compiler-nix-name }}${{ inputs.target-platform }}-iog + build-args: | + PLATFORM=${{ inputs.platform }} + TARGET_PLATFORM=${{ inputs.target-platform }} + COMPILER_NIX_NAME=${{ inputs.compiler-nix-name }} + MINIMAL=${{ inputs.minimal }} + IOG=${{ inputs.iog }}