Skip to content
CI build

CI: Use a non-rate-limiting docker.io mirror #12639

Sign in to view logs

CI: Use a non-rate-limiting docker.io mirror

CI: Use a non-rate-limiting docker.io mirror #12639

Workflow file for this run

.github/workflows/ci.yml at 4864271
# Copyright (C) 2020 Dremio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Unifies main + PR workflow.
#
# The unified CI workflow consists of 2 "stages":
# - Checks - test, intTest, NesQuEIT, etc
# - Finalize - a "success" dummy job for PRs + a "save to github-cache" job for push-to-main
#
# Utilizes the Gradle build cache for all stages. The updated build cache
# of the jobs in the checks stage are saved as artifacts (with the minimum
# retention period). The updated build cache is pushed back to GigHub's
# cache when the checks have successfully finished.
name: CI build
on:
push:
branches:
- main
- release-*
paths-ignore:
- 'LICENSE'
- 'NOTICE'
- '**.md'
- '!site/**'
- '.github/renovate.json5'
- '.github/workflows/release*.yml'
- '.github/workflows/check*.yml'
- '.idea/**'
- '.editorconfig'
pull_request:
types: [labeled, opened, synchronize, reopened]
# For the main branch: let all CI runs complete, one after the other. This has a couple advantages:
# * Site deployments happen in commit-order
# * Saved Gradle cache are persisted in commit-order
# * (Potentially) more GH runners available for PRs
concurrency:
# PRs: 1 CI run concurrently / older ones are cancelled
# main branch: 1 CI run concurrently / no cancellation
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
code-checks:
name: CI Code Checks et al
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
# Needed for the Quarkus plugin - can likely go away once we use Quarkus 3 or newer
- name: Bump Gradle daemon heap
run: sed -i 's/-Xms.*/-Xms6G -Xmx6G -XX:MaxMetaspaceSize=1g \\/' gradle.properties
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Gradle / Compile
run: |
./gradlew \
spotlessCheck \
compileAll \
-x :nessie-quarkus:compileAll \
-x :nessie-server-admin-tool:compileAll \
-x :nessie-events-quarkus:compileAll \
-x :nessie-events-ri:compileAll \
--scan
- name: Gradle / Compile Quarkus
run: |
# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll :nessie-events-ri:compileAll --scan || \
./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll :nessie-events-ri:compileAll --scan || \
./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll :nessie-events-ri:compileAll --scan
- name: Gradle / Code checks
run: ./gradlew codeChecks --scan
- name: Gradle / Assemble
run: ./gradlew assemble --scan
- name: Gradle / Publish to Maven local
run: ./gradlew publishToMavenLocal --scan
# This is a rather quick one and uses the output of 'publishToMavenLocal', which uses the
# outputs of 'assemble'
- name: Gradle / build tools integration tests
run: ./gradlew buildToolsIntegrationTest
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
job-name: 'code-checks'
test:
name: CI Test
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Gradle / test
run: ./gradlew test :nessie-client:check -x :nessie-client:intTest -x :nessie-quarkus:test -x :nessie-server-admin-tool:test -x :nessie-events-quarkus:test -x :nessie-events-ri:test --scan
- name: Capture Test Reports
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
if: ${{ failure() }}
with:
name: ci-test-reports
path: |
**/build/reports/*
**/build/test-results/*
retention-days: 7
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
job-name: 'test'
test-quarkus:
name: CI Test Quarkus
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Gradle / Test Quarkus Server
run: |
# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
./gradlew :nessie-quarkus:test --scan || \
./gradlew :nessie-quarkus:test --scan || \
./gradlew :nessie-quarkus:test --scan
- name: Gradle / Test Quarkus Events
run: |
# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
./gradlew :nessie-events-quarkus:test --scan || \
./gradlew :nessie-events-quarkus:test --scan || \
./gradlew :nessie-events-quarkus:test --scan
- name: Gradle / Test Quarkus Events RI
run: |
# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
./gradlew :nessie-events-ri:test --scan || \
./gradlew :nessie-events-ri:test --scan || \
./gradlew :nessie-events-ri:test --scan
- name: Dump quarkus.log
if: ${{ failure() }}
run: |
find . -path "**/build/quarkus.log" | while read ql ; do
echo "::group::Quarkus build log $ql"
cat $ql
echo "::endgroup::"
done
- name: Capture Test Reports
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
if: ${{ failure() }}
with:
name: ci-test-quarkus-reports
path: |
**/build/quarkus.log
**/build/reports/*
**/build/test-results/*
retention-days: 7
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
job-name: 'test-quarkus'
int-test:
name: CI intTest
runs-on: ubuntu-24.04
timeout-minutes: 60
env:
SPARK_LOCAL_IP: localhost
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Gradle / intTest
run: |
echo "::group::Collect :nessie-versioned-storage projects"
./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-persist- --output ../persist-prjs.txt --exclude
echo "::endgroup::"
echo "::group::Collect :nessie-versioned-persist projects"
./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-storage- --output ../storage-prjs.txt --exclude
echo "::endgroup::"
echo "::group::Collect :nessie-spark-extensions projects"
./gradlew :listProjectsWithPrefix --prefix :nessie-spark-ext --output ../spark-prjs.txt --exclude
echo "::endgroup::"
./gradlew intTest \
-x :nessie-quarkus:intTest \
-x :nessie-server-admin-tool:intTest \
-x :nessie-events-quarkus:intTest \
$(cat ../persist-prjs.txt) \
$(cat ../storage-prjs.txt) \
$(cat ../spark-prjs.txt) \
--scan
- name: Capture Test Reports
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
if: ${{ failure() }}
with:
name: ci-inttest-reports
path: |
**/build/reports/*
**/build/test-results/*
retention-days: 7
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
job-name: 'int-test'
int-test-stores:
name: CI intTest versioned/stores
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Gradle / intTest versioned/stores
run: |
echo "::group::Collect :nessie-versioned-storage projects"
./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-storage- --output ../storage-prjs.txt
echo "::endgroup::"
echo "::group::Collect :nessie-versioned-persist projects"
./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-persist- --output ../persist-prjs.txt
echo "::endgroup::"
./gradlew $(cat ../persist-prjs.txt) $(cat ../storage-prjs.txt) --scan
- name: Capture Test Reports
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
if: ${{ failure() }}
with:
name: ci-inttest-stores-reports
path: |
**/build/reports/*
**/build/test-results/*
retention-days: 7
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
job-name: 'int-test-stores'
int-test-integrations:
name: CI intTest integrations
runs-on: ubuntu-24.04
timeout-minutes: 60
env:
SPARK_LOCAL_IP: localhost
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
with:
# Need Java 17 in addition to the default Java 21
additional-java-version: 17
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Gradle / intTest integrations
run: |
echo "::group::Collect :nessie-spark-extensions projects"
./gradlew :listProjectsWithPrefix --prefix :nessie-spark-ext --output ../spark-prjs.txt
echo "::endgroup::"
./gradlew $(cat ../spark-prjs.txt) --scan
- name: Capture Test Reports
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
if: ${{ failure() }}
with:
name: ci-inttest-integrations-reports
path: |
**/build/reports/*
**/build/test-results/*
retention-days: 7
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
job-name: 'int-test-integrations'
int-test-quarkus-server:
name: CI intTest Quarkus Server
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Gradle / intTest Quarkus Server
run:
# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
./gradlew :nessie-quarkus:intTest --scan || \
./gradlew :nessie-quarkus:intTest --scan || \
./gradlew :nessie-quarkus:intTest --scan
- name: Dump quarkus.log
if: ${{ failure() }}
run: |
find . -path "**/build/quarkus.log" | while read ql ; do
echo "::group::Quarkus build log $ql"
cat $ql
echo "::endgroup::"
done
- name: Capture Test Reports
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
if: ${{ failure() }}
with:
name: ci-inttest-quarkus-server-reports
path: |
**/build/quarkus.log
**/build/reports/*
**/build/test-results/*
retention-days: 7
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
job-name: 'int-test-quarkus-server'
int-test-quarkus-tool:
name: CI intTest Admin Tool
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Gradle / intTest Admin Tool
run:
# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
./gradlew :nessie-server-admin-tool:intTest --scan || \
./gradlew :nessie-server-admin-tool:intTest --scan || \
./gradlew :nessie-server-admin-tool:intTest --scan
- name: Dump quarkus.log
if: ${{ failure() }}
run: |
find . -path "**/build/quarkus.log" | while read ql ; do
echo "::group::Quarkus build log $ql"
cat $ql
echo "::endgroup::"
done
- name: Capture Test Reports
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
if: ${{ failure() }}
with:
name: ci-inttest-quarkus-tool-reports
path: |
**/build/quarkus.log
**/build/reports/*
**/build/test-results/*
retention-days: 7
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
job-name: 'int-test-quarkus-tool'
int-test-quarkus-events:
name: CI intTest Quarkus Events
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Gradle / intTest Quarkus Events
run:
# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
./gradlew :nessie-events-quarkus:intTest --scan || \
./gradlew :nessie-events-quarkus:intTest --scan || \
./gradlew :nessie-events-quarkus:intTest --scan
- name: Dump quarkus.log
if: ${{ failure() }}
run: |
find . -path "**/build/quarkus.log" | while read ql ; do
echo "::group::Quarkus build log $ql"
cat $ql
echo "::endgroup::"
done
- name: Capture Test Reports
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4
if: ${{ failure() }}
with:
name: ci-inttest-quarkus-events-reports
path: |
**/build/quarkus.log
**/build/reports/*
**/build/test-results/*
retention-days: 7
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
job-name: 'int-test-quarkus-events'
determine-jobs:
name: CI Determine jobs
runs-on: ubuntu-24.04
timeout-minutes: 5
outputs:
# Each "conditional" job has a mapped output here, also a "non-PR" case and a "PR" case
# with label and globs (at the end of the script).
docker: ${{ steps.determine.outputs.docker }}
nesqueit: ${{ steps.determine.outputs.nesqueit }}
steps:
- name: install minimatch
shell: bash
run: npm install minimatch@"^7.4.3"
- name: Determine conditional jobs to run
id: determine
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
with:
script: |
const {Minimatch} = require("minimatch");
async function getChangedFiles(pull_request) {
const listFilesOptions = github.rest.pulls.listFiles.endpoint.merge({
owner: pull_request.base.repo.owner.login,
repo: pull_request.base.repo.name,
pull_number: pull_request.number,
});
return github.paginate(listFilesOptions).then(resp => resp.map(f => f.filename))
}
function isMatch(changedFile, matchers) {
for (const matcher of matchers) {
if (matcher.match(changedFile)) {
core.info("Match found for changed file " + changedFile);
return true;
}
}
core.info("No changed file matched the globs");
return false;
}
function filesMatchAnyGlob(changedFiles, globs) {
const matchers = globs.map(g => new Minimatch(g));
for (const changedFile of changedFiles) {
if (isMatch(changedFile, matchers)) {
return true;
}
}
return false;
}
function checkLabelOrGlobs(pr, changedFiles, labelName, globs) {
core.info("Checking for " + labelName);
// If the PR has the given label, then let the conditional job run.
for (const label of pr.labels) {
if (label.name === labelName) {
core.info("PR has label " + labelName);
return true;
}
}
// For pull request, check the list of changed files against the given globs.
// Let the conditional job run, if any of the globs matches and of the changed files.
core.info("Checking changed files against globs " + globs);
return filesMatchAnyGlob(changedFiles, globs);
}
///////////////////////////////////////////////////////////////////////////////////////
// "Non PR" case:
// Behavior of the conditional jobs when the event that triggered the workflow is not
// for a pull request (usually a push-to-main).
//
if (!context.payload || !context.payload.pull_request) {
core.info("Not a pull-request, enabling all jobs.");
// Yield default values for all event payload types, except pull_request
core.setOutput('docker', true);
core.setOutput('nesqueit', false); // handled in a scheduled job
return true;
}
///////////////////////////////////////////////////////////////////////////////////////
// "PR" case:
// Determine the whether the conditional jobs shall run, based on a label name, which
// has been manually added, or, if the label is not present, based on some globs to
// check.
const pr = context.payload.pull_request
core.info("pull-request #" + pr.number);
// Retrieves the full list of changed files for the PR.
const changedFiles = await getChangedFiles(pr);
core.startGroup('PR job check / Docker');
core.setOutput('docker', checkLabelOrGlobs(pr, changedFiles,
'pr-docker',
['gradle/wrapper/**', 'tools/dockerbuild/**', 'helm/**', '.github/**']));
core.endGroup();
core.startGroup('PR job check / NesQuEIT');
core.setOutput('nesqueit', checkLabelOrGlobs(pr, changedFiles,
'pr-integrations',
['gradle/wrapper/**', 'integrations/**', 'api/**', '.github/**']));
core.endGroup();
docker-testing:
name: CI Docker and Helm checks
runs-on: ubuntu-24.04
timeout-minutes: 60
needs:
- determine-jobs
if: needs.determine-jobs.outputs.docker == 'true'
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
- name: Extract version
run: |
VERSION="$(cat version.txt)"
DOCKER_VERSION="${VERSION%-SNAPSHOT}"
echo ${DOCKER_VERSION}
echo "DOCKER_VERSION=${DOCKER_VERSION}" >> ${GITHUB_ENV}
# Free disk space (minikube warning: "Docker is nearly out of disk space, which may cause
# deployments to fail! (85% of capacity)")
- name: Free disk space
uses: ./.github/actions/free-disk-space
- name: Setup Helm
uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4
with:
# https://helm.sh/docs/topics/version_skew/
version: 'v3.11.3'
- name: Setup chart-testing
uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1
- name: Setup Python
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
with:
python-version: '3.11'
- name: Setup & Start Minikube
uses: medyagh/setup-minikube@d8c0eb871f6f455542491d86a574477bd3894533 # v0.0.18
with:
# If required, use the matrix strategy against this option to test against multiple Kubernetes versions:
kubernetes-version: stable
cache: false
# This _should_ work, but doesn't somehow (the image push fails). It errors out looking for a coredns pod
# with these settings...
#addons: 'registry'
#insecure-registry: '192.168.0.0/16'
- name: Setup Docker registry
run: |
echo "::group::Get registry IP"
DOCKER_REGISTRY="$(minikube ip)"
echo "Registry IP is ${DOCKER_REGISTRY}"
echo "::endgroup::"
echo "::group::Update buildkitd_conf for 'docker buildx build'"
# Use 'http' instead of 'https' during 'docker buildx build' in 'tools/dockerbuild/build-push-images.sh'
cat <<EOF > ../buildkitd.toml
[registry."${DOCKER_REGISTRY}:5000"]
http = true
EOF
buildkitd_conf="$(pwd)/../buildkitd.toml"
echo "::endgroup::"
echo "::group::Update /etc/docker/daemon.json for 'docker pull'"
echo "{\"insecure-registries\": [\"${DOCKER_REGISTRY}:5000\"]}" | \
sudo tee /etc/docker/daemon.json
echo "::endgroup::"
# minikube restart, because:
# 1. required after docker daemon restart
# 2. tweak the "registry addon" into the start command
# 3. tweak the "insecure-registry" setting into the start command
# Must delete the minikube cluster to let the insecure-registry setting take effect.
# See 'Enabling Insecure Registries' in https://minikube.sigs.k8s.io/docs/handbook/registry/
echo "::group::Stop minikube"
minikube stop
echo "::endgroup::"
echo "::group::Delete minikube"
minikube delete
echo "::endgroup::"
echo "::group::Restart docker daemon"
sudo systemctl restart docker
echo "::endgroup::"
echo "::group::Start minikube"
minikube start --insecure-registry="${DOCKER_REGISTRY}:5000" --addons=registry
echo "::endgroup::"
echo "BUILDX_CONFIG=--config ${buildkitd_conf}" >> ${GITHUB_ENV}
echo "DOCKER_IMAGE=${DOCKER_REGISTRY}:5000/nessie-testing" >> ${GITHUB_ENV}
echo "DOCKER_GC_IMAGE=${DOCKER_REGISTRY}:5000/nessie-gc-testing" >> ${GITHUB_ENV}
echo "DOCKER_SERVER_ADMIN_IMAGE=${DOCKER_REGISTRY}:5000/nessie-server-admin-testing" >> ${GITHUB_ENV}
echo "DOCKER_CLI_IMAGE=${DOCKER_REGISTRY}:5000/nessie-cli-testing" >> ${GITHUB_ENV}
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Docker images publishing
run: |
tools/dockerbuild/build-push-images.sh \
-g ":nessie-quarkus" \
-p "servers/quarkus-server" \
-d "Dockerfile-server" \
${DOCKER_IMAGE}
tools/dockerbuild/build-push-images.sh \
-g ":nessie-gc-tool" \
-p "gc/gc-tool" \
-d "Dockerfile-gctool" \
${DOCKER_GC_IMAGE}
tools/dockerbuild/build-push-images.sh \
-g ":nessie-server-admin-tool" \
-p "tools/server-admin" \
-d "Dockerfile-admintool" \
${DOCKER_SERVER_ADMIN_IMAGE}
tools/dockerbuild/build-push-images.sh \
-g ":nessie-cli" \
-p "cli/cli" \
-d "Dockerfile-cli" \
${DOCKER_CLI_IMAGE}
- name: Cleanup buildx
run: |
docker buildx use default
docker buildx rm nessiebuild
- name: Check if expected Docker images exist
run: |
docker pull ${DOCKER_IMAGE}:latest
docker pull ${DOCKER_IMAGE}:latest-java
docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION}
docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION}-java
docker pull ${DOCKER_GC_IMAGE}:latest
docker pull ${DOCKER_GC_IMAGE}:latest-java
docker pull ${DOCKER_GC_IMAGE}:${DOCKER_VERSION}
docker pull ${DOCKER_GC_IMAGE}:${DOCKER_VERSION}-java
docker pull ${DOCKER_SERVER_ADMIN_IMAGE}:latest
docker pull ${DOCKER_SERVER_ADMIN_IMAGE}:latest-java
docker pull ${DOCKER_SERVER_ADMIN_IMAGE}:${DOCKER_VERSION}
docker pull ${DOCKER_SERVER_ADMIN_IMAGE}:${DOCKER_VERSION}-java
docker pull ${DOCKER_CLI_IMAGE}:latest
docker pull ${DOCKER_CLI_IMAGE}:latest-java
docker pull ${DOCKER_CLI_IMAGE}:${DOCKER_VERSION}
docker pull ${DOCKER_CLI_IMAGE}:${DOCKER_VERSION}-java
cat <<! >> $GITHUB_STEP_SUMMARY
## Docker images
\`\`\`
$(docker images)
\`\`\`
!
- name: Check if Server Docker Java image works
run: |
docker run --rm --detach --name nessie ${DOCKER_IMAGE}:latest-java
echo "Let Nessie Java Docker image run for one minute (to make sure it starts up fine)..."
for i in {1..60}; do
STATUS="$(docker container inspect nessie | jq -r '.[0].State.Status')"
if [[ ${STATUS} != "running" ]] ; then
echo "Nessie Java Docker image stopped on its own ... a bug?" > /dev/stderr
docker logs nessie
cat <<! >> $GITHUB_STEP_SUMMARY
## Nessie Java Docker image FAILED
\`\`\`
$(docker logs nessie)
\`\`\`
!
exit 1
fi
sleep 1
done
echo "## Nessie Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY
echo "Nessie Java Docker image smoke test: PASSED"
docker stop nessie
- name: Check if GC Tool Docker Java image works
run: |
if docker run --rm --name nessie-gc ${DOCKER_GC_IMAGE}:latest-java --help | grep -q "Usage: nessie-gc.jar"; then
echo "## GC Tool Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY
echo "GC Tool Java Docker image smoke test: PASSED"
else
echo "GC Tool Java Docker image smoke test: FAILED" > /dev/stderr
cat <<! >> $GITHUB_STEP_SUMMARY
## GC Tool Java Docker image FAILED
\`\`\`
$(docker logs nessie-gc)
\`\`\`
!
exit 1
fi
- name: Check if Server Admin Tool Docker Java image works
run: |
if docker run --rm --name nessie-server-admin ${DOCKER_SERVER_ADMIN_IMAGE}:latest-java --help | grep -q "Usage: nessie-server-admin"; then
echo "## Server Admin Tool Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY
echo "Server Admin Tool Java Docker image smoke test: PASSED"
else
echo "Server Admin Tool Java Docker image smoke test: FAILED" > /dev/stderr
cat <<! >> $GITHUB_STEP_SUMMARY
## Server Admin Tool Java Docker image FAILED
\`\`\`
$(docker logs nessie-server-admin)
\`\`\`
!
exit 1
fi
- name: Check if CLI Docker Java image works
run: |
if docker run --rm --name nessie-cli ${DOCKER_CLI_IMAGE}:latest-java --help | grep -q "Usage: nessie-cli.jar"; then
echo "## CLI Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY
echo "CLI Java Docker image smoke test: PASSED"
else
echo "CLI Java Docker image smoke test: FAILED" > /dev/stderr
cat <<! >> $GITHUB_STEP_SUMMARY
## CLI Java Docker image FAILED
\`\`\`
$(docker logs nessie-cli)
\`\`\`
!
exit 1
fi
- name: Run chart-testing (list-changed)
run: |
ct list-changed --target-branch ${{ github.event.repository.default_branch }}
- name: Run 'helm template' validation
run: |
cd helm/nessie
for f in values.yaml ci/*.yaml; do
echo "::group::helm template $f"
helm template --debug --namespace nessie-ns --values $f .
echo "::endgroup::"
done
- name: Run Helm unit tests
run: |
helm plugin install https://github.com/helm-unittest/helm-unittest.git || true
helm unittest helm/nessie
- name: Run chart-testing (lint)
run: ct lint --debug --charts ./helm/nessie
- name: Show pods
run: kubectl get pods -A
- name: Install secrets
run: |
kubectl create namespace nessie-ns
kubectl apply --namespace nessie-ns $(find helm/nessie/ci/fixtures -name "*.yaml" -exec echo -n "-f {} " \;)
- name: Run chart-testing (install)
run: |
echo "Using image: ${DOCKER_IMAGE}"
echo " tag: ${DOCKER_VERSION}"
ct install \
--namespace nessie-ns \
--helm-extra-set-args "--set=image.repository=${DOCKER_IMAGE} --set=image.tag=${DOCKER_VERSION}" \
--debug --charts ./helm/nessie
nesqueit:
name: CI NesQuEIT
runs-on: ubuntu-24.04
timeout-minutes: 60
needs:
- determine-jobs
# Only run NesQuEIT tests for PRs, if requested. This job can easily run for 30+ minutes.
if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.nesqueit == 'true'
env:
NESSIE_DIR: included-builds/nessie
NESSIE_PATCH_REPOSITORY: ''
NESSIE_PATCH_BRANCH: ''
NESQUEIT_REPOSITORY: projectnessie/query-engine-integration-tests
NESQUEIT_BRANCH: main
ICEBERG_DIR: included-builds/iceberg
ICEBERG_MAIN_REPOSITORY: apache/iceberg
ICEBERG_MAIN_BRANCH: main
ICEBERG_PATCH_REPOSITORY: snazy/iceberg
ICEBERG_PATCH_BRANCH: iceberg-nesqueit
SPARK_LOCAL_IP: localhost
steps:
- name: Prepare Git
run: |
git config --global user.email "integrations-testing@projectnessie.org"
git config --global user.name "Integrations Testing [Bot]"
- name: Checkout NeQuEIT repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
repository: ${{env.NESQUEIT_REPOSITORY}}
ref: ${{env.NESQUEIT_BRANCH}}
- name: Free disk space
uses: ./.github/actions/free-disk-space
- name: Setup runner
uses: ./.github/actions/setup-runner
with:
more-memory: 'true'
- name: Checkout and patch Nessie PR
uses: ./.github/actions/patch-git
with:
name: Nessie
local-dir: ${{env.NESSIE_DIR}}
main-repository: ${{ env.GITHUB_REPOSITORY }}
patch-repository: ${{env.NESSIE_PATCH_REPOSITORY}}
patch-branch: ${{env.NESSIE_PATCH_BRANCH}}
work-branch: nessie-integration-patched
- name: Checkout and patch Iceberg
uses: ./.github/actions/patch-git
with:
name: Nessie
local-dir: ${{env.ICEBERG_DIR}}
main-repository: ${{env.ICEBERG_MAIN_REPOSITORY}}
main-branch: ${{env.ICEBERG_MAIN_BRANCH}}
patch-repository: ${{env.ICEBERG_PATCH_REPOSITORY}}
patch-branch: ${{env.ICEBERG_PATCH_BRANCH}}
work-branch: iceberg-integration-patched
- name: Setup JDK
uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
with:
distribution: 'temurin'
# Java 17 or 21 required for Nessie build, Java 17 required for Iceberg build, Java 11 required for Flink & Presto
java-version: |
11
17
21
- name: Setup Gradle
uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 # v4
with:
cache-read-only: true
validate-wrappers: false
- name: Iceberg Nessie test
run: ./gradlew :iceberg:iceberg-nessie:test --scan
- name: Nessie Spark 3.3 / 2.12 Extensions test
run: ./gradlew :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:intTest --scan
- name: Nessie Spark 3.4 / 2.13 Extensions test
run: ./gradlew :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.13:intTest --scan
- name: Nessie Spark 3.5 / 2.13 Extensions test
run: ./gradlew :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:intTest --scan
#- name: Publish Nessie + Iceberg to local Maven repo
# run: ./gradlew publishLocal --scan
#
#- name: Gather locally published versions
# run: |
# NESSIE_VERSION="$(cat included-builds/nessie/version.txt)"
# ICEBERG_VERSION="$(cat included-builds/iceberg/build/iceberg-build.properties | grep '^git.build.version=' | cut -d= -f2)"
# echo "NESSIE_VERSION=${NESSIE_VERSION}" >> ${GITHUB_ENV}
# echo "ICEBERG_VERSION=${ICEBERG_VERSION}" >> ${GITHUB_ENV}
# cat <<! >> $GITHUB_STEP_SUMMARY
# ## Published versions
# | Published Nessie version | Published Iceberg version |
# | ------------------------ | ------------------------- |
# | ${NESSIE_VERSION} | ${ICEBERG_VERSION} |
# !
- name: Tools & Integrations tests
run: ./gradlew intTest --scan
site:
name: CI Website
runs-on: ubuntu-24.04
timeout-minutes: 20
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Setup Python
uses: ./.github/actions/dev-tool-python
with:
python-version: '3.11'
- name: Generate Static Site
run: make build
working-directory: ./site
- name: Check code block indentation
run: python3 check_code_indent.py
working-directory: ./site
- name: Deploy Static Site to GitHub
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'projectnessie/nessie'
uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4
with:
external_repository: projectnessie/projectnessie.github.io
publish_branch: main
deploy_key: ${{ secrets.NESSIE_SITE_DEPLOY_KEY }}
publish_dir: ./site/site
cname: projectnessie.org
store-cache:
# Store the Gradle cache to GH cache as soon as all relevant Nessie/Gradle jobs have finished.
name: CI Store Cache
runs-on: ubuntu-24.04
timeout-minutes: 30
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
needs:
# Only include jobs that use Nessie's Gradle cache, especially excluding NesQuEIT, which
# is a "very special" citizen and also not run for "main" CI, which does
# not add anything to the Gradle cache that's not already produced by other jobs.
- code-checks
- test
- test-quarkus
- int-test
- int-test-stores
- int-test-integrations
- int-test-quarkus-server
- int-test-quarkus-tool
- int-test-quarkus-events
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup runner
uses: ./.github/actions/setup-runner
- name: Setup Java, Gradle
uses: ./.github/actions/dev-tool-java
- name: Collect partial Gradle build caches
uses: ./.github/actions/ci-incr-build-cache-prepare
with:
cache-read-only: false
- name: Trigger Gradle home cleanup
run: ./gradlew --no-daemon :showVersion
# Note: the "Post Gradle invocation" archives the updated build cache.