diff --git a/.devcontainer/build-rapids.sh b/.devcontainer/build-rapids.sh index 48b0530a..1a76cb7f 100755 --- a/.devcontainer/build-rapids.sh +++ b/.devcontainer/build-rapids.sh @@ -47,7 +47,7 @@ build_rapids() { ( echo "building cuDF"; clean-cudf; - build-cudf -DBUILD_BENCHMARKS=ON --verbose + build-cudf -DBUILD_BENCHMARKS=ON -DNVBench_ENABLE_CUPTI=OFF --verbose sccache -s; ) 2>&1 | maybe_write_build_log cudf; diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index 7c2332c5..5aa82c5f 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -5,7 +5,7 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.04-cpp-cuda11.8-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.02-cpp-cuda11.8-mambaforge-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index ae4b06a3..94212b75 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,7 +5,7 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.04-cpp-llvm16-cuda11.8-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.02-cpp-llvm16-cuda11.8-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.0-conda/devcontainer.json index 09cfc9ce..40c6b889 100644 --- a/.devcontainer/cuda12.0-conda/devcontainer.json +++ b/.devcontainer/cuda12.0-conda/devcontainer.json @@ -5,7 +5,7 @@ "args": { "CUDA": "12.0", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.04-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.02-cpp-mambaforge-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json index 9dd89cb4..186498d1 100644 --- a/.devcontainer/cuda12.0-pip/devcontainer.json +++ b/.devcontainer/cuda12.0-pip/devcontainer.json @@ -5,7 +5,7 @@ "args": { "CUDA": "12.0", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.04-cpp-llvm16-cuda12.0-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.02-cpp-llvm16-cuda12.0-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, diff --git a/.github/actions/build-and-test-feature/action.yml b/.github/actions/build-and-test-feature/action.yml index d844418c..4c02bcd8 100644 --- a/.github/actions/build-and-test-feature/action.yml +++ b/.github/actions/build-and-test-feature/action.yml @@ -8,8 +8,6 @@ inputs: vault_host: {type: string, defaut: '', required: false} rw_sccache_bucket: {type: string, defaut: '', required: false} rw_sccache_region: {type: string, defaut: '', required: false} - ro_sccache_bucket: {type: string, defaut: '', required: false} - ro_sccache_region: {type: string, defaut: '', required: false} runs: using: composite @@ -34,5 +32,3 @@ runs: vault_host: "${{ inputs.vault_host }}" rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}" rw_sccache_region: "${{ inputs.rw_sccache_region }}" - ro_sccache_bucket: "${{ inputs.ro_sccache_bucket }}" - ro_sccache_region: "${{ inputs.ro_sccache_region }}" diff --git a/.github/actions/setup-runner-env/action.yml b/.github/actions/setup-runner-env/action.yml index ae701557..b5248736 100644 --- a/.github/actions/setup-runner-env/action.yml +++ b/.github/actions/setup-runner-env/action.yml @@ -5,30 +5,27 @@ description: Setup self-hosted runner environment runs: using: composite steps: - - name: Dump env - shell: bash -eo pipefail {0} - run: echo "${{ toJSON(env) }}" - - if: env.RUNNER_ENVIRONMENT != 'self-hosted' + - if: env.RUNNER_ENVIRONMENT == 'github-hosted' name: Free up disk space uses: ./.github/actions/free-disk-space with: tool_cache: "${{ runner.tool_cache }}" - - if: env.RUNNER_ENVIRONMENT == 'self-hosted' + - if: env.RUNNER_ENVIRONMENT != 'github-hosted' name: Setup self-hosted runner environment shell: bash -eo pipefail {0} run: | echo "HOME=${{ runner.workspace }}" >> $GITHUB_ENV; echo "TMPDIR=${{ runner.temp }}" >> $GITHUB_ENV; - - if: env.RUNNER_ENVIRONMENT == 'self-hosted' + - if: env.RUNNER_ENVIRONMENT != 'github-hosted' name: Setup Node.js uses: actions/setup-node@v3 with: node-version: '16' - - if: env.RUNNER_ENVIRONMENT != 'self-hosted' + - if: env.RUNNER_ENVIRONMENT == 'github-hosted' name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -36,15 +33,7 @@ runs: shell: bash run: docker context create builder - - if: env.RUNNER_ENVIRONMENT != 'self-hosted' - name: Setup docker buildx on github-hosted runners - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: --debug - endpoint: builder - - - if: env.RUNNER_ENVIRONMENT == 'self-hosted' - name: Setup docker buildx on self-hosted runners + - name: Setup docker buildx uses: docker/setup-buildx-action@v2 with: buildkitd-flags: --debug --config /etc/buildkit/buildkitd.toml diff --git a/.github/workflows/build-and-test-feature.yml b/.github/workflows/build-and-test-feature.yml index e94ae089..33868d1f 100644 --- a/.github/workflows/build-and-test-feature.yml +++ b/.github/workflows/build-and-test-feature.yml @@ -39,5 +39,3 @@ jobs: vault_host: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }}" rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}" rw_sccache_region: "${{ vars.AWS_REGION }}" - ro_sccache_bucket: rapids-sccache-east - ro_sccache_region: "${{ vars.AWS_REGION }}" diff --git a/USAGE.md b/USAGE.md index f1379831..bc202075 100644 --- a/USAGE.md +++ b/USAGE.md @@ -19,11 +19,11 @@ The features that comprise the image are noted in the image tags. If no version The pre-built images can be used as the `"image"`, or as the base of a Dockerfile in `"build"`, in `devcontainer.json`: -
devcontainer.json using pre-built image
{
  "image": "rapidsai/devcontainers:24.04-cpp-llvm16-cuda12.0-nvhpc23.5-ubuntu22.04",
  "hostRequirements": { "gpu": true },
  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind"
}
+
devcontainer.json using pre-built image
{
  "image": "rapidsai/devcontainers:24.02-cpp-llvm16-cuda12.0-nvhpc23.5-ubuntu22.04",
  "hostRequirements": { "gpu": true },
  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind"
}
You can also build a custom devcontainer by composing individual features: -
devcontainer.json using individual features
{
  "image": "ubuntu:22.04",
  "features": {
    "ghcr.io/rapidsai/devcontainers/features/cmake:24.04": {},
    "ghcr.io/rapidsai/devcontainers/features/ninja:24.04": {},
    "ghcr.io/rapidsai/devcontainers/features/sccache:24.04": {
      "version": "0.5.4"
    }
  },
  "overrideFeatureInstallOrder": [
    "ghcr.io/rapidsai/devcontainers/features/cmake",
    "ghcr.io/rapidsai/devcontainers/features/ninja",
    "ghcr.io/rapidsai/devcontainers/features/sccache"
  ],
  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind"
}
+
devcontainer.json using individual features
{
  "image": "ubuntu:22.04",
  "features": {
    "ghcr.io/rapidsai/devcontainers/features/cmake:24.02": {},
    "ghcr.io/rapidsai/devcontainers/features/ninja:24.02": {},
    "ghcr.io/rapidsai/devcontainers/features/sccache:24.02": {
      "version": "0.5.4"
    }
  },
  "overrideFeatureInstallOrder": [
    "ghcr.io/rapidsai/devcontainers/features/cmake",
    "ghcr.io/rapidsai/devcontainers/features/ninja",
    "ghcr.io/rapidsai/devcontainers/features/sccache"
  ],
  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind"
}
> **NOTE:** Feature updates published since your most recent image build will invalidate your docker image layer cache, meaning it can take the [devcontainers CLI](https://github.com/devcontainers/cli) longer to initialize containers composed from individual features. diff --git a/features/src/cccl-dev/devcontainer-feature.json b/features/src/cccl-dev/devcontainer-feature.json index b75458b4..58df0dd8 100644 --- a/features/src/cccl-dev/devcontainer-feature.json +++ b/features/src/cccl-dev/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA CCCL development utilities", "id": "cccl-dev", - "version": "24.4.0", + "version": "24.2.2", "description": "A feature to install NVIDIA CCCL development utilities", "options": { "litVersion": { diff --git a/features/src/cmake/devcontainer-feature.json b/features/src/cmake/devcontainer-feature.json index d3159ddd..66943ae5 100644 --- a/features/src/cmake/devcontainer-feature.json +++ b/features/src/cmake/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "CMake", "id": "cmake", - "version": "24.4.0", + "version": "24.2.0", "description": "A feature to install CMake", "options": { "version": { diff --git a/features/src/cuda/devcontainer-feature.json b/features/src/cuda/devcontainer-feature.json index e4c4bead..fd686841 100644 --- a/features/src/cuda/devcontainer-feature.json +++ b/features/src/cuda/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "CUDA Toolkit", "id": "cuda", - "version": "24.4.0", + "version": "24.2.2", "description": "A feature to install the NVIDIA CUDA Toolkit", "options": { "version": { @@ -23,94 +23,94 @@ "description": "Version of the CUDA Toolkit to install." }, "installCompilers": { - "type": "boolean", - "default": true, - "description": "Install NVIDIA CUDA Compiler (nvcc)" + "type": "boolean", + "default": true, + "description": "Install NVIDIA CUDA Compiler (nvcc)" }, "installProfilers": { - "type": "boolean", - "default": true, - "description": "Install NVIDIA NSight Systems Profiler (nsys)" + "type": "boolean", + "default": true, + "description": "Install NVIDIA NSight Systems Profiler (nsys)" }, "installCTKLibraries": { - "type": "boolean", - "default": true, - "description": "Shortcut to install all CUDA Toolkit Libraries" + "type": "boolean", + "default": true, + "description": "Shortcut to install all CUDA Toolkit Libraries" }, "installDevPackages": { - "type": "boolean", - "default": true, - "description": "Whether to install the CUDA dev packages (static libraries + headers)" + "type": "boolean", + "default": true, + "description": "Whether to install the CUDA dev packages (static libraries + headers)" }, "installcuDNN": { - "type": "boolean", - "default": false, - "description": "Install CUDA Deep Neural Network Library (cuDNN)" + "type": "boolean", + "default": false, + "description": "Install CUDA Deep Neural Network Library (cuDNN)" }, "installcuTensor": { - "type": "boolean", - "default": false, - "description": "Install CUDA Tensor Linear Algebra Library (cuTensor)" + "type": "boolean", + "default": false, + "description": "Install CUDA Tensor Linear Algebra Library (cuTensor)" }, "installNCCL": { - "type": "boolean", - "default": true, - "description": "Install NVIDIA Collective Communications Library (NCCL)" + "type": "boolean", + "default": true, + "description": "Install NVIDIA Collective Communications Library (NCCL)" }, "installCUDARuntime": { - "type": "boolean", - "default": true, - "description": "Install CUDA Runtime Library (cudart)" + "type": "boolean", + "default": true, + "description": "Install CUDA Runtime Library (cudart)" }, "installNVRTC": { - "type": "boolean", - "default": false, - "description": "Install NVIDIA Runtime Compilation Library (NVRTC)" + "type": "boolean", + "default": false, + "description": "Install NVIDIA Runtime Compilation Library (NVRTC)" }, "installOpenCL": { - "type": "boolean", - "default": false, - "description": "Install NVIDIA CUDA OpenCL Library" + "type": "boolean", + "default": false, + "description": "Install NVIDIA CUDA OpenCL Library" }, "installcuBLAS": { - "type": "boolean", - "default": false, - "description": "Install CUDA Basic Linear Algebra Library (cuBLAS)" + "type": "boolean", + "default": false, + "description": "Install CUDA Basic Linear Algebra Library (cuBLAS)" }, "installcuSPARSE": { - "type": "boolean", - "default": false, - "description": "Install CUDA Basic Linear Algebra for Sparse Matrices Library (cuSPARSE)" + "type": "boolean", + "default": false, + "description": "Install CUDA Basic Linear Algebra for Sparse Matrices Library (cuSPARSE)" }, "installcuFFT": { - "type": "boolean", - "default": false, - "description": "Install CUDA Fast Fourier Transform Library (cuFFT)" + "type": "boolean", + "default": false, + "description": "Install CUDA Fast Fourier Transform Library (cuFFT)" }, "installcuFile": { - "type": "boolean", - "default": false, - "description": "Install CUDA GPUDirect Storage API Library (cuFile)" + "type": "boolean", + "default": false, + "description": "Install CUDA GPUDirect Storage API Library (cuFile)" }, "installcuRAND": { - "type": "boolean", - "default": false, - "description": "Install CUDA Random Number Generation Library (cuRAND)" + "type": "boolean", + "default": false, + "description": "Install CUDA Random Number Generation Library (cuRAND)" }, "installcuSOLVER": { - "type": "boolean", - "default": false, - "description": "Install CUDA Direct Linear Solvers Library (cuSOLVER)" + "type": "boolean", + "default": false, + "description": "Install CUDA Direct Linear Solvers Library (cuSOLVER)" }, "installNPP": { - "type": "boolean", - "default": false, - "description": "Install NVIDIA Performance Primitives (NPP)" + "type": "boolean", + "default": false, + "description": "Install NVIDIA Performance Primitives (NPP)" }, "installnvJPEG": { - "type": "boolean", - "default": false, - "description": "Install NVIDIA JPEG decoder, encoder, and transcoder library (nvJPEG)" + "type": "boolean", + "default": false, + "description": "Install NVIDIA JPEG decoder, encoder, and transcoder library (nvJPEG)" }, "pruneStaticLibs": { "type": "boolean", diff --git a/features/src/gcc/devcontainer-feature.json b/features/src/gcc/devcontainer-feature.json index 0aa1deb5..8b609468 100644 --- a/features/src/gcc/devcontainer-feature.json +++ b/features/src/gcc/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "GCC", "id": "gcc", - "version": "24.4.0", + "version": "24.2.1", "description": "A feature to install gcc", "options": { "version": { diff --git a/features/src/gitlab-cli/devcontainer-feature.json b/features/src/gitlab-cli/devcontainer-feature.json index 9562d59f..0642d4ec 100644 --- a/features/src/gitlab-cli/devcontainer-feature.json +++ b/features/src/gitlab-cli/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "gitlab-cli", - "version": "24.4.0", + "version": "24.2.0", "name": "GitLab CLI", "documentationURL": "https://github.com/rapidsai/devcontainers/features/tree/main/src/gitlab-cli", "description": "Installs the GitLab CLI. Auto-detects latest version and installs needed dependencies.", diff --git a/features/src/llvm/devcontainer-feature.json b/features/src/llvm/devcontainer-feature.json index 45fad8f4..62dd3b96 100644 --- a/features/src/llvm/devcontainer-feature.json +++ b/features/src/llvm/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "LLVM compilers and tools", "id": "llvm", - "version": "24.4.0", + "version": "24.2.3", "description": "A feature to install LLVM compilers and tools", "options": { "version": { diff --git a/features/src/mambaforge/devcontainer-feature.json b/features/src/mambaforge/devcontainer-feature.json index 77013cb0..4bcff969 100644 --- a/features/src/mambaforge/devcontainer-feature.json +++ b/features/src/mambaforge/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "Mambaforge", "id": "mambaforge", - "version": "24.4.2", + "version": "24.2.3", "description": "A feature to install mambaforge", "options": { "version": { diff --git a/features/src/ninja/devcontainer-feature.json b/features/src/ninja/devcontainer-feature.json index 0df8f2f0..a3b7ca1f 100644 --- a/features/src/ninja/devcontainer-feature.json +++ b/features/src/ninja/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "Ninja build", "id": "ninja", - "version": "24.4.0", + "version": "24.2.0", "description": "A feature to install ninja-build", "options": { "version": { diff --git a/features/src/nvhpc/devcontainer-feature.json b/features/src/nvhpc/devcontainer-feature.json index 7019155e..ec292fc2 100644 --- a/features/src/nvhpc/devcontainer-feature.json +++ b/features/src/nvhpc/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVHPC SDK", "id": "nvhpc", - "version": "24.4.0", + "version": "24.2.2", "description": "A feature to install the NVHPC SDK", "options": { "version": { diff --git a/features/src/oneapi/devcontainer-feature.json b/features/src/oneapi/devcontainer-feature.json index 8230fad3..3ee97e75 100644 --- a/features/src/oneapi/devcontainer-feature.json +++ b/features/src/oneapi/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "Intel oneapi toolchain", "id": "oneapi", - "version": "24.4.0", + "version": "24.2.0", "description": "A feature to install the Intel oneapi toolchain", "options": { "version": { diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index de316729..cbe64799 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "24.4.0", + "version": "24.2.9", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/install.sh b/features/src/rapids-build-utils/install.sh index 49b9fc0d..f6f82704 100644 --- a/features/src/rapids-build-utils/install.sh +++ b/features/src/rapids-build-utils/install.sh @@ -11,20 +11,21 @@ check_packages bc jq sudo wget gettext-base bash-completion ca-certificates; # Install yq if not installed if ! type yq >/dev/null 2>&1; then - YQ_VERSION=latest; - find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq; - YQ_BINARY="yq"; YQ_BINARY+="_$(uname -s | tr '[:upper:]' '[:lower:]')"; YQ_BINARY+="_${TARGETARCH:-$(dpkg --print-architecture | awk -F'-' '{print $NF}')}"; - wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" \ - | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; + YQ_VERSION=latest; + find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq; + while ! wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; do + echo "(!) YQ version ${YQ_VERSION} failed to download. Attempting to fall back one version to retry..."; + find_prev_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq; + done fi # Install the rapids dependency file generator and conda-merge if type python >/dev/null 2>&1; then - python -m pip install rapids-dependency-file-generator conda-merge toml; + python -m pip install 'rapids-dependency-file-generator<1.14' conda-merge toml; fi # Install RAPIDS build utility scripts to /opt/ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index 3d57e10f..0e2dc4e2 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -1,6 +1,6 @@ x-git-defaults: &git_defaults host: github - tag: branch-24.04 + tag: branch-24.02 upstream: rapidsai repos: @@ -24,7 +24,7 @@ repos: git: repo: ucxx <<: *git_defaults - tag: branch-0.37 + tag: branch-0.36 cpp: - name: ucxx sub_dir: "cpp" @@ -158,9 +158,6 @@ repos: - name: cugraph sub_dir: cpp depends: [rmm, raft, cugraph-ops] - - name: cugraph_etl - sub_dir: cpp/libcugraph_etl - depends: [cugraph, cudf] python: - name: pylibcugraph sub_dir: python/pylibcugraph @@ -174,9 +171,6 @@ repos: - name: cugraph-dgl sub_dir: python/cugraph-dgl depends: [cugraph] - - name: cugraph-equivariant - sub_dir: python/cugraph-equivariant - depends: [cugraph] - name: cugraph_pyg sub_dir: python/cugraph-pyg depends: [cugraph] diff --git a/features/src/rust/devcontainer-feature.json b/features/src/rust/devcontainer-feature.json index 80f27fca..03429f32 100644 --- a/features/src/rust/devcontainer-feature.json +++ b/features/src/rust/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "rust", - "version": "24.4.0", + "version": "24.2.0", "name": "Rust", "documentationURL": "https://github.com/rapidsai/devcontainers/features/tree/main/src/rust", "description": "Installs Rust, common Rust utilities, and their required dependencies", diff --git a/features/src/sccache/devcontainer-feature.json b/features/src/sccache/devcontainer-feature.json index cddd2ba8..49146b5f 100644 --- a/features/src/sccache/devcontainer-feature.json +++ b/features/src/sccache/devcontainer-feature.json @@ -1,14 +1,14 @@ { "name": "sccache", "id": "sccache", - "version": "24.4.0", + "version": "24.2.2", "description": "A feature to install sccache", "options": { "version": { "type": "string", "proposals": [ "latest", - "0.7.4", + "0.7.7", "0.4.0", "0.3.1", "0.3.0", diff --git a/features/src/ucx/devcontainer-feature.json b/features/src/ucx/devcontainer-feature.json index 6915c84c..6f21df04 100644 --- a/features/src/ucx/devcontainer-feature.json +++ b/features/src/ucx/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "UCX", "id": "ucx", - "version": "24.4.0", + "version": "24.2.0", "description": "A feature to install UCX", "options": { "version": { diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index d319c04f..49a7fc6b 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "devcontainer-utils", "id": "utils", - "version": "24.4.4", + "version": "24.2.8", "description": "A feature to install RAPIDS devcontainer utility scripts", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/utils/install.sh b/features/src/utils/install.sh index 4681cd6a..53cbfcc9 100644 --- a/features/src/utils/install.sh +++ b/features/src/utils/install.sh @@ -32,15 +32,16 @@ fi # Install yq if not installed if ! type yq >/dev/null 2>&1; then - YQ_VERSION=latest; - find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq; - YQ_BINARY="yq"; YQ_BINARY+="_$(uname -s | tr '[:upper:]' '[:lower:]')"; YQ_BINARY+="_${TARGETARCH:-$(dpkg --print-architecture | awk -F'-' '{print $NF}')}"; - wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" \ - | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; + YQ_VERSION=latest; + find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq; + while ! wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; do + echo "(!) YQ version ${YQ_VERSION} failed to download. Attempting to fall back one version to retry..."; + find_prev_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq; + done fi # Remove built-in anacron configs diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/persist.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/persist.sh index bbd78d51..d9886982 100755 --- a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/persist.sh +++ b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/persist.sh @@ -38,7 +38,6 @@ store_s3_creds() { local region=; local no_bucket=; local no_region=; - local no_credentials=; local aws_access_key_id=; local aws_session_token=; local aws_secret_access_key=; @@ -50,7 +49,6 @@ store_s3_creds() { region | no_bucket | no_region | - no_credentials | aws_access_key_id | aws_session_token | aws_secret_access_key | @@ -64,7 +62,6 @@ store_s3_creds() { reset_envvar "AWS_ACCESS_KEY_ID"; reset_envvar "AWS_SESSION_TOKEN"; reset_envvar "AWS_SECRET_ACCESS_KEY"; - reset_envvar "SCCACHE_S3_NO_CREDENTIALS"; mkdir -p ~/.aws; rm -f ~/.aws/{config,credentials}; @@ -92,41 +89,37 @@ ________EOF fi if test -f ~/.aws/config; then - cat <<________EOF > ~/.aws/config + cat <<________EOF > ~/.aws/config2 && mv ~/.aws/config{2,} [default] $(cat ~/.aws/config) ________EOF fi - if ! grep -qE "^$" <<< "${no_credentials:-}"; then - export_envvar "SCCACHE_S3_NO_CREDENTIALS" "1"; - else - if ! grep -qE "^$" <<< "${aws_access_key_id:-}"; then - cat <<____________EOF >> ~/.aws/credentials + if ! grep -qE "^$" <<< "${aws_access_key_id:-}"; then + cat <<________EOF >> ~/.aws/credentials aws_access_key_id=${aws_access_key_id} -____________EOF - fi +________EOF + fi - if ! grep -qE "^$" <<< "${aws_session_token:-}"; then - cat <<____________EOF >> ~/.aws/credentials + if ! grep -qE "^$" <<< "${aws_session_token:-}"; then + cat <<________EOF >> ~/.aws/credentials aws_session_token=${aws_session_token} -____________EOF - fi +________EOF + fi - if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then - cat <<____________EOF >> ~/.aws/credentials + if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then + cat <<________EOF >> ~/.aws/credentials aws_secret_access_key=${aws_secret_access_key} -____________EOF - fi +________EOF + fi - if test -f ~/.aws/credentials; then - cat <<____________EOF > ~/.aws/credentials + if test -f ~/.aws/credentials; then + cat <<________EOF > ~/.aws/credentials2 && mv ~/.aws/credentials{2,} [default] $(cat ~/.aws/credentials) -____________EOF - chmod 0600 ~/.aws/credentials; - fi +________EOF + chmod 0600 ~/.aws/credentials; fi } diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/propagate.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/propagate.sh index 941905dd..e0d00085 100755 --- a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/propagate.sh +++ b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/propagate.sh @@ -16,7 +16,7 @@ ensure_s3_creds_have_propagated() { while true; do - if SCCACHE_NO_DAEMON=1 sccache --show-stats >/dev/null 2>&1; then + if sccache --start-server >/dev/null 2>&1; then if [ "${num_restarts}" -gt "0" ]; then echo "Success!"; fi exit 0; fi diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/test.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/test.sh index 85a1f46f..dc2fe7ad 100755 --- a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/test.sh +++ b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/test.sh @@ -17,47 +17,25 @@ test_aws_creds() { fi fi - local bucket="${SCCACHE_BUCKET:-"$(grep 'bucket=' ~/.aws/config 2>/dev/null | sed 's/bucket=//' || echo)"}"; + local bucket="${SCCACHE_BUCKET:-"$(sed -n 's/bucket=//p' ~/.aws/config 2>/dev/null)"}"; if [ -z "${bucket:-}" ]; then exit 1; fi - local region="${SCCACHE_REGION:-"${AWS_DEFAULT_REGION:-"$(grep 'region=' ~/.aws/config 2>/dev/null | sed 's/region=//' || echo)"}"}"; - local aws_access_key_id="${AWS_ACCESS_KEY_ID:-"$(grep 'aws_access_key_id=' ~/.aws/credentials 2>/dev/null | sed 's/aws_access_key_id=//' || echo)"}"; - local aws_session_token="${AWS_SESSION_TOKEN:-"$(grep 'aws_session_token=' ~/.aws/credentials 2>/dev/null | sed 's/aws_session_token=//' || echo)"}"; - local aws_secret_access_key="${AWS_SECRET_ACCESS_KEY:-"$(grep 'aws_secret_access_key=' ~/.aws/credentials 2>/dev/null | sed 's/aws_secret_access_key=//' || echo)"}"; + local region="${SCCACHE_REGION:-"${AWS_DEFAULT_REGION:-"$(sed -n 's/region=//p' ~/.aws/config 2>/dev/null)"}"}"; + local aws_access_key_id="${AWS_ACCESS_KEY_ID:-"$(sed -n 's/aws_access_key_id=//p' ~/.aws/credentials 2>/dev/null)"}"; + local aws_session_token="${AWS_SESSION_TOKEN:-"$(sed -n 's/aws_session_token=//p' ~/.aws/credentials 2>/dev/null)"}"; + local aws_secret_access_key="${AWS_SECRET_ACCESS_KEY:-"$(sed -n 's/aws_secret_access_key=//p' ~/.aws/credentials 2>/dev/null)"}"; if test -n "$(pgrep sccache || echo)"; then sccache --stop-server >/dev/null 2>&1 || true; fi - export SCCACHE_NO_DAEMON="1"; - export SCCACHE_BUCKET="${bucket:-}"; - export SCCACHE_REGION="${region:-}"; - export AWS_ACCESS_KEY_ID="${aws_access_key_id:-}"; - export AWS_SESSION_TOKEN="${aws_session_token:-}"; - export AWS_SECRET_ACCESS_KEY="${aws_secret_access_key:-}"; - - if ! sccache --show-stats 2>&1 | grep -qE 'Cache location \s+ s3'; then - - export SCCACHE_S3_NO_CREDENTIALS="1"; - - export AWS_ACCESS_KEY_ID=; - export AWS_SESSION_TOKEN=; - export AWS_SECRET_ACCESS_KEY=; - export -n AWS_ACCESS_KEY_ID; - export -n AWS_SESSION_TOKEN; - export -n AWS_SECRET_ACCESS_KEY; - unset AWS_ACCESS_KEY_ID; - unset AWS_SESSION_TOKEN; - unset AWS_SECRET_ACCESS_KEY; - - if sccache --show-stats 2>&1 | grep -qE 'Cache location \s+ s3'; then - exit 2; - fi - - exit 1; - fi - - exit 0; + SCCACHE_BUCKET="${bucket:-}" \ + SCCACHE_REGION="${region:-}" \ + AWS_ACCESS_KEY_ID="${aws_access_key_id:-}" \ + AWS_SESSION_TOKEN="${aws_session_token:-}" \ + AWS_SECRET_ACCESS_KEY="${aws_secret_access_key:-}" \ + sccache --start-server >/dev/null 2>&1; + sccache --show-stats | grep -qE 'Cache location \s+ s3'; } if test -n "${devcontainer_utils_debug:-}"; then diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/init.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/init.sh index bcabe313..3592c2f2 100755 --- a/features/src/utils/opt/devcontainer/bin/vault/s3/init.sh +++ b/features/src/utils/opt/devcontainer/bin/vault/s3/init.sh @@ -9,9 +9,9 @@ ____EOF s3_bucket_auth() { cat <<____EOF - --aws_access_key_id='$(grep 'aws_access_key_id=' ~/.aws/credentials 2>/dev/null | sed 's/aws_access_key_id=//' || echo)' - --aws_session_token='$(grep 'aws_session_token=' ~/.aws/credentials 2>/dev/null | sed 's/aws_session_token=//' || echo)' - --aws_secret_access_key='$(grep 'aws_secret_access_key=' ~/.aws/credentials 2>/dev/null | sed 's/aws_secret_access_key=//' || echo)' + --aws_access_key_id='$(sed -n 's/aws_access_key_id=//p' ~/.aws/credentials 2>/dev/null)' + --aws_session_token='$(sed -n 's/aws_session_token=//p' ~/.aws/credentials 2>/dev/null)' + --aws_secret_access_key='$(sed -n 's/aws_secret_access_key=//p' ~/.aws/credentials 2>/dev/null)' ____EOF } @@ -25,7 +25,7 @@ init_vault_s3_creds() { && grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}" ; then if test -n "${VAULT_HOST:-}" ; then # Generate S3 creds if they don't exist (or are expired) - if devcontainer-utils-vault-s3-creds-test 2>&1 >/dev/null\ + if devcontainer-utils-vault-s3-creds-test \ || devcontainer-utils-vault-s3-creds-generate; then # Persist creds in ~/.aws dir devcontainer-utils-vault-s3-creds-persist <<< " @@ -37,34 +37,21 @@ init_vault_s3_creds() { else devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region"; fi + elif devcontainer-utils-vault-s3-creds-test; then + # bucket is read + write with the current credentials + devcontainer-utils-vault-s3-creds-persist <<< " + $(s3_bucket_args) + $(s3_bucket_auth) + "; else - # If credentials have been mounted in, ensure they're used - case $(devcontainer-utils-vault-s3-creds-test; echo $?) in - # bucket is read + write with the current credentials - [0] ) - devcontainer-utils-vault-s3-creds-persist <<< " - $(s3_bucket_args) - $(s3_bucket_auth) - ";; - # bucket is read-only and should be accessed without credentials - [2] ) - devcontainer-utils-vault-s3-creds-persist <<< " - --no_credentials - $(s3_bucket_args) - ";; - # bucket is inaccessible - * ) - devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region";; - esac + # bucket is inaccessible + devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region"; fi - elif devcontainer-utils-vault-s3-creds-propagate; then - # Block until the new temporary AWS S3 credentials propagate - echo -n ""; + elif ! devcontainer-utils-vault-s3-creds-propagate; then + # bucket is inaccessible + devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region"; fi fi - . /etc/profile.d/*-devcontainer-utils.sh; - # start the sccache server - sccache --start-server >/dev/null 2>&1 || true; fi } diff --git a/features/test/utils/scenarios.json b/features/test/utils/scenarios.json index e823e52c..4546ca7c 100644 --- a/features/test/utils/scenarios.json +++ b/features/test/utils/scenarios.json @@ -7,9 +7,7 @@ "vault_host": "${localEnv:vault_host}", "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}", "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}", - "rw_sccache_region": "${localEnv:rw_sccache_region}", - "ro_sccache_bucket": "${localEnv:ro_sccache_bucket}", - "ro_sccache_region": "${localEnv:ro_sccache_region}" + "rw_sccache_region": "${localEnv:rw_sccache_region}" }, "features": { "ghcr.io/devcontainers/features/common-utils:2": { @@ -29,7 +27,7 @@ "cmake": {}, "ninja": {}, "sccache": { - "version": "0.7.4" + "version": "0.7.7" }, "utils": {} }, @@ -53,9 +51,7 @@ "vault_host": "${localEnv:vault_host}", "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}", "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}", - "rw_sccache_region": "${localEnv:rw_sccache_region}", - "ro_sccache_bucket": "${localEnv:ro_sccache_bucket}", - "ro_sccache_region": "${localEnv:ro_sccache_region}" + "rw_sccache_region": "${localEnv:rw_sccache_region}" }, "features": { "ghcr.io/devcontainers/features/common-utils:2": { @@ -75,7 +71,7 @@ "cmake": {}, "ninja": {}, "sccache": { - "version": "0.7.4" + "version": "0.7.7" }, "utils": {} }, @@ -99,9 +95,7 @@ "vault_host": "${localEnv:vault_host}", "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}", "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}", - "rw_sccache_region": "${localEnv:rw_sccache_region}", - "ro_sccache_bucket": "${localEnv:ro_sccache_bucket}", - "ro_sccache_region": "${localEnv:ro_sccache_region}" + "rw_sccache_region": "${localEnv:rw_sccache_region}" }, "features": { "ghcr.io/devcontainers/features/common-utils:2": { @@ -121,7 +115,7 @@ "cmake": {}, "ninja": {}, "sccache": { - "version": "0.7.4" + "version": "0.7.7" }, "utils": {} }, diff --git a/features/test/utils/ubuntu18.04.sh b/features/test/utils/ubuntu18.04.sh index bb4f96c4..d83f684d 100644 --- a/features/test/utils/ubuntu18.04.sh +++ b/features/test/utils/ubuntu18.04.sh @@ -126,19 +126,6 @@ if test -n "${rw_sccache_bucket:-}"; then check "bad creds with SCCACHE_BUCKET and no VAULT_HOST uses local disk cache" bad_creds_with_sccache_bucket_and_no_vault_host_uses_local_disk_cache; fi -if test -n "${ro_sccache_bucket:-}"; then - - readonly_sccache_bucket_uses_s3_cache() { - reset_state; - SCCACHE_BUCKET="${ro_sccache_bucket}" \ - SCCACHE_REGION="${ro_sccache_region}" \ - devcontainer-utils-post-attach-command; - expect_s3_cache_is_used; - } - - check "Readonly SCCACHE_BUCKET uses S3 cache" readonly_sccache_bucket_uses_s3_cache; -fi - if test -n "${gh_token:-}" \ && test -n "${vault_host:-}" \ && test -n "${rw_sccache_bucket:-}"; then @@ -173,9 +160,9 @@ if test -n "${gh_token:-}" \ reset_state; export SCCACHE_BUCKET="${rw_sccache_bucket}"; export SCCACHE_REGION="${rw_sccache_region}"; - export AWS_ACCESS_KEY_ID="$(grep 'aws_access_key_id=' /tmp/.aws/credentials 2>/dev/null | sed 's/aws_access_key_id=//' || echo)"; - export AWS_SESSION_TOKEN="$(grep 'aws_session_token=' /tmp/.aws/credentials 2>/dev/null | sed 's/aws_session_token=//' || echo)"; - export AWS_SECRET_ACCESS_KEY="$(grep 'aws_secret_access_key=' /tmp/.aws/credentials 2>/dev/null | sed 's/aws_secret_access_key=//' || echo)"; + export AWS_ACCESS_KEY_ID="$(sed -n 's/aws_access_key_id=//p' /tmp/.aws/credentials 2>/dev/null)"; + export AWS_SESSION_TOKEN="$(sed -n 's/aws_session_token=//p' /tmp/.aws/credentials 2>/dev/null)"; + export AWS_SECRET_ACCESS_KEY="$(sed -n 's/aws_secret_access_key=//p' /tmp/.aws/credentials 2>/dev/null)"; devcontainer-utils-post-attach-command; expect_s3_cache_is_used; } diff --git a/features/test/utils/ubuntu20.04.sh b/features/test/utils/ubuntu20.04.sh index bb4f96c4..d83f684d 100644 --- a/features/test/utils/ubuntu20.04.sh +++ b/features/test/utils/ubuntu20.04.sh @@ -126,19 +126,6 @@ if test -n "${rw_sccache_bucket:-}"; then check "bad creds with SCCACHE_BUCKET and no VAULT_HOST uses local disk cache" bad_creds_with_sccache_bucket_and_no_vault_host_uses_local_disk_cache; fi -if test -n "${ro_sccache_bucket:-}"; then - - readonly_sccache_bucket_uses_s3_cache() { - reset_state; - SCCACHE_BUCKET="${ro_sccache_bucket}" \ - SCCACHE_REGION="${ro_sccache_region}" \ - devcontainer-utils-post-attach-command; - expect_s3_cache_is_used; - } - - check "Readonly SCCACHE_BUCKET uses S3 cache" readonly_sccache_bucket_uses_s3_cache; -fi - if test -n "${gh_token:-}" \ && test -n "${vault_host:-}" \ && test -n "${rw_sccache_bucket:-}"; then @@ -173,9 +160,9 @@ if test -n "${gh_token:-}" \ reset_state; export SCCACHE_BUCKET="${rw_sccache_bucket}"; export SCCACHE_REGION="${rw_sccache_region}"; - export AWS_ACCESS_KEY_ID="$(grep 'aws_access_key_id=' /tmp/.aws/credentials 2>/dev/null | sed 's/aws_access_key_id=//' || echo)"; - export AWS_SESSION_TOKEN="$(grep 'aws_session_token=' /tmp/.aws/credentials 2>/dev/null | sed 's/aws_session_token=//' || echo)"; - export AWS_SECRET_ACCESS_KEY="$(grep 'aws_secret_access_key=' /tmp/.aws/credentials 2>/dev/null | sed 's/aws_secret_access_key=//' || echo)"; + export AWS_ACCESS_KEY_ID="$(sed -n 's/aws_access_key_id=//p' /tmp/.aws/credentials 2>/dev/null)"; + export AWS_SESSION_TOKEN="$(sed -n 's/aws_session_token=//p' /tmp/.aws/credentials 2>/dev/null)"; + export AWS_SECRET_ACCESS_KEY="$(sed -n 's/aws_secret_access_key=//p' /tmp/.aws/credentials 2>/dev/null)"; devcontainer-utils-post-attach-command; expect_s3_cache_is_used; } diff --git a/features/test/utils/ubuntu22.04.sh b/features/test/utils/ubuntu22.04.sh index bb4f96c4..d83f684d 100644 --- a/features/test/utils/ubuntu22.04.sh +++ b/features/test/utils/ubuntu22.04.sh @@ -126,19 +126,6 @@ if test -n "${rw_sccache_bucket:-}"; then check "bad creds with SCCACHE_BUCKET and no VAULT_HOST uses local disk cache" bad_creds_with_sccache_bucket_and_no_vault_host_uses_local_disk_cache; fi -if test -n "${ro_sccache_bucket:-}"; then - - readonly_sccache_bucket_uses_s3_cache() { - reset_state; - SCCACHE_BUCKET="${ro_sccache_bucket}" \ - SCCACHE_REGION="${ro_sccache_region}" \ - devcontainer-utils-post-attach-command; - expect_s3_cache_is_used; - } - - check "Readonly SCCACHE_BUCKET uses S3 cache" readonly_sccache_bucket_uses_s3_cache; -fi - if test -n "${gh_token:-}" \ && test -n "${vault_host:-}" \ && test -n "${rw_sccache_bucket:-}"; then @@ -173,9 +160,9 @@ if test -n "${gh_token:-}" \ reset_state; export SCCACHE_BUCKET="${rw_sccache_bucket}"; export SCCACHE_REGION="${rw_sccache_region}"; - export AWS_ACCESS_KEY_ID="$(grep 'aws_access_key_id=' /tmp/.aws/credentials 2>/dev/null | sed 's/aws_access_key_id=//' || echo)"; - export AWS_SESSION_TOKEN="$(grep 'aws_session_token=' /tmp/.aws/credentials 2>/dev/null | sed 's/aws_session_token=//' || echo)"; - export AWS_SECRET_ACCESS_KEY="$(grep 'aws_secret_access_key=' /tmp/.aws/credentials 2>/dev/null | sed 's/aws_secret_access_key=//' || echo)"; + export AWS_ACCESS_KEY_ID="$(sed -n 's/aws_access_key_id=//p' /tmp/.aws/credentials 2>/dev/null)"; + export AWS_SESSION_TOKEN="$(sed -n 's/aws_session_token=//p' /tmp/.aws/credentials 2>/dev/null)"; + export AWS_SECRET_ACCESS_KEY="$(sed -n 's/aws_secret_access_key=//p' /tmp/.aws/credentials 2>/dev/null)"; devcontainer-utils-post-attach-command; expect_s3_cache_is_used; } diff --git a/image/.devcontainer/devcontainer.json b/image/.devcontainer/devcontainer.json index 920de53d..63a5217f 100644 --- a/image/.devcontainer/devcontainer.json +++ b/image/.devcontainer/devcontainer.json @@ -27,7 +27,7 @@ "./features/src/cmake": {}, "./features/src/ninja": {}, "./features/src/sccache": { - "version": "0.7.4" + "version": "0.7.7" }, "./features/src/utils": {} },