From 2336d028e2d6e38886e7ce227a98f0496fbec5e4 Mon Sep 17 00:00:00 2001 From: Jinsong Ji Date: Wed, 17 Apr 2024 15:15:37 -0700 Subject: [PATCH] [CI] Factor out igc dev related code (#13441) Per discussion with @aelovikov-intel ,factor out all igc dev related code to new workflows to avoid necessary interaction with other docker images. --- .../workflows/sycl-containers-igc-dev.yaml | 52 +++++++++++++++++++ .github/workflows/sycl-containers.yaml | 8 +-- .../workflows/sycl-update-igc-dev-driver.yml | 35 +++++++++++++ .../ubuntu2204_intel_drivers.Dockerfile | 7 +-- ...buntu2204_intel_drivers_igc_dev.Dockerfile | 24 +++++++++ devops/dependencies-igc-dev.json | 11 ++++ devops/dependencies.json | 7 --- devops/scripts/install_drivers.sh | 9 ++-- devops/scripts/update_drivers.py | 40 +++++++++----- 9 files changed, 157 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/sycl-containers-igc-dev.yaml create mode 100644 .github/workflows/sycl-update-igc-dev-driver.yml create mode 100644 devops/containers/ubuntu2204_intel_drivers_igc_dev.Dockerfile create mode 100644 devops/dependencies-igc-dev.json diff --git a/.github/workflows/sycl-containers-igc-dev.yaml b/.github/workflows/sycl-containers-igc-dev.yaml new file mode 100644 index 0000000000000..8bf8def9d8148 --- /dev/null +++ b/.github/workflows/sycl-containers-igc-dev.yaml @@ -0,0 +1,52 @@ +name: IGC DEV CI Containers +on: + workflow_dispatch: + push: + branches: + - sycl + paths: + - 'devops/actions/build_container/**' + - 'devops/dependencies-igc-dev.json' + - '.github/workflows/sycl-containers-igc-dev.yaml' + pull_request: + paths: + - 'devops/actions/build_container/**' + - 'devops/dependencies-igc-dev.json' + - '.github/workflows/sycl-containers-igc-dev.yaml' + +permissions: read-all + +jobs: + build_and_push_images: + if: github.repository == 'intel/llvm' + name: Build and Push IGC Dev Docker Images + runs-on: ubuntu-22.04 + permissions: + packages: write + strategy: + matrix: + include: + - name: Intel Drivers Ubuntu 22.04 Docker image with dev IGC + dockerfile: ubuntu2204_intel_drivers_igc_dev + imagefile: ubuntu2204_intel_drivers + tag: devigc + build_args: | + "use_latest=false" + "use_igc_dev=true" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 2 + - name: Build and Push Container + uses: ./devops/actions/build_container + with: + push: ${{ github.event_name != 'pull_request' }} + file: ${{ matrix.dockerfile }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + tags: | + ghcr.io/${{ github.repository }}/${{ matrix.imagefile }}:${{ matrix.tag }}-${{ github.sha }} + ghcr.io/${{ github.repository }}/${{ matrix.imagefile }}:${{ matrix.tag }} + build-args: ${{ matrix.build_args }} + diff --git a/.github/workflows/sycl-containers.yaml b/.github/workflows/sycl-containers.yaml index b71030e328831..6228603bc62b6 100644 --- a/.github/workflows/sycl-containers.yaml +++ b/.github/workflows/sycl-containers.yaml @@ -8,6 +8,7 @@ on: branches: - sycl paths: + - 'devops/actions/build_container/**' - 'devops/containers/**' - 'devops/dependencies.json' - 'devops/scripts/install_drivers.sh' @@ -15,6 +16,7 @@ on: - '.github/workflows/sycl-containers.yaml' pull_request: paths: + - 'devops/actions/build_container/**' - 'devops/containers/**' - 'devops/dependencies.json' - 'devops/scripts/install_drivers.sh' @@ -45,12 +47,6 @@ jobs: file: ubuntu2204_intel_drivers tag: latest build_args: "use_latest=false" - - name: Intel Drivers Ubuntu 22.04 Docker image with dev IGC - file: ubuntu2204_intel_drivers - tag: devigc - build_args: | - "use_latest=false" - "use_igc_dev=true" - name: Intel Drivers (unstable) Ubuntu 22.04 Docker image file: ubuntu2204_intel_drivers tag: unstable diff --git a/.github/workflows/sycl-update-igc-dev-driver.yml b/.github/workflows/sycl-update-igc-dev-driver.yml new file mode 100644 index 0000000000000..8a323a1e05221 --- /dev/null +++ b/.github/workflows/sycl-update-igc-dev-driver.yml @@ -0,0 +1,35 @@ +name: Update IGC dev driver + +on: + schedule: + - cron: '0 3 * * 1,4' + workflow_dispatch: + +permissions: read-all + +jobs: + update_driver_linux: + permissions: + contents: write # for Git to git push + runs-on: ubuntu-20.04 + if: github.repository == 'intel/llvm' + steps: + - uses: actions/checkout@v4 + - name: Update dependencies file + run: | + version="$(python3 devops/scripts/update_drivers.py linux --igc-dev-only)" + echo 'NEW_DRIVER_VERSION='$version >> $GITHUB_ENV + - name: Create Pull Request + env: + BRANCH: ci/update_gpu_driver-linux-${{ env.NEW_DRIVER_VERSION }} + GITHUB_TOKEN: ${{ secrets.LLVM_MAIN_SYNC_BBSYCL_TOKEN }} + run: | + cd $GITHUB_WORKSPACE + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + git checkout -B $BRANCH + git add -u + git commit -m "[GHA] Uplift Linux IGC Dev RT version to $NEW_DRIVER_VERSION" || exit 0 # exit if commit is empty + git show + git push https://$GITHUB_TOKEN@github.com/${{ github.repository }} ${BRANCH} + gh pr create --head $BRANCH --title "[GHA] Uplift Linux IGC Dev RT version to $NEW_DRIVER_VERSION" --body "Scheduled igc dev drivers uplift" diff --git a/devops/containers/ubuntu2204_intel_drivers.Dockerfile b/devops/containers/ubuntu2204_intel_drivers.Dockerfile index 4ef9009da4744..fb018f195a48f 100644 --- a/devops/containers/ubuntu2204_intel_drivers.Dockerfile +++ b/devops/containers/ubuntu2204_intel_drivers.Dockerfile @@ -6,10 +6,8 @@ FROM $base_image:$base_tag ENV DEBIAN_FRONTEND=noninteractive ARG use_latest=true -ARG use_igc_dev=false -RUN apt update && apt install -yqq wget \ - && if [ "$use_igc_dev" = "true" ]; then apt-get install -yqq libllvm14; fi +RUN apt update && apt install -yqq wget COPY scripts/get_release.py / COPY scripts/install_drivers.sh / @@ -22,9 +20,6 @@ RUN --mount=type=secret,id=github_token \ install_driver_opt=" --use-latest"; \ else \ install_driver_opt=" dependencies.json"; \ - if [ "$use_igc_dev" = "true" ]; then \ - install_driver_opt="$install_driver_opt --use-dev-igc"; \ - fi; \ fi && \ GITHUB_TOKEN=$(cat /run/secrets/github_token) /install_drivers.sh $install_driver_opt --all diff --git a/devops/containers/ubuntu2204_intel_drivers_igc_dev.Dockerfile b/devops/containers/ubuntu2204_intel_drivers_igc_dev.Dockerfile new file mode 100644 index 0000000000000..3a90699e23030 --- /dev/null +++ b/devops/containers/ubuntu2204_intel_drivers_igc_dev.Dockerfile @@ -0,0 +1,24 @@ +ARG base_tag=latest +ARG base_image=ghcr.io/intel/llvm/ubuntu2204_base + +FROM $base_image:$base_tag + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update && apt install -yqq libllvm14 + +COPY scripts/get_release.py / +COPY scripts/install_drivers.sh / +COPY dependencies.json / +COPY dependencies-igc-dev.json / + +RUN mkdir /runtimes +ENV INSTALL_LOCATION=/runtimes +RUN --mount=type=secret,id=github_token \ + install_driver_opt="dependencies.json dependencies-igc-dev.json --use-dev-igc"; \ + GITHUB_TOKEN=$(cat /run/secrets/github_token) /install_drivers.sh $install_driver_opt --all + +COPY scripts/drivers_entrypoint.sh /drivers_entrypoint.sh + +ENTRYPOINT ["/bin/bash", "/drivers_entrypoint.sh"] + diff --git a/devops/dependencies-igc-dev.json b/devops/dependencies-igc-dev.json new file mode 100644 index 0000000000000..e2d2d07e21dbc --- /dev/null +++ b/devops/dependencies-igc-dev.json @@ -0,0 +1,11 @@ +{ + "linux": { + "igc_dev": { + "github_tag": "igc-dev-498324a", + "version": "498324a", + "updated_at": "2024-04-17T13:44:27Z", + "url": "https://api.github.com/repos/intel/intel-graphics-compiler/actions/artifacts/1422168296/zip", + "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu" + } + } +} diff --git a/devops/dependencies.json b/devops/dependencies.json index d330bf3b7449d..cb5b843ba03d2 100644 --- a/devops/dependencies.json +++ b/devops/dependencies.json @@ -12,13 +12,6 @@ "url": "https://github.com/intel/intel-graphics-compiler/releases/tag/igc-1.0.15985.7", "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu" }, - "igc_dev": { - "github_tag": "igc-dev-db4de5f", - "version": "db4de5f", - "updated_at": "2024-04-12T11:06:26Z", - "url": "https://api.github.com/repos/intel/intel-graphics-compiler/actions/artifacts/1409219375/zip", - "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu" - }, "cm": { "github_tag": "cmclang-1.0.144", "version": "1.0.144", diff --git a/devops/scripts/install_drivers.sh b/devops/scripts/install_drivers.sh index f27fb2b6d3979..805ffe1fcfff0 100755 --- a/devops/scripts/install_drivers.sh +++ b/devops/scripts/install_drivers.sh @@ -8,14 +8,17 @@ if [ -f "$1" ]; then CONFIG_FILE=$1 CR_TAG=$(jq -r '.linux.compute_runtime.github_tag' $CONFIG_FILE) IGC_TAG=$(jq -r '.linux.igc.github_tag' $CONFIG_FILE) - IGC_DEV_TAG=$(jq -r '.linux.igc_dev.github_tag' $CONFIG_FILE) - IGC_DEV_VER=$(jq -r '.linux.igc_dev.version' $CONFIG_FILE) - IGC_DEV_URL=$(jq -r '.linux.igc_dev.url' $CONFIG_FILE) CM_TAG=$(jq -r '.linux.cm.github_tag' $CONFIG_FILE) L0_TAG=$(jq -r '.linux.level_zero.github_tag' $CONFIG_FILE) TBB_TAG=$(jq -r '.linux.tbb.github_tag' $CONFIG_FILE) FPGA_TAG=$(jq -r '.linux.fpgaemu.github_tag' $CONFIG_FILE) CPU_TAG=$(jq -r '.linux.oclcpu.github_tag' $CONFIG_FILE) + if [[ "$*" == *"--use-dev-igc"* ]]; then + CONFIG_FILE_IGC_DEV=$2 + IGC_DEV_TAG=$(jq -r '.linux.igc_dev.github_tag' $CONFIG_FILE_IGC_DEV) + IGC_DEV_VER=$(jq -r '.linux.igc_dev.version' $CONFIG_FILE_IGC_DEV) + IGC_DEV_URL=$(jq -r '.linux.igc_dev.url' $CONFIG_FILE_IGC_DEV) + fi elif [[ "$*" == *"--use-latest"* ]]; then CR_TAG=latest IGC_TAG=latest diff --git a/devops/scripts/update_drivers.py b/devops/scripts/update_drivers.py index cd042c4ca5b9b..4c3cbb791c851 100644 --- a/devops/scripts/update_drivers.py +++ b/devops/scripts/update_drivers.py @@ -3,6 +3,7 @@ import sys import os import re +import argparse def get_latest_release(repo): @@ -28,7 +29,19 @@ def get_artifacts_download_url(repo, name): return json.loads(artifacts)["artifacts"][0]["archive_download_url"] -def uplift_linux_igfx_driver(config, platform_tag): +def uplift_linux_igfx_driver(config, platform_tag, igc_dev_only): + + if igc_dev_only: + igc_dev = get_latest_workflow_runs("intel/intel-graphics-compiler", "build-IGC") + igcdevver = igc_dev["head_sha"][:7] + config[platform_tag]["igc_dev"]["github_tag"] = "igc-dev-" + igcdevver + config[platform_tag]["igc_dev"]["version"] = igcdevver + config[platform_tag]["igc_dev"]["updated_at"] = igc_dev["updated_at"] + config[platform_tag]["igc_dev"]["url"] = get_artifacts_download_url( + "intel/intel-graphics-compiler", "IGC_Ubuntu22.04_llvm14_clang-" + igcdevver + ) + return config + compute_runtime = get_latest_release('intel/compute-runtime') config[platform_tag]['compute_runtime']['github_tag'] = compute_runtime['tag_name'] @@ -46,15 +59,6 @@ def uplift_linux_igfx_driver(config, platform_tag): config[platform_tag]['igc']['url'] = 'https://github.com/intel/intel-graphics-compiler/releases/tag/igc-' + ver break - igc_dev = get_latest_workflow_runs("intel/intel-graphics-compiler", "build-IGC") - igcdevver = igc_dev["head_sha"][:7] - config[platform_tag]["igc_dev"]["github_tag"] = "igc-dev-" + igcdevver - config[platform_tag]["igc_dev"]["version"] = igcdevver - config[platform_tag]["igc_dev"]["updated_at"] = igc_dev["updated_at"] - config[platform_tag]["igc_dev"]["url"] = get_artifacts_download_url( - "intel/intel-graphics-compiler", "IGC_Ubuntu22.04_llvm14_clang-" + igcdevver - ) - cm = get_latest_release('intel/cm-compiler') config[platform_tag]['cm']['github_tag'] = cm['tag_name'] config[platform_tag]['cm']['version'] = cm['tag_name'].replace('cmclang-', '') @@ -68,22 +72,30 @@ def uplift_linux_igfx_driver(config, platform_tag): return config -def main(platform_tag): +def main(platform_tag, igc_dev_only): script = os.path.dirname(os.path.realpath(__file__)) config_name = os.path.join(script, '..', 'dependencies.json') + if igc_dev_only: + config_name = os.path.join(script, "..", "dependencies-igc-dev.json") config = {} with open(config_name, "r") as f: config = json.loads(f.read()) - config = uplift_linux_igfx_driver(config, platform_tag) + config = uplift_linux_igfx_driver(config, platform_tag, igc_dev_only) with open(config_name, "w") as f: json.dump(config, f, indent=2) f.write('\n') + if igc_dev_only: + return config[platform_tag]["igc_dev"]["github_tag"] + return config[platform_tag]['compute_runtime']['version'] if __name__ == '__main__': - platform_tag = sys.argv[1] if len(sys.argv) > 1 else "ERROR_PLATFORM" - sys.stdout.write(main(platform_tag) + '\n') + parser = argparse.ArgumentParser() + parser.add_argument("platform_tag") + parser.add_argument("--igc-dev-only", action="store_true") + args = parser.parse_args() + sys.stdout.write(main(args.platform_tag, args.igc_dev_only) + "\n")