From ffdee6f32dd0883dc6a7e4617a1e4eaa18949a8c Mon Sep 17 00:00:00 2001 From: YAN Wenkun Date: Tue, 22 Oct 2024 18:46:59 +0800 Subject: [PATCH] 'rocm' stop auto-building GitHub: "No space left on device" --- .github/workflows/build-rocm.yml | 18 ++++----- rocm/Dockerfile | 65 +++++++++++++++----------------- rocm/README.adoc | 35 ++++++++++------- rocm/README.zh.adoc | 24 +++++++++--- rocm/runner-scripts/download.sh | 10 ++--- 5 files changed, 84 insertions(+), 68 deletions(-) diff --git a/.github/workflows/build-rocm.yml b/.github/workflows/build-rocm.yml index e0903f1..288e20d 100644 --- a/.github/workflows/build-rocm.yml +++ b/.github/workflows/build-rocm.yml @@ -2,15 +2,15 @@ name: Publish 'rocm' to Docker Hub on: workflow_dispatch: # Can be manually triggered - schedule: # Runs on every Monday, 07:00 UTC - - cron: '0 7 * * 1' - push: - branches: [ main ] - paths-ignore: - - '**' - - '!rocm/Dockerfile' - - '!rocm/runner-scripts/**' - - '!.github/workflows/build-rocm.yml' + # schedule: # Runs on every Monday, 07:00 UTC + # - cron: '0 7 * * 1' + # push: + # branches: [ main ] + # paths-ignore: + # - '**' + # - '!rocm/Dockerfile' + # - '!rocm/runner-scripts/**' + # - '!.github/workflows/build-rocm.yml' jobs: diff --git a/rocm/Dockerfile b/rocm/Dockerfile index 8a1c69a..ed09ad3 100644 --- a/rocm/Dockerfile +++ b/rocm/Dockerfile @@ -32,14 +32,13 @@ RUN --mount=type=cache,target=/var/cache/zypp \ RUN --mount=type=cache,target=/root/.cache/pip \ pip install --break-system-packages \ - --upgrade pip wheel setuptools \ - && df -h + --upgrade pip wheel setuptools # Install PyTorch && ONNX from AMD repo # https://rocm.docs.amd.com/projects/radeon/en/latest/docs/install/native_linux/install-pytorch.html # https://rocm.docs.amd.com/projects/radeon/en/latest/docs/install/native_linux/install-onnx.html # Using torchaudio CPU for compatibility -# The extra index of PyTorch nightly has no use here, just a fail-safe. +# The extra index of PyTorch has no use here, just a fail-safe. RUN --mount=type=cache,target=/root/.cache/pip \ pip install --break-system-packages \ https://repo.radeon.com/rocm/manylinux/rocm-rel-6.2.3/torch-2.3.0%2Brocm6.2.3-cp310-cp310-linux_x86_64.whl \ @@ -49,11 +48,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \ https://repo.radeon.com/rocm/manylinux/rocm-rel-6.2/jaxlib-0.4.23%2Brocm620-cp310-cp310-manylinux2014_x86_64.whl \ https://repo.radeon.com/rocm/manylinux/rocm-rel-6.2/rocpydecode-1.0.0.0-py3-none-manylinux_2_28_x86_64.whl \ https://download.pytorch.org/whl/cpu/torchaudio-2.3.0%2Bcpu-cp310-cp310-linux_x86_64.whl \ - --extra-index-url https://download.pytorch.org/whl/nightly/rocm6.2 \ - && df -h + --extra-index-url https://download.pytorch.org/whl/rocm6.2 # Dependencies for frequently-used -# Due to GitHub CI's "out of disk space" error, reducing the list. RUN --mount=type=cache,target=/root/.cache/pip \ pip install --break-system-packages \ -r https://raw.githubusercontent.com/comfyanonymous/ComfyUI/master/requirements.txt \ @@ -61,37 +58,35 @@ RUN --mount=type=cache,target=/root/.cache/pip \ -r https://raw.githubusercontent.com/cubiq/ComfyUI_essentials/main/requirements.txt \ -r https://raw.githubusercontent.com/Fannovel16/comfyui_controlnet_aux/main/requirements.txt \ -r https://raw.githubusercontent.com/jags111/efficiency-nodes-comfyui/main/requirements.txt \ - --extra-index-url https://download.pytorch.org/whl/nightly/rocm6.2 + --extra-index-url https://download.pytorch.org/whl/rocm6.2 -# When you build your own image, you can uncomment the code blocks below. - -# # Dependencies for more, with few hand-pick: -# # 'compel lark' for smZNodes -# # 'torchdiffeq' for DepthFM -# # 'fairscale' for APISR -# RUN --mount=type=cache,target=/root/.cache/pip \ -# pip install --break-system-packages \ -# -r https://raw.githubusercontent.com/crystian/ComfyUI-Crystools/main/requirements.txt \ -# -r https://raw.githubusercontent.com/cubiq/ComfyUI_FaceAnalysis/main/requirements.txt \ -# -r https://raw.githubusercontent.com/cubiq/ComfyUI_InstantID/main/requirements.txt \ -# -r https://raw.githubusercontent.com/Fannovel16/ComfyUI-Frame-Interpolation/main/requirements-no-cupy.txt \ -# -r https://raw.githubusercontent.com/FizzleDorf/ComfyUI_FizzNodes/main/requirements.txt \ -# -r https://raw.githubusercontent.com/kijai/ComfyUI-KJNodes/main/requirements.txt \ -# -r https://raw.githubusercontent.com/ltdrdata/ComfyUI-Impact-Pack/Main/requirements.txt \ -# -r https://raw.githubusercontent.com/ltdrdata/ComfyUI-Impact-Subpack/main/requirements.txt \ -# -r https://raw.githubusercontent.com/ltdrdata/ComfyUI-Inspire-Pack/main/requirements.txt \ -# -r https://raw.githubusercontent.com/melMass/comfy_mtb/main/requirements.txt \ -# -r https://raw.githubusercontent.com/storyicon/comfyui_segment_anything/main/requirements.txt \ -# -r https://raw.githubusercontent.com/ZHO-ZHO-ZHO/ComfyUI-InstantID/main/requirements.txt \ -# compel lark torchdiffeq fairscale \ -# python-ffmpeg \ -# --extra-index-url https://download.pytorch.org/whl/nightly/rocm6.2 +# Dependencies for more, with few hand-pick: +# 'compel lark' for smZNodes +# 'torchdiffeq' for DepthFM +# 'fairscale' for APISR +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --break-system-packages \ + -r https://raw.githubusercontent.com/crystian/ComfyUI-Crystools/main/requirements.txt \ + -r https://raw.githubusercontent.com/cubiq/ComfyUI_FaceAnalysis/main/requirements.txt \ + -r https://raw.githubusercontent.com/cubiq/ComfyUI_InstantID/main/requirements.txt \ + -r https://raw.githubusercontent.com/Fannovel16/ComfyUI-Frame-Interpolation/main/requirements-no-cupy.txt \ + -r https://raw.githubusercontent.com/FizzleDorf/ComfyUI_FizzNodes/main/requirements.txt \ + -r https://raw.githubusercontent.com/kijai/ComfyUI-KJNodes/main/requirements.txt \ + -r https://raw.githubusercontent.com/ltdrdata/ComfyUI-Impact-Pack/Main/requirements.txt \ + -r https://raw.githubusercontent.com/ltdrdata/ComfyUI-Impact-Subpack/main/requirements.txt \ + -r https://raw.githubusercontent.com/ltdrdata/ComfyUI-Inspire-Pack/main/requirements.txt \ + -r https://raw.githubusercontent.com/melMass/comfy_mtb/main/requirements.txt \ + -r https://raw.githubusercontent.com/storyicon/comfyui_segment_anything/main/requirements.txt \ + -r https://raw.githubusercontent.com/ZHO-ZHO-ZHO/ComfyUI-InstantID/main/requirements.txt \ + compel lark torchdiffeq fairscale \ + python-ffmpeg \ + --extra-index-url https://download.pytorch.org/whl/rocm6.2 -# # Fix MediaPipe's broken dep (protobuf<4). -# RUN --mount=type=cache,target=/root/.cache/pip \ -# pip install --break-system-packages \ -# mediapipe \ -# && pip list +# Fix MediaPipe's broken dep (protobuf<4). +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --break-system-packages \ + mediapipe \ + && pip list RUN df -h \ && du -ah /root \ diff --git a/rocm/README.adoc b/rocm/README.adoc index 3d99784..5fcc03f 100644 --- a/rocm/README.adoc +++ b/rocm/README.adoc @@ -6,14 +6,11 @@ image:https://github.com/YanWenKun/ComfyUI-Docker/actions/workflows/build-rocm.y https://hub.docker.com/r/yanwk/comfyui-boot/tags?name=rocm[View on ] -## Note - not all-in-one image +## Note: Image building is required -Due to GitHub CI error "No space left on device", -I have to shorten the list for installing Python dependencies and the list for downloading Custom Nodes. - -If you build this image locally, -you can uncomment the commented lines in `Dockerfile` and `download.sh`, -to build an all-in-one image. +This Docker image is too big to build on GitHub Actions (will throw error "No space left on device"). +So before running, a building step (basically downloading) is needed. +The commands below contain the steps. ## Prepare @@ -21,11 +18,17 @@ to build an all-in-one image. https://rocm.docs.amd.com/projects/radeon/en/latest/docs/install/native_linux/install-radeon.html[Radeon software for Linux with ROCm] is installed on your Linux host. -## Run +## Build & Run -.Run with Docker +.With Docker [source,sh] ---- +git clone https://github.com/YanWenKun/ComfyUI-Docker.git + +cd ComfyUI-Docker/rocm + +docker build . -t yanwk/comfyui-boot:rocm + mkdir -p storage docker run -it --rm \ @@ -40,9 +43,15 @@ docker run -it --rm \ yanwk/comfyui-boot:rocm ---- -.Run with Podman +.With Podman [source,sh] ---- +git clone https://github.com/YanWenKun/ComfyUI-Docker.git + +cd ComfyUI-Docker/rocm + +podman build . -t yanwk/comfyui-boot:rocm + mkdir -p storage podman run -it --rm \ @@ -54,7 +63,7 @@ podman run -it --rm \ -p 8188:8188 \ -v "$(pwd)"/storage:/root \ -e CLI_ARGS="" \ - docker.io/yanwk/comfyui-boot:rocm + yanwk/comfyui-boot:rocm ---- Once the app is loaded, visit http://localhost:8188/ @@ -62,7 +71,7 @@ Once the app is loaded, visit http://localhost:8188/ [[hint]] ## If you want to dive in... -(Just notes. Nothing to do with this Docker image) +(Just side notes. Nothing to do with this Docker image) ROCm has a PyTorch image: @@ -100,7 +109,7 @@ It's big, but if you find it hard to run the container, it may be helpful. As it ## Additional notes for Windows users -(Just notes. Nothing to do with this Docker image) +(Just side notes. Nothing to do with this Docker image) WSL2 supports ROCm and DirectML. diff --git a/rocm/README.zh.adoc b/rocm/README.zh.adoc index 21faacf..4c368ac 100644 --- a/rocm/README.zh.adoc +++ b/rocm/README.zh.adoc @@ -4,22 +4,28 @@ image:https://github.com/YanWenKun/ComfyUI-Docker/actions/workflows/build-rocm.y https://hub.docker.com/r/yanwk/comfyui-boot/tags?name=rocm[在 上查看] -## 备注 +## 注意：需要自行构建镜像 -由于 GitHub CI 总是报错“存储空间不足”，不得已裁剪镜像尺寸，少安装一些 Python 依赖项，因此也少下载若干扩展。 +由于 ROCm PyTorch 太大， GitHub Actions 总是报错“存储空间不足”，导致无法利用 GitHub 自动构建发布镜像，所以不再提供成品镜像。用户运行前需要先构建镜像（基本就是下载，加上少量编译）。 -如果你在本地构建本镜像，可以在 `Dockerfile` 与 `download.sh` 中将对应代码取消注释，以获得一个更加“全能”的镜像。 +如将来 AMD 缩减 PyTorch 包大小，会将再次启用自动构建。 ## 准备工作 * 确保 Linux 宿主机上正确安装了 https://rocm.docs.amd.com/projects/radeon/en/latest/docs/install/native_linux/install-radeon.html[Radeon software for Linux with ROCm]。 -## 运行 +## 构建并运行 .使用 Docker [source,sh] ---- +git clone https://github.com/YanWenKun/ComfyUI-Docker.git + +cd ComfyUI-Docker/rocm + +docker build . -t yanwk/comfyui-boot:rocm + mkdir -p storage docker run -it --rm \ @@ -37,6 +43,12 @@ docker run -it --rm \ .使用 Podman [source,sh] ---- +git clone https://github.com/YanWenKun/ComfyUI-Docker.git + +cd ComfyUI-Docker/rocm + +podman build . -t yanwk/comfyui-boot:rocm + mkdir -p storage podman run -it --rm \ @@ -48,7 +60,7 @@ podman run -it --rm \ -p 8188:8188 \ -v "$(pwd)"/storage:/root \ -e CLI_ARGS="" \ - docker.io/yanwk/comfyui-boot:rocm + yanwk/comfyui-boot:rocm ---- 启动完成后，访问 http://localhost:8188/ @@ -90,7 +102,7 @@ pip install -r ComfyUI/requirements.txt python ComfyUI/main.py --listen --port 8188 ---- -这个镜像很大，大到我没办法用免费的 GitHub CI 来构建。但如果你运行遇到困难，可以尝试用这个镜像手动安装运行 ComfyUI。 +这个镜像很大，但如果你运行遇到困难，可以尝试用这个镜像手动安装运行 ComfyUI。 它已经安装好了最重要的 PyTorch，你只需要再安装少量 Python 包即可运行 ComfyUI。 ## 备注： Windows 用户 diff --git a/rocm/runner-scripts/download.sh b/rocm/runner-scripts/download.sh index 779937c..0d1d7c1 100644 --- a/rocm/runner-scripts/download.sh +++ b/rocm/runner-scripts/download.sh @@ -39,8 +39,8 @@ cd /root/ComfyUI/custom_nodes # Workspace clone_or_pull https://github.com/AIGODLIKE/AIGODLIKE-ComfyUI-Translation.git -# clone_or_pull https://github.com/crystian/ComfyUI-Crystools.git -# clone_or_pull https://github.com/crystian/ComfyUI-Crystools-save.git +clone_or_pull https://github.com/crystian/ComfyUI-Crystools.git +clone_or_pull https://github.com/crystian/ComfyUI-Crystools-save.git # General clone_or_pull https://github.com/chrisgoringe/cg-use-everywhere.git @@ -52,12 +52,12 @@ clone_or_pull https://github.com/pythongosssss/ComfyUI-Custom-Scripts.git clone_or_pull https://github.com/cubiq/ComfyUI_IPAdapter_plus.git clone_or_pull https://github.com/Fannovel16/comfyui_controlnet_aux.git clone_or_pull https://github.com/Kosinkadink/ComfyUI-Advanced-ControlNet.git -# clone_or_pull https://github.com/ltdrdata/ComfyUI-Impact-Pack.git -# clone_or_pull https://github.com/ltdrdata/ComfyUI-Inspire-Pack.git +clone_or_pull https://github.com/ltdrdata/ComfyUI-Impact-Pack.git +clone_or_pull https://github.com/ltdrdata/ComfyUI-Inspire-Pack.git clone_or_pull https://github.com/twri/sdxl_prompt_styler.git # Video -# clone_or_pull https://github.com/Fannovel16/ComfyUI-Frame-Interpolation.git +clone_or_pull https://github.com/Fannovel16/ComfyUI-Frame-Interpolation.git clone_or_pull https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved.git clone_or_pull https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git