From 6767f414df18601244af8e9937e162eb75011fde Mon Sep 17 00:00:00 2001 From: TheSpad Date: Sat, 9 Mar 2024 14:44:41 +0000 Subject: [PATCH 1/4] Build Playwright node components from source --- Dockerfile | 27 +++++++++++++++++++++++++-- Dockerfile.aarch64 | 27 +++++++++++++++++++++++++-- README.md | 17 ++++++++++------- readme-vars.yml | 23 ++++------------------- 4 files changed, 64 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9101148..3bb689c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,12 +19,15 @@ RUN \ libc-dev \ libffi-dev \ libxslt-dev \ + npm \ openssl-dev \ python3-dev \ + zip \ zlib-dev && \ apk add --update --no-cache \ libjpeg \ libxslt \ + nodejs \ poppler-utils \ python3 && \ echo "**** install changedetection.io ****" && \ @@ -45,16 +48,36 @@ RUN \ pip \ wheel && \ pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.19/ -r /app/changedetection/requirements.txt && \ - PLAYWRIGHT_RELEASE=$(curl -sX GET "https://api.github.com/repos/microsoft/playwright-python/releases/latest" \ + echo "**** install playwright ****" && \ + PLAYWRIGHT_RELEASE=$(curl -sX GET "https://api.github.com/repos/microsoft/playwright/releases/latest" \ | awk '/tag_name/{print $4;exit}' FS='[""]'); \ git clone --depth 1 --branch "${PLAYWRIGHT_RELEASE}" https://github.com/microsoft/playwright-python /tmp/playwright-python && \ cd /tmp/playwright-python && \ pip install -U --no-cache-dir . && \ + git clone --depth 1 --branch "${PLAYWRIGHT_RELEASE}" https://github.com/microsoft/playwright /tmp/playwright && \ + cd /tmp/playwright && \ + npm ci && \ + npm run build && \ + # Don't build for other platforms + sed -i '/-darwin-x64/d' ./utils/build/build-playwright-driver.sh && \ + sed -i '/-darwin-arm64/d' ./utils/build/build-playwright-driver.sh && \ + sed -i '/-linux-arm64/d' ./utils/build/build-playwright-driver.sh && \ + sed -i '/-win-x64/d' ./utils/build/build-playwright-driver.sh && \ + # Don't download a static node binary, use the OS install + sed -i '/curl ${NODE_URL}/d' ./utils/build/build-playwright-driver.sh && \ + sed -i '/elif \[\[ "${ARCHIVE}" == "tar.gz" \]\]; then/,/else/d' ./utils/build/build-playwright-driver.sh && \ + sed -i '/cp .\/output\/${NODE_DIR}\/LICENSE .\/output\/playwright-${SUFFIX}\//d' ./utils/build/build-playwright-driver.sh && \ + sed -i 's/"..\/..\/${NODE_DIR}\/${NPM_PATH}"/\/usr\/lib\/node_modules\/npm\/bin\/npm-cli.js/' ./utils/build/build-playwright-driver.sh && \ + ./utils/build/build-playwright-driver.sh && \ + rm -rf /lsiopy/lib/python3.11/site-packages/playwright/driver/* && \ + cp -R ./utils/build/output/playwright-linux/* /lsiopy/lib/python3.11/site-packages/playwright/driver && \ + ln -s /usr/bin/node /lsiopy/lib/python3.11/site-packages/playwright/driver/node && \ apk del --purge \ build-dependencies && \ rm -rf \ /tmp/* \ - $HOME/.cache + $HOME/.cache \ + $HOME/.npm COPY root/ / diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 index b51bf48..1e51708 100644 --- a/Dockerfile.aarch64 +++ b/Dockerfile.aarch64 @@ -19,12 +19,15 @@ RUN \ libc-dev \ libffi-dev \ libxslt-dev \ + npm \ openssl-dev \ python3-dev \ + zip \ zlib-dev && \ apk add --update --no-cache \ libjpeg \ libxslt \ + nodejs \ poppler-utils \ python3 && \ echo "**** install changedetection.io ****" && \ @@ -45,16 +48,36 @@ RUN \ pip \ wheel && \ pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.19/ -r /app/changedetection/requirements.txt && \ - PLAYWRIGHT_RELEASE=$(curl -sX GET "https://api.github.com/repos/microsoft/playwright-python/releases/latest" \ + echo "**** install playwright ****" && \ + PLAYWRIGHT_RELEASE=$(curl -sX GET "https://api.github.com/repos/microsoft/playwright/releases/latest" \ | awk '/tag_name/{print $4;exit}' FS='[""]'); \ git clone --depth 1 --branch "${PLAYWRIGHT_RELEASE}" https://github.com/microsoft/playwright-python /tmp/playwright-python && \ cd /tmp/playwright-python && \ pip install -U --no-cache-dir . && \ + git clone --depth 1 --branch "${PLAYWRIGHT_RELEASE}" https://github.com/microsoft/playwright /tmp/playwright && \ + cd /tmp/playwright && \ + npm ci && \ + npm run build && \ + # Don't build for other platforms + sed -i '/-darwin-x64/d' ./utils/build/build-playwright-driver.sh && \ + sed -i '/-darwin-arm64/d' ./utils/build/build-playwright-driver.sh && \ + sed -i '/-linux-x64/d' ./utils/build/build-playwright-driver.sh && \ + sed -i '/-win-x64/d' ./utils/build/build-playwright-driver.sh && \ + # Don't download a static node binary, use the OS install + sed -i '/curl ${NODE_URL}/d' ./utils/build/build-playwright-driver.sh && \ + sed -i '/elif \[\[ "${ARCHIVE}" == "tar.gz" \]\]; then/,/else/d' ./utils/build/build-playwright-driver.sh && \ + sed -i '/cp .\/output\/${NODE_DIR}\/LICENSE .\/output\/playwright-${SUFFIX}\//d' ./utils/build/build-playwright-driver.sh && \ + sed -i 's/"..\/..\/${NODE_DIR}\/${NPM_PATH}"/\/usr\/lib\/node_modules\/npm\/bin\/npm-cli.js/' ./utils/build/build-playwright-driver.sh && \ + ./utils/build/build-playwright-driver.sh && \ + rm -rf /lsiopy/lib/python3.11/site-packages/playwright/driver/* && \ + cp -R ./utils/build/output/playwright-linux/* /lsiopy/lib/python3.11/site-packages/playwright/driver && \ + ln -s /usr/bin/node /lsiopy/lib/python3.11/site-packages/playwright/driver/node && \ apk del --purge \ build-dependencies && \ rm -rf \ /tmp/* \ - $HOME/.cache + $HOME/.cache \ + $HOME/.npm COPY root/ / diff --git a/README.md b/README.md index 7cf7274..0e6aac1 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ The architectures supported by this image are: Webui is accessible at http://SERVERIP:PORT -Please note that as of version 0.45.15 this image contains the [Playwright content fetcher](https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher#docker-compose-based). +Please note that as of version 0.45.15 this image contains the Playwright content fetcher. For more info read [the wiki](https://github.com/dgtlmoon/changedetection.io/wiki). @@ -73,18 +73,18 @@ To help you get started creating a container from this image you can either use ```yaml --- -version: "2.1" services: - changedetection: + changedetection.io: image: lscr.io/linuxserver/changedetection.io:latest - container_name: changedetection + container_name: changedetection.io environment: - PUID=1000 - PGID=1000 - - TZ=Europe/London + - TZ=Etc/UTC - BASE_URL= #optional + - PLAYWRIGHT_DRIVER_URL= #optional volumes: - - /path/to/appdata/config:/config + - /path/to/changedetection.io/config:/config ports: - 5000:5000 restart: unless-stopped @@ -99,6 +99,7 @@ docker run -d \ -e PGID=1000 \ -e TZ=Etc/UTC \ -e BASE_URL= `#optional` \ + -e PLAYWRIGHT_DRIVER_URL= `#optional` \ -p 5000:5000 \ -v /path/to/changedetection.io/config:/config \ --restart unless-stopped \ @@ -116,6 +117,7 @@ Containers are configured using parameters passed at runtime (such as those abov | `-e PGID=1000` | for GroupID - see below for explanation | | `-e TZ=Etc/UTC` | specify a timezone to use, see this [list](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List). | | `-e BASE_URL=` | Specify the full URL (including protocol) when running behind a reverse proxy | +| `-e PLAYWRIGHT_DRIVER_URL=` | Specify the full URL to your chrome driver instance. See the [wiki](https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher) for details. | | `-v /config` | Persistent config files | ## Environment variables from files (Docker secrets) @@ -279,7 +281,8 @@ Once registered you can define the dockerfile to use with `-f Dockerfile.aarch64 ## Versions -* **08.03.24:** - Build Playwright from source, add libjpeg. +* **09.03.24:** - Build Playwright from source because Microsoft's build and packaging process is awful. +* **08.03.24:** - Build Playwright-python from source, add libjpeg. * **23.12.23:** - Rebase to Alpine 3.19. * **10.08.23:** - Add poppler-utils for pdf conversion tools. * **11.06.23:** - Rebase to Alpine 3.18, deprecate armhf. diff --git a/readme-vars.yml b/readme-vars.yml index 1273bc2..55595ef 100644 --- a/readme-vars.yml +++ b/readme-vars.yml @@ -33,36 +33,21 @@ param_env_vars: opt_param_usage_include_env: true opt_param_env_vars: - { env_var: "BASE_URL", env_value: "", desc: "Specify the full URL (including protocol) when running behind a reverse proxy" } + - { env_var: "PLAYWRIGHT_DRIVER_URL", env_value: "", desc: "Specify the full URL to your chrome driver instance. See the [wiki](https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher) for details." } -custom_compose: | - --- - version: "2.1" - services: - changedetection: - image: lscr.io/linuxserver/changedetection.io:latest - container_name: changedetection - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/London - - BASE_URL= #optional - volumes: - - /path/to/appdata/config:/config - ports: - - 5000:5000 - restart: unless-stopped # application setup block app_setup_block_enabled: true app_setup_block: | Webui is accessible at http://SERVERIP:PORT - Please note that as of version 0.45.15 this image contains the [Playwright content fetcher](https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher#docker-compose-based). + Please note that as of version 0.45.16 this image contains the Playwright content fetcher. For more info read [the wiki](https://github.com/dgtlmoon/changedetection.io/wiki). # changelog changelogs: - - { date: "08.03.24:", desc: "Build Playwright from source, add libjpeg."} + - { date: "09.03.24:", desc: "Build Playwright from source because Microsoft's build and packaging process is awful."} + - { date: "08.03.24:", desc: "Build Playwright-python from source, add libjpeg."} - { date: "23.12.23:", desc: "Rebase to Alpine 3.19."} - { date: "10.08.23:", desc: "Add poppler-utils for pdf conversion tools." } - { date: "11.06.23:", desc: "Rebase to Alpine 3.18, deprecate armhf." } From 2e917f1c2d89e19ad90c2b601127026219f03a7d Mon Sep 17 00:00:00 2001 From: TheSpad Date: Sat, 9 Mar 2024 16:07:13 +0000 Subject: [PATCH 2/4] Invert node/python release logic --- Dockerfile | 7 +++---- Dockerfile.aarch64 | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3bb689c..4c631c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,13 +48,12 @@ RUN \ pip \ wheel && \ pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.19/ -r /app/changedetection/requirements.txt && \ - echo "**** install playwright ****" && \ - PLAYWRIGHT_RELEASE=$(curl -sX GET "https://api.github.com/repos/microsoft/playwright/releases/latest" \ + PLAYWRIGHT_PY_RELEASE=$(curl -sX GET "https://api.github.com/repos/microsoft/playwright-python/releases/latest" \ | awk '/tag_name/{print $4;exit}' FS='[""]'); \ - git clone --depth 1 --branch "${PLAYWRIGHT_RELEASE}" https://github.com/microsoft/playwright-python /tmp/playwright-python && \ + git clone --depth 1 --branch "${PLAYWRIGHT_PY_RELEASE}" https://github.com/microsoft/playwright-python /tmp/playwright-python && \ cd /tmp/playwright-python && \ pip install -U --no-cache-dir . && \ - git clone --depth 1 --branch "${PLAYWRIGHT_RELEASE}" https://github.com/microsoft/playwright /tmp/playwright && \ + git clone --depth 1 --branch "${PLAYWRIGHT_PY_RELEASE}" https://github.com/microsoft/playwright /tmp/playwright && \ cd /tmp/playwright && \ npm ci && \ npm run build && \ diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 index 1e51708..ab4fd17 100644 --- a/Dockerfile.aarch64 +++ b/Dockerfile.aarch64 @@ -49,12 +49,12 @@ RUN \ wheel && \ pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.19/ -r /app/changedetection/requirements.txt && \ echo "**** install playwright ****" && \ - PLAYWRIGHT_RELEASE=$(curl -sX GET "https://api.github.com/repos/microsoft/playwright/releases/latest" \ + PLAYWRIGHT_PY_RELEASE=$(curl -sX GET "https://api.github.com/repos/microsoft/playwright-python/releases/latest" \ | awk '/tag_name/{print $4;exit}' FS='[""]'); \ - git clone --depth 1 --branch "${PLAYWRIGHT_RELEASE}" https://github.com/microsoft/playwright-python /tmp/playwright-python && \ + git clone --depth 1 --branch "${PLAYWRIGHT_PY_RELEASE}" https://github.com/microsoft/playwright-python /tmp/playwright-python && \ cd /tmp/playwright-python && \ pip install -U --no-cache-dir . && \ - git clone --depth 1 --branch "${PLAYWRIGHT_RELEASE}" https://github.com/microsoft/playwright /tmp/playwright && \ + git clone --depth 1 --branch "${PLAYWRIGHT_PY_RELEASE}" https://github.com/microsoft/playwright /tmp/playwright && \ cd /tmp/playwright && \ npm ci && \ npm run build && \ From 2b9f8428b775c1c5a5d7b5b852d075fa63c40041 Mon Sep 17 00:00:00 2001 From: TheSpad Date: Sat, 9 Mar 2024 17:07:02 +0000 Subject: [PATCH 3/4] Fix output folder name for arm --- Dockerfile.aarch64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 index ab4fd17..645d0b8 100644 --- a/Dockerfile.aarch64 +++ b/Dockerfile.aarch64 @@ -70,7 +70,7 @@ RUN \ sed -i 's/"..\/..\/${NODE_DIR}\/${NPM_PATH}"/\/usr\/lib\/node_modules\/npm\/bin\/npm-cli.js/' ./utils/build/build-playwright-driver.sh && \ ./utils/build/build-playwright-driver.sh && \ rm -rf /lsiopy/lib/python3.11/site-packages/playwright/driver/* && \ - cp -R ./utils/build/output/playwright-linux/* /lsiopy/lib/python3.11/site-packages/playwright/driver && \ + cp -R ./utils/build/output/playwright-linux-arm64/* /lsiopy/lib/python3.11/site-packages/playwright/driver && \ ln -s /usr/bin/node /lsiopy/lib/python3.11/site-packages/playwright/driver/node && \ apk del --purge \ build-dependencies && \ From f3860f69a1b775207c0d92b30dd53b0896c21dca Mon Sep 17 00:00:00 2001 From: TheSpad Date: Sun, 10 Mar 2024 17:28:41 +0000 Subject: [PATCH 4/4] Restore custom compose --- README.md | 6 +++--- readme-vars.yml | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0e6aac1..0aaff0e 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ The architectures supported by this image are: Webui is accessible at http://SERVERIP:PORT -Please note that as of version 0.45.15 this image contains the Playwright content fetcher. +Please note that as of version 0.45.16 this image contains the Playwright content fetcher. For more info read [the wiki](https://github.com/dgtlmoon/changedetection.io/wiki). @@ -76,7 +76,7 @@ To help you get started creating a container from this image you can either use services: changedetection.io: image: lscr.io/linuxserver/changedetection.io:latest - container_name: changedetection.io + container_name: changedetection environment: - PUID=1000 - PGID=1000 @@ -84,7 +84,7 @@ services: - BASE_URL= #optional - PLAYWRIGHT_DRIVER_URL= #optional volumes: - - /path/to/changedetection.io/config:/config + - /path/to/changedetection/config:/config ports: - 5000:5000 restart: unless-stopped diff --git a/readme-vars.yml b/readme-vars.yml index 55595ef..1230e83 100644 --- a/readme-vars.yml +++ b/readme-vars.yml @@ -35,6 +35,24 @@ opt_param_env_vars: - { env_var: "BASE_URL", env_value: "", desc: "Specify the full URL (including protocol) when running behind a reverse proxy" } - { env_var: "PLAYWRIGHT_DRIVER_URL", env_value: "", desc: "Specify the full URL to your chrome driver instance. See the [wiki](https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher) for details." } +custom_compose: | + --- + services: + changedetection.io: + image: lscr.io/linuxserver/changedetection.io:latest + container_name: changedetection + environment: + - PUID=1000 + - PGID=1000 + - TZ=Etc/UTC + - BASE_URL= #optional + - PLAYWRIGHT_DRIVER_URL= #optional + volumes: + - /path/to/changedetection/config:/config + ports: + - 5000:5000 + restart: unless-stopped + # application setup block app_setup_block_enabled: true app_setup_block: |