From 20cad80042b733fbcab96e7196a5846a7272a3b2 Mon Sep 17 00:00:00 2001 From: Sudeep Pillai Date: Sat, 4 May 2024 10:43:27 -0700 Subject: [PATCH] Move base images to python 3.10 --- .github/workflows/benchmark.yml | 2 +- .github/workflows/ci.yml | 4 +- .github/workflows/docs.yml | 2 +- .github/workflows/integrations.yml | 4 +- .github/workflows/release-tests.yml | 2 +- .pre-commit-config.yaml | 2 +- docker/Dockerfile.cpu | 4 +- docker/Dockerfile.gpu | 4 +- docker/agibuild.cpu.yaml | 2 +- docker/agibuild.cu121.yaml | 2 +- docker/agibuild.gpu.yaml | 2 +- docs/OVERVIEW.md | 8 +-- examples/discord/app/docker-compose.yml | 2 - examples/docker/docker-compose.cpu.yml | 2 - .../docker-compose.gpu-with-gateway.yml | 2 - examples/docker/docker-compose.gpu.yml | 2 - .../00-quickstart-welcome-to-nos.ipynb | 4 +- examples/skypilot/sky.yaml | 2 +- makefiles/Makefile.agibuild.mk | 50 ++----------------- makefiles/Makefile.base.mk | 2 +- nos/cli/templates/docker-compose.serve.yml.j2 | 2 - nos/common/shm.py | 2 +- nos/telemetry.py | 4 +- pyproject.toml | 8 ++- tests/common/test_common.py | 2 +- 25 files changed, 34 insertions(+), 88 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 9f14adaa..5cc69535 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -15,7 +15,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] # , "macos-latest", "windows-latest" - python-version: ["3.9"] + python-version: ["3.10"] defaults: run: shell: bash -el {0} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b47d1b04..3dffc54f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: fail-fast: true matrix: os: ["ubuntu-latest"] - python-version: ["3.8"] + python-version: ["3.10"] defaults: run: shell: bash -el {0} @@ -62,7 +62,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: "3.10" - uses: actions/cache@v2 with: key: ${{ github.ref }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8fe96712..fac3f27f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: "3.10" - uses: actions/cache@v2 with: key: ${{ github.ref }} diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 09c4622c..aa1bf07d 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -19,7 +19,7 @@ jobs: fail-fast: true matrix: os: ["ubuntu-latest"] - python-version: ["3.8"] + python-version: ["3.10"] defaults: run: shell: bash -el {0} @@ -40,7 +40,7 @@ jobs: fail-fast: true matrix: os: ["macos-latest"] - python-version: ["3.8"] + python-version: ["3.10"] defaults: run: shell: bash -el {0} diff --git a/.github/workflows/release-tests.yml b/.github/workflows/release-tests.yml index 712e5938..2bb659fa 100644 --- a/.github/workflows/release-tests.yml +++ b/.github/workflows/release-tests.yml @@ -13,7 +13,7 @@ jobs: fail-fast: true matrix: os: ["ubuntu-latest"] - python-version: ["3.8"] + python-version: ["3.10"] defaults: run: shell: bash -el {0} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 650775be..6fa5c795 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_language_version: - python: python3.8 + python: python3.10 ci: autofix_prs: true diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu index 6774c798..b07f5504 100644 --- a/docker/Dockerfile.cpu +++ b/docker/Dockerfile.cpu @@ -4,11 +4,11 @@ FROM debian:buster-slim AS base-cpu # Setup environment variables ENV AGIPACK_PROJECT nos -ENV AGIPACK_PYENV nos-py38 +ENV AGIPACK_PYENV nos-py310 ENV AGIPACK_PATH /opt/agi-pack ENV DEBIAN_FRONTEND="noninteractive" -ENV PYTHON_VERSION 3.8.15 +ENV PYTHON_VERSION 3.10.11 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 ENV PYTHONWARNINGS ignore diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 1c0235bf..4f14159d 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -4,11 +4,11 @@ FROM nvidia/cuda:11.8.0-base-ubuntu22.04 AS base-gpu # Setup environment variables ENV AGIPACK_PROJECT nos -ENV AGIPACK_PYENV nos-py38 +ENV AGIPACK_PYENV nos-py310 ENV AGIPACK_PATH /opt/agi-pack ENV DEBIAN_FRONTEND="noninteractive" -ENV PYTHON_VERSION 3.8.15 +ENV PYTHON_VERSION 3.10.11 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 ENV PYTHONWARNINGS ignore diff --git a/docker/agibuild.cpu.yaml b/docker/agibuild.cpu.yaml index a1172b7c..340c1166 100644 --- a/docker/agibuild.cpu.yaml +++ b/docker/agibuild.cpu.yaml @@ -2,7 +2,7 @@ images: base-cpu: base: debian:buster-slim name: nos - python: "3.8" + python: "3.10" system: - gnupg2 - sudo diff --git a/docker/agibuild.cu121.yaml b/docker/agibuild.cu121.yaml index e39ee3b2..b7cded62 100644 --- a/docker/agibuild.cu121.yaml +++ b/docker/agibuild.cu121.yaml @@ -2,7 +2,7 @@ images: base-gpu: base: nvidia/cuda:12.1.1-base-ubuntu22.04 name: nos - python: "3.8" + python: "3.10" system: - gnupg2 - sudo diff --git a/docker/agibuild.gpu.yaml b/docker/agibuild.gpu.yaml index a8774746..d346be22 100644 --- a/docker/agibuild.gpu.yaml +++ b/docker/agibuild.gpu.yaml @@ -2,7 +2,7 @@ images: base-gpu: base: nvidia/cuda:11.8.0-base-ubuntu22.04 name: nos - python: "3.8" + python: "3.10" system: - gnupg2 - sudo diff --git a/docs/OVERVIEW.md b/docs/OVERVIEW.md index c27030af..118cc03c 100644 --- a/docs/OVERVIEW.md +++ b/docs/OVERVIEW.md @@ -34,8 +34,8 @@ To quickly get started with a light-weight NOS client, run the following command: ```shell - $ conda create -n nos-py38 python=3.8 - $ conda activate nos-py38 + $ conda create -n nos-py310 python=3.10 + $ conda activate nos-py310 $ pip install torch-nos ``` @@ -44,8 +44,8 @@ In the setup above, the client pulls and utilizes the NOS docker server that run If you're interested in developing or contributing new models to NOS, consider installing the full NOS server via pip/conda: ```shell - $ conda create -n nos-py38 python=3.8 - $ conda activate nos-py38 + $ conda create -n nos-py310 python=3.10 + $ conda activate nos-py310 $ conda install pytorch>=2.0.1 torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia $ pip install torch-nos[server] ``` diff --git a/examples/discord/app/docker-compose.yml b/examples/discord/app/docker-compose.yml index 336f093e..9e0ba776 100644 --- a/examples/discord/app/docker-compose.yml +++ b/examples/discord/app/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: bot: image: autonomi/nos:latest-discord-app diff --git a/examples/docker/docker-compose.cpu.yml b/examples/docker/docker-compose.cpu.yml index 7186f449..f6e54db9 100644 --- a/examples/docker/docker-compose.cpu.yml +++ b/examples/docker/docker-compose.cpu.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: server-cpu: image: autonomi/nos:latest-cpu diff --git a/examples/docker/docker-compose.gpu-with-gateway.yml b/examples/docker/docker-compose.gpu-with-gateway.yml index bb4e75bd..038aedef 100644 --- a/examples/docker/docker-compose.gpu-with-gateway.yml +++ b/examples/docker/docker-compose.gpu-with-gateway.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: server: image: autonomi/nos:latest-gpu diff --git a/examples/docker/docker-compose.gpu.yml b/examples/docker/docker-compose.gpu.yml index ce8052ac..373aff19 100644 --- a/examples/docker/docker-compose.gpu.yml +++ b/examples/docker/docker-compose.gpu.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: server-gpu: image: autonomi/nos:latest-gpu diff --git a/examples/notebook/00-quickstart-welcome-to-nos.ipynb b/examples/notebook/00-quickstart-welcome-to-nos.ipynb index d74ce67d..aca142c4 100644 --- a/examples/notebook/00-quickstart-welcome-to-nos.ipynb +++ b/examples/notebook/00-quickstart-welcome-to-nos.ipynb @@ -71,8 +71,8 @@ "NOS is available on PyPI and can be installed via pip. We recommend using a virtual environment or conda to install NOS:\n", "\n", "```bash\n", - "$ conda create -n nos-py38 python=3.8\n", - "$ conda activate nos-py38\n", + "$ conda create -n nos-py310 python=3.10\n", + "$ conda activate nos-py310\n", "$ pip install torch-nos\n", "```\n", "\n", diff --git a/examples/skypilot/sky.yaml b/examples/skypilot/sky.yaml index 3153f8da..5a4b5ad5 100644 --- a/examples/skypilot/sky.yaml +++ b/examples/skypilot/sky.yaml @@ -12,7 +12,7 @@ resources: setup: | # Setup conda environment conda init bash - conda create -n nos python=3.8 -y + conda create -n nos python=3.10 -y conda activate nos # Install docker compose plugin diff --git a/makefiles/Makefile.agibuild.mk b/makefiles/Makefile.agibuild.mk index d791ad22..42e1fdf5 100644 --- a/makefiles/Makefile.agibuild.mk +++ b/makefiles/Makefile.agibuild.mk @@ -1,54 +1,30 @@ # Copyright 2023 Autonomi AI, Inc. All rights reserved. AGIPACK_ARGS := -agi-build-cpu: # equivalent to agi-build-py38-cpu (target=base-cpu, cpu, test-cpu) +agi-build-cpu: # equivalent to agi-build-py310-cpu (target=base-cpu, cpu, test-cpu) agi-pack build ${AGIPACK_ARGS} \ --target cpu \ -c docker/agibuild.cpu.yaml \ -o docker/Dockerfile.cpu \ - -p 3.8.15 \ + -p 3.10.11 \ -b debian:buster-slim \ -t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-cpu' docker tag \ ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-cpu \ ${DOCKER_IMAGE_NAME}:latest-cpu -agi-build-gpu: # equivalent to agi-build-py38-cu118 (target=base-gpu, gpu, test-gpu) +agi-build-gpu: # equivalent to agi-build-py310-cu118 (target=base-gpu, gpu, test-gpu) agi-pack build ${AGIPACK_ARGS} \ --target gpu \ -c docker/agibuild.gpu.yaml \ -o docker/Dockerfile.gpu \ - -p 3.8.15 \ + -p 3.10.11 \ -b nvidia/cuda:11.8.0-base-ubuntu22.04 \ -t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-gpu' docker tag \ ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-gpu \ ${DOCKER_IMAGE_NAME}:latest-gpu -agi-build-py38-cu118: - agi-pack build ${AGIPACK_ARGS} \ - --target gpu \ - -c docker/agibuild.gpu.yaml \ - -o docker/Dockerfile.py38-cu118 \ - -p 3.8.15 \ - -b nvidia/cuda:11.8.0-base-ubuntu22.04 \ - -t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py38-cu118' - docker tag \ - ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py38-cu118 \ - ${DOCKER_IMAGE_NAME}:latest-py38-cu118 - -agi-build-py39-cu118: - agi-pack build ${AGIPACK_ARGS} \ - --target gpu \ - -c docker/agibuild.gpu.yaml \ - -o docker/Dockerfile.py39-cu118 \ - -p 3.9.13 \ - -b nvidia/cuda:11.8.0-base-ubuntu22.04 \ - -t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py39-cu118' - docker tag \ - ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py39-cu118 \ - ${DOCKER_IMAGE_NAME}:latest-py39-cu118 - agi-build-py310-cu118: agi-pack build ${AGIPACK_ARGS} \ --target gpu \ @@ -61,18 +37,6 @@ agi-build-py310-cu118: ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py310-cu118 \ ${DOCKER_IMAGE_NAME}:latest-py310-cu118 -agi-build-py38-cu121: - agi-pack build ${AGIPACK_ARGS} \ - --target gpu \ - -c docker/agibuild.cu121.yaml \ - -o docker/Dockerfile.py38-cu121 \ - -p 3.8.15 \ - -b nvidia/cuda:12.1.1-base-ubuntu22.04 \ - -t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py38-cu121' - docker tag \ - ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py38-cu121 \ - ${DOCKER_IMAGE_NAME}:latest-py38-cu121 - agi-build-py310-cu121: agi-pack build ${AGIPACK_ARGS} \ --target gpu \ @@ -86,18 +50,12 @@ agi-build-py310-cu121: ${DOCKER_IMAGE_NAME}:latest-py310-cu121 agi-build-cu118: \ - agi-build-py38-cu118 \ - agi-build-py39-cu118 \ agi-build-py310-cu118 agi-build-cu121: \ agi-build-py310-cu121 agi-push-cu118: - docker push ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py38-cu118 - docker push ${DOCKER_IMAGE_NAME}:latest-py38-cu118 - docker push ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py39-cu118 - docker push ${DOCKER_IMAGE_NAME}:latest-py39-cu118 docker push ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py310-cu118 docker push ${DOCKER_IMAGE_NAME}:latest-py310-cu118 diff --git a/makefiles/Makefile.base.mk b/makefiles/Makefile.base.mk index d05c88bc..266baab9 100644 --- a/makefiles/Makefile.base.mk +++ b/makefiles/Makefile.base.mk @@ -57,7 +57,7 @@ docker-build-and-push-multiplatform-cpu: agi-pack generate ${AGIPACK_ARGS} \ -c docker/agibuild.cpu.yaml \ -o docker/Dockerfile.multiplatform.cpu \ - -p 3.8.15 \ + -p 3.10.11 \ -b debian:buster-slim \ -t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-{target}' make .docker-build-and-push-multiplatform \ diff --git a/nos/cli/templates/docker-compose.serve.yml.j2 b/nos/cli/templates/docker-compose.serve.yml.j2 index be8eabe9..e78c3567 100644 --- a/nos/cli/templates/docker-compose.serve.yml.j2 +++ b/nos/cli/templates/docker-compose.serve.yml.j2 @@ -1,5 +1,3 @@ -version: "3.8" - services: {%- if http %} nos-http-gateway: diff --git a/nos/common/shm.py b/nos/common/shm.py index e1002bff..16ba943b 100644 --- a/nos/common/shm.py +++ b/nos/common/shm.py @@ -253,7 +253,7 @@ def copy(shm_map: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]: shm_map[key] ), f"Shared memory already initialized with length={len(shm_map[key])}, provided input with length={len(data[key])}." # Move data from the data dict value to the shared memory segment. - for item, shm in zip(data[key], shm_map[key]): + for item, shm in zip(data[key], shm_map[key], strict=False): shm.copy_from(item) elif isinstance(data[key], np.ndarray): # Move data from the data dict value to the shared memory segment. diff --git a/nos/telemetry.py b/nos/telemetry.py index 663b0972..bdc8ed44 100644 --- a/nos/telemetry.py +++ b/nos/telemetry.py @@ -27,11 +27,11 @@ def _init_telemetry_logger() -> None: debug=SENTRY_DEBUG, # Set traces_sample_rate to 1.0 to capture 100% # of transactions for performance monitoring. - traces_sample_rate=1.0, + traces_sample_rate=0.05, # Set profiles_sample_rate to 1.0 to profile 100% # of sampled transactions. # We recommend adjusting this value in production. - profiles_sample_rate=1.0, + profiles_sample_rate=0.05, release=__version__, integrations=[ sentry_loguru, diff --git a/pyproject.toml b/pyproject.toml index 0571aa5b..7457fb09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "torch-nos" description = "Nitrous Oxide for your AI Infrastructure." license = {file = "LICENSE"} readme = "README.md" -requires-python = ">=3.8.10" +requires-python = ">=3.10" classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", @@ -21,8 +21,6 @@ classifiers = [ "Topic :: Scientific/Engineering :: Image Processing", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -85,7 +83,7 @@ exclude = [ [tool.black] line-length = 119 -target-version = ['py38'] +target-version = ['py310'] exclude = ''' /( \.data @@ -119,7 +117,7 @@ ignore = [ "B904", # B904: check for raise from None ] line-length = 119 -target-version = "py38" +target-version = "py310" # Allow autofix for all enabled rules (when `--fix`) is provided. fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] diff --git a/tests/common/test_common.py b/tests/common/test_common.py index 4d996093..ae85b354 100644 --- a/tests/common/test_common.py +++ b/tests/common/test_common.py @@ -12,7 +12,7 @@ def test_common_tqdm(): import time # Use tqdm as a regular progress bar - for i1, i2 in zip(range(10), tqdm(range(10))): + for i1, i2 in zip(range(10), tqdm(range(10)), strict=False): assert i1 == i2 # Use tqdm as a timer