Move base images to python 3.10 (#550)

## Checks - [x] `make lint`: I've run `make lint` to lint the changes in this PR. - [x] `make test`: I've made sure the tests (`make test-cpu` or `make test`) are passing. - Additional tests: - [ ] Benchmark tests (when contributing new models) - [ ] GPU/HW tests
autonomi-ai · May 4, 2024 · e9a4c2c · e9a4c2c
1 parent 909c4ce
commit e9a4c2c
Show file tree

Hide file tree

Showing 25 changed files with 34 additions and 88 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
-        python-version: ["3.9"]
+        python-version: ["3.10"]
     defaults:
       run:
         shell: bash -el {0}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -19,7 +19,7 @@ jobs:
       fail-fast: true
       matrix:
         os: ["ubuntu-latest"]
-        python-version: ["3.8"]
+        python-version: ["3.10"]
     defaults:
       run:
         shell: bash -el {0}
@@ -62,7 +62,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - uses: actions/cache@v2
         with:
           key: ${{ github.ref }}

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - uses: actions/cache@v2
         with:
           key: ${{ github.ref }}

diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml
@@ -19,7 +19,7 @@ jobs:
       fail-fast: true
       matrix:
         os: ["ubuntu-latest"]
-        python-version: ["3.8"]
+        python-version: ["3.10"]
     defaults:
       run:
         shell: bash -el {0}
@@ -40,7 +40,7 @@ jobs:
       fail-fast: true
       matrix:
         os: ["macos-latest"]
-        python-version: ["3.8"]
+        python-version: ["3.10"]
     defaults:
       run:
         shell: bash -el {0}

diff --git a/.github/workflows/release-tests.yml b/.github/workflows/release-tests.yml
@@ -13,7 +13,7 @@ jobs:
       fail-fast: true
       matrix:
         os: ["ubuntu-latest"]
-        python-version: ["3.8"]
+        python-version: ["3.10"]
     defaults:
       run:
         shell: bash -el {0}

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,5 +1,5 @@
 default_language_version:
-  python: python3.8
+  python: python3.10
 
 ci:
   autofix_prs: true

diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu
@@ -4,11 +4,11 @@ FROM debian:buster-slim AS base-cpu
 
 # Setup environment variables
 ENV AGIPACK_PROJECT nos
-ENV AGIPACK_PYENV nos-py38
+ENV AGIPACK_PYENV nos-py310
 ENV AGIPACK_PATH /opt/agi-pack
 
 ENV DEBIAN_FRONTEND="noninteractive"
-ENV PYTHON_VERSION 3.8.15
+ENV PYTHON_VERSION 3.10.11
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONWARNINGS ignore

diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu
@@ -4,11 +4,11 @@ FROM nvidia/cuda:11.8.0-base-ubuntu22.04 AS base-gpu
 
 # Setup environment variables
 ENV AGIPACK_PROJECT nos
-ENV AGIPACK_PYENV nos-py38
+ENV AGIPACK_PYENV nos-py310
 ENV AGIPACK_PATH /opt/agi-pack
 
 ENV DEBIAN_FRONTEND="noninteractive"
-ENV PYTHON_VERSION 3.8.15
+ENV PYTHON_VERSION 3.10.11
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONWARNINGS ignore

diff --git a/docker/agibuild.cpu.yaml b/docker/agibuild.cpu.yaml
@@ -2,7 +2,7 @@ images:
   base-cpu:
     base: debian:buster-slim
     name: nos
-    python: "3.8"
+    python: "3.10"
     system:
       - gnupg2
       - sudo

diff --git a/docker/agibuild.cu121.yaml b/docker/agibuild.cu121.yaml
@@ -2,7 +2,7 @@ images:
   base-gpu:
     base: nvidia/cuda:12.1.1-base-ubuntu22.04
     name: nos
-    python: "3.8"
+    python: "3.10"
     system:
       - gnupg2
       - sudo

diff --git a/docker/agibuild.gpu.yaml b/docker/agibuild.gpu.yaml
@@ -2,7 +2,7 @@ images:
   base-gpu:
     base: nvidia/cuda:11.8.0-base-ubuntu22.04
     name: nos
-    python: "3.8"
+    python: "3.10"
     system:
       - gnupg2
       - sudo

diff --git a/docs/OVERVIEW.md b/docs/OVERVIEW.md
@@ -34,8 +34,8 @@
 To quickly get started with a light-weight NOS client, run the following command:
 
   ```shell
-  $ conda create -n nos-py38 python=3.8
-  $ conda activate nos-py38
+  $ conda create -n nos-py310 python=3.10
+  $ conda activate nos-py310
   $ pip install torch-nos
   ```
 
@@ -44,8 +44,8 @@ In the setup above, the client pulls and utilizes the NOS docker server that run
 If you're interested in developing or contributing new models to NOS, consider installing the full NOS server via pip/conda:
 
   ```shell
-  $ conda create -n nos-py38 python=3.8
-  $ conda activate nos-py38
+  $ conda create -n nos-py310 python=3.10
+  $ conda activate nos-py310
   $ conda install pytorch>=2.0.1 torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia
   $ pip install torch-nos[server]
   ```

diff --git a/examples/discord/app/docker-compose.yml b/examples/discord/app/docker-compose.yml
@@ -1,5 +1,3 @@
-version: "3.8"
-
 services:
   bot:
     image: autonomi/nos:latest-discord-app

diff --git a/examples/docker/docker-compose.cpu.yml b/examples/docker/docker-compose.cpu.yml
@@ -1,5 +1,3 @@
-version: "3.8"
-
 services:
   server-cpu:
     image: autonomi/nos:latest-cpu

diff --git a/examples/docker/docker-compose.gpu-with-gateway.yml b/examples/docker/docker-compose.gpu-with-gateway.yml
@@ -1,5 +1,3 @@
-version: "3.8"
-
 services:
   server:
     image: autonomi/nos:latest-gpu

diff --git a/examples/docker/docker-compose.gpu.yml b/examples/docker/docker-compose.gpu.yml
@@ -1,5 +1,3 @@
-version: "3.8"
-
 services:
   server-gpu:
     image: autonomi/nos:latest-gpu

diff --git a/examples/notebook/00-quickstart-welcome-to-nos.ipynb b/examples/notebook/00-quickstart-welcome-to-nos.ipynb
@@ -71,8 +71,8 @@
     "NOS is available on PyPI and can be installed via pip. We recommend using a virtual environment or conda to install NOS:\n",
     "\n",
     "```bash\n",
-    "$ conda create -n nos-py38 python=3.8\n",
-    "$ conda activate nos-py38\n",
+    "$ conda create -n nos-py310 python=3.10\n",
+    "$ conda activate nos-py310\n",
     "$ pip install torch-nos\n",
     "```\n",
     "\n",

diff --git a/examples/skypilot/sky.yaml b/examples/skypilot/sky.yaml
@@ -12,7 +12,7 @@ resources:
 setup: |
   # Setup conda environment
   conda init bash
-  conda create -n nos python=3.8 -y
+  conda create -n nos python=3.10 -y
   conda activate nos
 
   # Install docker compose plugin

diff --git a/makefiles/Makefile.agibuild.mk b/makefiles/Makefile.agibuild.mk
@@ -1,54 +1,30 @@
 # Copyright 2023 Autonomi AI, Inc. All rights reserved.
 AGIPACK_ARGS :=
 
-agi-build-cpu:  # equivalent to agi-build-py38-cpu (target=base-cpu, cpu, test-cpu)
+agi-build-cpu:  # equivalent to agi-build-py310-cpu (target=base-cpu, cpu, test-cpu)
 	agi-pack build ${AGIPACK_ARGS} \
 		--target cpu \
 		-c docker/agibuild.cpu.yaml \
 		-o docker/Dockerfile.cpu \
-		-p 3.8.15 \
+		-p 3.10.11 \
 		-b debian:buster-slim \
 		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-cpu'
 	docker tag \
 		${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-cpu \
 		${DOCKER_IMAGE_NAME}:latest-cpu
 
-agi-build-gpu:  # equivalent to agi-build-py38-cu118 (target=base-gpu, gpu, test-gpu)
+agi-build-gpu:  # equivalent to agi-build-py310-cu118 (target=base-gpu, gpu, test-gpu)
 	agi-pack build ${AGIPACK_ARGS} \
 		--target gpu \
 		-c docker/agibuild.gpu.yaml \
 		-o docker/Dockerfile.gpu \
-		-p 3.8.15 \
+		-p 3.10.11 \
 		-b nvidia/cuda:11.8.0-base-ubuntu22.04 \
 		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-gpu'
 	docker tag \
 		${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-gpu \
 		${DOCKER_IMAGE_NAME}:latest-gpu
 
-agi-build-py38-cu118:
-	agi-pack build ${AGIPACK_ARGS} \
-		--target gpu \
-		-c docker/agibuild.gpu.yaml \
-		-o docker/Dockerfile.py38-cu118 \
-		-p 3.8.15 \
-		-b nvidia/cuda:11.8.0-base-ubuntu22.04 \
-		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py38-cu118'
-	docker tag \
-		${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py38-cu118 \
-		${DOCKER_IMAGE_NAME}:latest-py38-cu118
-
-agi-build-py39-cu118:
-	agi-pack build ${AGIPACK_ARGS} \
-		--target gpu \
-		-c docker/agibuild.gpu.yaml \
-		-o docker/Dockerfile.py39-cu118 \
-		-p 3.9.13 \
-		-b nvidia/cuda:11.8.0-base-ubuntu22.04 \
-		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py39-cu118'
-	docker tag \
-		${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py39-cu118 \
-		${DOCKER_IMAGE_NAME}:latest-py39-cu118
-
 agi-build-py310-cu118:
 	agi-pack build ${AGIPACK_ARGS} \
 		--target gpu \
@@ -61,18 +37,6 @@ agi-build-py310-cu118:
 		${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py310-cu118 \
 		${DOCKER_IMAGE_NAME}:latest-py310-cu118
 
-agi-build-py38-cu121:
-	agi-pack build ${AGIPACK_ARGS} \
-		--target gpu \
-		-c docker/agibuild.cu121.yaml \
-		-o docker/Dockerfile.py38-cu121 \
-		-p 3.8.15 \
-		-b nvidia/cuda:12.1.1-base-ubuntu22.04 \
-		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py38-cu121'
-	docker tag \
-		${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py38-cu121 \
-		${DOCKER_IMAGE_NAME}:latest-py38-cu121
-
 agi-build-py310-cu121:
 	agi-pack build ${AGIPACK_ARGS} \
 		--target gpu \
@@ -86,18 +50,12 @@ agi-build-py310-cu121:
 		${DOCKER_IMAGE_NAME}:latest-py310-cu121
 
 agi-build-cu118: \
-	agi-build-py38-cu118 \
-	agi-build-py39-cu118 \
 	agi-build-py310-cu118
 
 agi-build-cu121: \
 	agi-build-py310-cu121
 
 agi-push-cu118:
-	docker push ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py38-cu118
-	docker push ${DOCKER_IMAGE_NAME}:latest-py38-cu118
-	docker push ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py39-cu118
-	docker push ${DOCKER_IMAGE_NAME}:latest-py39-cu118
 	docker push ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-py310-cu118
 	docker push ${DOCKER_IMAGE_NAME}:latest-py310-cu118
 

diff --git a/makefiles/Makefile.base.mk b/makefiles/Makefile.base.mk
@@ -57,7 +57,7 @@ docker-build-and-push-multiplatform-cpu:
 	agi-pack generate ${AGIPACK_ARGS} \
 		-c docker/agibuild.cpu.yaml \
 		-o docker/Dockerfile.multiplatform.cpu \
-		-p 3.8.15 \
+		-p 3.10.11 \
 		-b debian:buster-slim \
 		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-{target}'
 	make .docker-build-and-push-multiplatform \

diff --git a/nos/cli/templates/docker-compose.serve.yml.j2 b/nos/cli/templates/docker-compose.serve.yml.j2
@@ -1,5 +1,3 @@
-version: "3.8"
-
 services:
 {%- if http %}
   nos-http-gateway:

diff --git a/nos/common/shm.py b/nos/common/shm.py
@@ -253,7 +253,7 @@ def copy(shm_map: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]:
                     shm_map[key]
                 ), f"Shared memory already initialized with length={len(shm_map[key])}, provided input with length={len(data[key])}."
                 # Move data from the data dict value to the shared memory segment.
-                for item, shm in zip(data[key], shm_map[key]):
+                for item, shm in zip(data[key], shm_map[key], strict=False):
                     shm.copy_from(item)
             elif isinstance(data[key], np.ndarray):
                 # Move data from the data dict value to the shared memory segment.

diff --git a/nos/telemetry.py b/nos/telemetry.py
@@ -27,11 +27,11 @@ def _init_telemetry_logger() -> None:
             debug=SENTRY_DEBUG,
             # Set traces_sample_rate to 1.0 to capture 100%
             # of transactions for performance monitoring.
-            traces_sample_rate=1.0,
+            traces_sample_rate=0.05,
             # Set profiles_sample_rate to 1.0 to profile 100%
             # of sampled transactions.
             # We recommend adjusting this value in production.
-            profiles_sample_rate=1.0,
+            profiles_sample_rate=0.05,
             release=__version__,
             integrations=[
                 sentry_loguru,

diff --git a/pyproject.toml b/pyproject.toml
@@ -7,7 +7,7 @@ name = "torch-nos"
 description = "Nitrous Oxide for your AI Infrastructure."
 license = {file = "LICENSE"}
 readme = "README.md"
-requires-python = ">=3.8.10"
+requires-python = ">=3.10"
 classifiers = [
     "Development Status :: 4 - Beta",
     "Programming Language :: Python",
@@ -21,8 +21,6 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Image Processing",
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
 
@@ -85,7 +83,7 @@ exclude = [
 
 [tool.black]
 line-length = 119
-target-version = ['py38']
+target-version = ['py310']
 exclude = '''
     /(
         \.data
@@ -119,7 +117,7 @@ ignore = [
     "B904",  # B904: check for raise from None
 ]
 line-length = 119
-target-version = "py38"
+target-version = "py310"
 
 # Allow autofix for all enabled rules (when `--fix`) is provided.
 fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]

diff --git a/tests/common/test_common.py b/tests/common/test_common.py
@@ -12,7 +12,7 @@ def test_common_tqdm():
     import time
 
     # Use tqdm as a regular progress bar
-    for i1, i2 in zip(range(10), tqdm(range(10))):
+    for i1, i2 in zip(range(10), tqdm(range(10)), strict=False):
         assert i1 == i2
 
     # Use tqdm as a timer