From cbbdf335b599f72e6b05d0f2cab2b8b5d05d2fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 11:47:42 +0100 Subject: [PATCH 01/12] Use gha cache --- .github/workflows/tox.yaml | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index a4903af..b85888f 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -1,39 +1,17 @@ -name: Tox -on: [push] +on: + push: + pull_request: jobs: - # Based on https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#registry-cache docker: runs-on: ubuntu-latest steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Run tests in docker build - uses: docker/build-push-action@v2 + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: docker/build-push-action@v5 with: context: . file: Dockerfile target: tests push: false - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + cache-from: type=gha + cache-to: type=gha,mode=max From 10540e2a1aeaeb21319b2d7ce2b8bc855c1aa7f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 11:48:17 +0100 Subject: [PATCH 02/12] on: push --- .github/workflows/tox.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index b85888f..ed61b43 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -1,6 +1,4 @@ -on: - push: - pull_request: +on: push jobs: docker: runs-on: ubuntu-latest From f0b1002e41ac336b93ee228cf3c969b338d14409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 11:49:30 +0100 Subject: [PATCH 03/12] Cancel in progress --- .github/workflows/tox.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index ed61b43..ed3c0df 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -1,4 +1,7 @@ on: push +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true jobs: docker: runs-on: ubuntu-latest From 80e08a8925fd25068fc86075ed1085f0179e2768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 11:58:03 +0100 Subject: [PATCH 04/12] Bump to Python 3.8 (oldest non-EOL) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 735c88b..69e4c4b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36 +envlist = py38 [testenv] ignore_errors = true From 1e3c4e460e8d1673e32831fd3bd362755e356730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 11:58:43 +0100 Subject: [PATCH 05/12] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 24a2839..dd51043 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6.9 AS main +FROM python:3.8 AS main # Install binary dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ From 425b28d51b396159d4e02fafeb73d9edeaa1c419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 12:05:06 +0100 Subject: [PATCH 06/12] Create dependabot.yaml --- .github/dependabot.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/dependabot.yaml diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..ff7a00b --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,14 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "monthly" From 65aab1a851c0d463771933b356ac9a32bb67dbcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 12:06:34 +0100 Subject: [PATCH 07/12] Default is to not push --- .github/workflows/tox.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index ed3c0df..e07d20e 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -13,6 +13,5 @@ jobs: context: . file: Dockerfile target: tests - push: false cache-from: type=gha cache-to: type=gha,mode=max From 2e89edba233bb8ec7cdceb778cb449034f496c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 12:44:57 +0100 Subject: [PATCH 08/12] Add dill (expected by TFDS testing utils) --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 69e4c4b..73581b4 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,7 @@ deps = mypy pydocstyle pytest + dill commands = # Check formatting From 8210cd31397ff167b873aad37a4bbd1c0a850ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 12:45:21 +0100 Subject: [PATCH 09/12] Migrate from deprecated functions --- src/pmqd/torch.py | 9 +++++---- tests/test_torch.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/pmqd/torch.py b/src/pmqd/torch.py index 98d2ec6..526f6c9 100644 --- a/src/pmqd/torch.py +++ b/src/pmqd/torch.py @@ -9,7 +9,8 @@ import pandas as pd import torchaudio from torch.utils.data import Dataset -from torchaudio.datasets.utils import download_url, extract_archive +from torch.hub import download_url_to_file as download_url +from torchaudio.datasets.utils import _extract_tar from . import checksums @@ -56,10 +57,10 @@ def __init__( if download: if not self._audio_path.exists(): if not archive.is_file(): - download_url(url_audio, root, hash_value=CHECKSUM_AUDIO) - extract_archive(archive) + download_url(url_audio, str(root), hash_prefix=CHECKSUM_AUDIO) + _extract_tar(str(archive)) if not self._metadata_path.is_file(): - download_url(url_metadata, root, hash_value=CHECKSUM_METADATA) + download_url(url_metadata, str(root), hash_prefix=CHECKSUM_METADATA) if not self._metadata_path.is_file(): raise FileNotFoundError( diff --git a/tests/test_torch.py b/tests/test_torch.py index 6b77d7e..4890caf 100644 --- a/tests/test_torch.py +++ b/tests/test_torch.py @@ -17,7 +17,7 @@ def test_torchaudio_no_dowload(tmp_path: Path): def test_torchaudio_download(tmp_path: Path, dummy_data: Dict[str, Path]): - def mock_download_url(url: str, download_folder: str, hash_value: str) -> None: + def mock_download_url(url: str, download_folder: str, hash_prefix: str) -> None: filename = os.path.basename(url) shutil.copy(dummy_data[filename], download_folder) From 2bef0750193d56d61d1d859f5ebd3c8c2b3768bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 12:49:19 +0100 Subject: [PATCH 10/12] isort --- src/pmqd/torch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pmqd/torch.py b/src/pmqd/torch.py index 526f6c9..113e201 100644 --- a/src/pmqd/torch.py +++ b/src/pmqd/torch.py @@ -8,8 +8,8 @@ import pandas as pd import torchaudio -from torch.utils.data import Dataset from torch.hub import download_url_to_file as download_url +from torch.utils.data import Dataset from torchaudio.datasets.utils import _extract_tar from . import checksums From 8998f7faf439ee38eddc9806ce7030c77d6b0f7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 12:52:11 +0100 Subject: [PATCH 11/12] Patch first-party --- tests/test_torch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_torch.py b/tests/test_torch.py index 4890caf..0464b9c 100644 --- a/tests/test_torch.py +++ b/tests/test_torch.py @@ -35,6 +35,6 @@ def mock_download_url(url: str, download_folder: str, hash_prefix: str) -> None: assert isinstance(example["rating"], float) # Check that files are not downloaded again - with patch("torchaudio.datasets.utils.download_url") as patched: + with patch("pmqd.torch.download_url") as patched: dataset = PMQD(root=tmp_path, download=True) patched.assert_not_called() From a0c38ac541e046e7a4f1b216e50187acf91bb5ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl=20Thom=C3=A9?= Date: Thu, 9 Nov 2023 12:58:51 +0100 Subject: [PATCH 12/12] Add types-all --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 73581b4..8024108 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,7 @@ deps = pydocstyle pytest dill + types-all commands = # Check formatting