Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use gha cache and repair tests #1

Merged
merged 12 commits into from
Nov 9, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/dependabot.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
- package-ecosystem: "pip"
directory: "/"
schedule:
interval: "monthly"
- package-ecosystem: "docker"
directory: "/"
schedule:
interval: "monthly"
40 changes: 9 additions & 31 deletions .github/workflows/tox.yaml
Original file line number Diff line number Diff line change
@@ -1,39 +1,17 @@
name: Tox
on: [push]
on: push
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
jobs:
# Based on https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#registry-cache
docker:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
-
name: Run tests in docker build
uses: docker/build-push-action@v2
- uses: actions/checkout@v4
- uses: docker/setup-buildx-action@v3
- uses: docker/build-push-action@v5
with:
context: .
file: Dockerfile
target: tests
push: false
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
-
# Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
name: Move cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
cache-from: type=gha
cache-to: type=gha,mode=max
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM python:3.6.9 AS main
FROM python:3.8 AS main

# Install binary dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
Expand Down
9 changes: 5 additions & 4 deletions src/pmqd/torch.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@

import pandas as pd
import torchaudio
from torch.hub import download_url_to_file as download_url
from torch.utils.data import Dataset
from torchaudio.datasets.utils import download_url, extract_archive
from torchaudio.datasets.utils import _extract_tar

from . import checksums

Expand Down Expand Up @@ -56,10 +57,10 @@ def __init__(
if download:
if not self._audio_path.exists():
if not archive.is_file():
download_url(url_audio, root, hash_value=CHECKSUM_AUDIO)
extract_archive(archive)
download_url(url_audio, str(root), hash_prefix=CHECKSUM_AUDIO)
_extract_tar(str(archive))
if not self._metadata_path.is_file():
download_url(url_metadata, root, hash_value=CHECKSUM_METADATA)
download_url(url_metadata, str(root), hash_prefix=CHECKSUM_METADATA)

if not self._metadata_path.is_file():
raise FileNotFoundError(
Expand Down
4 changes: 2 additions & 2 deletions tests/test_torch.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def test_torchaudio_no_dowload(tmp_path: Path):


def test_torchaudio_download(tmp_path: Path, dummy_data: Dict[str, Path]):
def mock_download_url(url: str, download_folder: str, hash_value: str) -> None:
def mock_download_url(url: str, download_folder: str, hash_prefix: str) -> None:
filename = os.path.basename(url)
shutil.copy(dummy_data[filename], download_folder)

Expand All @@ -35,6 +35,6 @@ def mock_download_url(url: str, download_folder: str, hash_value: str) -> None:
assert isinstance(example["rating"], float)

# Check that files are not downloaded again
with patch("torchaudio.datasets.utils.download_url") as patched:
with patch("pmqd.torch.download_url") as patched:
dataset = PMQD(root=tmp_path, download=True)
patched.assert_not_called()
4 changes: 3 additions & 1 deletion tox.ini
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[tox]
envlist = py36
envlist = py38

[testenv]
ignore_errors = true
Expand All @@ -15,6 +15,8 @@ deps =
mypy
pydocstyle
pytest
dill
types-all

commands =
# Check formatting
Expand Down