From 517aa5b6effd43dce75b8006908a16334d6ff62b Mon Sep 17 00:00:00 2001 From: Your GitHub Username Date: Thu, 8 Aug 2024 09:05:29 +0100 Subject: [PATCH 01/10] initial commit --- .github/workflows/ci.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..133006f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: CI +on: [ push ] + +jobs: + build-and-run-tests: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up environment + run: | + git lfs fetch + git lfs pull + pip install uv + uv venv venv + source venv/bin/activate + uv pip install -r requirements.txt + + - name: Run tests + run: | + source venv/bin/activate + python -m pytest -rsx tests/ --data-path=/home/runner/work/fair-mast/fair-mast/tests/mock_data/index + + ruff-code-check: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: chartboost/ruff-action@v1 \ No newline at end of file From 4d578c18c60e45a2fe733e2f29cd8476652c3b63 Mon Sep 17 00:00:00 2001 From: Your GitHub Username Date: Thu, 8 Aug 2024 09:09:04 +0100 Subject: [PATCH 02/10] ruff fixes --- src/workflow.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/workflow.py b/src/workflow.py index a41737d..96cb7f3 100644 --- a/src/workflow.py +++ b/src/workflow.py @@ -107,8 +107,6 @@ def __init__( def __call__(self, shot: int): self.data_dir.mkdir(exist_ok=True, parents=True) - local_path = self.data_dir / f"{shot}.{self.file_format}" - create = CreateDatasetTask( self.metadata_dir, @@ -123,7 +121,8 @@ def __call__(self, shot: int): try: create() except Exception as e: - import traceback; traceback.print_exc(); + import traceback + traceback.print_exc() print(traceback.format_exc()) logging.error(f"Failed to run workflow with error {type(e)}: {e}") @@ -145,7 +144,6 @@ def run_workflows(self, shot_list: list[int], parallel=True): def _run_workflows_serial(self, shot_list: list[int]): n = len(shot_list) for i, shot in enumerate(shot_list): - result = self.workflow(shot) logging.info(f"Done shot {i+1}/{n} = {(i+1)/n*100:.2f}%") def _run_workflows_parallel(self, shot_list: list[int]): From c83856dacf0373ed7732e240640874567949c279 Mon Sep 17 00:00:00 2001 From: Your GitHub Username Date: Thu, 8 Aug 2024 09:36:42 +0100 Subject: [PATCH 03/10] install libopenmpi-dev --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 133006f..280cbbf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,7 @@ jobs: git lfs fetch git lfs pull pip install uv + sudo apt install libopenmpi-dev uv venv venv source venv/bin/activate uv pip install -r requirements.txt From 9506f3f2202c7f3d4784072936a08274f49e59d5 Mon Sep 17 00:00:00 2001 From: Your GitHub Username Date: Thu, 8 Aug 2024 09:39:42 +0100 Subject: [PATCH 04/10] removed arg from pytest --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 280cbbf..7934c83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: - name: Run tests run: | source venv/bin/activate - python -m pytest -rsx tests/ --data-path=/home/runner/work/fair-mast/fair-mast/tests/mock_data/index + python -m pytest -rsx tests/ ruff-code-check: runs-on: ubuntu-latest From 839bbea6d8703922e59ebaa1cd7f7f33c68287a2 Mon Sep 17 00:00:00 2001 From: Your GitHub Username Date: Thu, 8 Aug 2024 09:57:17 +0100 Subject: [PATCH 05/10] skipped tests that use MAST client --- tests/test_reader.py | 14 +++++++------- tests/test_task.py | 16 ++++++++-------- tests/test_transforms.py | 2 +- tests/test_writer.py | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_reader.py b/tests/test_reader.py index 8046207..8aa3ae3 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -4,13 +4,13 @@ import pytest pyuda_import = pytest.importorskip("pyuda") -from src.archive.reader import ( # noqa: E402 +from src.reader import ( # noqa: E402 DatasetReader, # noqa: E402 SignalMetadataReader, # noqa: E402 SourceMetadataReader, # noqa: E402 ) # noqa: E402 - +@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") def test_list_signals(): shot = 30420 reader = DatasetReader(shot) @@ -22,7 +22,7 @@ def test_list_signals(): info = signals[0] assert info.name == "abm/calib_shot" - +@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") def test_list_signals_exclude_raw(): shot = 30420 reader = DatasetReader(shot) @@ -34,7 +34,7 @@ def test_list_signals_exclude_raw(): info = signals[0] assert info.name == "abm/calib_shot" - +@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") def test_read_signal(): shot = 30420 reader = DatasetReader(shot) @@ -47,7 +47,7 @@ def test_read_signal(): assert dataset.attrs["name"] == "abm/calib_shot" assert dataset["time"].shape == (1,) - +@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") def test_read_image(): shot = 30420 reader = DatasetReader(shot) @@ -64,7 +64,7 @@ def test_read_image(): assert dataset["data"].shape == (186, 912, 768) assert list(dataset.dims.keys()) == ["time", "height", "width"] - +@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") def test_read_signals_metadata(): shot = 30420 reader = SignalMetadataReader(shot) @@ -72,7 +72,7 @@ def test_read_signals_metadata(): assert isinstance(df, pd.DataFrame) - +@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") def test_read_sources_metadata(): shot = 30420 reader = SourceMetadataReader(shot) diff --git a/tests/test_task.py b/tests/test_task.py index 09927b0..363c7d8 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -2,14 +2,14 @@ import zarr import xarray as xr import subprocess -from src.archive.uploader import UploadConfig +from src.uploader import UploadConfig from pathlib import Path import os import pytest pyuda_import = pytest.importorskip("pyuda") -from src.archive.writer import DatasetWriter # noqa: E402 -from src.archive.task import ( # noqa: E402 +from src.writer import DatasetWriter # noqa: E402 +from src.task import ( # noqa: E402 CreateDatasetTask, # noqa: E402 CleanupDatasetTask, # noqa: E402 UploadDatasetTask, # noqa: E402 @@ -17,7 +17,7 @@ CreateSignalMetadataTask, # noqa: E402 ) # noqa: E402 - +@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") def test_create_dataset_task(tmpdir, mocker): metadata_dir = tmpdir / "uda" shot = 30420 @@ -47,7 +47,7 @@ def test_create_dataset_task(tmpdir, mocker): ds = xr.open_zarr(dataset_path, group="abm") assert len(ds.data_vars) == 3 - +@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") @pytest.mark.usefixtures("fake_dataset") def test_write_cleanup(tmpdir, fake_dataset): shot = 30420 @@ -59,7 +59,7 @@ def test_write_cleanup(tmpdir, fake_dataset): task() assert not writer.dataset_path.exists() - +@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") def test_upload_dataset(mocker): mocker.patch("subprocess.run") @@ -93,7 +93,7 @@ def test_upload_dataset(mocker): env=env, ) - +@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") def test_source_metadata_reader(tmpdir): shot = 30420 task = CreateSourceMetadataTask(data_dir=tmpdir, shot=shot) @@ -104,7 +104,7 @@ def test_source_metadata_reader(tmpdir): df = pd.read_parquet(path) assert isinstance(df, pd.DataFrame) - +@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") def test_signal_metadata_reader(tmpdir): shot = 30420 task = CreateSignalMetadataTask(data_dir=tmpdir, shot=shot) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index b373166..d95225e 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -1,5 +1,5 @@ import xarray as xr -from src.archive.transforms import ( +from src.transforms import ( AddXSXCameraParams, DropDatasets, DropZeroDimensions, diff --git a/tests/test_writer.py b/tests/test_writer.py index 3ec5841..2bab06f 100644 --- a/tests/test_writer.py +++ b/tests/test_writer.py @@ -1,7 +1,7 @@ import pytest import zarr import xarray as xr -from src.archive.writer import DatasetWriter +from src.writer import DatasetWriter pyuda_import = pytest.importorskip("pyuda") From 87adcd2128d41c4f7f2b67cf87ca5e48ae8027ba Mon Sep 17 00:00:00 2001 From: Your GitHub Username Date: Thu, 8 Aug 2024 10:41:54 +0100 Subject: [PATCH 06/10] attempt to fix numpy error in worker --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7934c83..0166f96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: [ push ] jobs: - build-and-run-tests: + run-tests: runs-on: ubuntu-latest steps: @@ -17,6 +17,7 @@ jobs: uv venv venv source venv/bin/activate uv pip install -r requirements.txt + uv pip install --upgrade numpy - name: Run tests run: | From 6328463ba8f7c0a27a9da91b18ca03b9c5aa8b33 Mon Sep 17 00:00:00 2001 From: Your GitHub Username Date: Thu, 8 Aug 2024 10:57:49 +0100 Subject: [PATCH 07/10] attempt to fix numpy error in worker --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0166f96..afad88e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,8 @@ jobs: uv venv venv source venv/bin/activate uv pip install -r requirements.txt - uv pip install --upgrade numpy + uv pip uninstall numpy + uv pip install "numpy<2.0" - name: Run tests run: | From c7c5d3d30d2c55a1a4c3855d6a3c3e031c2b8bd8 Mon Sep 17 00:00:00 2001 From: Your GitHub Username Date: Thu, 8 Aug 2024 11:19:41 +0100 Subject: [PATCH 08/10] cleaned up steps --- .github/workflows/ci.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afad88e..86808ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,15 +10,13 @@ jobs: - name: Set up environment run: | - git lfs fetch - git lfs pull + git lfs fetch && git lfs pull + sudo apt update && sudo apt install -y libopenmpi-dev pip install uv - sudo apt install libopenmpi-dev uv venv venv source venv/bin/activate uv pip install -r requirements.txt - uv pip uninstall numpy - uv pip install "numpy<2.0" + uv pip install --upgrade --force-reinstall "numpy<2.0" - name: Run tests run: | From ffc683a111bc5be50a41ef463f03ba917c7f67a6 Mon Sep 17 00:00:00 2001 From: Your GitHub Username Date: Thu, 8 Aug 2024 11:23:46 +0100 Subject: [PATCH 09/10] changed to pull request --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86808ee..03db490 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ name: CI -on: [ push ] +on: [ pull_request ] jobs: run-tests: From 20881138289688647fddb0a020804d570d58d043 Mon Sep 17 00:00:00 2001 From: Your GitHub Username Date: Mon, 12 Aug 2024 14:43:48 +0100 Subject: [PATCH 10/10] removed mention of csd3, clearer skip message --- tests/test_reader.py | 12 ++++++------ tests/test_task.py | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_reader.py b/tests/test_reader.py index 8aa3ae3..60754ac 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -10,7 +10,7 @@ SourceMetadataReader, # noqa: E402 ) # noqa: E402 -@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") +@pytest.mark.skip(reason="Pyuda client unavailable") def test_list_signals(): shot = 30420 reader = DatasetReader(shot) @@ -22,7 +22,7 @@ def test_list_signals(): info = signals[0] assert info.name == "abm/calib_shot" -@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") +@pytest.mark.skip(reason="Pyuda client unavailable") def test_list_signals_exclude_raw(): shot = 30420 reader = DatasetReader(shot) @@ -34,7 +34,7 @@ def test_list_signals_exclude_raw(): info = signals[0] assert info.name == "abm/calib_shot" -@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") +@pytest.mark.skip(reason="Pyuda client unavailable") def test_read_signal(): shot = 30420 reader = DatasetReader(shot) @@ -47,7 +47,7 @@ def test_read_signal(): assert dataset.attrs["name"] == "abm/calib_shot" assert dataset["time"].shape == (1,) -@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") +@pytest.mark.skip(reason="Pyuda client unavailable") def test_read_image(): shot = 30420 reader = DatasetReader(shot) @@ -64,7 +64,7 @@ def test_read_image(): assert dataset["data"].shape == (186, 912, 768) assert list(dataset.dims.keys()) == ["time", "height", "width"] -@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") +@pytest.mark.skip(reason="Pyuda client unavailable") def test_read_signals_metadata(): shot = 30420 reader = SignalMetadataReader(shot) @@ -72,7 +72,7 @@ def test_read_signals_metadata(): assert isinstance(df, pd.DataFrame) -@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") +@pytest.mark.skip(reason="Pyuda client unavailable") def test_read_sources_metadata(): shot = 30420 reader = SourceMetadataReader(shot) diff --git a/tests/test_task.py b/tests/test_task.py index 363c7d8..1cafaea 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -17,7 +17,7 @@ CreateSignalMetadataTask, # noqa: E402 ) # noqa: E402 -@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") +@pytest.mark.skip(reason="Pyuda client unavailable") def test_create_dataset_task(tmpdir, mocker): metadata_dir = tmpdir / "uda" shot = 30420 @@ -47,7 +47,7 @@ def test_create_dataset_task(tmpdir, mocker): ds = xr.open_zarr(dataset_path, group="abm") assert len(ds.data_vars) == 3 -@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") +@pytest.mark.skip(reason="Pyuda client unavailable") @pytest.mark.usefixtures("fake_dataset") def test_write_cleanup(tmpdir, fake_dataset): shot = 30420 @@ -59,7 +59,7 @@ def test_write_cleanup(tmpdir, fake_dataset): task() assert not writer.dataset_path.exists() -@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") +@pytest.mark.skip(reason="Pyuda client unavailable") def test_upload_dataset(mocker): mocker.patch("subprocess.run") @@ -93,7 +93,7 @@ def test_upload_dataset(mocker): env=env, ) -@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") +@pytest.mark.skip(reason="Pyuda client unavailable") def test_source_metadata_reader(tmpdir): shot = 30420 task = CreateSourceMetadataTask(data_dir=tmpdir, shot=shot) @@ -104,7 +104,7 @@ def test_source_metadata_reader(tmpdir): df = pd.read_parquet(path) assert isinstance(df, pd.DataFrame) -@pytest.mark.skip(reason="Uses MAST Client, not available on CSD3") +@pytest.mark.skip(reason="Pyuda client unavailable") def test_signal_metadata_reader(tmpdir): shot = 30420 task = CreateSignalMetadataTask(data_dir=tmpdir, shot=shot)