diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 6875da8b..e4575efb 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -29,7 +29,7 @@ jobs: run: | sudo apt-get update python -m pip install --upgrade pip - pip install -e .[dev] + pip install -e .[dev,full] if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: List dependencies run: | diff --git a/.github/workflows/testing-and-coverage.yml b/.github/workflows/testing-and-coverage.yml index 20fa704f..2fadf770 100644 --- a/.github/workflows/testing-and-coverage.yml +++ b/.github/workflows/testing-and-coverage.yml @@ -27,7 +27,7 @@ jobs: run: | sudo apt-get update python -m pip install --upgrade pip - pip install -e .[dev] + pip install -e .[dev,full] if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Run unit tests with pytest run: | diff --git a/.github/workflows/testing-windows.yml b/.github/workflows/testing-windows.yml new file mode 100644 index 00000000..c27bcc58 --- /dev/null +++ b/.github/workflows/testing-windows.yml @@ -0,0 +1,34 @@ +# This workflow will install Python dependencies and run tests in a Windows environment. +# This is intended to catch any file-system specific issues, and so runs less +# frequently than other test suites. + +name: Windows unit test + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: windows-latest + strategy: + matrix: + python-version: ['3.10'] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[dev] + pip install -r requirements.txt + - name: Run unit tests with pytest + run: | + python -m pytest tests \ No newline at end of file diff --git a/.setup_dev.sh b/.setup_dev.sh index d8cd955c..8fd4a5cb 100644 --- a/.setup_dev.sh +++ b/.setup_dev.sh @@ -31,7 +31,7 @@ echo "Installing package and runtime dependencies in local environment" python -m pip install -e . > /dev/null echo "Installing developer dependencies in local environment" -python -m pip install -e .'[dev]' > /dev/null +python -m pip install -e .'[dev,full]' > /dev/null if [ -f docs/requirements.txt ]; then python -m pip install -r docs/requirements.txt; fi echo "Installing pre-commit" diff --git a/docs/requirements.txt b/docs/requirements.txt index 6a734805..8efa5a0e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,6 +3,7 @@ ipykernel ipython ipywidgets jupytext +lsst-sphgeom nbconvert nbsphinx sphinx diff --git a/pyproject.toml b/pyproject.toml index 17d1a1f2..1011d91e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ dependencies = [ "dask[complete]", "deprecated", "hats>=0.4.4", - "lsst-sphgeom", # To handle spherical sky polygons "nested-dask>=0.3.0", "nested-pandas", "pyarrow", @@ -49,6 +48,7 @@ full = [ "fsspec[full]", # complete file system specs. "ipykernel", # Support for Jupyter notebooks "ipywidgets", # useful for tqdm in notebooks. + "lsst-sphgeom", # To handle spherical sky polygons ] [build-system] @@ -62,6 +62,9 @@ build-backend = "setuptools.build_meta" write_to = "src/lsdb/_version.py" [tool.pytest.ini_options] +markers = [ + "sphgeom: mark tests as having a runtime dependency on lsst-sphgeom", +] testpaths = [ "tests", ] diff --git a/src/lsdb/core/search/polygon_search.py b/src/lsdb/core/search/polygon_search.py index fe7e3207..5fe1cd57 100644 --- a/src/lsdb/core/search/polygon_search.py +++ b/src/lsdb/core/search/polygon_search.py @@ -3,7 +3,6 @@ import numpy as np from hats.catalog import TableProperties from hats.pixel_math.validators import validate_polygon -from lsst.sphgeom import ConvexPolygon, UnitVector3d from lsdb.core.search.abstract_search import AbstractSearch from lsdb.types import HCCatalogTypeVar @@ -31,9 +30,7 @@ def search_points(self, frame: npd.NestedFrame, metadata: TableProperties) -> np return polygon_filter(frame, self.polygon, metadata) -def polygon_filter( - data_frame: npd.NestedFrame, polygon: ConvexPolygon, metadata: TableProperties -) -> npd.NestedFrame: +def polygon_filter(data_frame: npd.NestedFrame, polygon, metadata: TableProperties) -> npd.NestedFrame: """Filters a dataframe to only include points within the specified polygon. Args: @@ -51,7 +48,8 @@ def polygon_filter( return data_frame -def get_cartesian_polygon(vertices: list[tuple[float, float]]) -> ConvexPolygon: +# pylint: disable=import-outside-toplevel +def get_cartesian_polygon(vertices: list[tuple[float, float]]): """Creates the convex polygon to filter pixels with. It transforms the vertices, provided in sky coordinates of ra and dec, to their respective cartesian representation on the unit sphere. @@ -63,6 +61,8 @@ def get_cartesian_polygon(vertices: list[tuple[float, float]]) -> ConvexPolygon: Returns: The convex polygon object. """ + from lsst.sphgeom import ConvexPolygon, UnitVector3d # pylint: disable=import-error + vertices_xyz = hp.ang2vec(*np.array(vertices).T) edge_vectors = [UnitVector3d(x, y, z) for x, y, z in vertices_xyz] return ConvexPolygon(edge_vectors) diff --git a/tests/conftest.py b/tests/conftest.py index 853ecc5e..046595f8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -278,6 +278,27 @@ def cone_search_margin_expected(cone_search_expected_dir): return pd.read_csv(cone_search_expected_dir / "margin.csv", index_col=SPATIAL_INDEX_COLUMN) +# pylint: disable=import-outside-toplevel +def pytest_collection_modifyitems(items): + """Modify tests that use the `lsst-sphgeom` package to only run when that + package has been installed in the development environment. + + If we detect that we can import `lsst-sphgeom`, this method exits early + and does not modify any test items. + """ + try: + # pylint: disable=unused-import + from lsst.sphgeom import ConvexPolygon + + return + except ImportError: + pass + + for item in items: + if any(item.iter_markers(name="sphgeom")): + item.add_marker(pytest.mark.skip(reason="lsst-sphgeom is not installed")) + + @pytest.fixture def assert_divisions_are_correct(): def assert_divisions_are_correct(catalog): diff --git a/tests/lsdb/catalog/test_catalog.py b/tests/lsdb/catalog/test_catalog.py index 88271694..e04ac61c 100644 --- a/tests/lsdb/catalog/test_catalog.py +++ b/tests/lsdb/catalog/test_catalog.py @@ -695,9 +695,6 @@ def test_filtered_catalog_has_undetermined_len(small_sky_order1_catalog, small_s len(small_sky_order1_catalog.query("ra > 300")) with pytest.raises(ValueError, match="undetermined"): len(small_sky_order1_catalog.cone_search(0, -80, 1)) - with pytest.raises(ValueError, match="undetermined"): - vertices = [(300, -50), (300, -55), (272, -55), (272, -50)] - len(small_sky_order1_catalog.polygon_search(vertices)) with pytest.raises(ValueError, match="undetermined"): len(small_sky_order1_catalog.box_search(ra=(280, 300), dec=(0, 30))) with pytest.raises(ValueError, match="undetermined"): @@ -711,6 +708,14 @@ def test_filtered_catalog_has_undetermined_len(small_sky_order1_catalog, small_s len(small_sky_order1_catalog.dropna()) +@pytest.mark.sphgeom +def test_filtered_catalog_has_undetermined_len_polygon(small_sky_order1_catalog): + """Tests that filtered catalogs have an undetermined number of rows""" + with pytest.raises(ValueError, match="undetermined"): + vertices = [(300, -50), (300, -55), (272, -55), (272, -50)] + len(small_sky_order1_catalog.polygon_search(vertices)) + + def test_joined_catalog_has_undetermined_len( small_sky_order1_catalog, small_sky_xmatch_catalog, small_sky_order1_source_with_margin ): diff --git a/tests/lsdb/catalog/test_polygon_search.py b/tests/lsdb/catalog/test_polygon_search.py index 18e14fec..00c78f93 100644 --- a/tests/lsdb/catalog/test_polygon_search.py +++ b/tests/lsdb/catalog/test_polygon_search.py @@ -8,6 +8,7 @@ from lsdb.core.search.polygon_search import get_cartesian_polygon +@pytest.mark.sphgeom def test_polygon_search_filters_correct_points(small_sky_order1_catalog, assert_divisions_are_correct): vertices = [(300, -50), (300, -55), (272, -55), (272, -50)] polygon = get_cartesian_polygon(vertices) @@ -25,6 +26,7 @@ def test_polygon_search_filters_correct_points(small_sky_order1_catalog, assert_ assert_divisions_are_correct(polygon_search_catalog) +@pytest.mark.sphgeom def test_polygon_search_filters_correct_points_margin( small_sky_order1_source_with_margin, assert_divisions_are_correct ): @@ -53,6 +55,7 @@ def test_polygon_search_filters_correct_points_margin( assert_divisions_are_correct(polygon_search_catalog.margin) +@pytest.mark.sphgeom def test_polygon_search_filters_partitions(small_sky_order1_catalog): vertices = [(300, -50), (300, -55), (272, -55), (272, -50)] hc_polygon_search = small_sky_order1_catalog.hc_structure.filter_by_polygon(vertices) @@ -63,6 +66,7 @@ def test_polygon_search_filters_partitions(small_sky_order1_catalog): assert pixel in polygon_search_catalog._ddf_pixel_map +@pytest.mark.sphgeom def test_polygon_search_coarse_versus_fine(small_sky_order1_catalog): vertices = [(300, -50), (300, -55), (272, -55), (272, -50)] coarse_polygon_search = small_sky_order1_catalog.polygon_search(vertices, fine=False) @@ -99,6 +103,7 @@ def test_polygon_search_invalid_polygon(small_sky_order1_catalog): small_sky_order1_catalog.polygon_search(vertices) +@pytest.mark.sphgeom def test_polygon_search_wrapped_right_ascension(): """Tests the scenario where the polygon edges intersect the discontinuity of the RA [0,360] degrees range. For the same @@ -132,6 +137,7 @@ def test_polygon_search_wrapped_right_ascension(): npt.assert_allclose(polygon.getVertices(), polygon_2.getVertices(), rtol=1e-7) +@pytest.mark.sphgeom def test_empty_polygon_search_with_margin(small_sky_order1_source_with_margin): vertices = [(80, 0), (100, 30), (120, 0)] polygon = small_sky_order1_source_with_margin.polygon_search(vertices) diff --git a/tests/lsdb/loaders/hats/test_read_hats.py b/tests/lsdb/loaders/hats/test_read_hats.py index 74b638e0..11f8c27b 100644 --- a/tests/lsdb/loaders/hats/test_read_hats.py +++ b/tests/lsdb/loaders/hats/test_read_hats.py @@ -185,6 +185,7 @@ def test_read_hats_subset_with_box_search(small_sky_order1_dir, small_sky_order1 pd.testing.assert_frame_equal(box_search_catalog.compute(), box_search_catalog_2.compute()) +@pytest.mark.sphgeom def test_read_hats_subset_with_polygon_search(small_sky_order1_dir, small_sky_order1_catalog): vertices = [(300, -50), (300, -55), (272, -55), (272, -50)] polygon_search = PolygonSearch(vertices)