Test on Windows OS. (#522)

* Test on Windows OS.

* Make lsst-sphgeom optional.

* pylint warning, and docs build.

* PYLINT **shakes fist**

.github/workflows/smoke-test.yml

@@ -29,7 +29,7 @@ jobs:
       run: |
         sudo apt-get update
         python -m pip install --upgrade pip
-        pip install -e .[dev]
+        pip install -e .[dev,full]
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: List dependencies
       run: |

.github/workflows/testing-and-coverage.yml

@@ -27,7 +27,7 @@ jobs:
       run: |
         sudo apt-get update
         python -m pip install --upgrade pip
-        pip install -e .[dev]
+        pip install -e .[dev,full]
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Run unit tests with pytest
       run: |

.github/workflows/testing-windows.yml

@@ -0,0 +1,34 @@
+# This workflow will install Python dependencies and run tests in a Windows environment.
+# This is intended to catch any file-system specific issues, and so runs less
+# frequently than other test suites.
+
+name: Windows unit test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .[dev]
+        pip install -r requirements.txt
+    - name: Run unit tests with pytest
+      run: |
+        python -m pytest tests

.setup_dev.sh

@@ -31,7 +31,7 @@ echo "Installing package and runtime dependencies in local environment"
 python -m pip install -e . > /dev/null
 
 echo "Installing developer dependencies in local environment"
-python -m pip install -e .'[dev]' > /dev/null
+python -m pip install -e .'[dev,full]' > /dev/null
 if [ -f docs/requirements.txt ]; then python -m pip install -r docs/requirements.txt; fi
 
 echo "Installing pre-commit"

docs/requirements.txt

@@ -3,6 +3,7 @@ ipykernel
 ipython
 ipywidgets
 jupytext
+lsst-sphgeom
 nbconvert
 nbsphinx
 sphinx

pyproject.toml

@@ -22,7 +22,6 @@ dependencies = [
     "dask[complete]",
     "deprecated",
     "hats>=0.4.4",
-    "lsst-sphgeom", # To handle spherical sky polygons
     "nested-dask>=0.3.0",
     "nested-pandas",
     "pyarrow",
@@ -49,6 +48,7 @@ full = [
     "fsspec[full]", # complete file system specs.
     "ipykernel", # Support for Jupyter notebooks
     "ipywidgets", # useful for tqdm in notebooks.
+    "lsst-sphgeom", # To handle spherical sky polygons
 ]
 
 [build-system]
@@ -62,6 +62,9 @@ build-backend = "setuptools.build_meta"
 write_to = "src/lsdb/_version.py"
 
 [tool.pytest.ini_options]
+markers = [
+    "sphgeom: mark tests as having a runtime dependency on lsst-sphgeom",
+]
 testpaths = [
     "tests",
 ]

src/lsdb/core/search/polygon_search.py

@@ -3,7 +3,6 @@
 import numpy as np
 from hats.catalog import TableProperties
 from hats.pixel_math.validators import validate_polygon
-from lsst.sphgeom import ConvexPolygon, UnitVector3d
 
 from lsdb.core.search.abstract_search import AbstractSearch
 from lsdb.types import HCCatalogTypeVar
@@ -31,9 +30,7 @@ def search_points(self, frame: npd.NestedFrame, metadata: TableProperties) -> np
         return polygon_filter(frame, self.polygon, metadata)
 
 
-def polygon_filter(
-    data_frame: npd.NestedFrame, polygon: ConvexPolygon, metadata: TableProperties
-) -> npd.NestedFrame:
+def polygon_filter(data_frame: npd.NestedFrame, polygon, metadata: TableProperties) -> npd.NestedFrame:
     """Filters a dataframe to only include points within the specified polygon.
 
     Args:
@@ -51,7 +48,8 @@ def polygon_filter(
     return data_frame
 
 
-def get_cartesian_polygon(vertices: list[tuple[float, float]]) -> ConvexPolygon:
+# pylint: disable=import-outside-toplevel
+def get_cartesian_polygon(vertices: list[tuple[float, float]]):
     """Creates the convex polygon to filter pixels with. It transforms the
     vertices, provided in sky coordinates of ra and dec, to their respective
     cartesian representation on the unit sphere.
@@ -63,6 +61,8 @@ def get_cartesian_polygon(vertices: list[tuple[float, float]]) -> ConvexPolygon:
     Returns:
         The convex polygon object.
     """
+    from lsst.sphgeom import ConvexPolygon, UnitVector3d  # pylint: disable=import-error
+
     vertices_xyz = hp.ang2vec(*np.array(vertices).T)
     edge_vectors = [UnitVector3d(x, y, z) for x, y, z in vertices_xyz]
     return ConvexPolygon(edge_vectors)

tests/conftest.py

@@ -278,6 +278,27 @@ def cone_search_margin_expected(cone_search_expected_dir):
     return pd.read_csv(cone_search_expected_dir / "margin.csv", index_col=SPATIAL_INDEX_COLUMN)
 
 
+# pylint: disable=import-outside-toplevel
+def pytest_collection_modifyitems(items):
+    """Modify tests that use the `lsst-sphgeom` package to only run when that
+    package has been installed in the development environment.
+
+    If we detect that we can import `lsst-sphgeom`, this method exits early
+    and does not modify any test items.
+    """
+    try:
+        # pylint: disable=unused-import
+        from lsst.sphgeom import ConvexPolygon
+
+        return
+    except ImportError:
+        pass
+
+    for item in items:
+        if any(item.iter_markers(name="sphgeom")):
+            item.add_marker(pytest.mark.skip(reason="lsst-sphgeom is not installed"))
+
+
 @pytest.fixture
 def assert_divisions_are_correct():
     def assert_divisions_are_correct(catalog):

tests/lsdb/catalog/test_catalog.py

@@ -695,9 +695,6 @@ def test_filtered_catalog_has_undetermined_len(small_sky_order1_catalog, small_s
         len(small_sky_order1_catalog.query("ra > 300"))
     with pytest.raises(ValueError, match="undetermined"):
         len(small_sky_order1_catalog.cone_search(0, -80, 1))
-    with pytest.raises(ValueError, match="undetermined"):
-        vertices = [(300, -50), (300, -55), (272, -55), (272, -50)]
-        len(small_sky_order1_catalog.polygon_search(vertices))
     with pytest.raises(ValueError, match="undetermined"):
         len(small_sky_order1_catalog.box_search(ra=(280, 300), dec=(0, 30)))
     with pytest.raises(ValueError, match="undetermined"):
@@ -711,6 +708,14 @@ def test_filtered_catalog_has_undetermined_len(small_sky_order1_catalog, small_s
         len(small_sky_order1_catalog.dropna())
 
 
+@pytest.mark.sphgeom
+def test_filtered_catalog_has_undetermined_len_polygon(small_sky_order1_catalog):
+    """Tests that filtered catalogs have an undetermined number of rows"""
+    with pytest.raises(ValueError, match="undetermined"):
+        vertices = [(300, -50), (300, -55), (272, -55), (272, -50)]
+        len(small_sky_order1_catalog.polygon_search(vertices))
+
+
 def test_joined_catalog_has_undetermined_len(
     small_sky_order1_catalog, small_sky_xmatch_catalog, small_sky_order1_source_with_margin
 ):

tests/lsdb/catalog/test_polygon_search.py

@@ -8,6 +8,7 @@
 from lsdb.core.search.polygon_search import get_cartesian_polygon
 
 
+@pytest.mark.sphgeom
 def test_polygon_search_filters_correct_points(small_sky_order1_catalog, assert_divisions_are_correct):
     vertices = [(300, -50), (300, -55), (272, -55), (272, -50)]
     polygon = get_cartesian_polygon(vertices)
@@ -25,6 +26,7 @@ def test_polygon_search_filters_correct_points(small_sky_order1_catalog, assert_
     assert_divisions_are_correct(polygon_search_catalog)
 
 
+@pytest.mark.sphgeom
 def test_polygon_search_filters_correct_points_margin(
     small_sky_order1_source_with_margin, assert_divisions_are_correct
 ):
@@ -53,6 +55,7 @@ def test_polygon_search_filters_correct_points_margin(
     assert_divisions_are_correct(polygon_search_catalog.margin)
 
 
+@pytest.mark.sphgeom
 def test_polygon_search_filters_partitions(small_sky_order1_catalog):
     vertices = [(300, -50), (300, -55), (272, -55), (272, -50)]
     hc_polygon_search = small_sky_order1_catalog.hc_structure.filter_by_polygon(vertices)
@@ -63,6 +66,7 @@ def test_polygon_search_filters_partitions(small_sky_order1_catalog):
         assert pixel in polygon_search_catalog._ddf_pixel_map
 
 
+@pytest.mark.sphgeom
 def test_polygon_search_coarse_versus_fine(small_sky_order1_catalog):
     vertices = [(300, -50), (300, -55), (272, -55), (272, -50)]
     coarse_polygon_search = small_sky_order1_catalog.polygon_search(vertices, fine=False)
@@ -99,6 +103,7 @@ def test_polygon_search_invalid_polygon(small_sky_order1_catalog):
         small_sky_order1_catalog.polygon_search(vertices)
 
 
+@pytest.mark.sphgeom
 def test_polygon_search_wrapped_right_ascension():
     """Tests the scenario where the polygon edges intersect the
     discontinuity of the RA [0,360] degrees range. For the same
@@ -132,6 +137,7 @@ def test_polygon_search_wrapped_right_ascension():
         npt.assert_allclose(polygon.getVertices(), polygon_2.getVertices(), rtol=1e-7)
 
 
+@pytest.mark.sphgeom
 def test_empty_polygon_search_with_margin(small_sky_order1_source_with_margin):
     vertices = [(80, 0), (100, 30), (120, 0)]
     polygon = small_sky_order1_source_with_margin.polygon_search(vertices)

tests/lsdb/loaders/hats/test_read_hats.py

@@ -185,6 +185,7 @@ def test_read_hats_subset_with_box_search(small_sky_order1_dir, small_sky_order1
     pd.testing.assert_frame_equal(box_search_catalog.compute(), box_search_catalog_2.compute())
 
 
+@pytest.mark.sphgeom
 def test_read_hats_subset_with_polygon_search(small_sky_order1_dir, small_sky_order1_catalog):
     vertices = [(300, -50), (300, -55), (272, -55), (272, -50)]
     polygon_search = PolygonSearch(vertices)