install fugue and run distributed tests

Nixtla · Sep 13, 2024 · 4546517 · 4546517
1 parent 2cd2e37
commit 4546517
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 13 deletions.
diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml
@@ -32,7 +32,7 @@ jobs:
           set -ux
           python -m pip install --upgrade pip
           pip install -Uq nbdev
-          pip install -e ".[dev]"
+          pip install ".[dev]" fugue[dask,spark]>=0.8.1
           mkdir nbs/_extensions
           cp -r docs-scripts/mintlify/ nbs/_extensions/
           python docs-scripts/update-quarto.py

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -27,18 +27,15 @@ jobs:
       - name: Clone repo
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - name: Set up environment
-        uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 # v1.9.0
+      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
-          environment-file: environment.yml
-          create-args: python=${{ matrix.python-version }}
-          cache-environment: true
+          python-version: ${{ matrix.python-version }}
 
       - name: Install the library
-        run: pip install ./
+        run: pip install uv && uv pip install --system ".[dev]" fugue[dask,spark]>=0.8.1
 
       - name: Run tests
-        run: nbdev_test --do_print --timing --flags 'matplotlib polars pyarrow scipy'
+        run: nbdev_test --do_print --timing --flags 'distributed matplotlib polars pyarrow scipy'
 
   windows-tests:
     runs-on: windows-latest
@@ -50,8 +47,7 @@ jobs:
       - name: Clone repo
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - name: Set up environment
-        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/nbs/evaluation.ipynb b/nbs/evaluation.ipynb
@@ -964,16 +964,26 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "42460fe5-d803-4f93-8d63-91776db03566",
+   "id": "f74f5d78-0a39-4056-b99a-582a6e4e8261",
    "metadata": {},
    "outputs": [],
    "source": [
     "#| hide\n",
     "#| distributed\n",
     "import dask.dataframe as dd\n",
     "import fugue.api as fa\n",
-    "from pyspark.sql import SparkSession\n",
-    "\n",
+    "from pyspark.sql import SparkSession"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "106b5c5d-bac0-43e2-9ce1-fa9fb0c8b4da",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| hide\n",
+    "#| distributed\n",
     "spark = SparkSession.builder.getOrCreate()\n",
     "spark.sparkContext.setLogLevel('FATAL')\n",
     "dask_df = dd.from_pandas(series, npartitions=2)\n",