🔖 0.7.0 (#97)

* Support modin as backend (#96) * ✨ Support `modin` as backend * 🩹 Use modin.pandas.read_csv to read datasets if backend is modin * 📌 Update poetry.lock * 👷 Make show versions to include modin version when backend is modin * ✨ Add `_return` argument for `datar.options()` * 📝 Add doc about backends * 🐛 Fix `tidyr.expand()` when `nesting(f.name)` as argument * 🔖 0.7.0
pwwang · Mar 24, 2022 · a03f1c0 · a03f1c0
1 parent 7357907
commit a03f1c0
Show file tree

Hide file tree

Showing 161 changed files with 704 additions and 410 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -38,11 +38,10 @@ jobs:
           python -m pip install flake8
           python -m pip install poetry
           poetry config virtualenvs.create false
-          poetry install -v
+          poetry install -E pdtypes -v
           pip install wcwidth
           pip install scipy
           pip install python-slugify
-          pip install pdtypes
           # reinstall pandas to specific version
           pip install -U $PANDAS
         env:
@@ -62,13 +61,54 @@ jobs:
         if: ${{ always() }}
       - name: Run codacy-coverage-reporter
         uses: codacy/codacy-coverage-reporter-action@master
-        if: matrix.python-version == 3.8 && matrix.pandas == 'pandas'
+        if: matrix.python-version == 3.9 && matrix.pandas == 'pandas'
         with:
           project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
           coverage-reports: cov.xml
 
+  build-modin:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # can't build numpy for 3.10 in CI
+        # python-version: [3.7, 3.8, 3.9, "3.10"]
+        python-version: [3.7, 3.8, 3.9]
+        modin: [
+          modin==0.10.2,
+          modin # lastest
+        ]
+        exclude:
+          - python-version: 3.7
+            modin: modin==0.10.2
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python # Set Python version
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install flake8
+          python -m pip install poetry
+          poetry config virtualenvs.create false
+          poetry install -E modin -E pdtypes -v
+          pip install wcwidth
+          pip install scipy
+          pip install python-slugify
+          # reinstall modin to specific version
+          pip install -U $MODIN
+        env:
+          MODIN: ${{ matrix.modin }}
+      - name: Show versions
+        run: python -c 'import datar; datar.options(backend="modin"); datar.get_versions()'
+      - name: Test with pytest
+        run: pytest tests/ --backend modin --junitxml=junit/test-results-${{ matrix.python-version }}.xml
+
   deploy:
-    needs: build
+    needs: [build, build-modin]
     runs-on: ubuntu-latest
     if: github.event_name == 'release'
     strategy:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -28,9 +28,16 @@ repos:
               if [[ $v1 == $v2 ]]; then exit 0; else exit 1; fi
         pass_filenames: false
         files: ^pyproject\.toml|datar/__init__\.py$
-    -   id: pytest
-        name: Run pytest
-        entry: pytest
+    -   id: pytest-pandas
+        name: Run pytest with pandas backend
+        entry: pytest --backend pandas
+        language: system
+        args: [tests/]
+        pass_filenames: false
+        files: ^tests/.+$|^datar/.+$
+    -   id: pytest-modin
+        name: Run pytest with modin backend
+        entry: pytest --backend modin
         language: system
         args: [tests/]
         pass_filenames: false

diff --git a/README.md b/README.md
@@ -17,11 +17,14 @@ A Grammar of Data Manipulation in python
 
 ```shell
 pip install -U datar
-```
-or
-```shell
-conda install -c conda-forge datar
-# mamba install -c conda-forge datar
+
+# install pdtypes support
+pip install -U datar[pdtypes]
+
+# install dependencies for modin as backend
+pip install -U datar[modin]
+# you may also need to install dependencies for modin engines
+# pip install -U modin[ray]
 ```
 
 ## Example usage

diff --git a/datar/__init__.py b/datar/__init__.py
@@ -1,6 +1,3 @@
-# register operator
-from collections import namedtuple as _namedtuple
-
 from .core import operator as _
 from .core import f, options_context, options, add_option, get_option, logger
 
@@ -16,24 +13,11 @@
 
 options(enable_pdtypes=True)
 
-_VersionsTuple = _namedtuple(
-    "_VersionsTuple",
-    [
-        "python",
-        "datar",
-        "numpy",
-        "pandas",
-        "pipda",
-        "executing",
-        "varname",
-    ],
-)
-
 __all__ = ("f", "get_versions")
-__version__ = "0.6.4"
+__version__ = "0.7.0"
 
 
-def get_versions(prnt: bool = True) -> _VersionsTuple:
+def get_versions(prnt: bool = True):
     """Print or return related versions which help for bug reporting.
 
     Args:
@@ -45,25 +29,33 @@ def get_versions(prnt: bool = True) -> _VersionsTuple:
     import sys
 
     import numpy
-    import pandas
     import pipda
     import executing
     import varname
+    from diot import Diot
 
-    out = _VersionsTuple(
+    out = Diot(
         python=sys.version,
         datar=__version__,
         numpy=numpy.__version__,
-        pandas=pandas.__version__,
         pipda=pipda.__version__,
         executing=executing.__version__,
         varname=varname.__version__,
     )
+
+    backend = get_option("backend")
+    if backend == "pandas":
+        import pandas
+        out["pandas"] = pandas.__version__
+    elif backend == "modin":  # pragma: no cover
+        import modin
+        out["modin"] = modin.__version__
+
     if not prnt:
         return out
 
-    keylen = max(map(len, out._fields))
-    for key in out._fields:
+    keylen = max(map(len, out))
+    for key in out:
         ver = getattr(out, key)
         verlines = ver.splitlines()
         print(f"{key.ljust(keylen)}: {verlines.pop(0)}")

diff --git a/datar/base/arithmetic.py b/datar/base/arithmetic.py
@@ -4,10 +4,10 @@
 from typing import TYPE_CHECKING, Union
 
 import numpy as np
-import pandas as pd
-from pandas import DataFrame, Series
-from pandas.api.types import is_scalar
-from pandas.core.groupby import SeriesGroupBy, GroupBy
+from ..core.backends import pandas as pd
+from ..core.backends.pandas import DataFrame, Series
+from ..core.backends.pandas.api.types import is_scalar
+from ..core.backends.pandas.core.groupby import SeriesGroupBy, GroupBy
 
 from ..core.factory import func_factory, verb_factory
 from ..core.tibble import Tibble, TibbleGrouped

diff --git a/datar/base/casting.py b/datar/base/casting.py
@@ -1,10 +1,12 @@
 """Cast values between types"""
 import numpy as np
-import pandas as pd
-from pandas.api.types import is_scalar, is_categorical_dtype
-from pandas.core.groupby import SeriesGroupBy
 from pipda import register_func
 
+from ..core.backends import pandas as pd
+
+from ..core.backends.pandas.api.types import is_scalar, is_categorical_dtype
+from ..core.backends.pandas.core.groupby import SeriesGroupBy
+
 from ..core.contexts import Context
 
 from .factor import _ensure_categorical

diff --git a/datar/base/complex.py b/datar/base/complex.py
@@ -1,8 +1,9 @@
 """Functions related to complex numbers"""
 import numpy as np
-from pandas.api.types import is_complex_dtype
 from pipda import register_func
 
+from ..core.backends.pandas.api.types import is_complex_dtype
+
 from ..core.contexts import Context
 from ..core.factory import func_factory
 

diff --git a/datar/base/cum.py b/datar/base/cum.py
@@ -1,9 +1,9 @@
 """Cumulative functions"""
 import numpy as np
-from pandas.core.groupby import GroupBy
 
-from datar.core.tibble import TibbleGrouped
+from ..core.backends.pandas.core.groupby import GroupBy
 
+from ..core.tibble import TibbleGrouped
 from ..core.factory import func_factory
 from .arithmetic import SINGLE_ARG_SIGNATURE
 

diff --git a/datar/base/date.py b/datar/base/date.py
@@ -3,10 +3,9 @@
 import functools
 
 import numpy as np
-import pandas as pd
-# from pandas import Series
-from pandas.api.types import is_scalar, is_integer
 
+from ..core.backends import pandas as pd
+from ..core.backends.pandas.api.types import is_scalar, is_integer
 from ..core.factory import func_factory
 
 

diff --git a/datar/base/factor.py b/datar/base/factor.py
@@ -4,10 +4,11 @@
 R's factors support NAs in levels but Categorical cannot have NAs in categories.
 """
 import numpy as np
-from pandas import Categorical, Series
-from pandas.api.types import is_categorical_dtype, is_scalar
 from pipda import register_func
 
+from ..core.backends.pandas import Categorical, Series
+from ..core.backends.pandas.api.types import is_categorical_dtype, is_scalar
+
 from ..core.contexts import Context
 
 

diff --git a/datar/base/funs.py b/datar/base/funs.py
@@ -6,11 +6,12 @@
 import itertools
 
 import numpy as np
-import pandas as pd
-from pandas.api.types import is_scalar
-from pandas.core.groupby import SeriesGroupBy
 from pipda import register_func
 
+from ..core.backends import pandas as pd
+from ..core.backends.pandas.api.types import is_scalar
+from ..core.backends.pandas.core.groupby import SeriesGroupBy
+
 from ..core.middlewares import WithDataEnv
 from ..core.contexts import Context
 from ..core.factory import func_factory

diff --git a/datar/base/logical.py b/datar/base/logical.py
@@ -1,8 +1,8 @@
 """Logical/Boolean functions"""
 import numpy as np
-from pandas.api.types import is_scalar, is_bool_dtype
 from pipda import register_func
 
+from ..core.backends.pandas.api.types import is_scalar, is_bool_dtype
 from ..core.contexts import Context
 
 from .testing import _register_type_testing

diff --git a/datar/base/na.py b/datar/base/na.py
@@ -1,7 +1,7 @@
 """NA related constants or functions"""
 import numpy as np
-import pandas as pd
 
+from ..core.backends import pandas as pd
 from ..core.defaults import NA_REPR
 from ..core.factory import func_factory
 

diff --git a/datar/base/rep.py b/datar/base/rep.py
@@ -1,12 +1,13 @@
 from functools import singledispatch
 
 import numpy as np
-import pandas as pd
-from pandas import DataFrame, Series, Categorical
-from pandas.api.types import is_scalar, is_integer
-from pandas.core.groupby import SeriesGroupBy
 from pipda import register_func
 
+from ..core.backends import pandas as pd
+from ..core.backends.pandas import DataFrame, Series, Categorical
+from ..core.backends.pandas.api.types import is_scalar, is_integer
+from ..core.backends.pandas.core.groupby import SeriesGroupBy
+
 from ..core.contexts import Context
 from ..core.tibble import TibbleGrouped, reconstruct_tibble
 from ..core.utils import ensure_nparray, logger

diff --git a/datar/base/seq.py b/datar/base/seq.py
@@ -1,9 +1,10 @@
 import numpy as np
-from pandas import DataFrame, Series
-from pandas.api.types import is_scalar
-from pandas.core.groupby import SeriesGroupBy, GroupBy
 from pipda import register_func
 
+from ..core.backends.pandas import DataFrame, Series
+from ..core.backends.pandas.api.types import is_scalar
+from ..core.backends.pandas.core.groupby import SeriesGroupBy, GroupBy
+
 from ..core.utils import logger, regcall
 from ..core.factory import func_factory
 from ..core.contexts import Context

diff --git a/datar/base/special.py b/datar/base/special.py
@@ -3,7 +3,8 @@
 from pipda import register_func
 
 import numpy as np
-from pandas.api.types import is_scalar
+
+from ..core.backends.pandas.api.types import is_scalar
 from ..core.contexts import Context
 from .bessel import _get_special_func_from_scipy
 

diff --git a/datar/base/stats.py b/datar/base/stats.py
@@ -3,11 +3,11 @@
 from functools import singledispatch
 
 import numpy as np
-from pandas import DataFrame, Series
-from pandas.api.types import is_scalar
 
-from datar.core.tibble import TibbleGrouped, TibbleRowwise
+from ..core.backends.pandas import DataFrame, Series
+from ..core.backends.pandas.api.types import is_scalar
 
+from ..core.tibble import TibbleGrouped, TibbleRowwise
 from ..core.factory import func_factory
 
 

diff --git a/datar/base/string.py b/datar/base/string.py
@@ -2,13 +2,13 @@
 import re
 
 import numpy as np
-import pandas as pd
-from pandas import Series
-from pandas.core.base import PandasObject
-from pandas.core.groupby import SeriesGroupBy
-from pandas.api.types import is_string_dtype, is_scalar
 from pipda import register_func
 
+from ..core.backends import pandas as pd
+from ..core.backends.pandas import Series
+from ..core.backends.pandas.core.base import PandasObject
+from ..core.backends.pandas.core.groupby import SeriesGroupBy
+from ..core.backends.pandas.api.types import is_string_dtype, is_scalar
 
 from ..core.tibble import TibbleGrouped, TibbleRowwise
 from ..core.contexts import Context

diff --git a/datar/base/table.py b/datar/base/table.py
@@ -1,10 +1,11 @@
 """Port `table` function from r-base"""
 import numpy as np
-import pandas as pd
-from pandas import DataFrame, Series
-from pandas.api.types import is_scalar, is_categorical_dtype
 from pipda import register_func
 
+from ..core.backends import pandas as pd
+from ..core.backends.pandas import DataFrame, Series
+from ..core.backends.pandas.api.types import is_scalar, is_categorical_dtype
+
 from ..core.contexts import Context
 from ..core.utils import ensure_nparray, regcall
 from ..core.defaults import NA_REPR