Merge pull request #127 from capitalone/develop

Release v0.8.0
capitalone · Mar 2, 2022 · fa94a4f · fa94a4f
2 parents 89170ef + d4f7e4c
commit fa94a4f
Show file tree

Hide file tree

Showing 10 changed files with 116 additions and 46 deletions.
diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml
@@ -0,0 +1,32 @@
+# This workflow builds and publishes the latest docs to
+# the `gh-pages` branch.
+# For more details: https://github.com/marketplace/actions/deploy-to-github-pages
+name: Publish docs
+
+on:
+  release:
+    types: [created]
+  workflow_dispatch:
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        # fetch all tags so `versioneer` can properly determine current version
+        fetch-depth: 0
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+    - name: Install dependencies
+      run: python -m pip install .[dev]
+    - name: Build
+      run: make sphinx
+      shell: bash
+    - name: Publish
+      uses: JamesIves/github-pages-deploy-action@4.0.0
+      with:
+        branch: gh-pages
+        folder: docs/build/html
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -16,16 +16,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
-        spark: [2.4.8, 3.0.3, 3.1.2]
-        exclude:
-          - python-version: 3.8
-            spark: 2.4.8
-          - python-version: 3.9
-            spark: 2.4.8
+        python-version: [3.7, 3.8, 3.9, '3.10']
+        spark-version: [3.0.3, 3.1.2, 3.2.0]
+        hadoop: [3.2]
+        include:
+          - python-version: 3.7
+            spark-version: 2.4.8
+            hadoop: 2.7
     env:
       PYTHON_VERSION: ${{ matrix.python-version }} 
-      SPARK_VERSION: ${{ matrix.spark }}
+      SPARK_VERSION: ${{ matrix.spark-version }}
 
     steps:
     - uses: actions/checkout@v2
@@ -40,38 +40,19 @@ jobs:
       with:
         java-version: 1.8
 
-    - name: Install Spark (2.4.x)
-      if: matrix.spark == '2.4.8'
+    - name: Install Spark
       run: |
-        wget -q -O spark.tgz https://archive.apache.org/dist/spark/spark-${{ matrix.spark }}/spark-${{ matrix.spark }}-bin-hadoop2.7.tgz
+        wget -q -O spark.tgz https://archive.apache.org/dist/spark/spark-${{ matrix.spark-version }}/spark-${{ matrix.spark-version }}-bin-hadoop${{ matrix.hadoop }}.tgz
         tar xzf spark.tgz
         rm spark.tgz
-        echo "SPARK_HOME=${{ runner.workspace }}/datacompy/spark-${{ matrix.spark }}-bin-hadoop2.7" >> $GITHUB_ENV
-        echo "${{ runner.workspace }}/datacompy/spark-${{ matrix.spark }}-bin-hadoop2.7/bin" >> $GITHUB_PATH
-
-    - name: Install Spark (3.0.x)
-      if: matrix.spark == '3.0.3'
-      run: |
-        wget -q -O spark.tgz https://archive.apache.org/dist/spark/spark-${{ matrix.spark }}/spark-${{ matrix.spark }}-bin-hadoop3.2.tgz
-        tar xzf spark.tgz
-        rm spark.tgz
-        echo "SPARK_HOME=${{ runner.workspace }}/datacompy/spark-${{ matrix.spark }}-bin-hadoop3.2" >> $GITHUB_ENV
-        echo "${{ runner.workspace }}/datacompy/spark-${{ matrix.spark }}-bin-hadoop3.2/bin" >> $GITHUB_PATH
-
-    - name: Install Spark (3.1.x)
-      if: matrix.spark == '3.1.2'
-      run: |
-        wget -q -O spark.tgz https://archive.apache.org/dist/spark/spark-${{ matrix.spark }}/spark-${{ matrix.spark }}-bin-hadoop3.2.tgz
-        tar xzf spark.tgz
-        rm spark.tgz
-        echo "SPARK_HOME=${{ runner.workspace }}/datacompy/spark-${{ matrix.spark }}-bin-hadoop3.2" >> $GITHUB_ENV
-        echo "${{ runner.workspace }}/datacompy/spark-${{ matrix.spark }}-bin-hadoop3.2/bin" >> $GITHUB_PATH
+        echo "SPARK_HOME=${{ runner.workspace }}/datacompy/spark-${{ matrix.spark-version }}-bin-hadoop${{ matrix.hadoop }}" >> $GITHUB_ENV
+        echo "${{ runner.workspace }}/datacompy/spark-${{ matrix.spark-version }}-bin-hadoop${{ matrix.hadoop }}/bin" >> $GITHUB_PATH
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         python -m pip install pytest pytest-spark pypandoc
-        python -m pip install pyspark==${{ matrix.spark }}
+        python -m pip install pyspark==${{ matrix.spark-version }}
         python -m pip install .[dev,spark]
     
     - name: Test with pytest

diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,7 @@ dist/
 *.egg-info/
 */__pycache__/
 .idea/
+.vscode/
 *.pyc
 .DS_Store
 .pytest_cache/
@@ -18,3 +19,6 @@ derby.log
 # Sphinx documentation
 docs/_build/
 docs/source/api/
+
+#edgetest
+.edgetest/
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -1 +1 @@
-*       @fdosani @elzzhu
+*       @fdosani @elzzhu @ak-gupta
diff --git a/datacompy/_version.py b/datacompy/_version.py
@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.7.3"
+__version__ = "0.8.0"
diff --git a/docs/source/developer_instructions.rst b/docs/source/developer_instructions.rst
@@ -50,6 +50,38 @@ Requirements of the project should be added to ``requirements.txt``.  Optional r
 documentation, or code quality are added to ``setup.py`` and ``EXTRAS_REQUIRE``
 
 
+
+edgetest
+--------
+
+edgetest is a utility to help keep requirements up to date and ensure a subset of testing requirements still work.
+More on edgetest `here <https://github.com/capitalone/edgetest>`_.
+
+The ``setup.cfg`` has configuration details on how to run edgetest. This process can be automated via GitHub Actions.
+(A future addition, which will come soon).
+
+In order to execute edgetest locally you can run the following after install ``edgetest``:
+
+.. code-block:: bash
+
+    edgetest -c setup.cfg -r requirements.txt --export
+
+This should return output like the following and also updating ``requirements.txt``:
+
+.. code-block:: bash
+
+    =============  ===============  ===================  =================
+    Environment    Passing tests    Upgraded packages    Package version
+    =============  ===============  ===================  =================
+    core           True             boto3                1.21.7
+    core           True             pandas               1.3.5
+    core           True             PyYAML               6.0
+    =============  ===============  ===================  =================
+    No PEP-517 style requirements in setup.cfg to update. Updating requirements.txt
+
+
+
+
 Release Guide
 -------------
 

diff --git a/pytest.ini b/pytest.ini
@@ -0,0 +1,5 @@
+[pytest]
+
+spark_options =
+    spark.sql.catalogImplementation: in-memory
+    spark.master: local
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,3 @@
-pandas>=0.25.0
-numpy>=1.11.3
-ordered-set>=4.0.2
+pandas<=1.4.1,>=0.25.0
+numpy<=1.22.2,>=1.11.3
+ordered-set<=4.1.0,>=4.0.2
diff --git a/setup.cfg b/setup.cfg
@@ -1,6 +1,24 @@
 [isort]
-multi_line_output=3
-include_trailing_comma=True
-force_grid_wrap=0
-use_parentheses=True
-line_length=88
+multi_line_output = 3
+include_trailing_comma = True
+force_grid_wrap = 0
+use_parentheses = True
+line_length = 88
+
+[options]
+install_requires = 
+
+[edgetest.envs.core]
+python_version = 3.9
+conda_install = 
+	openjdk=8
+extras = 
+	tests
+	spark
+command = 
+	pytest tests -m 'not integration'
+upgrade = 
+	pandas
+	numpy
+	ordered-set
+
diff --git a/setup.py b/setup.py
@@ -22,16 +22,14 @@
 EXTRAS_REQUIRE = {
     "spark": ["pyspark>=2.2.0"],
     "docs": ["sphinx", "sphinx_rtd_theme"],
-    "tests": [
-        "pytest",
-        "pytest-cov",
-    ],
+    "tests": ["pytest", "pytest-cov", "pytest-spark"],
     "qa": [
         "pre-commit",
         "black",
         "isort",
     ],
     "build": ["twine", "wheel"],
+    "edgetest": ["edgetest", "edgetest-conda"],
 }
 
 EXTRAS_REQUIRE["dev"] = (