Refactor requirements (#407)

* Refactor requirements

* Removing requirements files

* Adjust reqs

* Adjust tox

* remove requirements

* Remove redundant requirements/docs.txt files

* Remove deprecate extras and redundancy in testenv deps

* Restore jsonschema min version

* Fix tox testenv:min

* Move optional dependencies into extra[monitoring] for backwards compatibility

* Fix incorrect coverage file being uploaded to codecov

---------

Co-authored-by: mauricio.barg <>
Co-authored-by: Víctor Ruiz <victorruiz@zyte.com>

.github/workflows/main.yml

@@ -34,7 +34,9 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install tox codecov
-      - name: Run tests
+      - name: Run base tests
+        run: tox -e base
+      - name: Run all test
         run: tox -e ${{ matrix.tox || 'py' }}
       - name: Run docs and formatting tests
         if: ${{ matrix.python-version == 3.11 }}

docs/requirements-docs.txt

@@ -0,0 +1,3 @@
+sphinx
+sphinx-rtd-theme
+s3cmd

setup.py

@@ -1,21 +1,5 @@
 from setuptools import find_packages, setup
 
-test_requirements = [
-    "pytest>=2.7.0",
-    "pytest-cov",
-    "pytest-mock",
-    "jinja2",
-    "boto3",
-    "lxml",
-    "premailer",
-    "scrapinghub",
-    "scrapinghub-entrypoint-scrapy",
-    "scrapy",
-    "slack-sdk",
-    "twisted>=19.7.0",
-    "itemadapter",
-]
-
 setup(
     name="spidermon",
     version="1.19.0",
@@ -29,24 +13,25 @@
     package_data={"spidermon": ["VERSION"]},
     zip_safe=False,
     include_package_data=True,
-    install_requires=["jsonschema[format]>=3.2.0", "python-slugify"],
-    tests_require=test_requirements,
+    install_requires=[
+        "jsonschema[format]>=3.2.0",
+        "python-slugify",
+    ],
     extras_require={
         # Specific monitors and tools to support notifications and reports
         "monitoring": [
-            "scrapy",
             "Jinja2",
-            "scrapinghub",
-            "slack-sdk",
             "boto",
             "boto3",
+            "itemadapter",
             "premailer",
-            "sentry-sdk",
-        ],
-        # Tools to run the tests
-        "tests": test_requirements,
-        # Tools to build and publish the documentation
-        "docs": ["sphinx", "sphinx-rtd-theme", "s3cmd"],
+            "requests",
+            "scrapinghub",
+            "scrapinghub-entrypoint-scrapy",
+            "scrapy",
+            "sentry_sdk",
+            "slack_sdk",
+        ]
     },
     classifiers=[
         "Development Status :: 5 - Production/Stable",

tox.ini

@@ -3,19 +3,31 @@ envlist = min,py39,py310,py311
 skip_missing_interpreters = True
 
 [testenv]
+deps =
+    pytest
+    pytest-cov
+    pytest-mock
 extras =
-    tests
-    validation
+    monitoring
 commands = pytest -s --cov=spidermon --cov-report= {posargs:tests}
 
+[testenv:base]
+deps =
+    {[testenv]deps}
+    scrapy
+    Jinja2
+commands = pytest -s --ignore=./tests/contrib --ignore=./tests/utils/test_zyte.py {posargs:tests}
+
 [testenv:min]
 basepython = python3.8
 deps =
+    {[testenv]deps}
     jsonschema[format]==3.2.0
 
 [testenv:docs]
+deps = 
+    -r {toxinidir}/docs/requirements-docs.txt
 changedir = docs
-extras = docs
 commands =
     sphinx-build -nW -b html source/ {envtmpdir}/html
     sphinx-build -W -b linkcheck source/ {envtmpdir}/linkcheck