diff --git a/README.md b/README.md index 491c13a7..2627c66c 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,22 @@ Please See [Documentation](https://gokart.readthedocs.io/en/latest/) . Have a good gokart life. +# Contributing to gokart + +install + +```bash +git clone https://github.com/m3dev/gokart.git # or your own fork + +poetry install --extras=s3 --extras=gcs --with=dev-s3 +``` + +test + +```bash +poetry run pytest # see `tox.ini` for specific extras +``` + # Achievements Gokart is a proven product. diff --git a/docs/intro_to_gokart.rst b/docs/intro_to_gokart.rst index ca49e65e..99957b39 100644 --- a/docs/intro_to_gokart.rst +++ b/docs/intro_to_gokart.rst @@ -10,6 +10,10 @@ Within the activated Python environment, use the following command to install go .. code:: sh pip install gokart + # or + pip install gokart[s3] # to use `s3://` + # or + pip install gokart[gcs] # to use `gs://` diff --git a/gokart/file_processor.py b/gokart/file_processor.py index 218d0dfa..f53d6e2c 100644 --- a/gokart/file_processor.py +++ b/gokart/file_processor.py @@ -5,7 +5,6 @@ from logging import getLogger import luigi -import luigi.contrib.s3 import luigi.format import numpy as np import pandas as pd diff --git a/gokart/gcs_config.py b/gokart/gcs_config.py index 01aca1a2..026fdea9 100644 --- a/gokart/gcs_config.py +++ b/gokart/gcs_config.py @@ -1,3 +1,9 @@ +try: + import googleapiclient # noqa: F401 +except ImportError: + # sentinal: this file should not be imported if [gcs] extra is not installed. + raise + import json import os diff --git a/gokart/object_storage.py b/gokart/object_storage.py index f867c82e..37091806 100644 --- a/gokart/object_storage.py +++ b/gokart/object_storage.py @@ -1,19 +1,47 @@ from datetime import datetime import luigi -import luigi.contrib.gcs -import luigi.contrib.s3 from luigi.format import Format -from gokart.gcs_config import GCSConfig -from gokart.gcs_zip_client import GCSZipClient -from gokart.s3_config import S3Config -from gokart.s3_zip_client import S3ZipClient from gokart.zip_client import ZipClient +try: + from gokart.gcs_config import GCSConfig + from gokart.gcs_zip_client import GCSZipClient + + # to avoid warning, import here which means gcs dependencies are exist + import luigi.contrib.gcs # isort: skip + GCS_AVAILABLE = True +except ImportError: + GCS_AVAILABLE = False + +try: + from gokart.s3_config import S3Config + from gokart.s3_zip_client import S3ZipClient + + # to avoid warning, import here which means s3 dependencies are exist + import luigi.contrib.s3 # isort: skip + S3_AVAILABLE = True +except ImportError: + S3_AVAILABLE = False + object_storage_path_prefix = ['s3://', 'gs://'] +def assert_gcs_available(): + if GCS_AVAILABLE: + return + + raise ImportError('gs:// is not available. You may need `pip install gokart[gcs]`') + + +def assert_s3_available(): + if S3_AVAILABLE: + return + + raise ImportError('s3:// is not available. You may need `pip install gokart[s3]`') + + class ObjectStorage(object): @staticmethod @@ -26,8 +54,10 @@ def if_object_storage_path(path: str) -> bool: @staticmethod def get_object_storage_target(path: str, format: Format) -> luigi.Target: if path.startswith('s3://'): + assert_s3_available() return luigi.contrib.s3.S3Target(path, client=S3Config().get_s3_client(), format=format) elif path.startswith('gs://'): + assert_gcs_available() return luigi.contrib.gcs.GCSTarget(path, client=GCSConfig().get_gcs_client(), format=format) else: raise @@ -35,8 +65,10 @@ def get_object_storage_target(path: str, format: Format) -> luigi.Target: @staticmethod def exists(path: str) -> bool: if path.startswith('s3://'): + assert_s3_available() return S3Config().get_s3_client().exists(path) elif path.startswith('gs://'): + assert_gcs_available() return GCSConfig().get_gcs_client().exists(path) else: raise @@ -44,8 +76,10 @@ def exists(path: str) -> bool: @staticmethod def get_timestamp(path: str) -> datetime: if path.startswith('s3://'): + assert_s3_available() return S3Config().get_s3_client().get_key(path).last_modified elif path.startswith('gs://'): + assert_gcs_available() # for gcs object # should PR to luigi bucket, obj = GCSConfig().get_gcs_client()._path_to_bucket_and_key(path) @@ -57,12 +91,14 @@ def get_timestamp(path: str) -> datetime: @staticmethod def get_zip_client(file_path: str, temporary_directory: str) -> ZipClient: if file_path.startswith('s3://'): + assert_s3_available() return S3ZipClient(file_path=file_path, temporary_directory=temporary_directory) elif file_path.startswith('gs://'): + assert_gcs_available() return GCSZipClient(file_path=file_path, temporary_directory=temporary_directory) else: raise @staticmethod def is_buffered_reader(file: object): - return not isinstance(file, luigi.contrib.s3.ReadableS3File) + return not (S3_AVAILABLE and isinstance(file, luigi.contrib.s3.ReadableS3File)) diff --git a/gokart/s3_config.py b/gokart/s3_config.py index 7c05f296..bf0c3d58 100644 --- a/gokart/s3_config.py +++ b/gokart/s3_config.py @@ -1,3 +1,9 @@ +try: + import boto3 # noqa: F401 +except ImportError: + # sentinal: this file should not be imported if [s3] extra is not installed. + raise + import os import luigi diff --git a/poetry.lock b/poetry.lock index 48c860f5..85559b36 100644 --- a/poetry.lock +++ b/poetry.lock @@ -496,6 +496,20 @@ files = [ {file = "docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b"}, ] +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "fakeredis" version = "2.20.1" @@ -617,7 +631,7 @@ woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] name = "google-api-core" version = "2.15.0" description = "Google API client core library" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "google-api-core-2.15.0.tar.gz", hash = "sha256:abc978a72658f14a2df1e5e12532effe40f94f868f6e23d95133bd6abcca35ca"}, @@ -639,7 +653,7 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] name = "google-api-python-client" version = "2.113.0" description = "Google API Client Library for Python" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "google-api-python-client-2.113.0.tar.gz", hash = "sha256:bcffbc8ffbad631f699cf85aa91993f3dc03060b234ca9e6e2f9135028bd9b52"}, @@ -657,7 +671,7 @@ uritemplate = ">=3.0.1,<5" name = "google-auth" version = "2.26.2" description = "Google Authentication Library" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "google-auth-2.26.2.tar.gz", hash = "sha256:97327dbbf58cccb58fc5a1712bba403ae76668e64814eb30f7316f7e27126b81"}, @@ -680,7 +694,7 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] name = "google-auth-httplib2" version = "0.2.0" description = "Google Authentication Library: httplib2 transport" -optional = false +optional = true python-versions = "*" files = [ {file = "google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05"}, @@ -695,7 +709,7 @@ httplib2 = ">=0.19.0" name = "googleapis-common-protos" version = "1.62.0" description = "Common protobufs used in Google APIs" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, @@ -712,7 +726,7 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] name = "httplib2" version = "0.22.0" description = "A comprehensive HTTP client library." -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ {file = "httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, @@ -770,6 +784,17 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-ruff", "zipp (>=3.17)"] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "isort" version = "5.13.2" @@ -1074,16 +1099,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1526,7 +1541,7 @@ testing = ["pytest", "pytest-benchmark"] name = "protobuf" version = "4.25.2" description = "" -optional = false +optional = true python-versions = ">=3.8" files = [ {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, @@ -1594,7 +1609,7 @@ numpy = ">=1.16.6" name = "pyasn1" version = "0.5.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, @@ -1605,7 +1620,7 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, @@ -1696,6 +1711,28 @@ files = [ flake8 = "6.1.0" tomli = {version = "*", markers = "python_version < \"3.11\""} +[[package]] +name = "pytest" +version = "7.4.4" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + [[package]] name = "python-daemon" version = "3.0.1" @@ -1753,6 +1790,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1760,8 +1798,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1778,6 +1823,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1785,6 +1831,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1852,7 +1899,7 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" -optional = false +optional = true python-versions = ">=3.6,<4" files = [ {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, @@ -2238,7 +2285,11 @@ files = [ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] +[extras] +gcs = ["google-api-python-client"] +s3 = ["boto3"] + [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "1c87c1615797403371c0338f7384b6b8dfb57cf41b5ec7a9cf614d2972eb9011" +content-hash = "28219fd0098228bdda6dc5dbdd03eada85848129587e1fbc90c84bdfb1595aaa" diff --git a/pyproject.toml b/pyproject.toml index 771c613d..8225f186 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,15 +18,14 @@ pattern = "^(?P\\d+\\.\\d+\\.\\d+)" [tool.poetry.dependencies] python = ">=3.9,<3.13" luigi = "*" -boto3 = "*" +boto3 = {version = "*", optional = true} slack-sdk = "^3" pandas = "*" numpy = "*" tqdm = "*" -google-auth = "*" pyarrow = "*" uritemplate = "*" -google-api-python-client = "*" +google-api-python-client = {version = "*", optional = true} APScheduler = "*" redis = "*" matplotlib = "*" @@ -34,7 +33,6 @@ matplotlib = "*" [tool.poetry.group.dev.dependencies] pyproject-flake8 = "^6.1.0" tox = "*" -moto = "*" testfixtures = "*" coverage = "*" isort = "^5.7" @@ -44,6 +42,26 @@ lupa = "*" fakeredis = "*" mypy = "*" types-redis = "*" +pytest = "*" + +[tool.poetry.group.dev-s3] +optional = true +[tool.poetry.group.dev-s3.dependencies] +moto = "*" + +[tool.poetry.extras] +s3 = ["boto3"] +gcs = ["google-api-python-client"] + +[tool.pytest.ini_options] +addopts = "--strict-markers -m 'not no_gcs and not no_s3'" +testpaths = "test" +markers = [ + "gcs", + "s3", + "no_gcs", + "no_s3" +] [tool.flake8] # B006: Do not use mutable data structures for argument defaults. They are created during function definition time. All calls to the function reuse this one instance of that data structure, persisting changes between them. diff --git a/test/helpers.py b/test/helpers.py new file mode 100644 index 00000000..2b2aebd0 --- /dev/null +++ b/test/helpers.py @@ -0,0 +1,19 @@ +def safe_mock_s3(func): + """ + Annotations will be evaluated even if `pytes -m "not s3"` is specified. So + ``` + @pytest.mark.s3 + class TestS3(unittest.TestCase): + @mock_s3 + def test_foo(): + ``` + will raise an error if moto is not installed. + This decorator is used to avoid this error. + """ + + def wrapper(*args, **kwargs): + from moto import mock_s3 + with mock_s3(): + return func(*args, **kwargs) + + return wrapper diff --git a/test/test_gcs_config.py b/test/test_gcs_config.py index cb8b306d..aa8b4393 100644 --- a/test/test_gcs_config.py +++ b/test/test_gcs_config.py @@ -2,9 +2,15 @@ import unittest from unittest.mock import MagicMock, patch -from gokart.gcs_config import GCSConfig +import pytest +try: + from gokart.gcs_config import GCSConfig +except ImportError: + pass + +@pytest.mark.gcs class TestGCSConfig(unittest.TestCase): def test_get_gcs_client_without_gcs_credential_name(self): diff --git a/test/test_pyproject_extra.py b/test/test_pyproject_extra.py new file mode 100644 index 00000000..c4adac1e --- /dev/null +++ b/test/test_pyproject_extra.py @@ -0,0 +1,36 @@ +import unittest + +import pytest + + +class TestPyprojectExtra(unittest.TestCase): + + @pytest.mark.gcs + def test_gcs_installed(self): + try: + import googleapiclient # noqa: F401 + except ImportError: + raise Exception('googleapiclient should be installed') + + @pytest.mark.no_gcs + def test_no_gcs(self): + try: + import googleapiclient # noqa: F401 + raise Exception('googleapiclient should not be installed') + except ImportError: + pass + + @pytest.mark.s3 + def test_s3_installed(self): + try: + import boto3 # noqa: F401 + except ImportError: + raise Exception('boto3 should be installed') + + @pytest.mark.no_s3 + def test_no_s3(self): + try: + import boto3 # noqa: F401 + raise Exception('boto3 should not be installed') + except ImportError: + pass diff --git a/test/test_s3_config.py b/test/test_s3_config.py index dd74b911..2ad865ff 100644 --- a/test/test_s3_config.py +++ b/test/test_s3_config.py @@ -1,8 +1,14 @@ import unittest -from gokart.s3_config import S3Config +import pytest +try: + from gokart.s3_config import S3Config +except ImportError: + pass + +@pytest.mark.s3 class TestS3Config(unittest.TestCase): def test_get_same_s3_client(self): diff --git a/test/test_s3_zip_client.py b/test/test_s3_zip_client.py index c8bd8025..6f55f28f 100644 --- a/test/test_s3_zip_client.py +++ b/test/test_s3_zip_client.py @@ -2,22 +2,29 @@ import shutil import unittest -import boto3 -from moto import mock_s3 +import pytest -from gokart.s3_zip_client import S3ZipClient +from .helpers import safe_mock_s3 + +try: + import boto3 + + from gokart.s3_zip_client import S3ZipClient +except ImportError: + pass def _get_temporary_directory(): return os.path.abspath(os.path.join(os.path.dirname(__name__), 'temporary')) +@pytest.mark.s3 class TestS3ZipClient(unittest.TestCase): def tearDown(self): shutil.rmtree(_get_temporary_directory(), ignore_errors=True) - @mock_s3 + @safe_mock_s3 def test_make_archive(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') @@ -34,7 +41,7 @@ def test_make_archive(self): os.makedirs(temporary_directory, exist_ok=True) zip_client.make_archive() - @mock_s3 + @safe_mock_s3 def test_unpack_archive(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') diff --git a/test/test_target.py b/test/test_target.py index 781eb0cf..13c428b3 100644 --- a/test/test_target.py +++ b/test/test_target.py @@ -5,15 +5,21 @@ from datetime import datetime from unittest.mock import patch -import boto3 import numpy as np import pandas as pd +import pytest from matplotlib import pyplot -from moto import mock_s3 from gokart.file_processor import _ChunkedLargeFileReader from gokart.target import make_model_target, make_target +from .helpers import safe_mock_s3 + +try: + import boto3 +except ImportError: + pass + def _get_temporary_directory(): return os.path.abspath(os.path.join(os.path.dirname(__name__), 'temporary')) @@ -171,9 +177,10 @@ def test_dump_without_lock(self): wrap_with_lock_mock.assert_not_called() +@pytest.mark.s3 class S3TargetTest(unittest.TestCase): - @mock_s3 + @safe_mock_s3 def test_save_on_s3(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') @@ -187,7 +194,7 @@ def test_save_on_s3(self): self.assertEqual(loaded, obj) - @mock_s3 + @safe_mock_s3 def test_last_modified_time(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') @@ -200,7 +207,7 @@ def test_last_modified_time(self): t = target.last_modification_time() self.assertIsInstance(t, datetime) - @mock_s3 + @safe_mock_s3 def test_last_modified_time_without_file(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') @@ -238,7 +245,8 @@ def test_model_target_on_local(self): self.assertEqual(loaded, obj) - @mock_s3 + @pytest.mark.s3 + @safe_mock_s3 def test_model_target_on_s3(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') diff --git a/tox.ini b/tox.ini index 29fcdddb..b143585a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,19 @@ [tox] -envlist = py{39,310,311,312},yapf,isort,flake8,mypy +envlist = py{39,310,311,312}-{core,s3,gcs},yapf,isort,flake8,mypy isolated_build = true [testenv] -allowlist_externals = coverage +allowlist_externals = poetry, pytest skip_install = true -commands = coverage run -m unittest discover -s test +commands = + s3: poetry install --extras=s3 --with=dev-s3 + s3: pytest -v -m "s3" + + gcs: poetry install --extras=gcs + gcs: pytest -v -m "gcs" + + core: poetry install + core: pytest -v -m "not gcs and not s3" [testenv:yapf] allowlist_externals = yapf