From 11ec73aaf686a987ff34513dc61d97c93c017940 Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 15 Dec 2023 00:04:43 +0900 Subject: [PATCH 01/12] optional deps --- poetry.lock | 56 +++++++++++++++++++++++++------------------------- pyproject.toml | 7 +++---- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/poetry.lock b/poetry.lock index 65c95272..3d28f615 100644 --- a/poetry.lock +++ b/poetry.lock @@ -72,7 +72,7 @@ tzdata = ["tzdata"] name = "boto3" version = "1.26.126" description = "The AWS SDK for Python" -optional = false +optional = true python-versions = ">= 3.7" files = [ {file = "boto3-1.26.126-py3-none-any.whl", hash = "sha256:a0a049d16dabf559bd86d4622a985549154c6918f779f373fb7588bd3cf1272d"}, @@ -91,7 +91,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.29.126" description = "Low-level, data-driven core of boto 3." -optional = false +optional = true python-versions = ">= 3.7" files = [ {file = "botocore-1.29.126-py3-none-any.whl", hash = "sha256:a2c5cedb26b364386d02f9ed0a3492da6a322a82e539e54600fbc7190a49660d"}, @@ -121,7 +121,7 @@ files = [ name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." -optional = false +optional = true python-versions = ">=3.6" files = [ {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, @@ -219,7 +219,7 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false +optional = true python-versions = ">=3.7.0" files = [ {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, @@ -604,7 +604,7 @@ woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] name = "google-api-core" version = "2.11.0" description = "Google API client core library" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "google-api-core-2.11.0.tar.gz", hash = "sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22"}, @@ -626,7 +626,7 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] name = "google-api-python-client" version = "2.86.0" description = "Google API Client Library for Python" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "google-api-python-client-2.86.0.tar.gz", hash = "sha256:3ca4e93821f4e9ac29b91ab0d9df168b42c8ad0fb8bff65b8c2ccb2d462b0464"}, @@ -644,7 +644,7 @@ uritemplate = ">=3.0.1,<5" name = "google-auth" version = "2.17.3" description = "Google Authentication Library" -optional = false +optional = true python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" files = [ {file = "google-auth-2.17.3.tar.gz", hash = "sha256:ce311e2bc58b130fddf316df57c9b3943c2a7b4f6ec31de9663a9333e4064efc"}, @@ -668,7 +668,7 @@ requests = ["requests (>=2.20.0,<3.0.0dev)"] name = "google-auth-httplib2" version = "0.1.0" description = "Google Authentication Library: httplib2 transport" -optional = false +optional = true python-versions = "*" files = [ {file = "google-auth-httplib2-0.1.0.tar.gz", hash = "sha256:a07c39fd632becacd3f07718dfd6021bf396978f03ad3ce4321d060015cc30ac"}, @@ -684,7 +684,7 @@ six = "*" name = "googleapis-common-protos" version = "1.59.0" description = "Common protobufs used in Google APIs" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "googleapis-common-protos-1.59.0.tar.gz", hash = "sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44"}, @@ -701,7 +701,7 @@ grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] name = "httplib2" version = "0.22.0" description = "A comprehensive HTTP client library." -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ {file = "httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, @@ -715,7 +715,7 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -optional = false +optional = true python-versions = ">=3.5" files = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, @@ -761,7 +761,7 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, @@ -778,7 +778,7 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, @@ -988,7 +988,7 @@ files = [ name = "markupsafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, @@ -1120,7 +1120,7 @@ files = [ name = "moto" version = "4.1.8" description = "" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "moto-4.1.8-py2.py3-none-any.whl", hash = "sha256:df5b52eff70bf125ee03ea72c4e01f2daff243796f984a534c3dee92a0b93522"}, @@ -1435,7 +1435,7 @@ testing = ["pytest", "pytest-benchmark"] name = "protobuf" version = "4.22.3" description = "" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "protobuf-4.22.3-cp310-abi3-win32.whl", hash = "sha256:8b54f56d13ae4a3ec140076c9d937221f887c8f64954673d46f63751209e839a"}, @@ -1505,7 +1505,7 @@ numpy = ">=1.16.6" name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, @@ -1516,7 +1516,7 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, @@ -1671,7 +1671,7 @@ tzdata = {version = "*", markers = "python_version >= \"3.6\""} name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -optional = false +optional = true python-versions = ">=3.6" files = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, @@ -1738,7 +1738,7 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)" name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, @@ -1759,7 +1759,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "responses" version = "0.23.1" description = "A utility library for mocking out the `requests` Python library." -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "responses-0.23.1-py3-none-any.whl", hash = "sha256:8a3a5915713483bf353b6f4079ba8b2a29029d1d1090a503c70b0dc5d9d0c7bd"}, @@ -1779,7 +1779,7 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" -optional = false +optional = true python-versions = ">=3.6,<4" files = [ {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, @@ -1793,7 +1793,7 @@ pyasn1 = ">=0.1.3" name = "s3transfer" version = "0.6.0" description = "An Amazon S3 Transfer Manager" -optional = false +optional = true python-versions = ">= 3.7" files = [ {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, @@ -1996,7 +1996,7 @@ cryptography = ">=35.0.0" name = "types-pyyaml" version = "6.0.12.9" description = "Typing stubs for PyYAML" -optional = false +optional = true python-versions = "*" files = [ {file = "types-PyYAML-6.0.12.9.tar.gz", hash = "sha256:c51b1bd6d99ddf0aa2884a7a328810ebf70a4262c292195d3f4f9a0005f9eeb6"}, @@ -2074,7 +2074,7 @@ files = [ name = "urllib3" version = "1.26.18" description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, @@ -2110,7 +2110,7 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "coverage-enable-subprocess name = "werkzeug" version = "3.0.1" description = "The comprehensive WSGI web application library." -optional = false +optional = true python-versions = ">=3.8" files = [ {file = "werkzeug-3.0.1-py3-none-any.whl", hash = "sha256:90a285dc0e42ad56b34e696398b8122ee4c681833fb35b8334a095d82c56da10"}, @@ -2127,7 +2127,7 @@ watchdog = ["watchdog (>=2.3)"] name = "xmltodict" version = "0.13.0" description = "Makes working with XML feel like you are working with JSON" -optional = false +optional = true python-versions = ">=3.4" files = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, @@ -2166,4 +2166,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "fbf0da16b562a99eed1f060a19087065d5bb1ffc9204f6fd57c897e018892bbe" +content-hash = "4336bfed27a5b95e8a369c933862b6497e2095adb4d3629fbb42def643c6fe14" diff --git a/pyproject.toml b/pyproject.toml index 7010e26d..67c7b95e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,15 +18,14 @@ pattern = "^(?P\\d+\\.\\d+\\.\\d+)" [tool.poetry.dependencies] python = ">=3.8,<3.12" luigi = "*" -boto3 = "*" +boto3 = {version = "*", optional = true} slack-sdk = "^3" pandas = "*" numpy = "*" tqdm = "*" -google-auth = "*" pyarrow = "*" uritemplate = "*" -google-api-python-client = "*" +google-api-python-client = {version = "*", optional = true} APScheduler = "*" redis = "*" matplotlib = "*" @@ -34,7 +33,7 @@ matplotlib = "*" [tool.poetry.group.dev.dependencies] pyproject-flake8 = "5.0.4" tox = "*" -moto = "*" +moto = {version = "*", optional = true} testfixtures = "*" coverage = "*" isort = "^5.7" From 5cf26e96bf20b54ae15fe761e16cbde6cebe72ac Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 15 Dec 2023 00:06:38 +0900 Subject: [PATCH 02/12] handle import error --- gokart/file_processor.py | 1 - gokart/gcs_config.py | 6 +++++ gokart/object_storage.py | 50 ++++++++++++++++++++++++++++++++++------ gokart/s3_config.py | 6 +++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/gokart/file_processor.py b/gokart/file_processor.py index 218d0dfa..f53d6e2c 100644 --- a/gokart/file_processor.py +++ b/gokart/file_processor.py @@ -5,7 +5,6 @@ from logging import getLogger import luigi -import luigi.contrib.s3 import luigi.format import numpy as np import pandas as pd diff --git a/gokart/gcs_config.py b/gokart/gcs_config.py index 01aca1a2..480dfc72 100644 --- a/gokart/gcs_config.py +++ b/gokart/gcs_config.py @@ -1,3 +1,9 @@ +try: + import google.auth + import httplib2 +except ImportError: + raise + import json import os diff --git a/gokart/object_storage.py b/gokart/object_storage.py index f867c82e..1c5d00f5 100644 --- a/gokart/object_storage.py +++ b/gokart/object_storage.py @@ -1,19 +1,47 @@ from datetime import datetime import luigi -import luigi.contrib.gcs -import luigi.contrib.s3 from luigi.format import Format -from gokart.gcs_config import GCSConfig -from gokart.gcs_zip_client import GCSZipClient -from gokart.s3_config import S3Config -from gokart.s3_zip_client import S3ZipClient from gokart.zip_client import ZipClient +try: + from gokart.gcs_config import GCSConfig + from gokart.gcs_zip_client import GCSZipClient + + # to avoid warning, import here which means gcs dependencies are exist + import luigi.contrib.gcs # isort: skip + GCS_AVAILABLE = True +except ImportError: + GCS_AVAILABLE = False + +try: + from gokart.s3_config import S3Config + from gokart.s3_zip_client import S3ZipClient + + # to avoid warning, import here which means s3 dependencies are exist + import luigi.contrib.s3 # isort: skip + S3_AVAILABLE = True +except ImportError: + S3_AVAILABLE = False + object_storage_path_prefix = ['s3://', 'gs://'] +def assert_gcs_available(): + if GCS_AVAILABLE: + return + + raise ImportError('gs:// is not available. Please install gokart[gcs]') + + +def assert_s3_available(): + if S3_AVAILABLE: + return + + raise ImportError('s3:// is not available. Please install gokart[s3]') + + class ObjectStorage(object): @staticmethod @@ -26,8 +54,10 @@ def if_object_storage_path(path: str) -> bool: @staticmethod def get_object_storage_target(path: str, format: Format) -> luigi.Target: if path.startswith('s3://'): + assert_s3_available() return luigi.contrib.s3.S3Target(path, client=S3Config().get_s3_client(), format=format) elif path.startswith('gs://'): + assert_gcs_available() return luigi.contrib.gcs.GCSTarget(path, client=GCSConfig().get_gcs_client(), format=format) else: raise @@ -35,8 +65,10 @@ def get_object_storage_target(path: str, format: Format) -> luigi.Target: @staticmethod def exists(path: str) -> bool: if path.startswith('s3://'): + assert_s3_available() return S3Config().get_s3_client().exists(path) elif path.startswith('gs://'): + assert_gcs_available() return GCSConfig().get_gcs_client().exists(path) else: raise @@ -44,8 +76,10 @@ def exists(path: str) -> bool: @staticmethod def get_timestamp(path: str) -> datetime: if path.startswith('s3://'): + assert_s3_available() return S3Config().get_s3_client().get_key(path).last_modified elif path.startswith('gs://'): + assert_gcs_available() # for gcs object # should PR to luigi bucket, obj = GCSConfig().get_gcs_client()._path_to_bucket_and_key(path) @@ -57,12 +91,14 @@ def get_timestamp(path: str) -> datetime: @staticmethod def get_zip_client(file_path: str, temporary_directory: str) -> ZipClient: if file_path.startswith('s3://'): + assert_s3_available() return S3ZipClient(file_path=file_path, temporary_directory=temporary_directory) elif file_path.startswith('gs://'): + assert_gcs_available() return GCSZipClient(file_path=file_path, temporary_directory=temporary_directory) else: raise @staticmethod def is_buffered_reader(file: object): - return not isinstance(file, luigi.contrib.s3.ReadableS3File) + return not (S3_AVAILABLE and isinstance(file, luigi.contrib.s3.ReadableS3File)) diff --git a/gokart/s3_config.py b/gokart/s3_config.py index 7c05f296..23b90520 100644 --- a/gokart/s3_config.py +++ b/gokart/s3_config.py @@ -1,3 +1,9 @@ +try: + import botocore + from boto3.s3.transfer import TransferConfig +except ImportError: + raise + import os import luigi From 53dbd77975db375788b6d7a7b39e9a0b700ef3ca Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 15 Dec 2023 02:33:14 +0900 Subject: [PATCH 03/12] mark tests --- poetry.lock | 49 +++++++++++++++++++++++++++++++++++++- pyproject.toml | 9 +++++++ test/helpers.py | 24 +++++++++++++++++++ test/test_gcs_config.py | 8 ++++++- test/test_s3_config.py | 8 ++++++- test/test_s3_zip_client.py | 17 +++++++++---- test/test_target.py | 20 +++++++++++----- 7 files changed, 121 insertions(+), 14 deletions(-) create mode 100644 test/helpers.py diff --git a/poetry.lock b/poetry.lock index 3d28f615..8b1c59fc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -525,6 +525,20 @@ files = [ {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "fakeredis" version = "2.11.2" @@ -740,6 +754,17 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "isort" version = "5.12.0" @@ -1607,6 +1632,28 @@ files = [ flake8 = "5.0.4" tomli = {version = "*", markers = "python_version < \"3.11\""} +[[package]] +name = "pytest" +version = "7.4.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, + {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + [[package]] name = "python-daemon" version = "3.0.1" @@ -2166,4 +2213,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "4336bfed27a5b95e8a369c933862b6497e2095adb4d3629fbb42def643c6fe14" +content-hash = "7a636459b0bb807d529a886a02b9dc58955043940a8309d0b2f2b2c0dff21eb0" diff --git a/pyproject.toml b/pyproject.toml index 67c7b95e..49c22122 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ google-api-python-client = {version = "*", optional = true} APScheduler = "*" redis = "*" matplotlib = "*" +pytest = "*" [tool.poetry.group.dev.dependencies] pyproject-flake8 = "5.0.4" @@ -44,6 +45,14 @@ fakeredis = "*" mypy = "*" types-redis = "*" +[tool.pytest.ini_options] +addopts = "--strict-markers" +testpaths = "test" +markers = [ + "gcs", + "s3", +] + [tool.flake8] # B006: Do not use mutable data structures for argument defaults. They are created during function definition time. All calls to the function reuse this one instance of that data structure, persisting changes between them. # B008 Do not perform function calls in argument defaults. The call is performed only once at function definition time. All calls to your function will reuse the result of that definition-time function call. If this is intended, assign the function call to a module-level variable and use that variable as a default value. diff --git a/test/helpers.py b/test/helpers.py new file mode 100644 index 00000000..8ab8639e --- /dev/null +++ b/test/helpers.py @@ -0,0 +1,24 @@ +def safe_mock_s3(func): + """ + Annotations will be evaluated even if `pytes -m "not s3"` is specified. So + ``` + @pytest.mark.s3 + class TestS3(unittest.TestCase): + @mock_s3 + def test_foo(): + ``` + will raise an error if moto is not installed. + This decorator is used to avoid this error. + """ + + # try: + # from moto import mock_s3 + # return mock_s3(func) + # except ImportError: + # return func + def wrapper(*args, **kwargs): + from moto import mock_s3 + with mock_s3(): + return func(*args, **kwargs) + + return wrapper diff --git a/test/test_gcs_config.py b/test/test_gcs_config.py index cb8b306d..aa8b4393 100644 --- a/test/test_gcs_config.py +++ b/test/test_gcs_config.py @@ -2,9 +2,15 @@ import unittest from unittest.mock import MagicMock, patch -from gokart.gcs_config import GCSConfig +import pytest +try: + from gokart.gcs_config import GCSConfig +except ImportError: + pass + +@pytest.mark.gcs class TestGCSConfig(unittest.TestCase): def test_get_gcs_client_without_gcs_credential_name(self): diff --git a/test/test_s3_config.py b/test/test_s3_config.py index dd74b911..2ad865ff 100644 --- a/test/test_s3_config.py +++ b/test/test_s3_config.py @@ -1,8 +1,14 @@ import unittest -from gokart.s3_config import S3Config +import pytest +try: + from gokart.s3_config import S3Config +except ImportError: + pass + +@pytest.mark.s3 class TestS3Config(unittest.TestCase): def test_get_same_s3_client(self): diff --git a/test/test_s3_zip_client.py b/test/test_s3_zip_client.py index c8bd8025..6f55f28f 100644 --- a/test/test_s3_zip_client.py +++ b/test/test_s3_zip_client.py @@ -2,22 +2,29 @@ import shutil import unittest -import boto3 -from moto import mock_s3 +import pytest -from gokart.s3_zip_client import S3ZipClient +from .helpers import safe_mock_s3 + +try: + import boto3 + + from gokart.s3_zip_client import S3ZipClient +except ImportError: + pass def _get_temporary_directory(): return os.path.abspath(os.path.join(os.path.dirname(__name__), 'temporary')) +@pytest.mark.s3 class TestS3ZipClient(unittest.TestCase): def tearDown(self): shutil.rmtree(_get_temporary_directory(), ignore_errors=True) - @mock_s3 + @safe_mock_s3 def test_make_archive(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') @@ -34,7 +41,7 @@ def test_make_archive(self): os.makedirs(temporary_directory, exist_ok=True) zip_client.make_archive() - @mock_s3 + @safe_mock_s3 def test_unpack_archive(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') diff --git a/test/test_target.py b/test/test_target.py index 781eb0cf..10107b59 100644 --- a/test/test_target.py +++ b/test/test_target.py @@ -5,15 +5,21 @@ from datetime import datetime from unittest.mock import patch -import boto3 import numpy as np import pandas as pd +import pytest from matplotlib import pyplot -from moto import mock_s3 from gokart.file_processor import _ChunkedLargeFileReader from gokart.target import make_model_target, make_target +from .helpers import safe_mock_s3 + +try: + import boto3 +except: + pass + def _get_temporary_directory(): return os.path.abspath(os.path.join(os.path.dirname(__name__), 'temporary')) @@ -171,9 +177,10 @@ def test_dump_without_lock(self): wrap_with_lock_mock.assert_not_called() +@pytest.mark.s3 class S3TargetTest(unittest.TestCase): - @mock_s3 + @safe_mock_s3 def test_save_on_s3(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') @@ -187,7 +194,7 @@ def test_save_on_s3(self): self.assertEqual(loaded, obj) - @mock_s3 + @safe_mock_s3 def test_last_modified_time(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') @@ -200,7 +207,7 @@ def test_last_modified_time(self): t = target.last_modification_time() self.assertIsInstance(t, datetime) - @mock_s3 + @safe_mock_s3 def test_last_modified_time_without_file(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') @@ -238,7 +245,8 @@ def test_model_target_on_local(self): self.assertEqual(loaded, obj) - @mock_s3 + @pytest.mark.s3 + @safe_mock_s3 def test_model_target_on_s3(self): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket='test') From b3c4d535e39eaff1b4648bf2c97a2d3ef56f1bf8 Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 15 Dec 2023 02:48:11 +0900 Subject: [PATCH 04/12] pyproject extras --- poetry.lock | 6 +++++- pyproject.toml | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 8b1c59fc..80ccd49b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2210,7 +2210,11 @@ files = [ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +[extras] +gcs = ["google-api-python-client"] +s3 = ["boto3"] + [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "7a636459b0bb807d529a886a02b9dc58955043940a8309d0b2f2b2c0dff21eb0" +content-hash = "5a93c96e799aa19380253645587a829a56c210bd139aed3cc5d8ab6db592875f" diff --git a/pyproject.toml b/pyproject.toml index 49c22122..b36e3549 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,10 @@ fakeredis = "*" mypy = "*" types-redis = "*" +[tool.poetry.extras] +s3 = ["boto3"] +gcs = ["google-api-python-client"] + [tool.pytest.ini_options] addopts = "--strict-markers" testpaths = "test" From a942821b963ff5f8de3036ab99bc65f2bc4a4dcb Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 15 Dec 2023 02:48:39 +0900 Subject: [PATCH 05/12] fix tox --- .github/workflows/test.yml | 2 +- tox.ini | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 646db9c8..03986010 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,4 +27,4 @@ jobs: poetry run python -m pip install tox-gh-actions poetry install - name: Test with tox - run: poetry run tox + run: poetry run tox -v diff --git a/tox.ini b/tox.ini index 08a0801f..e38a7515 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,18 @@ [tox] -envlist = py{38,39,310,311},yapf,isort,flake8,mypy +envlist = py{38,39,310,311}-{core,s3,gcs},yapf,isort,flake8,mypy isolated_build = true [testenv] -allowlist_externals = coverage +allowlist_externals = poetry, pytest skip_install = true -commands = coverage run -m unittest discover -s test +commands = + s3: poetry install -E s3 + s3: pytest -v -m "s3" + + gcs: poetry install -E gcs + gcs: pytest -v -m "gcs" + + core: pytest -v -m "not gcs and not s3" [testenv:yapf] allowlist_externals = yapf From c1e718493431b4b9d7221fb7691c53b8ecb8aa19 Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 15 Dec 2023 03:06:34 +0900 Subject: [PATCH 06/12] fix s3 dev deps --- poetry.lock | 36 ++++++++++++++++++------------------ pyproject.toml | 6 +++++- tox.ini | 2 +- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/poetry.lock b/poetry.lock index 80ccd49b..015c297b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -72,7 +72,7 @@ tzdata = ["tzdata"] name = "boto3" version = "1.26.126" description = "The AWS SDK for Python" -optional = true +optional = false python-versions = ">= 3.7" files = [ {file = "boto3-1.26.126-py3-none-any.whl", hash = "sha256:a0a049d16dabf559bd86d4622a985549154c6918f779f373fb7588bd3cf1272d"}, @@ -91,7 +91,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.29.126" description = "Low-level, data-driven core of boto 3." -optional = true +optional = false python-versions = ">= 3.7" files = [ {file = "botocore-1.29.126-py3-none-any.whl", hash = "sha256:a2c5cedb26b364386d02f9ed0a3492da6a322a82e539e54600fbc7190a49660d"}, @@ -121,7 +121,7 @@ files = [ name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." -optional = true +optional = false python-versions = ">=3.6" files = [ {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, @@ -219,7 +219,7 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = true +optional = false python-versions = ">=3.7.0" files = [ {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, @@ -729,7 +729,7 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -optional = true +optional = false python-versions = ">=3.5" files = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, @@ -786,7 +786,7 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, @@ -803,7 +803,7 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, @@ -1013,7 +1013,7 @@ files = [ name = "markupsafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, @@ -1145,7 +1145,7 @@ files = [ name = "moto" version = "4.1.8" description = "" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "moto-4.1.8-py2.py3-none-any.whl", hash = "sha256:df5b52eff70bf125ee03ea72c4e01f2daff243796f984a534c3dee92a0b93522"}, @@ -1718,7 +1718,7 @@ tzdata = {version = "*", markers = "python_version >= \"3.6\""} name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -optional = true +optional = false python-versions = ">=3.6" files = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, @@ -1785,7 +1785,7 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)" name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, @@ -1806,7 +1806,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "responses" version = "0.23.1" description = "A utility library for mocking out the `requests` Python library." -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "responses-0.23.1-py3-none-any.whl", hash = "sha256:8a3a5915713483bf353b6f4079ba8b2a29029d1d1090a503c70b0dc5d9d0c7bd"}, @@ -1840,7 +1840,7 @@ pyasn1 = ">=0.1.3" name = "s3transfer" version = "0.6.0" description = "An Amazon S3 Transfer Manager" -optional = true +optional = false python-versions = ">= 3.7" files = [ {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, @@ -2043,7 +2043,7 @@ cryptography = ">=35.0.0" name = "types-pyyaml" version = "6.0.12.9" description = "Typing stubs for PyYAML" -optional = true +optional = false python-versions = "*" files = [ {file = "types-PyYAML-6.0.12.9.tar.gz", hash = "sha256:c51b1bd6d99ddf0aa2884a7a328810ebf70a4262c292195d3f4f9a0005f9eeb6"}, @@ -2121,7 +2121,7 @@ files = [ name = "urllib3" version = "1.26.18" description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = true +optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, @@ -2157,7 +2157,7 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "coverage-enable-subprocess name = "werkzeug" version = "3.0.1" description = "The comprehensive WSGI web application library." -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "werkzeug-3.0.1-py3-none-any.whl", hash = "sha256:90a285dc0e42ad56b34e696398b8122ee4c681833fb35b8334a095d82c56da10"}, @@ -2174,7 +2174,7 @@ watchdog = ["watchdog (>=2.3)"] name = "xmltodict" version = "0.13.0" description = "Makes working with XML feel like you are working with JSON" -optional = true +optional = false python-versions = ">=3.4" files = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, @@ -2217,4 +2217,4 @@ s3 = ["boto3"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "5a93c96e799aa19380253645587a829a56c210bd139aed3cc5d8ab6db592875f" +content-hash = "1144737be88961eadae4152c6d9ab1cf0ed62c0548ec24b8ff819f0535695a4a" diff --git a/pyproject.toml b/pyproject.toml index b36e3549..3cc6f4c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,6 @@ pytest = "*" [tool.poetry.group.dev.dependencies] pyproject-flake8 = "5.0.4" tox = "*" -moto = {version = "*", optional = true} testfixtures = "*" coverage = "*" isort = "^5.7" @@ -45,6 +44,11 @@ fakeredis = "*" mypy = "*" types-redis = "*" +[tool.poetry.group.dev-s3] +optional = true +[tool.poetry.group.dev-s3.dependencies] +moto = "*" + [tool.poetry.extras] s3 = ["boto3"] gcs = ["google-api-python-client"] diff --git a/tox.ini b/tox.ini index e38a7515..1597e187 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ isolated_build = true allowlist_externals = poetry, pytest skip_install = true commands = - s3: poetry install -E s3 + s3: poetry install -E s3 --with=dev-s3 s3: pytest -v -m "s3" gcs: poetry install -E gcs From cfc7932fa3b9bf205511014c81c171ba1b94ad0a Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 15 Dec 2023 03:28:38 +0900 Subject: [PATCH 07/12] fix lint --- gokart/gcs_config.py | 4 ++-- gokart/s3_config.py | 4 ++-- test/test_target.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gokart/gcs_config.py b/gokart/gcs_config.py index 480dfc72..026fdea9 100644 --- a/gokart/gcs_config.py +++ b/gokart/gcs_config.py @@ -1,7 +1,7 @@ try: - import google.auth - import httplib2 + import googleapiclient # noqa: F401 except ImportError: + # sentinal: this file should not be imported if [gcs] extra is not installed. raise import json diff --git a/gokart/s3_config.py b/gokart/s3_config.py index 23b90520..bf0c3d58 100644 --- a/gokart/s3_config.py +++ b/gokart/s3_config.py @@ -1,7 +1,7 @@ try: - import botocore - from boto3.s3.transfer import TransferConfig + import boto3 # noqa: F401 except ImportError: + # sentinal: this file should not be imported if [s3] extra is not installed. raise import os diff --git a/test/test_target.py b/test/test_target.py index 10107b59..13c428b3 100644 --- a/test/test_target.py +++ b/test/test_target.py @@ -17,7 +17,7 @@ try: import boto3 -except: +except ImportError: pass From 19401ab0c0dd65ca2e6885244cc67f75411ac5f7 Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 15 Dec 2023 03:36:36 +0900 Subject: [PATCH 08/12] clean --- .github/workflows/test.yml | 2 +- test/helpers.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03986010..646db9c8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,4 +27,4 @@ jobs: poetry run python -m pip install tox-gh-actions poetry install - name: Test with tox - run: poetry run tox -v + run: poetry run tox diff --git a/test/helpers.py b/test/helpers.py index 8ab8639e..2b2aebd0 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -11,11 +11,6 @@ def test_foo(): This decorator is used to avoid this error. """ - # try: - # from moto import mock_s3 - # return mock_s3(func) - # except ImportError: - # return func def wrapper(*args, **kwargs): from moto import mock_s3 with mock_s3(): From 6ef8a912bdf1debddc2301d59999f0ada72d1a92 Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 15 Dec 2023 08:26:19 +0900 Subject: [PATCH 09/12] test pyproject extra --- pyproject.toml | 4 +++- test/test_pyproject_extra.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 test/test_pyproject_extra.py diff --git a/pyproject.toml b/pyproject.toml index 3cc6f4c1..95426521 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,11 +54,13 @@ s3 = ["boto3"] gcs = ["google-api-python-client"] [tool.pytest.ini_options] -addopts = "--strict-markers" +addopts = "--strict-markers -m 'not no_gcs and not no_s3'" testpaths = "test" markers = [ "gcs", "s3", + "no_gcs", + "no_s3" ] [tool.flake8] diff --git a/test/test_pyproject_extra.py b/test/test_pyproject_extra.py new file mode 100644 index 00000000..c4adac1e --- /dev/null +++ b/test/test_pyproject_extra.py @@ -0,0 +1,36 @@ +import unittest + +import pytest + + +class TestPyprojectExtra(unittest.TestCase): + + @pytest.mark.gcs + def test_gcs_installed(self): + try: + import googleapiclient # noqa: F401 + except ImportError: + raise Exception('googleapiclient should be installed') + + @pytest.mark.no_gcs + def test_no_gcs(self): + try: + import googleapiclient # noqa: F401 + raise Exception('googleapiclient should not be installed') + except ImportError: + pass + + @pytest.mark.s3 + def test_s3_installed(self): + try: + import boto3 # noqa: F401 + except ImportError: + raise Exception('boto3 should be installed') + + @pytest.mark.no_s3 + def test_no_s3(self): + try: + import boto3 # noqa: F401 + raise Exception('boto3 should not be installed') + except ImportError: + pass From eae624f9868c2697e20d9e4f74348191cfea0cfb Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 15 Dec 2023 08:35:34 +0900 Subject: [PATCH 10/12] default install extra for dev --- pyproject.toml | 5 ++++- tox.ini | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 95426521..bb470400 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,8 +44,11 @@ fakeredis = "*" mypy = "*" types-redis = "*" +# for dev, extra is installed by default +[tool.poetry.group.dev-gcs] +[tool.poetry.group.dev-gcs.dependencies] +google-api-python-client = "*" [tool.poetry.group.dev-s3] -optional = true [tool.poetry.group.dev-s3.dependencies] moto = "*" diff --git a/tox.ini b/tox.ini index 1597e187..e27251e6 100644 --- a/tox.ini +++ b/tox.ini @@ -6,12 +6,13 @@ isolated_build = true allowlist_externals = poetry, pytest skip_install = true commands = - s3: poetry install -E s3 --with=dev-s3 + s3: poetry install -E s3 --without=dev-gcs s3: pytest -v -m "s3" - gcs: poetry install -E gcs + gcs: poetry install -E gcs --without=dev-gcs gcs: pytest -v -m "gcs" + core: poetry install --without=dev-s3 --without=dev-gcs core: pytest -v -m "not gcs and not s3" [testenv:yapf] From 84269d727d07631c6c3fc3ed2e4eb1ffec04d11e Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 15 Dec 2023 09:48:59 +0900 Subject: [PATCH 11/12] fix dev-extra --- poetry.lock | 22 +++++++++++----------- pyproject.toml | 5 +---- tox.ini | 6 +++--- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/poetry.lock b/poetry.lock index 015c297b..9e1b1925 100644 --- a/poetry.lock +++ b/poetry.lock @@ -618,7 +618,7 @@ woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] name = "google-api-core" version = "2.11.0" description = "Google API client core library" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "google-api-core-2.11.0.tar.gz", hash = "sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22"}, @@ -640,7 +640,7 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] name = "google-api-python-client" version = "2.86.0" description = "Google API Client Library for Python" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "google-api-python-client-2.86.0.tar.gz", hash = "sha256:3ca4e93821f4e9ac29b91ab0d9df168b42c8ad0fb8bff65b8c2ccb2d462b0464"}, @@ -658,7 +658,7 @@ uritemplate = ">=3.0.1,<5" name = "google-auth" version = "2.17.3" description = "Google Authentication Library" -optional = true +optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" files = [ {file = "google-auth-2.17.3.tar.gz", hash = "sha256:ce311e2bc58b130fddf316df57c9b3943c2a7b4f6ec31de9663a9333e4064efc"}, @@ -682,7 +682,7 @@ requests = ["requests (>=2.20.0,<3.0.0dev)"] name = "google-auth-httplib2" version = "0.1.0" description = "Google Authentication Library: httplib2 transport" -optional = true +optional = false python-versions = "*" files = [ {file = "google-auth-httplib2-0.1.0.tar.gz", hash = "sha256:a07c39fd632becacd3f07718dfd6021bf396978f03ad3ce4321d060015cc30ac"}, @@ -698,7 +698,7 @@ six = "*" name = "googleapis-common-protos" version = "1.59.0" description = "Common protobufs used in Google APIs" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "googleapis-common-protos-1.59.0.tar.gz", hash = "sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44"}, @@ -715,7 +715,7 @@ grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] name = "httplib2" version = "0.22.0" description = "A comprehensive HTTP client library." -optional = true +optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ {file = "httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, @@ -1460,7 +1460,7 @@ testing = ["pytest", "pytest-benchmark"] name = "protobuf" version = "4.22.3" description = "" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "protobuf-4.22.3-cp310-abi3-win32.whl", hash = "sha256:8b54f56d13ae4a3ec140076c9d937221f887c8f64954673d46f63751209e839a"}, @@ -1530,7 +1530,7 @@ numpy = ">=1.16.6" name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" -optional = true +optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, @@ -1541,7 +1541,7 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" -optional = true +optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, @@ -1826,7 +1826,7 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" -optional = true +optional = false python-versions = ">=3.6,<4" files = [ {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, @@ -2217,4 +2217,4 @@ s3 = ["boto3"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "1144737be88961eadae4152c6d9ab1cf0ed62c0548ec24b8ff819f0535695a4a" +content-hash = "d897197fe72338d13851c18b17c19e54d43e7297c4468cb3a969dc4dfc6b3f7c" diff --git a/pyproject.toml b/pyproject.toml index bb470400..8b019556 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,11 +45,8 @@ mypy = "*" types-redis = "*" # for dev, extra is installed by default -[tool.poetry.group.dev-gcs] -[tool.poetry.group.dev-gcs.dependencies] +[tool.poetry.group.dev-extra.dependencies] google-api-python-client = "*" -[tool.poetry.group.dev-s3] -[tool.poetry.group.dev-s3.dependencies] moto = "*" [tool.poetry.extras] diff --git a/tox.ini b/tox.ini index e27251e6..93ece72a 100644 --- a/tox.ini +++ b/tox.ini @@ -6,13 +6,13 @@ isolated_build = true allowlist_externals = poetry, pytest skip_install = true commands = - s3: poetry install -E s3 --without=dev-gcs + s3: poetry install -E s3 --without=dev-extra s3: pytest -v -m "s3" - gcs: poetry install -E gcs --without=dev-gcs + gcs: poetry install -E gcs --without=dev-extra gcs: pytest -v -m "gcs" - core: poetry install --without=dev-s3 --without=dev-gcs + core: poetry install --without=dev-extra core: pytest -v -m "not gcs and not s3" [testenv:yapf] From 3e060edb9c99fb389f21e2bb82ce6b6c0f032d5e Mon Sep 17 00:00:00 2001 From: yokomotod Date: Fri, 19 Jan 2024 16:20:30 +0900 Subject: [PATCH 12/12] fix --- README.md | 16 ++++++ docs/intro_to_gokart.rst | 4 ++ gokart/object_storage.py | 4 +- poetry.lock | 114 +++++++++++++++++++++------------------ pyproject.toml | 8 +-- tox.ini | 6 +-- 6 files changed, 91 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 491c13a7..2627c66c 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,22 @@ Please See [Documentation](https://gokart.readthedocs.io/en/latest/) . Have a good gokart life. +# Contributing to gokart + +install + +```bash +git clone https://github.com/m3dev/gokart.git # or your own fork + +poetry install --extras=s3 --extras=gcs --with=dev-s3 +``` + +test + +```bash +poetry run pytest # see `tox.ini` for specific extras +``` + # Achievements Gokart is a proven product. diff --git a/docs/intro_to_gokart.rst b/docs/intro_to_gokart.rst index ca49e65e..99957b39 100644 --- a/docs/intro_to_gokart.rst +++ b/docs/intro_to_gokart.rst @@ -10,6 +10,10 @@ Within the activated Python environment, use the following command to install go .. code:: sh pip install gokart + # or + pip install gokart[s3] # to use `s3://` + # or + pip install gokart[gcs] # to use `gs://` diff --git a/gokart/object_storage.py b/gokart/object_storage.py index 1c5d00f5..37091806 100644 --- a/gokart/object_storage.py +++ b/gokart/object_storage.py @@ -32,14 +32,14 @@ def assert_gcs_available(): if GCS_AVAILABLE: return - raise ImportError('gs:// is not available. Please install gokart[gcs]') + raise ImportError('gs:// is not available. You may need `pip install gokart[gcs]`') def assert_s3_available(): if S3_AVAILABLE: return - raise ImportError('s3:// is not available. Please install gokart[s3]') + raise ImportError('s3:// is not available. You may need `pip install gokart[s3]`') class ObjectStorage(object): diff --git a/poetry.lock b/poetry.lock index 9e1b1925..bc95498c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -618,7 +618,7 @@ woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] name = "google-api-core" version = "2.11.0" description = "Google API client core library" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "google-api-core-2.11.0.tar.gz", hash = "sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22"}, @@ -640,7 +640,7 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] name = "google-api-python-client" version = "2.86.0" description = "Google API Client Library for Python" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "google-api-python-client-2.86.0.tar.gz", hash = "sha256:3ca4e93821f4e9ac29b91ab0d9df168b42c8ad0fb8bff65b8c2ccb2d462b0464"}, @@ -658,7 +658,7 @@ uritemplate = ">=3.0.1,<5" name = "google-auth" version = "2.17.3" description = "Google Authentication Library" -optional = false +optional = true python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" files = [ {file = "google-auth-2.17.3.tar.gz", hash = "sha256:ce311e2bc58b130fddf316df57c9b3943c2a7b4f6ec31de9663a9333e4064efc"}, @@ -682,7 +682,7 @@ requests = ["requests (>=2.20.0,<3.0.0dev)"] name = "google-auth-httplib2" version = "0.1.0" description = "Google Authentication Library: httplib2 transport" -optional = false +optional = true python-versions = "*" files = [ {file = "google-auth-httplib2-0.1.0.tar.gz", hash = "sha256:a07c39fd632becacd3f07718dfd6021bf396978f03ad3ce4321d060015cc30ac"}, @@ -698,7 +698,7 @@ six = "*" name = "googleapis-common-protos" version = "1.59.0" description = "Common protobufs used in Google APIs" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "googleapis-common-protos-1.59.0.tar.gz", hash = "sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44"}, @@ -715,7 +715,7 @@ grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] name = "httplib2" version = "0.22.0" description = "A comprehensive HTTP client library." -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ {file = "httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, @@ -1460,7 +1460,7 @@ testing = ["pytest", "pytest-benchmark"] name = "protobuf" version = "4.22.3" description = "" -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "protobuf-4.22.3-cp310-abi3-win32.whl", hash = "sha256:8b54f56d13ae4a3ec140076c9d937221f887c8f64954673d46f63751209e839a"}, @@ -1530,7 +1530,7 @@ numpy = ">=1.16.6" name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, @@ -1541,7 +1541,7 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, @@ -1716,51 +1716,61 @@ tzdata = {version = "*", markers = "python_version >= \"3.6\""} [[package]] name = "pyyaml" -version = "6.0" +version = "6.0.1" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.6" files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] [[package]] @@ -1826,7 +1836,7 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" -optional = false +optional = true python-versions = ">=3.6,<4" files = [ {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, @@ -2217,4 +2227,4 @@ s3 = ["boto3"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "d897197fe72338d13851c18b17c19e54d43e7297c4468cb3a969dc4dfc6b3f7c" +content-hash = "46ae58cf3b2caba8e8a7a8915c2c6ee0712844b99c3c6735eb6d434a355a95a7" diff --git a/pyproject.toml b/pyproject.toml index 8b019556..cb690741 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,6 @@ google-api-python-client = {version = "*", optional = true} APScheduler = "*" redis = "*" matplotlib = "*" -pytest = "*" [tool.poetry.group.dev.dependencies] pyproject-flake8 = "5.0.4" @@ -43,10 +42,11 @@ lupa = "*" fakeredis = "*" mypy = "*" types-redis = "*" +pytest = "*" -# for dev, extra is installed by default -[tool.poetry.group.dev-extra.dependencies] -google-api-python-client = "*" +[tool.poetry.group.dev-s3] +optional = true +[tool.poetry.group.dev-s3.dependencies] moto = "*" [tool.poetry.extras] diff --git a/tox.ini b/tox.ini index 93ece72a..f45ace78 100644 --- a/tox.ini +++ b/tox.ini @@ -6,13 +6,13 @@ isolated_build = true allowlist_externals = poetry, pytest skip_install = true commands = - s3: poetry install -E s3 --without=dev-extra + s3: poetry install --extras=s3 --with=dev-s3 s3: pytest -v -m "s3" - gcs: poetry install -E gcs --without=dev-extra + gcs: poetry install --extras=gcs gcs: pytest -v -m "gcs" - core: poetry install --without=dev-extra + core: poetry install core: pytest -v -m "not gcs and not s3" [testenv:yapf]