From 93fc20eac1df3db2bf3d8f431ee36964f1497d5e Mon Sep 17 00:00:00 2001 From: yuji38kwmt Date: Tue, 28 May 2024 15:46:10 +0900 Subject: [PATCH] =?UTF-8?q?`login`=20API=E3=81=AB=E5=A4=B1=E6=95=97?= =?UTF-8?q?=E3=81=97=E3=81=9F=E3=81=A8=E3=81=8D=E3=81=AB=E3=80=81=E3=82=BB?= =?UTF-8?q?=E3=83=B3=E3=82=B7=E3=83=86=E3=82=A3=E3=83=96=E3=81=AA=E6=83=85?= =?UTF-8?q?=E5=A0=B1=E3=81=8C=E3=83=AD=E3=82=B0=E3=81=AB=E5=87=BA=E5=8A=9B?= =?UTF-8?q?=E3=81=95=E3=82=8C=E3=81=A6=E3=81=97=E3=81=BE=E3=81=86=E5=95=8F?= =?UTF-8?q?=E9=A1=8C=E3=82=92=E4=BF=AE=E6=AD=A3=E3=81=97=E3=81=BE=E3=81=97?= =?UTF-8?q?=E3=81=9F=E3=80=82=20(#651)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ログメッセージの修正 * 共通化 * poetry update * collectionのモジュールを変更 --- annofabapi/api.py | 107 ++++++++++++++++++++++++---------------------- poetry.lock | 23 +++++----- pyproject.toml | 2 +- 3 files changed, 69 insertions(+), 63 deletions(-) diff --git a/annofabapi/api.py b/annofabapi/api.py index 7495c51a..ceec9fec 100644 --- a/annofabapi/api.py +++ b/annofabapi/api.py @@ -4,7 +4,7 @@ import time from functools import wraps from json import JSONDecodeError -from typing import Any, Callable, Dict, Optional, Tuple +from typing import Any, Callable, Collection, Dict, Optional, Tuple import backoff import requests @@ -23,6 +23,34 @@ """HTTP Status Codeが429のときの、デフォルト（Retry-Afterヘッダがないとき）の待ち時間です。""" +def _mask_senritive_value_for_dict(data: Dict[str, Any], keys: Collection[str]) -> Dict[str, Any]: + """ + dictに含まれているセンシティブな情報を"***"でマスクします。 + + Args: + data: マスク対象のdict + keys: マスク対象の複数のkey + + Returns: + センシティブな情報がマスクされたdict。 + 1つ以上の値をマスクした場合は、複製されたdictが返ります。 + 1つもマスクしていない場合は、引数`data`そのものが返ります。 + + """ + MASKED_VALUE = "***" + diff_keys = set(keys) - set(data.keys()) + if len(diff_keys) == len(keys): + # マスク対象のキーがない + return data + + copied_data = copy.deepcopy(data) + for key in keys: + if key in copied_data: + copied_data[key] = MASKED_VALUE + + return copied_data + + def _raise_for_status(response: requests.Response) -> None: """ HTTP Status CodeがErrorの場合、``requests.exceptions.HTTPError`` を発生させる。 @@ -55,29 +83,34 @@ def _log_error_response(arg_logger: logging.Logger, response: requests.Response) """ - def mask_key(d, key: str): # noqa: ANN001, ANN202 - if key in d: - d[key] = "***" + def mask_str_rquest_body(str_request_body: str) -> Any: # noqa: ANN401 + """ + 文字列型であるrequest_bodyがJSON形式だとみなして、センシティブな情報をマスクします。 + """ + try: + # JSON文字列だとみなして、Pythonオブジェクトへの変換を試みる + json_request_body = json.loads(str_request_body) + except JSONDecodeError: + return str_request_body + + return _create_request_body_for_logger(json_request_body) if 400 <= response.status_code < 600: - headers = copy.deepcopy(response.request.headers) # logにAuthorizationを出力しないようにマスクする - mask_key(headers, "Authorization") + headers_for_logger = _mask_senritive_value_for_dict(dict(response.request.headers), {"Authorization"}) # request_bodyのpassword関係をマスクして、logに出力する - request_body = response.request.body - request_body_for_logger: Optional[Any] - if request_body is not None and request_body != "": - if isinstance(request_body, str): - try: - dict_request_body = json.loads(request_body) - request_body_for_logger = _create_request_body_for_logger(dict_request_body) - except JSONDecodeError: - request_body_for_logger = request_body - else: - request_body_for_logger = _create_request_body_for_logger(request_body) - else: - request_body_for_logger = request_body + request_body_for_logger: Optional[Any] = None + if isinstance(response.request.body, bytes): + try: + # 文字列への変換を試みる + str_request_body = response.request.body.decode() + request_body_for_logger = mask_str_rquest_body(str_request_body) + except UnicodeError: + request_body_for_logger = response.request.body + + elif isinstance(response.request.body, str): + request_body_for_logger = mask_str_rquest_body(response.request.body) arg_logger.error( "HTTP error occurred :: %s", @@ -90,7 +123,7 @@ def mask_key(d, key: str): # noqa: ANN001, ANN202 "http_method": response.request.method, "url": response.request.url, "body": request_body_for_logger, - "headers": headers, + "headers": headers_for_logger, }, }, ) @@ -109,28 +142,13 @@ def _create_request_body_for_logger(data: Any) -> Any: # noqa: ANN401 Returns: ログ出力用のrequest_body """ - - def mask_key(d, key: str): # noqa: ANN001, ANN202 - if key in d: - d[key] = "***" - if not isinstance(data, dict): return data elif isinstance(data, bytes): # bytes型のときは値を出力しても意味がないので、bytesであることが分かるようにする return "(bytes)" - MASKED_KEYS = {"password", "old_password", "new_password", "id_token", "refresh_token", "access_token"} - diff = MASKED_KEYS - set(data.keys()) - if len(diff) == len(MASKED_KEYS): - # マスク対象のキーがない - return data - - copied_data = copy.deepcopy(data) - for key in MASKED_KEYS: - mask_key(copied_data, key) - - return copied_data + return _mask_senritive_value_for_dict(data, keys={"password", "old_password", "new_password", "id_token", "refresh_token", "access_token"}) def _create_query_params_for_logger(params: Dict[str, Any]) -> Dict[str, Any]: @@ -144,22 +162,7 @@ def _create_query_params_for_logger(params: Dict[str, Any]) -> Dict[str, Any]: Returns: ログ出力用のparams """ - - def mask_key(d, key: str): # noqa: ANN001, ANN202 - if key in d: - d[key] = "***" - - MASKED_KEYS = {"X-Amz-Security-Token", "X-Amz-Credential"} - diff = MASKED_KEYS - set(params.keys()) - if len(diff) == len(MASKED_KEYS): - # マスク対象のキーがない - return params - - copied_params = copy.deepcopy(params) - for key in MASKED_KEYS: - mask_key(copied_params, key) - - return copied_params + return _mask_senritive_value_for_dict(params, keys={"X-Amz-Security-Token", "X-Amz-Credential"}) def _should_retry_with_status(status_code: int) -> bool: diff --git a/poetry.lock b/poetry.lock index 6faac2f6..343e0ab1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -359,8 +359,11 @@ name = "docutils" version = "0.20.1" description = "Docutils -- Python Documentation Utilities" optional = false -python-versions = "*" -files = [] +python-versions = ">=3.7" +files = [ + {file = "docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6"}, + {file = "docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b"}, +] [[package]] name = "exceptiongroup" @@ -1085,13 +1088,13 @@ testutils = ["gitpython (>3)"] [[package]] name = "pytest" -version = "7.4.4" +version = "8.2.1" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, - {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, + {file = "pytest-8.2.1-py3-none-any.whl", hash = "sha256:faccc5d332b8c3719f40283d0d44aa5cf101cec36f88cde9ed8f2bc0538612b1"}, + {file = "pytest-8.2.1.tar.gz", hash = "sha256:5046e5b46d8e4cac199c373041f26be56fdb81eb4e67dc11d4e10811fc3408fd"}, ] [package.dependencies] @@ -1099,11 +1102,11 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} +pluggy = ">=1.5,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-cov" @@ -1539,4 +1542,4 @@ segmentation = ["numpy", "numpy", "pillow"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "f56133f4caa2050c0b28abcf02a5026733a06230048ae3e0432902bc1d789a2a" +content-hash = "4caf6b3eb6a9083a8c774d91187bf5891521042b34b788d62b27ae2d45719219" diff --git a/pyproject.toml b/pyproject.toml index c4ff7b63..d5d77025 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ segmentation = ["numpy", "pillow"] [tool.poetry.group.test.dependencies] -pytest = "^7" +pytest = "^8" pytest-xdist = "*" pytest-cov = "*"