feat: expose retry settings

Mitigate shadowing retry cause.
Add typing and accept Path for log-dir.

Closes #113 and #60 

Co-authored-by: Maciej Adamiak <adamiak.maciek@gmail.com>

CHANGELOG.md

@@ -3,11 +3,14 @@
 ### Added
 
  - support for python 3.12
+ - custom [retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry) configuration
 
 ### Removed
 
- - support for python 3.8 and 3.9
- - support for geopandas <= 0.14, pandas <= 2.1 and urllib3 <= 2.1
+ - support for python < 3.10
+ - support for geopandas < 0.14
+ - support for pandas < 2.1
+ - support for urllib3 < 2.1
 
 ## 0.2.0
 

ohsome/clients.py

@@ -3,12 +3,15 @@
 
 """OhsomeClient classes to build and handle requests to ohsome API"""
 import json
-import os
+from pathlib import Path
+from typing import Union, Optional
 from urllib.parse import urljoin
 
 import requests
+from requests import Session
 from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
+from requests.exceptions import RetryError
+from urllib3 import Retry
 
 from ohsome import OhsomeException, OhsomeResponse
 from ohsome.constants import (
@@ -29,11 +32,12 @@
 class _OhsomeBaseClient:
     def __init__(
         self,
-        base_api_url=None,
-        log=DEFAULT_LOG,
-        log_dir=DEFAULT_LOG_DIR,
-        cache=None,
-        user_agent=None,
+        base_api_url: Optional[str] = None,
+        log: Optional[bool] = DEFAULT_LOG,
+        log_dir: Optional[Union[str, Path]] = DEFAULT_LOG_DIR,
+        cache: Optional[list] = None,
+        user_agent: Optional[str] = None,
+        retry: Optional[Retry] = None,
     ):
         """
         Initialize _OhsomeInfoClient object
@@ -42,12 +46,14 @@ def __init__(
         :param log_dir: Directory for log files, default: ./ohsome_log
         :param cache: Cache for endpoint components
         :param user_agent: User agent passed with the request to the ohsome API
+        :param retry: Set a custom retry mechanism for requests. Be aware that ohsome-py will call the API once more
+        after all retries have failed. This overcomes the problem that the cause of the retries is shadowed behind a
+        RetryError by the underlying library.
         """
         self.log = log
-        self.log_dir = log_dir
+        self.log_dir = Path(log_dir)
         if self.log:
-            if not os.path.exists(self.log_dir):
-                os.mkdir(self.log_dir)
+            self.log_dir.mkdir(parents=True, exist_ok=True)
         if base_api_url is not None:
             self._base_api_url = base_api_url.strip("/") + "/"
         else:
@@ -59,6 +65,20 @@ def __init__(
             agent_list.append(user_agent)
         self.user_agent = " ".join(agent_list)
 
+        if retry is None:
+            self.__retry = Retry(
+                total=3,
+                status_forcelist=[429, 500, 502, 503, 504],
+                allowed_methods=["GET", "POST"],
+                backoff_factor=1,
+            )
+        else:
+            self.__retry = retry or Retry(
+                total=3,
+                status_forcelist=[429, 500, 502, 503, 504],
+                allowed_methods=["GET", "POST"],
+                backoff_factor=1,
+            )
         self.__session = None
 
     def _session(self):
@@ -67,14 +87,8 @@ def _session(self):
         :return:
         """
         if self.__session is None:
-            retry_strategy = Retry(
-                total=3,
-                status_forcelist=[429, 500, 502, 503, 504],
-                allowed_methods=["GET", "POST"],
-                backoff_factor=1,
-            )
-            adapter = HTTPAdapter(max_retries=retry_strategy)
-            self.__session = requests.Session()
+            adapter = HTTPAdapter(max_retries=self.__retry)
+            self.__session = Session()
             self.__session.mount("https://", adapter)
             self.__session.mount("http://", adapter)
             self.__session.headers["user-agent"] = self.user_agent
@@ -89,11 +103,12 @@ class _OhsomeInfoClient(_OhsomeBaseClient):
 
     def __init__(
         self,
-        base_api_url=None,
-        log=DEFAULT_LOG,
-        log_dir=DEFAULT_LOG_DIR,
-        cache=None,
-        user_agent=None,
+        base_api_url: Optional[str] = None,
+        log: Optional[bool] = DEFAULT_LOG,
+        log_dir: Optional[Union[str, Path]] = DEFAULT_LOG_DIR,
+        cache: Optional[list] = None,
+        user_agent: Optional[str] = None,
+        retry: Optional[Retry] = None,
     ):
         """
         Initialize _OhsomeInfoClient object
@@ -102,9 +117,12 @@ def __init__(
         :param log_dir: Directory for log files, default: ./ohsome_log
         :param cache: Cache for endpoint components
         :param user_agent: User agent passed with the request to the ohsome API
+        :param retry: Set a custom retry mechanism for requests. Be aware that ohsome-py will call the API once more
+        after all retries have failed. This overcomes the problem that the cause of the retries is shadowed behind a
+        RetryError by the underlying library.
         """
         super(_OhsomeInfoClient, self).__init__(
-            base_api_url, log, log_dir, cache, user_agent
+            base_api_url, log, log_dir, cache, user_agent, retry
         )
         self._parameters = None
         self._metadata = None
@@ -183,11 +201,12 @@ class _OhsomePostClient(_OhsomeBaseClient):
 
     def __init__(
         self,
-        base_api_url=None,
-        log=DEFAULT_LOG,
-        log_dir=DEFAULT_LOG_DIR,
-        cache=None,
-        user_agent=None,
+        base_api_url: Optional[str] = None,
+        log: Optional[bool] = DEFAULT_LOG,
+        log_dir: Optional[Union[str, Path]] = DEFAULT_LOG_DIR,
+        cache: Optional[list] = None,
+        user_agent: Optional[str] = None,
+        retry: Optional[Retry] = None,
     ):
         """
         Initialize _OhsomePostClient object
@@ -196,9 +215,12 @@ def __init__(
         :param log_dir: Directory for log files, default: ./ohsome_log
         :param cache: Cache for endpoint components
         :param user_agent: User agent passed with the request to the ohsome API
+        :param retry: Set a custom retry mechanism for requests. Be aware that ohsome-py will call the API once more
+        after all retries have failed. This overcomes the problem that the cause of the retries is shadowed behind a
+        RetryError by the underlying library.
         """
         super(_OhsomePostClient, self).__init__(
-            base_api_url, log, log_dir, cache, user_agent
+            base_api_url, log, log_dir, cache, user_agent, retry
         )
         self._parameters = None
         self._metadata = None
@@ -317,6 +339,13 @@ def _handle_request(self) -> OhsomeResponse:
             )
 
         except requests.exceptions.RequestException as e:
+            if isinstance(e, RetryError):
+                # retry one last time without retries, this will raise the original error instead of a cryptic retry
+                # error (or succeed)
+                self._OhsomeBaseClient__session = None
+                self._OhsomeBaseClient__retry = False
+                self._handle_request()
+
             ohsome_exception = OhsomeException(
                 message=str(e),
                 url=self._url,

ohsome/constants.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """Constants and default values"""
+from pathlib import Path
 
 OHSOME_BASE_API_URL = "https://api.ohsome.org/v1/"
 DEFAULT_LOG = True
-DEFAULT_LOG_DIR = "./ohsome_log"
+DEFAULT_LOG_DIR = Path("./ohsome_log")
 # update version in pyproject.toml as well
 OHSOME_VERSION = "0.2.0"

ohsome/exceptions.py

@@ -5,7 +5,7 @@
 
 import datetime as dt
 import json
-import os
+from pathlib import Path
 
 
 class OhsomeException(Exception):
@@ -24,7 +24,7 @@ def __init__(
         self.response = response
         self.timestamp = dt.datetime.now().isoformat()
 
-    def log(self, log_dir):
+    def log(self, log_dir: Path):
         """
         Logs OhsomeException
         :return:
@@ -34,53 +34,45 @@ def log(self, log_dir):
         self.log_parameter(log_dir, log_file_name)
         if self.response is not None:
             self.log_response(log_dir, log_file_name)
-        # self.log_query(log_dir, log_file_name)
 
-    def log_response(self, log_dir, log_file_name):
+    def log_response(self, log_dir: Path, log_file_name: str):
         """
         Log raw response. This may duplicate much data but is helpful for debugging to know the exact raw answer by the
         ohsome-api.
         """
-        log_file = os.path.join(
-            log_dir,
-            f"{log_file_name}_raw.txt",
-        )
-        with open(log_file, "w") as dst:
+        log_file = log_dir / f"{log_file_name}_raw.txt"
+        with log_file.open(mode="w") as dst:
             dst.write(self.response.text)
 
-    def log_parameter(self, log_dir, log_file_name):
+    def log_parameter(self, log_dir: Path, log_file_name: str) -> None:
         """
         Log query parameters to file
         :param log_dir:
+        :param log_file_name:
         :return:
         """
-        log_file = os.path.join(
-            log_dir,
-            f"{log_file_name}.json",
-        )
+        log_file = log_dir / f"{log_file_name}.json"
+
         log = {
             "timestamp": self.timestamp,
             "status": self.error_code,
             "message": self.message,
             "requestUrl": self.url,
             "parameters": self.parameters,
         }
-        with open(log_file, "w") as dst:
+        with log_file.open(mode="w") as dst:
             json.dump(obj=log, fp=dst, indent=4)
 
-    def log_bpolys(self, log_dir, log_file_name):
+    def log_bpolys(self, log_dir: Path, log_file_name: str) -> None:
         """
         Log bpolys parameter to geojson file if it is included in the query
         :params log_file: Path to geojson file which should contain bpolys parameter
         :return:
         """
         if "bpolys" in self.parameters:
-            log_file_bpolys = os.path.join(
-                log_dir,
-                f"{log_file_name}_bpolys.geojson",
-            )
+            log_file_bpolys = log_dir / f"{log_file_name}_bpolys.geojson"
             bpolys = self.parameters.pop("bpolys")
-            with open(log_file_bpolys, "w") as dst:
+            with log_file_bpolys.open(mode="w") as dst:
                 json.dump(obj=json.loads(bpolys), fp=dst, indent=4)
 
     def __str__(self):

ohsome/helper.py

@@ -208,7 +208,7 @@ def find_groupby_names(url):
     return [name.strip("/") for name in url.split("groupBy")[1:]]
 
 
-def extract_error_message_from_invalid_json(responsetext) -> Tuple[int, str]:
+def extract_error_message_from_invalid_json(responsetext: str) -> Tuple[int, str]:
     """
     Extract error code and error message from invalid json returned from ohsome API
     Otherwise throws OhsomeException.

ohsome/test/cassettes/test_client/test_check_time_parameter_datetime.yaml

@@ -43,7 +43,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 16 Nov 2023 11:47:56 GMT
+      - Thu, 16 Nov 2023 12:52:49 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

ohsome/test/cassettes/test_client/test_check_time_parameter_list.yaml

@@ -44,7 +44,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 16 Nov 2023 11:47:55 GMT
+      - Thu, 16 Nov 2023 12:52:47 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

ohsome/test/cassettes/test_client/test_end_timestamp_as_time_input.yaml

@@ -42,7 +42,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 16 Nov 2023 11:47:56 GMT
+      - Thu, 16 Nov 2023 12:52:49 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

ohsome/test/cassettes/test_client/test_format_bboxes_dataframe.yaml

@@ -42,7 +42,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 16 Nov 2023 11:48:00 GMT
+      - Thu, 16 Nov 2023 12:52:52 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

ohsome/test/cassettes/test_client/test_format_bboxes_list.yaml

@@ -42,7 +42,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 16 Nov 2023 11:48:00 GMT
+      - Thu, 16 Nov 2023 12:52:53 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:
@@ -99,7 +99,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 16 Nov 2023 11:48:01 GMT
+      - Thu, 16 Nov 2023 12:52:53 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:
@@ -156,7 +156,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 16 Nov 2023 11:48:01 GMT
+      - Thu, 16 Nov 2023 12:52:55 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:
@@ -213,7 +213,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 16 Nov 2023 11:48:01 GMT
+      - Thu, 16 Nov 2023 12:52:55 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:
@@ -270,7 +270,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 16 Nov 2023 11:48:03 GMT
+      - Thu, 16 Nov 2023 12:52:55 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

ohsome/test/cassettes/test_client/test_format_bcircles_dataframe.yaml

@@ -45,7 +45,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 16 Nov 2023 11:47:57 GMT
+      - Thu, 16 Nov 2023 12:52:50 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server:

ohsome/test/cassettes/test_client/test_format_bcircles_geodataframe.yaml

@@ -51,7 +51,7 @@ interactions:
       Content-Type:
       - application/json
       Date:
-      - Thu, 16 Nov 2023 11:47:59 GMT
+      - Thu, 16 Nov 2023 12:52:51 GMT
       Keep-Alive:
       - timeout=5, max=100
       Server: