Merge pull request #33 from EBI-Metagenomics/bugfix/bad-responses-fro…

…m-ena-browser

Bugfix: handle bad responses from ENA browser API

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.10.3
+current_version = 0.10.4
 commit = True
 tag = True
 

.gitignore

@@ -65,3 +65,4 @@ target/
 #IntelliJ project structure files
 *.iml
 .idea
+venv*

README.md

@@ -124,6 +124,18 @@ erp001736 = OriginalMetadata('ERP001736')
 erp001736.fetch_metadata()
 ```
 
+
+Development setup
+=================
+Install the package in edit mode, and additional dev requirements (pre-commit hooks and version bumper).
+```shell
+pip install -e . -r requirements-dev.txt
+pre-commit install
+```
+
+You can bump the version with e.g. `bump2version patch`.
+
+
 Contributors
 ============
 

mg_toolkit/__init__.py

@@ -20,4 +20,4 @@
 
 __all__ = ["original_metadata", "sequence_search", "bulk_download"]
 
-__version__ = "0.10.3"
+__version__ = "0.10.4"

mg_toolkit/constants.py

@@ -28,3 +28,6 @@
 
 ENA_SEARCH_API_URL = "https://www.ebi.ac.uk/ena/portal/api/search"
 ENA_XML_VIEW_URL = "https://www.ebi.ac.uk/ena/browser/api/xml"
+
+EBI_URL_PREFIX = "https://www.ebi.ac.uk/"
+REQUESTS_RETRIES = 3

mg_toolkit/metadata.py

@@ -19,8 +19,16 @@
 
 import requests
 from pandas import DataFrame
+from requests import Session
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
 
-from .constants import ENA_SEARCH_API_URL, ENA_XML_VIEW_URL
+from .constants import (
+    EBI_URL_PREFIX,
+    ENA_SEARCH_API_URL,
+    ENA_XML_VIEW_URL,
+    REQUESTS_RETRIES,
+)
 
 try:
     from json.decoder import JSONDecodeError
@@ -48,18 +56,29 @@ class OriginalMetadata:
     """
 
     accession = None
+    session = None
 
     def __init__(self, accession, *args, **kwargs):
         self.accession = accession
 
+        self.session = Session()
+        retries = Retry(
+            total=REQUESTS_RETRIES,
+            backoff_factor=0.1,
+            status_forcelist=[500, 502, 503, 504],
+        )
+        self.session.mount(EBI_URL_PREFIX, HTTPAdapter(max_retries=retries))
+
     def get_metadata(self, sample_accession):
         """Get the sample metadata from ENA API."""
         return_meta = {}
 
-        response = requests.get(ENA_XML_VIEW_URL + "/" + sample_accession)
+        response = self.session.get(ENA_XML_VIEW_URL + "/" + sample_accession)
 
         if not response.ok:
-            logger.error("Metadata fetch failed for accession:" + self.accession)
+            logger.error(
+                "Metadata fetch failed for sample accession: " + sample_accession
+            )
             return
 
         metadata_xml = ET.fromstring(response.content)
@@ -88,7 +107,7 @@ def get_metadata(self, sample_accession):
 
     def fetch_metadata(self):
         """Get metadata from ENA API."""
-        response = requests.get(
+        response = self.session.get(
             ENA_SEARCH_API_URL,
             params={
                 "result": "read_run",

requirements.txt

@@ -2,7 +2,9 @@
 # install and create a virtual environment
 # run pip install -r requirements
 
-requests>=2.24.0
+requests>=2.31.0
 pandas==2.0.3
 jsonapi-client>=0.9.9
 tqdm>=4.49.0
+
+urllib3~=2.2.1

setup.py

@@ -44,7 +44,7 @@
     long_description=long_description,
     long_description_content_type="text/markdown",
     packages=find_packages(exclude=["ez_setup"]),
-    version="0.10.3",
+    version="0.10.4",
     python_requires=">=3.8",
     install_requires=install_requirements,
     setup_requires=["pytest-runner"],