diff --git a/argopy/data_fetchers/erddap_data.py b/argopy/data_fetchers/erddap_data.py index c0b27980..111d29c3 100644 --- a/argopy/data_fetchers/erddap_data.py +++ b/argopy/data_fetchers/erddap_data.py @@ -432,11 +432,11 @@ def _bgc_vlist_avail(self): for p in params: if p.lower() in self._bgc_vlist_erddap: results.append(p) - else: - log.error( - "Removed '%s' because it's not available on the erddap, but it must !" - % p - ) + # else: + # log.error( + # "Removed '%s' because it is not available on the erddap server (%s), but it should !" + # % (p, self._server) + # ) return results diff --git a/argopy/stores/filesystems.py b/argopy/stores/filesystems.py index 482fbcbd..9cf27acc 100644 --- a/argopy/stores/filesystems.py +++ b/argopy/stores/filesystems.py @@ -673,6 +673,7 @@ def download_url( n_attempt: int = 1, max_attempt: int = 5, cat_opts: dict = {}, + errors: str = 'raise', *args, **kwargs, ): @@ -683,19 +684,27 @@ def download_url( """ def make_request( - ffs, url, n_attempt: int = 1, max_attempt: int = 5, cat_opts: dict = {} + ffs, url, n_attempt: int = 1, max_attempt: int = 5, cat_opts: dict = {}, errors: str = 'raise', ): data = None if n_attempt <= max_attempt: try: data = ffs.cat_file(url, **cat_opts) + except FileNotFoundError as e: + if errors == 'raise': + raise e + elif errors == 'ignore': + log.error('FileNotFoundError raised from: %s' % url) except aiohttp.ClientResponseError as e: if e.status == 413: - log.debug( - "Error %i (Payload Too Large) raised with %s" - % (e.status, url) - ) - raise + if errors == 'raise': + raise e + elif errors == 'ignore': + log.error( + "Error %i (Payload Too Large) raised with %s" + % (e.status, url) + ) + elif e.status == 429: retry_after = int(e.headers.get("Retry-After", 5)) log.debug( @@ -707,14 +716,26 @@ def make_request( else: # Handle other client response errors print(f"Error: {e}") - except aiohttp.ClientError: - # Handle other request exceptions - # print(f"Error: {e}") - raise + + except aiohttp.ClientError as e: + if errors == 'raise': + raise e + elif errors == 'ignore': + log.error("Error: {e}") + + except fsspec.FSTimeoutError as e: + if errors == 'raise': + raise e + elif errors == 'ignore': + log.error("Error: {e}") else: - raise ValueError( - f"Error: All attempts failed to download this url: {url}" - ) + if errors == 'raise': + raise ValueError( + f"Error: All attempts failed to download this url: {url}" + ) + elif errors == 'ignore': + log.error("Error: All attempts failed to download this url: {url}") + return data, n_attempt url = self.curateurl(url) @@ -724,14 +745,18 @@ def make_request( n_attempt=n_attempt, max_attempt=max_attempt, cat_opts=cat_opts, + errors=errors, ) if data is None: - raise FileNotFoundError(url) + if errors == 'raise': + raise FileNotFoundError(url) + elif errors == 'ignore': + log.error("FileNotFoundError: %s" % url) return data - def open_dataset(self, url, **kwargs): + def open_dataset(self, url, errors: str = 'raise', **kwargs): """Open and decode a xarray dataset from an url Parameters @@ -749,6 +774,14 @@ def open_dataset(self, url, **kwargs): if "download_url_opts" in kwargs: dwn_opts.update(kwargs["download_url_opts"]) data = self.download_url(url, **dwn_opts) + log.info(dwn_opts) + + if data is None: + if errors == 'raise': + raise DataNotFound(url) + elif errors == 'ignore': + log.error("DataNotFound: %s" % url) + return None if data[0:3] != b"CDF": raise TypeError(