From 4070159236db17030c2e6234226b09cb33bb24fc Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 4 Sep 2024 10:35:38 +0200 Subject: [PATCH 01/19] Refactor option "dataset" into "ds" - for consistency with fetcher argument name "ds" --- argopy/data_fetchers/argovis_data.py | 2 +- argopy/data_fetchers/erddap_data.py | 2 +- argopy/data_fetchers/gdacftp_data.py | 2 +- argopy/data_fetchers/gdacftp_index.py | 2 +- argopy/fetchers.py | 6 +++--- argopy/options.py | 2 +- argopy/tests/test_options.py | 6 +++--- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/argopy/data_fetchers/argovis_data.py b/argopy/data_fetchers/argovis_data.py index ce6304fb..1fcf5686 100755 --- a/argopy/data_fetchers/argovis_data.py +++ b/argopy/data_fetchers/argovis_data.py @@ -87,7 +87,7 @@ def __init__( Argovis API request time out in seconds. Set to OPTIONS['api_timeout'] by default. """ self.definition = "Argovis Argo data fetcher" - self.dataset_id = OPTIONS["dataset"] if ds == "" else ds + self.dataset_id = OPTIONS["ds"] if ds == "" else ds self.user_mode = kwargs["mode"] if "mode" in kwargs else OPTIONS["mode"] self.server = kwargs["server"] if "server" in kwargs else api_server timeout = OPTIONS["api_timeout"] if api_timeout == 0 else api_timeout diff --git a/argopy/data_fetchers/erddap_data.py b/argopy/data_fetchers/erddap_data.py index 826acc1c..e7175ae0 100644 --- a/argopy/data_fetchers/erddap_data.py +++ b/argopy/data_fetchers/erddap_data.py @@ -128,7 +128,7 @@ def __init__( # noqa: C901 """ timeout = OPTIONS["api_timeout"] if api_timeout == 0 else api_timeout self.definition = "Ifremer erddap Argo data fetcher" - self.dataset_id = OPTIONS["dataset"] if ds == "" else ds + self.dataset_id = OPTIONS["ds"] if ds == "" else ds self.user_mode = kwargs["mode"] if "mode" in kwargs else OPTIONS["mode"] self.server = kwargs["server"] if "server" in kwargs else OPTIONS["erddap"] self.store_opts = { diff --git a/argopy/data_fetchers/gdacftp_data.py b/argopy/data_fetchers/gdacftp_data.py index d21af153..64a5ffeb 100644 --- a/argopy/data_fetchers/gdacftp_data.py +++ b/argopy/data_fetchers/gdacftp_data.py @@ -91,7 +91,7 @@ def __init__( FTP request time out in seconds. Set to OPTIONS['api_timeout'] by default. """ self.timeout = OPTIONS["api_timeout"] if api_timeout == 0 else api_timeout - self.dataset_id = OPTIONS["dataset"] if ds == "" else ds + self.dataset_id = OPTIONS["ds"] if ds == "" else ds self.user_mode = kwargs["mode"] if "mode" in kwargs else OPTIONS["mode"] self.server = OPTIONS["ftp"] if ftp == "" else ftp self.errors = errors diff --git a/argopy/data_fetchers/gdacftp_index.py b/argopy/data_fetchers/gdacftp_index.py index 7b0400a6..7c4fe1e3 100644 --- a/argopy/data_fetchers/gdacftp_index.py +++ b/argopy/data_fetchers/gdacftp_index.py @@ -79,7 +79,7 @@ def __init__( """ self.timeout = OPTIONS["api_timeout"] if api_timeout == 0 else api_timeout self.definition = "Ifremer GDAC ftp Argo index fetcher" - self.dataset_id = OPTIONS["dataset"] if ds == "" else ds + self.dataset_id = OPTIONS["ds"] if ds == "" else ds self.server = OPTIONS['ftp'] if ftp == "" else ftp self.errors = errors diff --git a/argopy/fetchers.py b/argopy/fetchers.py index 786d3d9c..ffcc919b 100755 --- a/argopy/fetchers.py +++ b/argopy/fetchers.py @@ -78,7 +78,7 @@ class ArgoDataFetcher: src: str, optional Source of the data to use. Eg: ``erddap``. Set to OPTIONS['src'] by default if empty. ds: str, optional - Name of the dataset to load. Eg: ``phy``. Set to OPTIONS['dataset'] by default if empty. + Name of the dataset to load. Eg: ``phy``. Set to OPTIONS['ds'] by default if empty. **fetcher_kwargs: optional Additional arguments passed on data source fetcher creation of each access points. @@ -102,7 +102,7 @@ def __init__(self, mode: str = "", src: str = "", ds: str = "", **fetcher_kwargs # Facade options : self._mode = OPTIONS["mode"] if mode == "" else mode - self._dataset_id = OPTIONS["dataset"] if ds == "" else ds + self._dataset_id = OPTIONS["ds"] if ds == "" else ds self._src = OPTIONS["src"] if src == "" else src if self._dataset_id == "bgc": @@ -812,7 +812,7 @@ def __init__( self, mode: str = OPTIONS["mode"], src: str = OPTIONS["src"], - ds: str = OPTIONS["dataset"], + ds: str = OPTIONS["ds"], **fetcher_kwargs, ): """Facade for Argo index fetchers diff --git a/argopy/options.py b/argopy/options.py index 23f13d47..60c30e72 100644 --- a/argopy/options.py +++ b/argopy/options.py @@ -21,7 +21,7 @@ DATA_SOURCE = "src" FTP = "ftp" ERDDAP = 'erddap' -DATASET = "dataset" +DATASET = "ds" CACHE_FOLDER = "cachedir" CACHE_EXPIRATION = "cache_expiration" USER_LEVEL = "mode" diff --git a/argopy/tests/test_options.py b/argopy/tests/test_options.py index c935189d..932df0ee 100644 --- a/argopy/tests/test_options.py +++ b/argopy/tests/test_options.py @@ -46,11 +46,11 @@ def test_opt_dataset(): with pytest.raises(OptionValueError): argopy.set_options(dataset="invalid_ds") with argopy.set_options(dataset="phy"): - assert OPTIONS["dataset"] == "phy" + assert OPTIONS["ds"] == "phy" with argopy.set_options(dataset="bgc"): - assert OPTIONS["dataset"] == "bgc" + assert OPTIONS["ds"] == "bgc" with argopy.set_options(dataset="ref"): - assert OPTIONS["dataset"] == "ref" + assert OPTIONS["ds"] == "ref" @pytest.mark.skipif(platform.system() == 'Windows', reason="Need to be debugged for Windows support") From 36fca9e4371f83adad73c0602e7f60a6fb717001 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 4 Sep 2024 10:47:45 +0200 Subject: [PATCH 02/19] Refactor option "dataset" into "ds" --- argopy/tests/test_options.py | 12 ++++++++---- docs/user-guide/fetching-argo-data/data_set.rst | 8 ++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/argopy/tests/test_options.py b/argopy/tests/test_options.py index 932df0ee..81e130a1 100644 --- a/argopy/tests/test_options.py +++ b/argopy/tests/test_options.py @@ -44,12 +44,16 @@ def test_opt_ifremer_erddap(mocked_httpserver): def test_opt_dataset(): with pytest.raises(OptionValueError): - argopy.set_options(dataset="invalid_ds") - with argopy.set_options(dataset="phy"): + argopy.set_options(ds="invalid_ds") + with argopy.set_options(ds="phy"): assert OPTIONS["ds"] == "phy" - with argopy.set_options(dataset="bgc"): + with argopy.set_options(ds="bgc"): assert OPTIONS["ds"] == "bgc" - with argopy.set_options(dataset="ref"): + with argopy.set_options(ds="bgc-s"): + assert OPTIONS["ds"] == "bgc-s" + with argopy.set_options(ds="bgc-b"): + assert OPTIONS["ds"] == "bgc-b" + with argopy.set_options(ds="ref"): assert OPTIONS["ds"] == "ref" diff --git a/docs/user-guide/fetching-argo-data/data_set.rst b/docs/user-guide/fetching-argo-data/data_set.rst index 406c485b..32111450 100644 --- a/docs/user-guide/fetching-argo-data/data_set.rst +++ b/docs/user-guide/fetching-argo-data/data_set.rst @@ -52,7 +52,7 @@ You have several ways to specify which dataset you want to use: :okwarning: import argopy - argopy.set_options(dataset='bgc') + argopy.set_options(ds='bgc') - **with an option in a temporary context**: @@ -60,7 +60,7 @@ You have several ways to specify which dataset you want to use: :okwarning: import argopy - with argopy.set_options(dataset='phy'): + with argopy.set_options(ds='phy'): argopy.DataFetcher().profile(6904241, 12) - **with the `ds` argument in the data fetcher**: @@ -111,7 +111,7 @@ By default, the ``params`` argument is set to the keyword ``all`` to return *all :okwarning: import argopy - with argopy.set_options(dataset='bgc', src='erddap', mode='expert'): + with argopy.set_options(ds='bgc', src='erddap', mode='expert'): params = 'all' # eg: 'DOXY' or ['DOXY', 'BBP700'] f = argopy.DataFetcher(params=params) f = f.region([-75, -45, 20, 30, 0, 10, '2021-01', '2021-06']) @@ -139,7 +139,7 @@ By default, the ``measured`` argument is set to ``None`` for unconstrained param :okwarning: import argopy - with argopy.set_options(dataset='bgc', src='erddap', mode='expert'): + with argopy.set_options(ds='bgc', src='erddap', mode='expert'): f = argopy.DataFetcher(params='all', measured=['DOXY', 'BBP700']) f = f.region([-75, -45, 20, 30, 0, 10, '2021-01', '2021-06']) f.load() From 942022db6ae3053630f5dfbb437c99ae2c618cec Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 6 Sep 2024 09:44:58 +0200 Subject: [PATCH 03/19] Update fetchers.py --- argopy/fetchers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/argopy/fetchers.py b/argopy/fetchers.py index 450c8368..6fa310be 100755 --- a/argopy/fetchers.py +++ b/argopy/fetchers.py @@ -112,9 +112,9 @@ def __init__(self, mode: str = "", src: str = "", ds: str = "", **fetcher_kwargs raise OptionValueError( f"option 'mode' given an invalid value: {self._mode}" ) - if not _VALIDATORS["dataset"](self._dataset_id): + if not _VALIDATORS["ds"](self._dataset_id): raise OptionValueError( - f"option 'dataset' given an invalid value: {self._dataset_id}" + f"option 'ds' given an invalid value: {self._dataset_id}" ) if not _VALIDATORS["src"](self._src): raise OptionValueError(f"option 'src' given an invalid value: {self._src}") From c379d550646df56c80df092e9c569b6b09f3abf0 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 10 Sep 2024 12:26:35 +0200 Subject: [PATCH 04/19] Refactor "gdacftp_*" modules into "gdac_*" --- argopy/data_fetchers/__init__.py | 8 ++++---- argopy/data_fetchers/{gdacftp_data.py => gdac_data.py} | 0 argopy/data_fetchers/{gdacftp_index.py => gdac_index.py} | 0 argopy/utils/lists.py | 6 +++--- docs/api-hidden.rst | 6 +++--- docs/api.rst | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) rename argopy/data_fetchers/{gdacftp_data.py => gdac_data.py} (100%) rename argopy/data_fetchers/{gdacftp_index.py => gdac_index.py} (100%) diff --git a/argopy/data_fetchers/__init__.py b/argopy/data_fetchers/__init__.py index 0876b7bc..ddd242a7 100644 --- a/argopy/data_fetchers/__init__.py +++ b/argopy/data_fetchers/__init__.py @@ -6,14 +6,14 @@ from . import erddap_data from . import erddap_index from . import argovis_data -from . import gdacftp_data -from . import gdacftp_index +from . import gdac_data +from . import gdac_index __all__ = ( "erddap_data", "erddap_index", "argovis_data", - "gdacftp_data", - "gdacftp_index", + "gdac_data.py", + "gdac_index.py", "CTDRefDataFetcher", ) diff --git a/argopy/data_fetchers/gdacftp_data.py b/argopy/data_fetchers/gdac_data.py similarity index 100% rename from argopy/data_fetchers/gdacftp_data.py rename to argopy/data_fetchers/gdac_data.py diff --git a/argopy/data_fetchers/gdacftp_index.py b/argopy/data_fetchers/gdac_index.py similarity index 100% rename from argopy/data_fetchers/gdacftp_index.py rename to argopy/data_fetchers/gdac_index.py diff --git a/argopy/utils/lists.py b/argopy/utils/lists.py index 2fec787a..bb6beb7e 100644 --- a/argopy/utils/lists.py +++ b/argopy/utils/lists.py @@ -45,7 +45,7 @@ def list_available_data_src() -> dict: pass try: - from ..data_fetchers import gdacftp_data as GDAC_Fetchers + from ..data_fetchers import gdac_data as GDAC_Fetchers # Ensure we're loading the gdac data fetcher with the current options: GDAC_Fetchers.api_server_check = OPTIONS["ftp"] @@ -86,7 +86,7 @@ def list_available_index_src() -> dict: pass try: - from ..data_fetchers import gdacftp_index as GDAC_Fetchers + from ..data_fetchers import gdac_index as GDAC_Fetchers # Ensure we're loading the gdac data fetcher with the current options: GDAC_Fetchers.api_server_check = OPTIONS["ftp"] @@ -203,7 +203,7 @@ def list_standard_variables(ds: str = 'phy') -> List[str]: ---------- ds: str, default='phy' - Return variables for one of the argopy ``dataset`` option possible values: + Return variables for one of the argopy ``ds`` option possible values: - ``phy`` is valid for the 🟑 core and πŸ”΅ deep missions variables - ``bgc`` is valid for the 🟒 BGC missions variables diff --git a/docs/api-hidden.rst b/docs/api-hidden.rst index bd2aa31d..da8f81fc 100644 --- a/docs/api-hidden.rst +++ b/docs/api-hidden.rst @@ -40,9 +40,9 @@ argopy.data_fetchers.erddap_data.Fetch_wmo argopy.data_fetchers.erddap_data.Fetch_box - argopy.data_fetchers.gdacftp_data.FTPArgoDataFetcher - argopy.data_fetchers.gdacftp_data.Fetch_wmo - argopy.data_fetchers.gdacftp_data.Fetch_box + argopy.data_fetchers.gdac_data.FTPArgoDataFetcher + argopy.data_fetchers.gdac_data.Fetch_wmo + argopy.data_fetchers.gdac_data.Fetch_box argopy.data_fetchers.argovis_data.ArgovisDataFetcher argopy.data_fetchers.argovis_data.Fetch_wmo diff --git a/docs/api.rst b/docs/api.rst index 9bfe3930..818511da 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -299,9 +299,9 @@ GDAC .. autosummary:: :toctree: generated/ - data_fetchers.gdacftp_data.FTPArgoDataFetcher - data_fetchers.gdacftp_data.Fetch_wmo - data_fetchers.gdacftp_data.Fetch_box + data_fetchers.gdac_data.FTPArgoDataFetcher + data_fetchers.gdac_data.Fetch_wmo + data_fetchers.gdac_data.Fetch_box Argovis From e2c25fc5449e7bf847272e3b0f2c561fa7e0b101 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 10 Sep 2024 12:45:17 +0200 Subject: [PATCH 05/19] upgrade pinned s3fs --- ci/requirements/py3.10-all-pinned.yml | 4 ++-- ci/requirements/py3.10-core-pinned.yml | 4 ++-- ci/requirements/py3.9-all-pinned.yml | 6 +++--- ci/requirements/py3.9-core-pinned.yml | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/requirements/py3.10-all-pinned.yml b/ci/requirements/py3.10-all-pinned.yml index f20410d2..3541691f 100644 --- a/ci/requirements/py3.10-all-pinned.yml +++ b/ci/requirements/py3.10-all-pinned.yml @@ -19,7 +19,7 @@ dependencies: # EXT.UTIL: - boto3 = 1.35.0 - gsw = 3.6.19 - - s3fs = 0.4.2 + - s3fs = 2024.6.1 - tqdm = 4.66.5 - zarr = 2.18.2 @@ -58,4 +58,4 @@ dependencies: # PIP: - pip: - - pytest-reportlog == 0.4.0 \ No newline at end of file + - pytest-reportlog == 0.4.0 \ No newline at end of file diff --git a/ci/requirements/py3.10-core-pinned.yml b/ci/requirements/py3.10-core-pinned.yml index 95e4e481..cee7ce7c 100644 --- a/ci/requirements/py3.10-core-pinned.yml +++ b/ci/requirements/py3.10-core-pinned.yml @@ -19,7 +19,7 @@ dependencies: # EXT.UTIL: # - boto3 = 1.35.0 # - gsw = 3.6.19 -# - s3fs = 0.4.2 +# - s3fs = 2024.6.1 # - tqdm = 4.66.5 # - zarr = 2.18.2 @@ -58,4 +58,4 @@ dependencies: # PIP: - pip: - - pytest-reportlog == 0.4.0 \ No newline at end of file + - pytest-reportlog == 0.4.0 \ No newline at end of file diff --git a/ci/requirements/py3.9-all-pinned.yml b/ci/requirements/py3.9-all-pinned.yml index 62ca9fa9..04372598 100644 --- a/ci/requirements/py3.9-all-pinned.yml +++ b/ci/requirements/py3.9-all-pinned.yml @@ -19,7 +19,7 @@ dependencies: # EXT.UTIL: - boto3 = 1.35.0 - gsw = 3.6.19 - - s3fs = 0.4.2 + - s3fs = 2024.6.1 - tqdm = 4.66.5 - zarr = 2.18.2 @@ -44,7 +44,7 @@ dependencies: - bottleneck = 1.4.0 - cfgrib = 0.9.14.0 - cftime = 1.6.4 - - codespell + - codespell = 2.3.0 - flake8 = 7.1.1 - numpy = 1.26.4 - pandas = 2.2.2 @@ -58,4 +58,4 @@ dependencies: # PIP: - pip: - - pytest-reportlog == 0.4.0 + - pytest-reportlog == 0.4.0 \ No newline at end of file diff --git a/ci/requirements/py3.9-core-pinned.yml b/ci/requirements/py3.9-core-pinned.yml index 47051c8b..dc2d7909 100644 --- a/ci/requirements/py3.9-core-pinned.yml +++ b/ci/requirements/py3.9-core-pinned.yml @@ -19,7 +19,7 @@ dependencies: # EXT.UTIL: # - boto3 = 1.35.0 # - gsw = 3.6.19 -# - s3fs = 0.4.2 +# - s3fs = 2024.6.1 # - tqdm = 4.66.5 # - zarr = 2.18.2 @@ -44,7 +44,7 @@ dependencies: - bottleneck = 1.4.0 - cfgrib = 0.9.14.0 - cftime = 1.6.4 - - codespell + - codespell = 2.3.0 - flake8 = 7.1.1 - numpy = 1.26.4 - pandas = 2.2.2 @@ -58,4 +58,4 @@ dependencies: # PIP: - pip: - - pytest-reportlog == 0.4.0 + - pytest-reportlog == 0.4.0 \ No newline at end of file From 07bc71df654b0b0e3ee6f6c4c1c214ee04a55747 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 10 Sep 2024 12:48:30 +0200 Subject: [PATCH 06/19] Refactor "ftp" vocab to "gdac" - option "ftp" renamed "gdac" - "FTPArgoDataFetcher" renamed "GDACArgoDataFetcher" - GDAC fetchers argument "ftp" renamed "gdac" - "validate_ftp" renamed "validate_gdac" in options.py --- argopy/data_fetchers/gdac_data.py | 36 +++++++++++++++--------------- argopy/data_fetchers/gdac_index.py | 36 +++++++++++++++--------------- argopy/fetchers.py | 5 ----- argopy/options.py | 14 ++++++------ 4 files changed, 43 insertions(+), 48 deletions(-) diff --git a/argopy/data_fetchers/gdac_data.py b/argopy/data_fetchers/gdac_data.py index 40037800..4b4f666f 100644 --- a/argopy/data_fetchers/gdac_data.py +++ b/argopy/data_fetchers/gdac_data.py @@ -1,5 +1,5 @@ """ -Argo data fetcher for remote GDAC FTP +Argo data fetcher for remote GDAC servers This is not intended to be used directly, only by the facade at fetchers.py @@ -18,18 +18,18 @@ from ..stores import ArgoIndex from .proto import ArgoDataFetcherProto -log = logging.getLogger("argopy.gdacftp.data") +log = logging.getLogger("argopy.gdac.data") access_points = ["wmo", "box"] exit_formats = ["xarray"] dataset_ids = ["phy", "bgc", "bgc-s", "bgc-b"] # First is default -api_server = OPTIONS["ftp"] # API root url +api_server = OPTIONS["gdac"] # API root url api_server_check = ( api_server # URL to check if the API is alive, used by isAPIconnected ) -class FTPArgoDataFetcher(ArgoDataFetcherProto): - """Manage access to Argo data from a remote GDAC FTP. +class GDACArgoDataFetcher(ArgoDataFetcherProto): + """Manage access to Argo data from a GDAC server Warnings -------- @@ -50,7 +50,7 @@ def init(self, *args, **kwargs): ### def __init__( self, - ftp: str = "", + gdac: str = "", ds: str = "", cache: bool = False, cachedir: str = "", @@ -66,8 +66,8 @@ def __init__( Parameters ---------- - ftp: str (optional) - Path to the remote FTP directory where the 'dac' folder is located. + gdac: str (optional) + Path to the local or remote directory where the 'dac' folder is located ds: str (optional) Dataset to load: 'phy' or 'bgc' cache: bool (optional) @@ -88,12 +88,12 @@ def __init__( progress: bool (optional) Show a progress bar or not when fetching data. api_timeout: int (optional) - FTP request time out in seconds. Set to OPTIONS['api_timeout'] by default. + Server request time out in seconds. Set to OPTIONS['api_timeout'] by default. """ self.timeout = OPTIONS["api_timeout"] if api_timeout == 0 else api_timeout self.dataset_id = OPTIONS["ds"] if ds == "" else ds self.user_mode = kwargs["mode"] if "mode" in kwargs else OPTIONS["mode"] - self.server = OPTIONS["ftp"] if ftp == "" else ftp + self.server = OPTIONS["gdac"] if gdac == "" else gdac self.errors = errors # Validate server, raise FtpPathError if not valid. @@ -137,10 +137,10 @@ def __init__( self.init(**kwargs) def __repr__(self): - summary = [""] + summary = [""] summary.append("Name: %s" % self.definition) summary.append("Index: %s" % self.indexfs.index_file) - summary.append("Host: %s" % self.server) + summary.append("Server: %s" % self.server) if hasattr(self, "BOX"): summary.append("Domain: %s" % self.cname()) else: @@ -446,8 +446,8 @@ def filter_researchmode(self, ds: xr.Dataset, *args, **kwargs) -> xr.Dataset: return ds -class Fetch_wmo(FTPArgoDataFetcher): - """Manage access to GDAC ftp Argo data for: a list of WMOs. +class Fetch_wmo(GDACArgoDataFetcher): + """Manage access to GDAC Argo data for: a list of WMOs. This class is instantiated when a call is made to these facade access points: @@ -474,7 +474,7 @@ def init(self, WMO: list = [], CYC=None, **kwargs): if "MAX_FILES" in kwargs: self._nrows = kwargs["MAX_FILES"] - self.definition = "Ifremer GDAC ftp Argo data fetcher" + self.definition = "Ifremer GDAC Argo data fetcher" if self.CYC is not None: self.definition = "%s for profiles" % self.definition else: @@ -503,8 +503,8 @@ def uri(self): return self._list_of_argo_files -class Fetch_box(FTPArgoDataFetcher): - """Manage access to GDAC ftp Argo data for: a rectangular space/time domain. +class Fetch_box(GDACArgoDataFetcher): + """Manage access to GDAC Argo data for: a rectangular space/time domain. This class is instantiated when a call is made to these facade access points: @@ -535,7 +535,7 @@ def init(self, box: list, nrows=None, **kwargs): if "MAX_FILES" in kwargs: self._nrows = kwargs["MAX_FILES"] - self.definition = "Ifremer GDAC ftp Argo data fetcher for a space/time region" + self.definition = "Ifremer GDAC Argo data fetcher for a space/time region" return self @property diff --git a/argopy/data_fetchers/gdac_index.py b/argopy/data_fetchers/gdac_index.py index 7c4fe1e3..a697f26c 100644 --- a/argopy/data_fetchers/gdac_index.py +++ b/argopy/data_fetchers/gdac_index.py @@ -1,5 +1,5 @@ """ -Argo index fetcher for remote GDAC FTP +Argo index fetcher for remote GDAC server This is not intended to be used directly, only by the facade at fetchers.py @@ -16,7 +16,7 @@ from ..plot import dashboard -log = logging.getLogger("argopy.gdacftp.index") +log = logging.getLogger("argopy.gdac.index") has_pyarrow = importlib.util.find_spec('pyarrow') is not None if has_pyarrow: @@ -30,12 +30,12 @@ access_points = ["wmo", "box"] exit_formats = ["xarray"] dataset_ids = ["phy", "bgc"] # First is default -api_server = OPTIONS['ftp'] # API root url +api_server = OPTIONS['gdac'] # API root url api_server_check = api_server # URL to check if the API is alive, used by isAPIconnected -class FTPArgoIndexFetcher(ABC): - """ Manage access to Argo index from a remote GDAC FTP """ +class GDACArgoIndexFetcher(ABC): + """Manage access to Argo index from a GDAC server""" # N_FILES = None ### @@ -51,7 +51,7 @@ def init(self, *args, **kwargs): ### def __init__( self, - ftp: str = "", + gdac: str = "", ds: str = "", cache: bool = False, cachedir: str = "", @@ -63,8 +63,8 @@ def __init__( Parameters ---------- - ftp: str (optional) - Path to the remote FTP directory where the 'dac' folder is located. + gdac: str (optional) + Path to the local or remote directory where the 'dac' folder is located ds: str (optional) Dataset to load: 'phy' or 'ref' or 'bgc' errors: str (optional) @@ -75,12 +75,12 @@ def __init__( cachedir: str (optional) Path to cache folder api_timeout: int (optional) - FTP request time out in seconds. Set to OPTIONS['api_timeout'] by default. + Server request time out in seconds. Set to OPTIONS['api_timeout'] by default. """ self.timeout = OPTIONS["api_timeout"] if api_timeout == 0 else api_timeout - self.definition = "Ifremer GDAC ftp Argo index fetcher" + self.definition = "Ifremer GDAC Argo index fetcher" self.dataset_id = OPTIONS["ds"] if ds == "" else ds - self.server = OPTIONS['ftp'] if ftp == "" else ftp + self.server = OPTIONS['gdac'] if gdac == "" else gdac self.errors = errors # Validate server, raise FtpPathError if not valid. @@ -107,10 +107,10 @@ def __init__( self.init(**kwargs) def __repr__(self): - summary = [""] + summary = [""] summary.append("Name: %s" % self.definition) summary.append("Index: %s" % self.indexfs.index_file) - summary.append("FTP: %s" % self.server) + summary.append("Server: %s" % self.server) summary.append("Domain: %s" % format_oneline(self.cname())) if hasattr(self.indexfs, 'index'): summary.append("Index loaded: True (%i records)" % self.N_RECORDS) @@ -177,8 +177,8 @@ def dashboard(self, **kw): warnings.warn("Dashboard only available for a single float or profile request") -class Fetch_wmo(FTPArgoIndexFetcher): - """ Manage access to GDAC ftp Argo index for: a list of WMOs, CYCs """ +class Fetch_wmo(GDACArgoIndexFetcher): + """ Manage access to GDAC Argo index for: a list of WMOs, CYCs """ def init(self, WMO: list = [], CYC=None, **kwargs): """ Create Argo index fetcher for WMOs @@ -217,8 +217,8 @@ def uri(self): return self._list_of_argo_files -class Fetch_box(FTPArgoIndexFetcher): - """ Manage access to GDAC ftp Argo index for: a rectangular space/time domain """ +class Fetch_box(GDACArgoIndexFetcher): + """ Manage access to GDAC Argo index for: a rectangular space/time domain """ def init(self, box: list, **kwargs): """ Create Argo index fetcher @@ -238,7 +238,7 @@ def init(self, box: list, **kwargs): """ # We use a full domain definition (x, y, z, t) as argument for compatibility with the other fetchers # but at this point, we internally work only with x, y and t. - # log.debug("Create FTPArgoIndexFetcher.Fetch_box instance with index BOX: %s" % box) + # log.debug("Create GDACArgoIndexFetcher.Fetch_box instance with index BOX: %s" % box) self.indexBOX = box.copy() self._nrows = None diff --git a/argopy/fetchers.py b/argopy/fetchers.py index 5fe8fdc5..4c26406f 100755 --- a/argopy/fetchers.py +++ b/argopy/fetchers.py @@ -832,11 +832,6 @@ def __init__( _VALIDATORS["src"](self._src) # Load data source access points: - if self._src == "localftp": - raise ValueError( - "The 'localftp' data source is deprecated. It's been replaced by 'gdac'." - ) - if self._src not in AVAILABLE_INDEX_SOURCES: raise InvalidFetcher( "Requested index fetcher '%s' not available ! " diff --git a/argopy/options.py b/argopy/options.py index 60c30e72..ed9daf60 100644 --- a/argopy/options.py +++ b/argopy/options.py @@ -19,7 +19,7 @@ # Define option names as seen by users: DATA_SOURCE = "src" -FTP = "ftp" +GDAC = "gdac" ERDDAP = 'erddap' DATASET = "ds" CACHE_FOLDER = "cachedir" @@ -35,7 +35,7 @@ # Define the list of available options and default values: OPTIONS = { DATA_SOURCE: "erddap", - FTP: "https://data-argo.ifremer.fr", + GDAC: "https://data-argo.ifremer.fr", ERDDAP: "https://erddap.ifremer.fr/erddap", DATASET: "phy", CACHE_FOLDER: os.path.expanduser(os.path.sep.join(["~", ".cache", "argopy"])), @@ -61,11 +61,11 @@ def _positive_integer(value): return isinstance(value, int) and value > 0 -def validate_ftp(this_path): +def validate_gdac(this_path): if this_path != "-": return check_gdac_path(this_path, errors='raise') else: - log.debug("OPTIONS['%s'] is not defined" % FTP) + log.debug("OPTIONS['%s'] is not defined" % GDAC) return False @@ -79,7 +79,7 @@ def validate_http(this_path): _VALIDATORS = { DATA_SOURCE: _DATA_SOURCE_LIST.__contains__, - FTP: validate_ftp, + GDAC: validate_gdac, ERDDAP: validate_http, DATASET: _DATASET_LIST.__contains__, CACHE_FOLDER: lambda x: os.access(x, os.W_OK), @@ -108,7 +108,7 @@ class set_options: - ``mode``: User mode. Default: ``standard``. Possible values: ``standard``, ``expert`` or ``research``. - - ``ftp``: Default path to be used by the GDAC fetchers and Argo index stores + - ``gdac``: Default path to be used by the GDAC fetchers and Argo index stores Default: https://data-argo.ifremer.fr - ``erddap``: Default server address to be used by the data and index erddap fetchers Default: https://erddap.ifremer.fr/erddap @@ -187,7 +187,7 @@ def check_erddap_path(path, errors='ignore'): def check_gdac_path(path, errors='ignore'): # noqa: C901 - """ Check if a path has the expected GDAC ftp structure + """ Check if a path has the expected GDAC server structure Check if a path is structured like: . From ea0dc614e4f9a8e53040157c4856dac8661490cb Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 10 Sep 2024 13:00:54 +0200 Subject: [PATCH 07/19] More 'ftp' to 'gdac' refactoring --- argopy/utils/lists.py | 8 ++--- .../fetching-argo-data/data_sources.rst | 35 ++++++++++--------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/argopy/utils/lists.py b/argopy/utils/lists.py index bb6beb7e..837361e2 100644 --- a/argopy/utils/lists.py +++ b/argopy/utils/lists.py @@ -48,8 +48,8 @@ def list_available_data_src() -> dict: from ..data_fetchers import gdac_data as GDAC_Fetchers # Ensure we're loading the gdac data fetcher with the current options: - GDAC_Fetchers.api_server_check = OPTIONS["ftp"] - GDAC_Fetchers.api_server = OPTIONS["ftp"] + GDAC_Fetchers.api_server_check = OPTIONS["gdac"] + GDAC_Fetchers.api_server = OPTIONS["gdac"] sources["gdac"] = GDAC_Fetchers except Exception: @@ -89,8 +89,8 @@ def list_available_index_src() -> dict: from ..data_fetchers import gdac_index as GDAC_Fetchers # Ensure we're loading the gdac data fetcher with the current options: - GDAC_Fetchers.api_server_check = OPTIONS["ftp"] - GDAC_Fetchers.api_server = OPTIONS["ftp"] + GDAC_Fetchers.api_server_check = OPTIONS["gdac"] + GDAC_Fetchers.api_server = OPTIONS["gdac"] sources["gdac"] = GDAC_Fetchers except Exception: diff --git a/docs/user-guide/fetching-argo-data/data_sources.rst b/docs/user-guide/fetching-argo-data/data_sources.rst index 4c03e9e4..41c6ca73 100644 --- a/docs/user-guide/fetching-argo-data/data_sources.rst +++ b/docs/user-guide/fetching-argo-data/data_sources.rst @@ -40,9 +40,9 @@ Available data sources Since this is the most efficient method to fetcher Argo data, it's the default data source in **argopy**. 2. 🌐 an Argo GDAC server or any other GDAC-compliant local folder. - You can fetch data from any of the 3 official GDAC online servers: the Ifremer https and ftp and the US ftp. - This data source can also point toward your own local copy of the `GDAC - ftp content `__. + You can fetch data from any of the 3 official GDAC online servers: the Ifremer https and ftp and the US https. + This data source can also point toward your own local copy of a `GDAC + server content `__. You can select this data source with the keyword ``gdac`` and methods described below. 3. πŸ‘ the `Argovis server `__. @@ -182,22 +182,22 @@ Fetched data and variables .. tabs:: - .. tab:: **GDAC** ftp + .. tab:: **GDAC** servers - Let's retrieve one float data from a local sample of the GDAC ftp (a sample GDAC ftp is downloaded automatically with the method :meth:`argopy.tutorial.open_dataset`): + Let's retrieve one float data from a local sample of the GDAC (a GDAC sample can be downloaded automatically with the method :meth:`argopy.tutorial.open_dataset`): .. ipython:: python :okwarning: - # Download ftp sample and get the ftp local path: - ftproot = argopy.tutorial.open_dataset('gdac')[0] + # Download a GDAC sample and get the corresponding local path: + gdacroot = argopy.tutorial.open_dataset('gdac')[0] # then fetch data: - with argopy.set_options(src='gdac', ftp=ftproot): + with argopy.set_options(src='gdac', gdac=gdacroot): ds = ArgoDataFetcher().float(1900857).load().data print(ds) - .. tab:: **erddap** + .. tab:: **erddap** API Let’s now retrieve the latest data for this float from the ``erddap``: @@ -208,7 +208,7 @@ Fetched data and variables ds = ArgoDataFetcher().float(1900857).load().data print(ds) - .. tab:: **argovis** + .. tab:: **argovis** API And with ``argovis``: @@ -247,21 +247,22 @@ Note that the :meth:`argopy.status` method has a ``refresh`` option to let you s Last, you can check out `the following argopy status webpage that monitors all important resources to the software `_. -Setting-up your own local copy of the GDAC ftp ----------------------------------------------- +Setting-up your own local copy of the GDAC https server +------------------------------------------------------- Data fetching with the ``gdac`` data source will require you to -specify the path toward your local copy of the GDAC ftp server with the -``ftp`` option. +specify the path toward your local copy of the GDAC server with the +``gdac`` option. This is not an issue for expert users, but standard users may wonder how to set this up. The primary distribution point for Argo data, the only one with full support from data centers and with nearly a 100% time -availability, is the GDAC ftp. One server is available (the US GODAE mirror has been discontinued in 2024): +availability, is a GDAC https server. Two servers are available: -- France Coriolis: ftp://ftp.ifremer.fr/ifremer/argo +- France Coriolis: https://data-argo.ifremer.fr +- USA GODAE: https://usgodae.org/pub/outgoing/argo -If you want to get your own copy of the ftp server content, you have 2 options detailed below. +If you want to get your own copy of the GDAC server content, you have 2 options detailed below. Copy with DOI reference From 691c48762533b0bb4dd860c7c255a2510714b0c9 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 10 Sep 2024 13:08:01 +0200 Subject: [PATCH 08/19] More 'ftp' to 'gdac' refactoring in documentation --- docs/metadata_fetching.rst | 3 +-- docs/user-guide/working-with-argo-data/owc_workflow_eg.py | 4 ++-- docs/whats-new.rst | 6 ++++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/metadata_fetching.rst b/docs/metadata_fetching.rst index 3148e579..f43d306f 100644 --- a/docs/metadata_fetching.rst +++ b/docs/metadata_fetching.rst @@ -31,8 +31,7 @@ You can use the Index fetcher with the ``region`` or ``float`` access points, si :suppress: import argopy - ftproot = argopy.tutorial.open_dataset('gdac')[0] - argopy.set_options(ftp=ftproot) + argopy.set_options(gdac=argopy.tutorial.open_dataset('gdac')[0]) .. ipython:: python :okwarning: diff --git a/docs/user-guide/working-with-argo-data/owc_workflow_eg.py b/docs/user-guide/working-with-argo-data/owc_workflow_eg.py index e74bb0a0..e2f05065 100644 --- a/docs/user-guide/working-with-argo-data/owc_workflow_eg.py +++ b/docs/user-guide/working-with-argo-data/owc_workflow_eg.py @@ -32,8 +32,8 @@ print(owc.configuration.print_cfg(USER_CONFIG)) # Create float source data with argopy: -fetcher_for_real = DataFetcher(src='localftp', cache=True, mode='expert').float(FLOAT_NAME) -fetcher_sample = DataFetcher(src='localftp', cache=True, mode='expert').profile(FLOAT_NAME, [1, +fetcher_for_real = DataFetcher(src='gdac', cache=True, mode='expert').float(FLOAT_NAME) +fetcher_sample = DataFetcher(src='gdac', cache=True, mode='expert').profile(FLOAT_NAME, [1, 2]) # To reduce execution time for demo ds = fetcher_sample.load().data ds.argo.create_float_source(path=USER_CONFIG['FLOAT_SOURCE_DIRECTORY'], force='default') diff --git a/docs/whats-new.rst b/docs/whats-new.rst index 2869c8f7..a001489a 100644 --- a/docs/whats-new.rst +++ b/docs/whats-new.rst @@ -26,8 +26,6 @@ Features and front-end API - **Improved support for BGC** - **argopy** now support `standard` and `research` user modes for the `erddap` data source with the `bgc` dataset. These new user modes follows the last available ADMT recommendations to bring users a finely tuned set of BGC parameters. - - - New BGC method :class:`Dataset.argo.canyon_med` to make CANYON-MED predictions of Water-Column Nutrient Concentrations and Carbonate System Variables in the Mediterranean Sea. This method can be used to predict PO4, NO3, DIC, SiOH4, AT and pHT. (:pr:`364`) by `G. Maze `_. .. currentmodule:: argopy @@ -90,6 +88,10 @@ Breaking changes .. currentmodule:: argopy +- The option name "ftp" is now renamed "gdac" (:pr:`389`) by `G. Maze `_ + +- The option name "dataset" is now renamed "ds" (:pr:`389`) by `G. Maze `_ + Coming up next in v0.1.17 ------------------------- From 1b1c35cb9ac39e422810836c6d6e9d49688b06c2 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 10 Sep 2024 16:32:20 +0200 Subject: [PATCH 09/19] Refactor "FtpPathError" into "GdacPathError" --- argopy/data_fetchers/gdac_data.py | 2 +- argopy/data_fetchers/gdac_index.py | 2 +- argopy/errors.py | 8 ++--- argopy/options.py | 8 ++--- argopy/stores/argo_index_proto.py | 8 ++--- argopy/tests/helpers/utils.py | 4 +-- argopy/tests/test_errors.py | 4 +-- argopy/tests/test_fetchers_data_gdac.py | 46 ++++++++++++------------ argopy/tests/test_fetchers_index_gdac.py | 4 +-- argopy/tests/test_options.py | 4 +-- argopy/tests/test_stores_index.py | 4 +-- argopy/tests/test_utils_checkers.py | 4 +-- argopy/utils/checkers.py | 8 ++--- 13 files changed, 53 insertions(+), 53 deletions(-) diff --git a/argopy/data_fetchers/gdac_data.py b/argopy/data_fetchers/gdac_data.py index 4b4f666f..ad7af97e 100644 --- a/argopy/data_fetchers/gdac_data.py +++ b/argopy/data_fetchers/gdac_data.py @@ -96,7 +96,7 @@ def __init__( self.server = OPTIONS["gdac"] if gdac == "" else gdac self.errors = errors - # Validate server, raise FtpPathError if not valid. + # Validate server, raise GdacPathError if not valid. check_gdac_path(self.server, errors="raise") index_file = "core" diff --git a/argopy/data_fetchers/gdac_index.py b/argopy/data_fetchers/gdac_index.py index a697f26c..83b48df5 100644 --- a/argopy/data_fetchers/gdac_index.py +++ b/argopy/data_fetchers/gdac_index.py @@ -83,7 +83,7 @@ def __init__( self.server = OPTIONS['gdac'] if gdac == "" else gdac self.errors = errors - # Validate server, raise FtpPathError if not valid. + # Validate server, raise GdacPathError if not valid. check_gdac_path(self.server, errors='raise') if self.dataset_id == 'phy': diff --git a/argopy/errors.py b/argopy/errors.py index 25abc8b4..4b185691 100644 --- a/argopy/errors.py +++ b/argopy/errors.py @@ -21,20 +21,20 @@ class NoDataLeft(NoData): pass -class FtpPathError(ValueError): - """Raise when a ftp path is not appropriate.""" +class GdacPathError(ValueError): + """Raise when a GDAC compliant path is not appropriate""" pass class ErddapPathError(ValueError): - """Raise when a erddap path is not appropriate.""" + """Raise when an erddap path is not appropriate""" pass class S3PathError(ValueError): - """Raise when a S3 path is not appropriate.""" + """Raise when a S3 path is not appropriate""" pass diff --git a/argopy/options.py b/argopy/options.py index ed9daf60..3cbf8b0d 100644 --- a/argopy/options.py +++ b/argopy/options.py @@ -11,7 +11,7 @@ from fsspec.core import split_protocol from socket import gaierror from urllib.parse import urlparse -from .errors import OptionValueError, FtpPathError, ErddapPathError +from .errors import OptionValueError, GdacPathError, ErddapPathError # Define a logger @@ -232,14 +232,14 @@ def check_gdac_path(path, errors='ignore'): # noqa: C901 fs = fsspec.filesystem('ftp', host=host, port=port) except gaierror: if errors == 'raise': - raise FtpPathError("Can't get address info (GAIerror) on '%s'" % host) + raise GdacPathError("Can't get address info (GAIerror) on '%s'" % host) elif errors == "warn": warnings.warn("Can't get address info (GAIerror) on '%s'" % host) return False else: return False else: - raise FtpPathError("Unknown protocol for an Argo GDAC host: %s" % split_protocol(path)[0]) + raise GdacPathError("Unknown protocol for an Argo GDAC host: %s" % split_protocol(path)[0]) # dacs = [ # "aoml", @@ -266,7 +266,7 @@ def check_gdac_path(path, errors='ignore'): # noqa: C901 return True elif errors == "raise": - raise FtpPathError("This path is not GDAC compliant (no `dac` folder with legitimate sub-folder):\n%s" % path) + raise GdacPathError("This path is not GDAC compliant (no `dac` folder with legitimate sub-folder):\n%s" % path) elif errors == "warn": warnings.warn("This path is not GDAC compliant:\n%s" % path) diff --git a/argopy/stores/argo_index_proto.py b/argopy/stores/argo_index_proto.py index fa2e9162..0ab95958 100644 --- a/argopy/stores/argo_index_proto.py +++ b/argopy/stores/argo_index_proto.py @@ -13,7 +13,7 @@ from pathlib import Path from ..options import OPTIONS -from ..errors import FtpPathError, S3PathError, InvalidDataset, OptionValueError +from ..errors import GdacPathError, S3PathError, InvalidDataset, OptionValueError from ..utils.checkers import isconnected, has_aws_credentials from ..utils.accessories import Registry from .filesystems import httpstore, memorystore, filestore, ftpstore, s3store @@ -144,7 +144,7 @@ def __init__( % host ) if not isconnected(host): - raise FtpPathError("This host (%s) is not alive !" % host) + raise GdacPathError("This host (%s) is not alive !" % host) self.fs["src"] = ftpstore( host=urlparse(host).hostname, # host eg: ftp.ifremer.fr port=0 if urlparse(host).port is None else urlparse(host).port, @@ -170,7 +170,7 @@ def __init__( self.skip_rows = 10 else: - raise FtpPathError( + raise GdacPathError( "Unknown protocol for an Argo index store: %s" % split_protocol(host)[0] ) @@ -217,7 +217,7 @@ def __init__( index_found = True break if not index_found: - raise FtpPathError("Index file does not exist: %s" % self.index_path) + raise GdacPathError("Index file does not exist: %s" % self.index_path) else: # Will init search with full index by default: self._nrows_index = None diff --git a/argopy/tests/helpers/utils.py b/argopy/tests/helpers/utils.py index 6fd7c6d1..c0becdd4 100644 --- a/argopy/tests/helpers/utils.py +++ b/argopy/tests/helpers/utils.py @@ -30,7 +30,7 @@ from typing import Generator, List from argopy.options import set_options -from argopy.errors import ErddapServerError, ArgovisServerError, DataNotFound, FtpPathError +from argopy.errors import ErddapServerError, ArgovisServerError, DataNotFound, GdacPathError from argopy.utils.lists import ( list_available_data_src, list_available_index_src, @@ -260,7 +260,7 @@ def func_wrapper(*args, **kwargs): msg = "\nServer didn't return the data:\n%s" % str(e.args) xmsg = "Failing because some file were not found, but should work" pass - except FtpPathError as e: + except GdacPathError as e: if 'xfail' in kwargs and kwargs['xfail']: # Some tests will expect this error to be raised, so we make sure it is, for pytest to catch it raise diff --git a/argopy/tests/test_errors.py b/argopy/tests/test_errors.py index 54e57371..804ce558 100644 --- a/argopy/tests/test_errors.py +++ b/argopy/tests/test_errors.py @@ -1,7 +1,7 @@ import pytest from argopy.errors import ( DataNotFound, - FtpPathError, + GdacPathError, ErddapPathError, NetCDF4FileNotFoundError, CacheFileNotFound, @@ -23,7 +23,7 @@ @pytest.mark.parametrize("error", [ DataNotFound, - FtpPathError, + GdacPathError, ErddapPathError, NetCDF4FileNotFoundError, CacheFileNotFound, diff --git a/argopy/tests/test_fetchers_data_gdac.py b/argopy/tests/test_fetchers_data_gdac.py index 48b64883..f9861046 100644 --- a/argopy/tests/test_fetchers_data_gdac.py +++ b/argopy/tests/test_fetchers_data_gdac.py @@ -1,5 +1,5 @@ """ -Test the "GDAC ftp" data fetcher backend +Test the GDAC data fetcher backend Here we try an approach based on fixtures and pytest parametrization to make more explicit the full list of scenario tested. @@ -18,7 +18,7 @@ from argopy.errors import ( CacheFileNotFound, FileSystemHasNoCache, - FtpPathError, + GdacPathError, ) from argopy.utils.checkers import isconnected, is_list_of_strings from utils import requires_gdac @@ -30,7 +30,7 @@ """ -List ftp hosts to be tested. +List GDAC hosts to be tested. Since the fetcher is compatible with host from local, http or ftp protocols, we try to test them all: """ @@ -102,14 +102,14 @@ def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): assert is_list_of_strings(core.cachepath) -def ftp_shortname(ftp): +def gdac_shortname(gdac): """Get a short name for scenarios IDs, given a FTP host""" - if ftp == 'MOCKFTP': + if gdac == 'MOCKFTP': return 'ftp_mocked' - elif 'localhost' in ftp or '127.0.0.1' in ftp: + elif 'localhost' in gdac or '127.0.0.1' in gdac: return 'http_mocked' else: - return (lambda x: 'file' if x == "" else x)(urlparse(ftp).scheme) + return (lambda x: 'file' if x == "" else x)(urlparse(gdac).scheme) """ Make a list of VALID host/dataset/access_points to be tested @@ -119,7 +119,7 @@ def ftp_shortname(ftp): for mode in USER_MODES: for ap in ACCESS_POINTS: VALID_ACCESS_POINTS.append({'host': host, 'ds': 'phy', 'mode': mode, 'access_point': ap}) - VALID_ACCESS_POINTS_IDS.append("host='%s', ds='%s', mode='%s', %s" % (ftp_shortname(host), 'phy', mode, ap)) + VALID_ACCESS_POINTS_IDS.append("host='%s', ds='%s', mode='%s', %s" % (gdac_shortname(host), 'phy', mode, ap)) @@ -136,21 +136,21 @@ def setup_class(self): # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests self.cachedir = tempfile.mkdtemp() - def _patch_ftp(self, ftp): + def _patch_gdac(self, gdac): """Patch Mocked FTP server keyword""" - if ftp == 'MOCKFTP': + if gdac == 'MOCKFTP': return pytest.MOCKFTP # this was set in conftest.py else: - return ftp + return gdac def _setup_fetcher(self, this_request, cached=False, parallel=False): """Helper method to set up options for a fetcher creation""" - ftp = this_request.param['host'] + gdac = this_request.param['host'] access_point = this_request.param['access_point'] - N_RECORDS = None if 'tutorial' in ftp or 'MOCK' in ftp else 100 # Make sure we're not going to load the full index + N_RECORDS = None if 'tutorial' in gdac or 'MOCK' in gdac else 100 # Make sure we're not going to load the full index fetcher_args = {"src": self.src, - "ftp": self._patch_ftp(ftp), + "gdac": self._patch_gdac(gdac), "ds": this_request.param['ds'], "mode": this_request.param['mode'], "cache": cached, @@ -167,8 +167,8 @@ def _setup_fetcher(self, this_request, cached=False, parallel=False): # parallel is False by default, so we don't need to clutter the arguments list del fetcher_args["parallel"] - if not isconnected(fetcher_args['ftp']): - pytest.xfail("Fails because %s not available" % fetcher_args['ftp']) + if not isconnected(fetcher_args['gdac']): + pytest.xfail("Fails because %s not available" % fetcher_args['gdac']) else: return fetcher_args, access_point @@ -194,21 +194,21 @@ def remove_test_dir(): # TESTS # ######### # def test_nocache(self, mocked_httpserver): - # this_fetcher = create_fetcher({"src": self.src, "ftp": self._patch_ftp(VALID_HOSTS[0]), "N_RECORDS": 10}, VALID_ACCESS_POINTS[0]) + # this_fetcher = create_fetcher({"src": self.src, "gdac": self._patch_gdac(VALID_HOSTS[0]), "N_RECORDS": 10}, VALID_ACCESS_POINTS[0]) # with pytest.raises(FileSystemHasNoCache): # this_fetcher.cachepath # @pytest.mark.parametrize("fetcher", VALID_HOSTS, # indirect=True, - # ids=["%s" % ftp_shortname(ftp) for ftp in VALID_HOSTS]) + # ids=["%s" % gdac_shortname(gdac) for gdac in VALID_HOSTS]) # def test_hosts(self, mocked_httpserver, fetcher): # assert (fetcher.N_RECORDS >= 1) - # @pytest.mark.parametrize("ftp_host", ['invalid', 'https://invalid_ftp', 'ftp://invalid_ftp'], indirect=False) - # def test_hosts_invalid(self, ftp_host): + # @pytest.mark.parametrize("gdac_host", ['invalid', 'https://invalid_gdac', 'ftp://invalid_ftp'], indirect=False) + # def test_hosts_invalid(self, gdac_host): # # Invalid servers: - # with pytest.raises(FtpPathError): - # create_fetcher({"src": self.src, "ftp": ftp_host}, VALID_ACCESS_POINTS[0]) + # with pytest.raises(GdacPathError): + # create_fetcher({"src": self.src, "gdac": gdac_host}, VALID_ACCESS_POINTS[0]) @pytest.mark.parametrize("fetcher", VALID_ACCESS_POINTS, indirect=True, ids=VALID_ACCESS_POINTS_IDS) def test_fetching(self, mocked_httpserver, fetcher): @@ -225,5 +225,5 @@ def test_fetching_cached(self, mocked_httpserver, cached_fetcher): def test_uri_mono2multi(self, mocked_httpserver): ap = [v for v in ACCESS_POINTS if 'region' in v.keys()][0] - f = create_fetcher({"src": self.src, "ftp": HOSTS[0], "N_RECORDS": 100}, ap).fetcher + f = create_fetcher({"src": self.src, "gdac": HOSTS[0], "N_RECORDS": 100}, ap).fetcher assert is_list_of_strings(f.uri_mono2multi(f.uri)) diff --git a/argopy/tests/test_fetchers_index_gdac.py b/argopy/tests/test_fetchers_index_gdac.py index d6c6cda9..8534c570 100644 --- a/argopy/tests/test_fetchers_index_gdac.py +++ b/argopy/tests/test_fetchers_index_gdac.py @@ -11,7 +11,7 @@ from argopy.errors import ( CacheFileNotFound, FileSystemHasNoCache, - FtpPathError + GdacPathError ) from argopy.utils.checkers import isconnected, is_list_of_strings from utils import requires_gdac @@ -183,7 +183,7 @@ def test_hosts(self, mocked_httpserver, _make_a_fetcher): @pytest.mark.parametrize("ftp_host", ['invalid', 'https://invalid_ftp', 'ftp://invalid_ftp'], indirect=False) def test_hosts_invalid(self, ftp_host): # Invalid servers: - with pytest.raises(FtpPathError): + with pytest.raises(GdacPathError): create_fetcher({"src": self.src, "ftp": ftp_host}, VALID_ACCESS_POINTS[0]) @pytest.mark.parametrize("_make_a_fetcher", scenarios, indirect=True, ids=scenarios_ids) diff --git a/argopy/tests/test_options.py b/argopy/tests/test_options.py index 81e130a1..bccde57e 100644 --- a/argopy/tests/test_options.py +++ b/argopy/tests/test_options.py @@ -3,7 +3,7 @@ import platform import argopy from argopy.options import OPTIONS -from argopy.errors import OptionValueError, FtpPathError, ErddapPathError +from argopy.errors import OptionValueError, GdacPathError, ErddapPathError from utils import requires_gdac, create_read_only_folder from mocked_http import mocked_httpserver, mocked_server_address import logging @@ -26,7 +26,7 @@ def test_opt_src(): @requires_gdac def test_opt_gdac_ftp(): - with pytest.raises(FtpPathError): + with pytest.raises(GdacPathError): argopy.set_options(ftp="invalid_path") local_ftp = argopy.tutorial.open_dataset("gdac")[0] diff --git a/argopy/tests/test_stores_index.py b/argopy/tests/test_stores_index.py index 711ab710..ffcdaf0f 100644 --- a/argopy/tests/test_stores_index.py +++ b/argopy/tests/test_stores_index.py @@ -12,7 +12,7 @@ import argopy from argopy.errors import ( - FtpPathError, + GdacPathError, OptionValueError, InvalidDatasetStructure, ) @@ -323,7 +323,7 @@ def test_hosts(self, mocked_httpserver, a_store): ) def test_hosts_invalid(self, ftp_host): # Invalid servers: - with pytest.raises(FtpPathError): + with pytest.raises(GdacPathError): self.indexstore(host=ftp_host) def test_index(self): diff --git a/argopy/tests/test_utils_checkers.py b/argopy/tests/test_utils_checkers.py index b8c2d53d..397750a2 100644 --- a/argopy/tests/test_utils_checkers.py +++ b/argopy/tests/test_utils_checkers.py @@ -5,7 +5,7 @@ requires_erddap, ) import argopy -from argopy.errors import FtpPathError +from argopy.errors import GdacPathError from argopy.utils.checkers import ( is_box, is_indexbox, check_wmo, is_wmo, @@ -187,7 +187,7 @@ def test_check_cyc(): def test_check_gdac_path(): assert check_gdac_path("dummy_path", errors='ignore') is False - with pytest.raises(FtpPathError): + with pytest.raises(GdacPathError): check_gdac_path("dummy_path", errors='raise') with pytest.warns(UserWarning): assert check_gdac_path("dummy_path", errors='warn') is False diff --git a/argopy/utils/checkers.py b/argopy/utils/checkers.py index 51fd20f6..1576e643 100644 --- a/argopy/utils/checkers.py +++ b/argopy/utils/checkers.py @@ -13,7 +13,7 @@ import importlib from ..options import OPTIONS -from ..errors import InvalidDatasetStructure, FtpPathError, InvalidFetcher +from ..errors import InvalidDatasetStructure, GdacPathError, InvalidFetcher from .lists import list_available_data_src, list_available_index_src from .casting import to_list @@ -489,14 +489,14 @@ def check_gdac_path(path, errors="ignore"): # noqa: C901 fs = fsspec.filesystem("ftp", host=host) except gaierror: if errors == "raise": - raise FtpPathError("Can't get address info (GAIerror) on '%s'" % host) + raise GdacPathError("Can't get address info (GAIerror) on '%s'" % host) elif errors == "warn": warnings.warn("Can't get address info (GAIerror) on '%s'" % host) return False else: return False else: - raise FtpPathError( + raise GdacPathError( "Unknown protocol for an Argo GDAC host: %s" % split_protocol(path)[0] ) @@ -523,7 +523,7 @@ def check_gdac_path(path, errors="ignore"): # noqa: C901 if check1: return True elif errors == "raise": - raise FtpPathError( + raise GdacPathError( "This path is not GDAC compliant (no `dac` folder with legitimate sub-folder):\n%s" % path ) From 91310fcd1a4315621902a483caee494fd2ebff5f Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 10 Sep 2024 16:37:48 +0200 Subject: [PATCH 10/19] Update checkers.py --- argopy/utils/checkers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/argopy/utils/checkers.py b/argopy/utils/checkers.py index 1576e643..497776b4 100644 --- a/argopy/utils/checkers.py +++ b/argopy/utils/checkers.py @@ -442,9 +442,9 @@ def check_index_cols(column_names: list, convention: str = "ar_index_global_prof def check_gdac_path(path, errors="ignore"): # noqa: C901 - """Check if a path has the expected GDAC ftp structure + """Check if a path has the expected GDAC structure - Expected GDAC ftp structure:: + Expected GDAC structure:: . └── dac @@ -458,9 +458,10 @@ def check_gdac_path(path, errors="ignore"): # noqa: C901 This check will return True if at least one DAC sub-folder is found under path/dac/ Examples:: + >>> check_gdac_path("https://data-argo.ifremer.fr") # True + >>> check_gdac_path("https://usgodae.org/pub/outgoing/argo") # True >>> check_gdac_path("ftp://ftp.ifremer.fr/ifremer/argo") # True - >>> check_gdac_path("ftp://usgodae.org/pub/outgoing/argo") # True >>> check_gdac_path("/home/ref-argo/gdac") # True >>> check_gdac_path("https://www.ifremer.fr") # False >>> check_gdac_path("ftp://usgodae.org/pub/outgoing") # False From 86f2edd18c8a93e0438b3d62ecd75c75bb096ad6 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 11 Sep 2024 11:20:06 +0200 Subject: [PATCH 11/19] More 'ftp' to 'gdac' refactoring --- argopy/options.py | 2 +- argopy/tests/test_fetchers_index_gdac.py | 40 ++++++++++++------------ argopy/tests/test_options.py | 10 +++--- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/argopy/options.py b/argopy/options.py index 3cbf8b0d..bc3dbe5c 100644 --- a/argopy/options.py +++ b/argopy/options.py @@ -99,7 +99,7 @@ class set_options: List of options: - - ``dataset``: Define the Dataset to work with. + - ``ds``: Define the Dataset to work with. Default: ``phy``. Possible values: ``phy``, ``bgc`` or ``ref``. - ``src``: Source of fetched data. diff --git a/argopy/tests/test_fetchers_index_gdac.py b/argopy/tests/test_fetchers_index_gdac.py index 8534c570..bf7ebc38 100644 --- a/argopy/tests/test_fetchers_index_gdac.py +++ b/argopy/tests/test_fetchers_index_gdac.py @@ -23,7 +23,7 @@ """ -List ftp hosts to be tested. +List GDAC hosts to be tested. Since the fetcher is compatible with host from local, http or ftp protocols, we try to test them all: """ @@ -89,14 +89,14 @@ def assert_fetcher(mocked_erddapserver, this_fetcher, cacheable=False): assert is_list_of_strings(this_fetcher.cachepath) -def ftp_shortname(ftp): +def gdac_shortname(gdac): """Get a short name for scenarios IDs, given a FTP host""" - if ftp == 'MOCKFTP': + if gdac == 'MOCKFTP': return 'ftp_mocked' - elif 'localhost' in ftp or '127.0.0.1' in ftp: + elif 'localhost' in gdac or '127.0.0.1' in gdac: return 'http_mocked' else: - return (lambda x: 'file' if x == "" else x)(urlparse(ftp).scheme) + return (lambda x: 'file' if x == "" else x)(urlparse(gdac).scheme) @requires_gdac @@ -108,7 +108,7 @@ class TestBackend: scenarios = [(h, ap) for h in VALID_HOSTS for ap in VALID_ACCESS_POINTS] scenarios_ids = [ - "%s, %s" % (ftp_shortname(fix[0]), list(fix[1].keys())[0]) for + "%s, %s" % (gdac_shortname(fix[0]), list(fix[1].keys())[0]) for fix in scenarios] @@ -122,29 +122,29 @@ def setup_class(self): # Create the cache folder here, so that it's not the same for the pandas and pyarrow tests self.cachedir = tempfile.mkdtemp() - def _patch_ftp(self, ftp): + def _patch_gdac(self, gdac): """Patch Mocked FTP server keyword""" - if ftp == 'MOCKFTP': + if gdac == 'MOCKFTP': return pytest.MOCKFTP # this was set in conftest.py else: - return ftp + return gdac def _setup_fetcher(self, this_request, cached=False): """Helper method to set up options for a fetcher creation""" if isinstance(this_request.param, tuple): - ftp = this_request.param[0] + gdac = this_request.param[0] access_point = this_request.param[1] else: - ftp = this_request.param + gdac = this_request.param access_point = VALID_ACCESS_POINTS[0] # Use 1st valid access point - N_RECORDS = None if 'tutorial' in ftp or 'MOCK' in ftp else 100 # Make sure we're not going to load the full index - fetcher_args = {"src": self.src, "ftp": self._patch_ftp(ftp), "cache": False, "N_RECORDS": N_RECORDS} + N_RECORDS = None if 'tutorial' in gdac or 'MOCK' in gdac else 100 # Make sure we're not going to load the full index + fetcher_args = {"src": self.src, "gdac": self._patch_gdac(gdac), "cache": False, "N_RECORDS": N_RECORDS} if cached: fetcher_args = {**fetcher_args, **{"cache": True, "cachedir": self.cachedir}} - if not isconnected(fetcher_args['ftp']): - pytest.xfail("Fails because %s not available" % fetcher_args['ftp']) + if not isconnected(fetcher_args['gdac']): + pytest.xfail("Fails because %s not available" % fetcher_args['gdac']) else: return fetcher_args, access_point @@ -170,21 +170,21 @@ def remove_test_dir(): # TESTS # ######### def test_nocache(self, mocked_httpserver): - this_fetcher = create_fetcher({"src": self.src, "ftp": self._patch_ftp(VALID_HOSTS[0]), "N_RECORDS": 10}, VALID_ACCESS_POINTS[0]) + this_fetcher = create_fetcher({"src": self.src, "gdac": self._patch_gdac(VALID_HOSTS[0]), "N_RECORDS": 10}, VALID_ACCESS_POINTS[0]) with pytest.raises(FileSystemHasNoCache): this_fetcher.cachepath @pytest.mark.parametrize("_make_a_fetcher", VALID_HOSTS, indirect=True, - ids=["%s" % ftp_shortname(ftp) for ftp in VALID_HOSTS]) + ids=["%s" % gdac_shortname(host) for host in VALID_HOSTS]) def test_hosts(self, mocked_httpserver, _make_a_fetcher): assert (_make_a_fetcher.N_RECORDS >= 1) - @pytest.mark.parametrize("ftp_host", ['invalid', 'https://invalid_ftp', 'ftp://invalid_ftp'], indirect=False) - def test_hosts_invalid(self, ftp_host): + @pytest.mark.parametrize("gdac_host", ['invalid', 'https://invalid_gdac', 'ftp://invalid_ftp'], indirect=False) + def test_hosts_invalid(self, gdac_host): # Invalid servers: with pytest.raises(GdacPathError): - create_fetcher({"src": self.src, "ftp": ftp_host}, VALID_ACCESS_POINTS[0]) + create_fetcher({"src": self.src, "gdac": gdac_host}, VALID_ACCESS_POINTS[0]) @pytest.mark.parametrize("_make_a_fetcher", scenarios, indirect=True, ids=scenarios_ids) def test_fetching(self, mocked_httpserver, _make_a_fetcher): diff --git a/argopy/tests/test_options.py b/argopy/tests/test_options.py index bccde57e..e521cecb 100644 --- a/argopy/tests/test_options.py +++ b/argopy/tests/test_options.py @@ -25,13 +25,13 @@ def test_opt_src(): @requires_gdac -def test_opt_gdac_ftp(): +def test_opt_gdac(): with pytest.raises(GdacPathError): - argopy.set_options(ftp="invalid_path") + argopy.set_options(gdac="invalid_path") - local_ftp = argopy.tutorial.open_dataset("gdac")[0] - with argopy.set_options(ftp=local_ftp): - assert OPTIONS["ftp"] == local_ftp + local_gdac = argopy.tutorial.open_dataset("gdac")[0] + with argopy.set_options(gdac=local_gdac): + assert OPTIONS["gdac"] == local_gdac def test_opt_ifremer_erddap(mocked_httpserver): From a7557049d8428ab32e214bea7ff1f5a96eac6a63 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 11 Sep 2024 14:20:02 +0200 Subject: [PATCH 12/19] Update test_fetchers_facade_data.py --- argopy/tests/test_fetchers_facade_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argopy/tests/test_fetchers_facade_data.py b/argopy/tests/test_fetchers_facade_data.py index a6edd17d..6d389506 100644 --- a/argopy/tests/test_fetchers_facade_data.py +++ b/argopy/tests/test_fetchers_facade_data.py @@ -45,7 +45,7 @@ class Test_Facade: # Use the first valid data source: src = 'gdac' - src_opts = {'ftp': argopy.tutorial.open_dataset("gdac")[0]} + src_opts = {'gdac': argopy.tutorial.open_dataset("gdac")[0]} def __get_fetcher(self, empty: bool = False, pt: str = 'profile'): f = ArgoDataFetcher(src=self.src, **self.src_opts) From 46a658615931c226af58443fc491b8483fe629cb Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 12 Sep 2024 10:22:25 +0200 Subject: [PATCH 13/19] more ftp to gdac fixing --- argopy/tests/test_plot_plot.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/argopy/tests/test_plot_plot.py b/argopy/tests/test_plot_plot.py index f05bdfe1..256cd9d4 100644 --- a/argopy/tests/test_plot_plot.py +++ b/argopy/tests/test_plot_plot.py @@ -83,7 +83,7 @@ def test_invalid_method(self): @requires_matplotlib class Test_plot_trajectory: src = "gdac" - local_ftp = argopy.tutorial.open_dataset("gdac")[0] + local_gdac = argopy.tutorial.open_dataset("gdac")[0] requests = { # "float": [[2901623], [2901623, 6901929, 5906072]], # "profile": [[2901623, 12], [6901929, [5, 45]]], @@ -117,7 +117,7 @@ def __test_traj_plot(self, df, opts): @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) def test_with_a_region(self, opts): - with argopy.set_options(src=self.src, ftp=self.local_ftp): + with argopy.set_options(src=self.src, gdac=self.local_gdac): for arg in self.requests["region"]: loader = ArgoDataFetcher(cache=True).region(arg).load() self.__test_traj_plot(loader.index, opts) @@ -127,7 +127,7 @@ def test_with_a_region(self, opts): @requires_matplotlib class Test_bar_plot: src = "gdac" - local_ftp = argopy.tutorial.open_dataset("gdac")[0] + local_gdac = argopy.tutorial.open_dataset("gdac")[0] requests = { # "float": [[2901623], [2901623, 6901929, 5906072]], # "profile": [[2901623, 12], [6901929, [5, 45]]], @@ -148,7 +148,7 @@ def __test_bar_plot(self, df, opts): @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) def test_with_a_region(self, opts): - with argopy.set_options(src=self.src, ftp=self.local_ftp): + with argopy.set_options(src=self.src, gdac=self.local_gdac): for arg in self.requests["region"]: loader = ArgoDataFetcher().region(arg) self.__test_bar_plot(loader.to_index(full=True), opts) @@ -159,7 +159,7 @@ def test_with_a_region(self, opts): @requires_cartopy class Test_scatter_map: src = "gdac" - local_ftp = argopy.tutorial.open_dataset("gdac")[0] + local_gdac = argopy.tutorial.open_dataset("gdac")[0] requests = { # "float": [[2901623], [2901623, 6901929, 5906072]], # "profile": [[2901623, 12], [6901929, [5, 45]]], @@ -188,7 +188,7 @@ def __test(self, data, axis, opts): @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) def test_with_a_dataset_of_points(self, opts): - with argopy.set_options(src=self.src, ftp=self.local_ftp): + with argopy.set_options(src=self.src, gdac=self.local_gdac): for arg in self.requests["region"]: loader = ArgoDataFetcher(cache=True).region(arg).load() with pytest.raises(InvalidDatasetStructure): @@ -196,7 +196,7 @@ def test_with_a_dataset_of_points(self, opts): @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) def test_with_a_dataset_of_profiles(self, opts): - with argopy.set_options(src=self.src, ftp=self.local_ftp): + with argopy.set_options(src=self.src, gdac=self.local_gdac): for arg in self.requests["region"]: loader = ArgoDataFetcher(cache=True).region(arg).load() dsp = loader.data.argo.point2profile() @@ -206,7 +206,7 @@ def test_with_a_dataset_of_profiles(self, opts): @pytest.mark.parametrize("opts", opts, indirect=False, ids=opts_ids) def test_with_a_dataframe_of_index(self, opts): - with argopy.set_options(src=self.src, ftp=self.local_ftp): + with argopy.set_options(src=self.src, gdac=self.local_gdac): for arg in self.requests["region"]: loader = ArgoDataFetcher(cache=True).region(arg).load() self.__test(loader.index, (None, None, None), opts) From a62dc9d0f8bf1e8a9889ad85b219c64133df9cbc Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 12 Sep 2024 10:22:32 +0200 Subject: [PATCH 14/19] Update test_xarray_accessor.py --- argopy/tests/test_xarray_accessor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/argopy/tests/test_xarray_accessor.py b/argopy/tests/test_xarray_accessor.py index cdf086df..fdca2cc1 100644 --- a/argopy/tests/test_xarray_accessor.py +++ b/argopy/tests/test_xarray_accessor.py @@ -188,7 +188,7 @@ def test_teos10_invalid_variable(self, ds_pts): @requires_gsw @requires_gdac class Test_create_float_source: - local_ftp = argopy.tutorial.open_dataset("gdac")[0] + local_gdac = argopy.tutorial.open_dataset("gdac")[0] def is_valid_mdata(self, this_mdata): """Validate structure of the output dataset """ @@ -209,13 +209,13 @@ def is_valid_mdata(self, this_mdata): return np.all(check) def test_error_user_mode(self): - with argopy.set_options(ftp=self.local_ftp): + with argopy.set_options(gdac=self.local_gdac): with pytest.raises(InvalidDatasetStructure): ds = ArgoDataFetcher(src="gdac", mode='standard').float([6901929, 2901623]).load().data ds.argo.create_float_source() def test_opt_force(self): - with argopy.set_options(ftp=self.local_ftp): + with argopy.set_options(gdac=self.local_gdac): expert_ds = ArgoDataFetcher(src="gdac", mode='expert').float([2901623]).load().data with pytest.raises(OptionValueError): @@ -232,7 +232,7 @@ def test_opt_force(self): assert np.all([self.is_valid_mdata(ds_float_source[k]) for k in ds_float_source.keys()]) def test_filecreate(self): - with argopy.set_options(ftp=self.local_ftp): + with argopy.set_options(gdac=self.local_gdac): expert_ds = ArgoDataFetcher(src="gdac", mode='expert').float([6901929, 2901623]).load().data N_file = len(np.unique(expert_ds['PLATFORM_NUMBER'])) From aa61927728d4f2846f9f256bfddb29c262d590c1 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 12 Sep 2024 10:22:47 +0200 Subject: [PATCH 15/19] Update argovis_data.py removing numpy datetime64 utc warning --- argopy/data_fetchers/argovis_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argopy/data_fetchers/argovis_data.py b/argopy/data_fetchers/argovis_data.py index 9a7298f3..3d828ea7 100755 --- a/argopy/data_fetchers/argovis_data.py +++ b/argopy/data_fetchers/argovis_data.py @@ -360,7 +360,7 @@ def to_xarray(self, errors: str = "ignore"): ds = ds.set_coords(coords) # Cast data types and add variable attributes (not available in the csv download): - ds["TIME"] = ds["TIME"].astype("datetime64[ns]") + ds["TIME"] = pd.to_datetime(ds["TIME"], utc=True) ds = self._add_attributes(ds) ds = ds.argo.cast_types() From c6bb528b1c037cb01f7a2c446e594edfec23d734 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Thu, 12 Sep 2024 16:15:21 +0200 Subject: [PATCH 16/19] Update static assets --- .../assets/admt_documentation_catalogue.json | 2 +- .../assets/api_coriolis_parameter_codes.json | 182 +++++++++- argopy/static/assets/data_types.json | 322 +++++++++--------- argopy/static/assets/institutions.json | 40 +-- argopy/static/assets/profilers.json | 78 ++--- .../assets/variables_bgc_synthetic.json | 2 +- 6 files changed, 402 insertions(+), 224 deletions(-) diff --git a/argopy/static/assets/admt_documentation_catalogue.json b/argopy/static/assets/admt_documentation_catalogue.json index 73764d40..a1c0250e 100644 --- a/argopy/static/assets/admt_documentation_catalogue.json +++ b/argopy/static/assets/admt_documentation_catalogue.json @@ -1,7 +1,7 @@ { "name": "ADMT documentation catalogue", "long_name": "Titles and DOIs of all the official ADMT documentation", - "last_update": "2023-12-21T15:21:36.774525+00:00", + "last_update": "2024-09-12T14:13:10.643203+00:00", "data": { "catalogue": [ { diff --git a/argopy/static/assets/api_coriolis_parameter_codes.json b/argopy/static/assets/api_coriolis_parameter_codes.json index cd956388..9cf81981 100644 --- a/argopy/static/assets/api_coriolis_parameter_codes.json +++ b/argopy/static/assets/api_coriolis_parameter_codes.json @@ -1,7 +1,7 @@ { "name": "parameters", "long_name": "All valid requests to https://api-coriolis.ifremer.fr/legacy/parameter?code={code}", - "last_update": "2023-10-20T12:58:00.620292+00:00", + "last_update": "2024-09-12T14:13:10.597080+00:00", "data": { "valid_codes": [ "9", @@ -520,7 +520,15 @@ "772", "773", "774", - "775" + "775", + "776", + "777", + "778", + "779", + "780", + "781", + "782", + "783" ], "valid_params": [ "DEPH", @@ -1029,7 +1037,15 @@ "SIGT", "VMSP", "FLUO_ADJUSTED", - "FLUO_ADJUSTED_ERROR" + "FLUO_ADJUSTED_ERROR", + "EWCT_WS", + "NSCT_WS", + "WS_TYPE_OF_PROCESSING", + "EXT_779", + "EXT_780", + "EXT_781", + "EXT_782", + "EXT_783" ], "codes": { "9": { @@ -6101,6 +6117,86 @@ "parent_code": "63", "label": "Chlorophyll-a fluorescence adjusted error", "param_type": "D" + }, + "776": { + "unit": "meter/second", + "time_sampling": "", + "code": "776", + "gf3_strict_name": "", + "gf3_extended_name": "EWCT_WS", + "parent_code": "13", + "label": "West-east current wind slippage correction at the drogue depth", + "param_type": "D" + }, + "777": { + "unit": "meter/second", + "time_sampling": "", + "code": "777", + "gf3_strict_name": "", + "gf3_extended_name": "NSCT_WS", + "parent_code": "20", + "label": "South-north current wind slippage correction at the drogue depth", + "param_type": "D" + }, + "778": { + "unit": "none", + "time_sampling": "", + "code": "778", + "gf3_strict_name": "", + "gf3_extended_name": "WS_TYPE_OF_PROCESSING", + "parent_code": "0", + "label": "Wind slippage correction method", + "param_type": "T" + }, + "779": { + "unit": "count", + "time_sampling": "", + "code": "779", + "gf3_strict_name": "", + "gf3_extended_name": "EXT_779", + "parent_code": "0", + "label": "Drifting buoy drogue sensor count", + "param_type": "T" + }, + "780": { + "unit": "count", + "time_sampling": "", + "code": "780", + "gf3_strict_name": "", + "gf3_extended_name": "EXT_780", + "parent_code": "0", + "label": "Drifting buoy sensor 4 count", + "param_type": "T" + }, + "781": { + "unit": "count", + "time_sampling": "", + "code": "781", + "gf3_strict_name": "", + "gf3_extended_name": "EXT_781", + "parent_code": "0", + "label": "Drifting buoy sensor 5 count", + "param_type": "T" + }, + "782": { + "unit": "count", + "time_sampling": "", + "code": "782", + "gf3_strict_name": "", + "gf3_extended_name": "EXT_782", + "parent_code": "0", + "label": "Drifting buoy sensor 6 count", + "param_type": "T" + }, + "783": { + "unit": "count", + "time_sampling": "", + "code": "783", + "gf3_strict_name": "", + "gf3_extended_name": "EXT_783", + "parent_code": "0", + "label": "Biodiversity phytoplankton count", + "param_type": "E" } }, "params": { @@ -11173,6 +11269,86 @@ "parent_code": "63", "label": "Chlorophyll-a fluorescence adjusted error", "param_type": "D" + }, + "EWCT_WS": { + "unit": "meter/second", + "time_sampling": "", + "code": "776", + "gf3_strict_name": "", + "gf3_extended_name": "EWCT_WS", + "parent_code": "13", + "label": "West-east current wind slippage correction at the drogue depth", + "param_type": "D" + }, + "NSCT_WS": { + "unit": "meter/second", + "time_sampling": "", + "code": "777", + "gf3_strict_name": "", + "gf3_extended_name": "NSCT_WS", + "parent_code": "20", + "label": "South-north current wind slippage correction at the drogue depth", + "param_type": "D" + }, + "WS_TYPE_OF_PROCESSING": { + "unit": "none", + "time_sampling": "", + "code": "778", + "gf3_strict_name": "", + "gf3_extended_name": "WS_TYPE_OF_PROCESSING", + "parent_code": "0", + "label": "Wind slippage correction method", + "param_type": "T" + }, + "EXT_779": { + "unit": "count", + "time_sampling": "", + "code": "779", + "gf3_strict_name": "", + "gf3_extended_name": "EXT_779", + "parent_code": "0", + "label": "Drifting buoy drogue sensor count", + "param_type": "T" + }, + "EXT_780": { + "unit": "count", + "time_sampling": "", + "code": "780", + "gf3_strict_name": "", + "gf3_extended_name": "EXT_780", + "parent_code": "0", + "label": "Drifting buoy sensor 4 count", + "param_type": "T" + }, + "EXT_781": { + "unit": "count", + "time_sampling": "", + "code": "781", + "gf3_strict_name": "", + "gf3_extended_name": "EXT_781", + "parent_code": "0", + "label": "Drifting buoy sensor 5 count", + "param_type": "T" + }, + "EXT_782": { + "unit": "count", + "time_sampling": "", + "code": "782", + "gf3_strict_name": "", + "gf3_extended_name": "EXT_782", + "parent_code": "0", + "label": "Drifting buoy sensor 6 count", + "param_type": "T" + }, + "EXT_783": { + "unit": "count", + "time_sampling": "", + "code": "783", + "gf3_strict_name": "", + "gf3_extended_name": "EXT_783", + "parent_code": "0", + "label": "Biodiversity phytoplankton count", + "param_type": "E" } } } diff --git a/argopy/static/assets/data_types.json b/argopy/static/assets/data_types.json index 9ae7185b..c75d14b2 100644 --- a/argopy/static/assets/data_types.json +++ b/argopy/static/assets/data_types.json @@ -1,7 +1,7 @@ { "name": "data_types", "long_name": "Expected data types of Argo variables", - "last_update": "2023-10-20T12:57:34.009529+00:00", + "last_update": "2024-09-12T14:12:40.949768+00:00", "data": { "str": [ "PLATFORM_NUMBER", @@ -80,204 +80,204 @@ "PREDEPLOYMENT_CALIB_EQUATION", "PREDEPLOYMENT_CALIB_COEFFICIENT", "PREDEPLOYMENT_CALIB_COMMENT", - "PROFILE_BBP_QC", - "PROFILE_BBP470_QC", - "PROFILE_BBP532_QC", - "PROFILE_BBP700_QC", - "PROFILE_BETA_BACKSCATTERING_QC", - "PROFILE_BETA_BACKSCATTERING470_QC", - "PROFILE_BETA_BACKSCATTERING532_QC", - "PROFILE_BETA_BACKSCATTERING700_QC", - "PROFILE_BISULFIDE_QC", - "PROFILE_BPHASE_DOXY_QC", - "PROFILE_C1PHASE_DOXY_QC", - "PROFILE_C2PHASE_DOXY_QC", - "PROFILE_CDOM_QC", - "PROFILE_CHLA_QC", + "PROFILE_NB_SAMPLE_MCOMS_QC", + "PROFILE_NB_SAMPLE_OPTODE_QC", "PROFILE_CHLA_FLUORESCENCE_QC", + "PROFILE_NB_SAMPLE_SUNA_QC", + "PROFILE_NB_SAMPLE_CYC_QC", + "PROFILE_NB_SAMPLE_DURAFET_QC", + "PROFILE_NB_SAMPLE_CTD_QC", + "PROFILE_NB_SAMPLE_OCR_QC", + "PROFILE_NB_SAMPLE_STM_QC", + "PROFILE_NB_SAMPLE_ECO_QC", + "PROFILE_NB_SAMPLE_CROVER_QC", + "PROFILE_NB_SAMPLE_ISUS_QC", + "PROFILE_TEMP_CNDC_QC", + "PROFILE_PH_IN_SITU_FREE_QC", + "PROFILE_PH_IN_SITU_TOTAL_QC", + "PROFILE_PH_IN_SITU_SEAWATER_QC", + "PROFILE_DOWNWELLING_PAR_QC", + "PROFILE_TEMP_VOLTAGE_DOXY_QC", + "PROFILE_VRS_PH_QC", + "PROFILE_VK_PH_QC", + "PROFILE_VOLTAGE_DOXY_QC", + "PROFILE_IK_PH_QC", + "PROFILE_IB_PH_QC", + "PROFILE_PHASE_DELAY_DOXY_QC", + "PROFILE_PPOX_DOXY_QC", + "PROFILE_PRES_QC", "PROFILE_CNDC_QC", - "PROFILE_COUNT_DOXY_QC", - "PROFILE_CP_QC", + "PROFILE_MTIME_QC", "PROFILE_CP660_QC", - "PROFILE_DOWNWELLING_PAR_QC", + "PROFILE_CP_QC", + "PROFILE_UP_RADIANCE555_QC", + "PROFILE_UP_RADIANCE_QC", + "PROFILE_UP_RADIANCE443_QC", + "PROFILE_UP_RADIANCE412_QC", + "PROFILE_UP_RADIANCE490_QC", + "PROFILE_CDOM_QC", "PROFILE_DOWN_IRRADIANCE_QC", - "PROFILE_DOWN_IRRADIANCE380_QC", "PROFILE_DOWN_IRRADIANCE412_QC", - "PROFILE_DOWN_IRRADIANCE443_QC", "PROFILE_DOWN_IRRADIANCE490_QC", + "PROFILE_DOWN_IRRADIANCE443_QC", + "PROFILE_DOWN_IRRADIANCE380_QC", "PROFILE_DOWN_IRRADIANCE555_QC", - "PROFILE_DOXY_QC", - "PROFILE_DPHASE_DOXY_QC", - "PROFILE_FIT_ERROR_NITRATE_QC", - "PROFILE_FLUORESCENCE_CDOM_QC", - "PROFILE_FLUORESCENCE_CHLA_QC", "PROFILE_FREQUENCY_DOXY_QC", + "PROFILE_BBP700_QC", + "PROFILE_BBP_QC", + "PROFILE_BBP532_QC", + "PROFILE_BBP470_QC", + "PROFILE_TEMP_SPECTROPHOTOMETER_NITRATE_QC", + "PROFILE_TEMP_DOXY_QC", + "PROFILE_TEMP_PH_QC", + "PROFILE_TEMP_NITRATE_QC", "PROFILE_HUMIDITY_NITRATE_QC", - "PROFILE_IB_PH_QC", - "PROFILE_IK_PH_QC", - "PROFILE_LED_FLASHING_COUNT_DOXY_QC", - "PROFILE_MLPL_DOXY_QC", + "PROFILE_C1PHASE_DOXY_QC", + "PROFILE_TPHASE_DOXY_QC", + "PROFILE_TILT_QC", + "PROFILE_C2PHASE_DOXY_QC", + "PROFILE_RPHASE_DOXY_QC", + "PROFILE_BPHASE_DOXY_QC", + "PROFILE_DPHASE_DOXY_QC", "PROFILE_MOLAR_DOXY_QC", "PROFILE_MOLAR_NITRATE_QC", - "PROFILE_MTIME_QC", - "PROFILE_NB_SAMPLE_QC", - "PROFILE_NB_SAMPLE_CROVER_QC", - "PROFILE_NB_SAMPLE_CTD_QC", - "PROFILE_NB_SAMPLE_CYC_QC", - "PROFILE_NB_SAMPLE_DURAFET_QC", - "PROFILE_NB_SAMPLE_ECO_QC", - "PROFILE_NB_SAMPLE_ISUS_QC", - "PROFILE_NB_SAMPLE_MCOMS_QC", - "PROFILE_NB_SAMPLE_OCR_QC", - "PROFILE_NB_SAMPLE_OPTODE_QC", - "PROFILE_NB_SAMPLE_STM_QC", - "PROFILE_NB_SAMPLE_SUNA_QC", - "PROFILE_NITRATE_QC", - "PROFILE_PHASE_DELAY_DOXY_QC", - "PROFILE_PH_IN_SITU_FREE_QC", - "PROFILE_PH_IN_SITU_SEAWATER_QC", - "PROFILE_PH_IN_SITU_TOTAL_QC", - "PROFILE_PPOX_DOXY_QC", - "PROFILE_PRES_QC", + "PROFILE_BETA_BACKSCATTERING700_QC", + "PROFILE_BETA_BACKSCATTERING470_QC", + "PROFILE_SIDE_SCATTERING_TURBIDITY_QC", "PROFILE_PSAL_QC", - "PROFILE_RAW_DOWNWELLING_IRRADIANCE_QC", + "PROFILE_TRANSMITTANCE_PARTICLE_BEAM_ATTENUATION_QC", + "PROFILE_RAW_UPWELLING_RADIANCE412_QC", + "PROFILE_COUNT_DOXY_QC", + "PROFILE_RAW_DOWNWELLING_IRRADIANCE490_QC", + "PROFILE_TRANSMITTANCE_PARTICLE_BEAM_ATTENUATION660_QC", + "PROFILE_UV_INTENSITY_DARK_NITRATE_QC", + "PROFILE_LED_FLASHING_COUNT_DOXY_QC", + "PROFILE_UV_INTENSITY_NITRATE_QC", + "PROFILE_FLUORESCENCE_CDOM_QC", + "PROFILE_RAW_UPWELLING_RADIANCE555_QC", + "PROFILE_NB_SAMPLE_QC", "PROFILE_RAW_DOWNWELLING_IRRADIANCE380_QC", "PROFILE_RAW_DOWNWELLING_IRRADIANCE412_QC", - "PROFILE_RAW_DOWNWELLING_IRRADIANCE443_QC", - "PROFILE_RAW_DOWNWELLING_IRRADIANCE490_QC", - "PROFILE_RAW_DOWNWELLING_IRRADIANCE555_QC", - "PROFILE_RAW_DOWNWELLING_PAR_QC", + "PROFILE_RAW_DOWNWELLING_IRRADIANCE_QC", + "PROFILE_BETA_BACKSCATTERING_QC", "PROFILE_RAW_UPWELLING_RADIANCE_QC", - "PROFILE_RAW_UPWELLING_RADIANCE412_QC", "PROFILE_RAW_UPWELLING_RADIANCE443_QC", - "PROFILE_RAW_UPWELLING_RADIANCE490_QC", - "PROFILE_RAW_UPWELLING_RADIANCE555_QC", - "PROFILE_RPHASE_DOXY_QC", - "PROFILE_SIDE_SCATTERING_TURBIDITY_QC", - "PROFILE_TEMP_QC", - "PROFILE_TEMP_CNDC_QC", + "PROFILE_RAW_DOWNWELLING_IRRADIANCE443_QC", + "PROFILE_UV_INTENSITY_DARK_SEAWATER_NITRATE_QC", + "PROFILE_FLUORESCENCE_CHLA_QC", + "PROFILE_FIT_ERROR_NITRATE_QC", + "PROFILE_RAW_DOWNWELLING_PAR_QC", + "PROFILE_RAW_DOWNWELLING_IRRADIANCE555_QC", "PROFILE_TEMP_COUNT_DOXY_QC", + "PROFILE_RAW_UPWELLING_RADIANCE490_QC", + "PROFILE_BETA_BACKSCATTERING532_QC", "PROFILE_TEMP_CPU_CHLA_QC", - "PROFILE_TEMP_DOXY_QC", - "PROFILE_TEMP_NITRATE_QC", - "PROFILE_TEMP_PH_QC", - "PROFILE_TEMP_SPECTROPHOTOMETER_NITRATE_QC", - "PROFILE_TEMP_VOLTAGE_DOXY_QC", - "PROFILE_TILT_QC", - "PROFILE_TPHASE_DOXY_QC", - "PROFILE_TRANSMITTANCE_PARTICLE_BEAM_ATTENUATION_QC", - "PROFILE_TRANSMITTANCE_PARTICLE_BEAM_ATTENUATION660_QC", + "PROFILE_DOXY_QC", + "PROFILE_NITRATE_QC", + "PROFILE_BISULFIDE_QC", + "PROFILE_MLPL_DOXY_QC", "PROFILE_TURBIDITY_QC", - "PROFILE_UP_RADIANCE_QC", - "PROFILE_UP_RADIANCE412_QC", - "PROFILE_UP_RADIANCE443_QC", - "PROFILE_UP_RADIANCE490_QC", - "PROFILE_UP_RADIANCE555_QC", - "PROFILE_UV_INTENSITY_DARK_NITRATE_QC", - "PROFILE_UV_INTENSITY_DARK_SEAWATER_NITRATE_QC", - "PROFILE_UV_INTENSITY_NITRATE_QC", - "PROFILE_VK_PH_QC", - "PROFILE_VOLTAGE_DOXY_QC", - "PROFILE_VRS_PH_QC", - "BBP_DATA_MODE", - "BBP470_DATA_MODE", - "BBP532_DATA_MODE", - "BBP700_DATA_MODE", - "BETA_BACKSCATTERING_DATA_MODE", - "BETA_BACKSCATTERING470_DATA_MODE", - "BETA_BACKSCATTERING532_DATA_MODE", - "BETA_BACKSCATTERING700_DATA_MODE", - "BISULFIDE_DATA_MODE", - "BPHASE_DOXY_DATA_MODE", - "C1PHASE_DOXY_DATA_MODE", - "C2PHASE_DOXY_DATA_MODE", - "CDOM_DATA_MODE", - "CHLA_DATA_MODE", + "PROFILE_CHLA_QC", + "PROFILE_TEMP_QC", + "NB_SAMPLE_MCOMS_DATA_MODE", + "NB_SAMPLE_OPTODE_DATA_MODE", "CHLA_FLUORESCENCE_DATA_MODE", + "NB_SAMPLE_SUNA_DATA_MODE", + "NB_SAMPLE_CYC_DATA_MODE", + "NB_SAMPLE_DURAFET_DATA_MODE", + "NB_SAMPLE_CTD_DATA_MODE", + "NB_SAMPLE_OCR_DATA_MODE", + "NB_SAMPLE_STM_DATA_MODE", + "NB_SAMPLE_ECO_DATA_MODE", + "NB_SAMPLE_CROVER_DATA_MODE", + "NB_SAMPLE_ISUS_DATA_MODE", + "TEMP_CNDC_DATA_MODE", + "PH_IN_SITU_FREE_DATA_MODE", + "PH_IN_SITU_TOTAL_DATA_MODE", + "PH_IN_SITU_SEAWATER_DATA_MODE", + "DOWNWELLING_PAR_DATA_MODE", + "TEMP_VOLTAGE_DOXY_DATA_MODE", + "VRS_PH_DATA_MODE", + "VK_PH_DATA_MODE", + "VOLTAGE_DOXY_DATA_MODE", + "IK_PH_DATA_MODE", + "IB_PH_DATA_MODE", + "PHASE_DELAY_DOXY_DATA_MODE", + "PPOX_DOXY_DATA_MODE", + "PRES_DATA_MODE", "CNDC_DATA_MODE", - "COUNT_DOXY_DATA_MODE", - "CP_DATA_MODE", + "MTIME_DATA_MODE", "CP660_DATA_MODE", - "DOWNWELLING_PAR_DATA_MODE", + "CP_DATA_MODE", + "UP_RADIANCE555_DATA_MODE", + "UP_RADIANCE_DATA_MODE", + "UP_RADIANCE443_DATA_MODE", + "UP_RADIANCE412_DATA_MODE", + "UP_RADIANCE490_DATA_MODE", + "CDOM_DATA_MODE", "DOWN_IRRADIANCE_DATA_MODE", - "DOWN_IRRADIANCE380_DATA_MODE", "DOWN_IRRADIANCE412_DATA_MODE", - "DOWN_IRRADIANCE443_DATA_MODE", "DOWN_IRRADIANCE490_DATA_MODE", + "DOWN_IRRADIANCE443_DATA_MODE", + "DOWN_IRRADIANCE380_DATA_MODE", "DOWN_IRRADIANCE555_DATA_MODE", - "DOXY_DATA_MODE", - "DPHASE_DOXY_DATA_MODE", - "FIT_ERROR_NITRATE_DATA_MODE", - "FLUORESCENCE_CDOM_DATA_MODE", - "FLUORESCENCE_CHLA_DATA_MODE", "FREQUENCY_DOXY_DATA_MODE", + "BBP700_DATA_MODE", + "BBP_DATA_MODE", + "BBP532_DATA_MODE", + "BBP470_DATA_MODE", + "TEMP_SPECTROPHOTOMETER_NITRATE_DATA_MODE", + "TEMP_DOXY_DATA_MODE", + "TEMP_PH_DATA_MODE", + "TEMP_NITRATE_DATA_MODE", "HUMIDITY_NITRATE_DATA_MODE", - "IB_PH_DATA_MODE", - "IK_PH_DATA_MODE", - "LED_FLASHING_COUNT_DOXY_DATA_MODE", - "MLPL_DOXY_DATA_MODE", + "C1PHASE_DOXY_DATA_MODE", + "TPHASE_DOXY_DATA_MODE", + "TILT_DATA_MODE", + "C2PHASE_DOXY_DATA_MODE", + "RPHASE_DOXY_DATA_MODE", + "BPHASE_DOXY_DATA_MODE", + "DPHASE_DOXY_DATA_MODE", "MOLAR_DOXY_DATA_MODE", "MOLAR_NITRATE_DATA_MODE", - "MTIME_DATA_MODE", - "NB_SAMPLE_DATA_MODE", - "NB_SAMPLE_CROVER_DATA_MODE", - "NB_SAMPLE_CTD_DATA_MODE", - "NB_SAMPLE_CYC_DATA_MODE", - "NB_SAMPLE_DURAFET_DATA_MODE", - "NB_SAMPLE_ECO_DATA_MODE", - "NB_SAMPLE_ISUS_DATA_MODE", - "NB_SAMPLE_MCOMS_DATA_MODE", - "NB_SAMPLE_OCR_DATA_MODE", - "NB_SAMPLE_OPTODE_DATA_MODE", - "NB_SAMPLE_STM_DATA_MODE", - "NB_SAMPLE_SUNA_DATA_MODE", - "NITRATE_DATA_MODE", - "PHASE_DELAY_DOXY_DATA_MODE", - "PH_IN_SITU_FREE_DATA_MODE", - "PH_IN_SITU_SEAWATER_DATA_MODE", - "PH_IN_SITU_TOTAL_DATA_MODE", - "PPOX_DOXY_DATA_MODE", - "PRES_DATA_MODE", + "BETA_BACKSCATTERING700_DATA_MODE", + "BETA_BACKSCATTERING470_DATA_MODE", + "SIDE_SCATTERING_TURBIDITY_DATA_MODE", "PSAL_DATA_MODE", - "RAW_DOWNWELLING_IRRADIANCE_DATA_MODE", + "TRANSMITTANCE_PARTICLE_BEAM_ATTENUATION_DATA_MODE", + "RAW_UPWELLING_RADIANCE412_DATA_MODE", + "COUNT_DOXY_DATA_MODE", + "RAW_DOWNWELLING_IRRADIANCE490_DATA_MODE", + "TRANSMITTANCE_PARTICLE_BEAM_ATTENUATION660_DATA_MODE", + "UV_INTENSITY_DARK_NITRATE_DATA_MODE", + "LED_FLASHING_COUNT_DOXY_DATA_MODE", + "UV_INTENSITY_NITRATE_DATA_MODE", + "FLUORESCENCE_CDOM_DATA_MODE", + "RAW_UPWELLING_RADIANCE555_DATA_MODE", + "NB_SAMPLE_DATA_MODE", "RAW_DOWNWELLING_IRRADIANCE380_DATA_MODE", "RAW_DOWNWELLING_IRRADIANCE412_DATA_MODE", - "RAW_DOWNWELLING_IRRADIANCE443_DATA_MODE", - "RAW_DOWNWELLING_IRRADIANCE490_DATA_MODE", - "RAW_DOWNWELLING_IRRADIANCE555_DATA_MODE", - "RAW_DOWNWELLING_PAR_DATA_MODE", + "RAW_DOWNWELLING_IRRADIANCE_DATA_MODE", + "BETA_BACKSCATTERING_DATA_MODE", "RAW_UPWELLING_RADIANCE_DATA_MODE", - "RAW_UPWELLING_RADIANCE412_DATA_MODE", "RAW_UPWELLING_RADIANCE443_DATA_MODE", - "RAW_UPWELLING_RADIANCE490_DATA_MODE", - "RAW_UPWELLING_RADIANCE555_DATA_MODE", - "RPHASE_DOXY_DATA_MODE", - "SIDE_SCATTERING_TURBIDITY_DATA_MODE", - "TEMP_DATA_MODE", - "TEMP_CNDC_DATA_MODE", + "RAW_DOWNWELLING_IRRADIANCE443_DATA_MODE", + "UV_INTENSITY_DARK_SEAWATER_NITRATE_DATA_MODE", + "FLUORESCENCE_CHLA_DATA_MODE", + "FIT_ERROR_NITRATE_DATA_MODE", + "RAW_DOWNWELLING_PAR_DATA_MODE", + "RAW_DOWNWELLING_IRRADIANCE555_DATA_MODE", "TEMP_COUNT_DOXY_DATA_MODE", + "RAW_UPWELLING_RADIANCE490_DATA_MODE", + "BETA_BACKSCATTERING532_DATA_MODE", "TEMP_CPU_CHLA_DATA_MODE", - "TEMP_DOXY_DATA_MODE", - "TEMP_NITRATE_DATA_MODE", - "TEMP_PH_DATA_MODE", - "TEMP_SPECTROPHOTOMETER_NITRATE_DATA_MODE", - "TEMP_VOLTAGE_DOXY_DATA_MODE", - "TILT_DATA_MODE", - "TPHASE_DOXY_DATA_MODE", - "TRANSMITTANCE_PARTICLE_BEAM_ATTENUATION_DATA_MODE", - "TRANSMITTANCE_PARTICLE_BEAM_ATTENUATION660_DATA_MODE", + "DOXY_DATA_MODE", + "NITRATE_DATA_MODE", + "BISULFIDE_DATA_MODE", + "MLPL_DOXY_DATA_MODE", "TURBIDITY_DATA_MODE", - "UP_RADIANCE_DATA_MODE", - "UP_RADIANCE412_DATA_MODE", - "UP_RADIANCE443_DATA_MODE", - "UP_RADIANCE490_DATA_MODE", - "UP_RADIANCE555_DATA_MODE", - "UV_INTENSITY_DARK_NITRATE_DATA_MODE", - "UV_INTENSITY_DARK_SEAWATER_NITRATE_DATA_MODE", - "UV_INTENSITY_NITRATE_DATA_MODE", - "VK_PH_DATA_MODE", - "VOLTAGE_DOXY_DATA_MODE", - "VRS_PH_DATA_MODE" + "CHLA_DATA_MODE", + "TEMP_DATA_MODE" ], "int": [ "PLATFORM_NUMBER", diff --git a/argopy/static/assets/institutions.json b/argopy/static/assets/institutions.json index 604d80e0..3d61bd7f 100644 --- a/argopy/static/assets/institutions.json +++ b/argopy/static/assets/institutions.json @@ -1,35 +1,35 @@ { "name": "institutions", "long_name": "Institution names from Argo reference table 4", - "last_update": "2023-10-20T12:57:34.039003+00:00", + "last_update": "2024-09-12T14:12:41.637174+00:00", "data": { "institutions": { - "AO": "AOML, USA", - "BO": "BODC, United Kingdom", - "CI": "Institute of Ocean Sciences, Canada", - "CS": "CSIRO, Australia", - "GE": "BSH, Germany", - "GT": "GTS : used for data coming from WMO GTS network", - "HZ": "CSIO, China Second Institute of Oceanography", - "IF": "Ifremer, France", - "IN": "INCOIS, India", + "PL": "IOPAN, Institute of Oceanology Polish Academy of Science", + "NA": "NAVO, USA", "JA": "JMA, Japan", + "RU": "Russia", + "UW": "University of Washington, USA", + "BO": "BODC, United Kingdom", + "SP": "Spain", + "VL": "Far Eastern Regional Hydrometeorological Research Institute of Vladivostock, Russia", + "KO": "KORDI, Korea", "JM": "Jamstec, Japan", + "AO": "AOML, USA", "KM": "KMA, Korea", - "KO": "KORDI, Korea", - "LV": "Laboratoire Oc\u00e9anographique de Villefranche, Sorbonne University", + "GT": "GTS : used for data coming from WMO GTS network", "MB": "MBARI, USA", - "ME": "MEDS, Canada", - "NA": "NAVO, USA", "NM": "NMDIS, China", - "PL": "IOPAN, Institute of Oceanology Polish Academy of Science", + "LV": "Laboratoire Oc\u00e9anographique de Villefranche, Sorbonne University", + "IF": "Ifremer, France", + "HZ": "CSIO, China Second Institute of Oceanography", "PM": "PMEL, USA", - "RU": "Russia", + "WH": "Woods Hole Oceanographic Institution, USA", + "GE": "BSH, Germany", + "IN": "INCOIS, India", "SI": "SIO, Scripps, USA", - "SP": "Spain", - "UW": "University of Washington, USA", - "VL": "Far Eastern Regional Hydrometeorological Research Institute of Vladivostock, Russia", - "WH": "Woods Hole Oceanographic Institution, USA" + "CI": "Institute of Ocean Sciences, Canada", + "CS": "CSIRO, Australia", + "ME": "MEDS, Canada" } } } \ No newline at end of file diff --git a/argopy/static/assets/profilers.json b/argopy/static/assets/profilers.json index b84fb385..ad967b86 100644 --- a/argopy/static/assets/profilers.json +++ b/argopy/static/assets/profilers.json @@ -1,58 +1,60 @@ { "name": "profilers", "long_name": "Profiler codes and description from Argo reference table 8", - "last_update": "2023-10-20T12:57:34.066062+00:00", + "last_update": "2024-09-12T14:12:42.118648+00:00", "data": { "profilers": { + "841": "PROVOR float with SBE conductivity sensor", + "838": "ARVOR-D deep float with SBE conductivity sensor", "831": "PALACE float", - "834": "PROVOR V SBE", + "887": "SOLO_BGC_MRV, RBR", + "880": "S2A float with RBR conductivity sensor", + "877": "APEX float with RBR conductivity sensor", + "881": "HM4000", + "851": "SOLO float with SBE conductivity sensor", + "847": "Teledyne Webb Research float with FSI conductivity sensor", + "848": "APEX-EM float with SBE conductivity sensor", "835": "PROVOR IV", - "836": "PROVOR III", "837": "ARVOR-C float with SBE conductivity sensor", - "838": "ARVOR-D deep float with SBE conductivity sensor", - "839": "PROVOR-II float with SBE conductivity sensor", - "840": "PROVOR float with no conductivity sensor", - "841": "PROVOR float with SBE conductivity sensor", - "842": "PROVOR float with FSI conductivity sensor", + "883": "APEX_D deep float with RBR conductivity sensor", + "879": "SOLO-II with RBR conductivity sensor", + "885": "SOLO_BGC, RBR", + "859": "NEMO float with no conductivity", + "861": "NEMO float with FSI conductivity sensor", + "858": "NINJA float with TSK conductivity sensor", + "836": "PROVOR III", "843": "POPS ice platform using PROVOR float with SBE conductivity sensor", + "878": "ARVOR float with RBR conductivity sensor", + "889": "PROVOR_V - Jumbo, RBR", + "888": "PROVOR_V - Jumbo, SBE", + "834": "PROVOR V SBE", + "863": "Navis-A float with SBE conductivity sensor", "844": "ARVOR float with SBE conductivity sensor", + "886": "SOLO_BGC_MRV, SBE", + "853": "SOLO-II float with SBE conductivity sensor", + "864": "NINJA-D deep float with SBE conductivity sensor", + "842": "PROVOR float with FSI conductivity sensor", "845": "Teledyne Webb Research float with no conductivity sensor", - "846": "Teledyne Webb Research float with SBE conductivity sensor", - "847": "Teledyne Webb Research float with FSI conductivity sensor", - "848": "APEX-EM float with SBE conductivity sensor", - "849": "APEX-D deep float with SBE conductivity sensor", - "850": "SOLO float with no conductivity sensor", - "851": "SOLO float with SBE conductivity sensor", + "890": "PROVOR_III - Jumbo, SBE", + "875": "ALTO float with RBR conductivity sensor", "852": "SOLO float with FSI conductivity sensor", - "853": "SOLO-II float with SBE conductivity sensor", - "854": "S2A float with SBE conductivity sensor", - "855": "NINJA float with no conductivity sensor", "856": "NINJA float with SBE conductivity sensor", - "857": "NINJA float with FSI conductivity sensor", - "858": "NINJA float with TSK conductivity sensor", - "859": "NEMO float with no conductivity", - "860": "NEMO float with SBE conductivity sensor", - "861": "NEMO float with FSI conductivity sensor", + "884": "SOLO_BGC, SBE", "862": "SOLO-D deep float with SBE conductivity sensor", - "863": "Navis-A float with SBE conductivity sensor", - "864": "NINJA-D deep float with SBE conductivity sensor", + "854": "S2A float with SBE conductivity sensor", + "840": "PROVOR float with no conductivity sensor", "865": "NOVA float with SBE conductivity sensor", - "875": "ALTO float with RBR conductivity sensor", + "872": "S2X", + "846": "Teledyne Webb Research float with SBE conductivity sensor", + "891": "PROVOR_III - Jumbo, RBR", + "850": "SOLO float with no conductivity sensor", + "839": "PROVOR-II float with SBE conductivity sensor", "876": "ALTO float with SBE conductivity sensor", - "877": "APEX float with RBR conductivity sensor", - "878": "ARVOR float with RBR conductivity sensor", - "879": "SOLO-II with RBR conductivity sensor", - "880": "S2A float with RBR conductivity sensor", - "881": "HM4000", "882": "XUANWU", - "884": "SOLO_BGC, SBE", - "885": "SOLO_BGC, RBR", - "886": "SOLO_BGC_MRV, SBE", - "887": "SOLO_BGC_MRV, RBR", - "888": "PROVOR_V - Jumbo, SBE", - "889": "PROVOR_V - Jumbo, RBR", - "890": "PROVOR_III - Jumbo, SBE", - "891": "PROVOR_III - Jumbo, RBR" + "857": "NINJA float with FSI conductivity sensor", + "860": "NEMO float with SBE conductivity sensor", + "849": "APEX-D deep float with SBE conductivity sensor", + "855": "NINJA float with no conductivity sensor" } } } \ No newline at end of file diff --git a/argopy/static/assets/variables_bgc_synthetic.json b/argopy/static/assets/variables_bgc_synthetic.json index 15020fa2..970c4cef 100644 --- a/argopy/static/assets/variables_bgc_synthetic.json +++ b/argopy/static/assets/variables_bgc_synthetic.json @@ -1,7 +1,7 @@ { "name": "BGC synthetic netcdf files variables", "long_name": "Variables from the Ifremer Erddap ArgoFloats-synthetic-BGC dataset based on GDAC synthetic netcdf files", - "last_update": "2024-06-11T07:34:02.639820+00:00", + "last_update": "2024-09-12T14:13:10.711233+00:00", "data": { "variables": [ "BBP470", From 87a996d351e32a377f28cc4b8cd5b6284fd3fb15 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Fri, 13 Sep 2024 14:34:15 +0200 Subject: [PATCH 17/19] Update options.py improve doctrings --- argopy/options.py | 85 ++++++++++++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/argopy/options.py b/argopy/options.py index bc3dbe5c..1f691c52 100644 --- a/argopy/options.py +++ b/argopy/options.py @@ -97,37 +97,58 @@ def validate_http(this_path): class set_options: """Set options for argopy - List of options: - - - ``ds``: Define the Dataset to work with. - Default: ``phy``. - Possible values: ``phy``, ``bgc`` or ``ref``. - - ``src``: Source of fetched data. - Default: ``erddap``. - Possible values: ``erddap``, ``gdac``, ``argovis`` - - ``mode``: User mode. - Default: ``standard``. - Possible values: ``standard``, ``expert`` or ``research``. - - ``gdac``: Default path to be used by the GDAC fetchers and Argo index stores - Default: https://data-argo.ifremer.fr - - ``erddap``: Default server address to be used by the data and index erddap fetchers - Default: https://erddap.ifremer.fr/erddap - - ``cachedir``: Absolute path to a local cache directory. - Default: ``~/.cache/argopy`` - - ``cache_expiration``: Expiration delay of cache files in seconds. - Default: 86400 - - ``api_timeout``: Define the time out of internet requests to web API, in seconds. - Default: 60 - - ``trust_env``: Allow for local environment variables to be used to connect to the internet. - Default: False. - Argopy will get proxies information from HTTP_PROXY / HTTPS_PROXY environment variables if this option is True and it can also get proxy credentials from ~/.netrc file if this file exists. - - ``user``/``password``: Username and password to use when a simple authentication is required. - Default: None, None - - ``server``: Other than expected/default server to be uses by a function/method. This is mostly intended to be used for unit testing - Default: None - - ``argovis_api_key``: The API key to use when fetching data from the `argovis` data source. You can get a free key at https://argovis-keygen.colorado.edu - Default: `guest` + Parameters + ---------- + ds: str, default: 'phy' + Define the Dataset to work with: ``phy``, ``bgc`` or ``ref`` + + src: str, default: 'erddap' + Source of fetched data: ``erddap``, ``gdac``, ``argovis`` + + mode: str, default: 'standard' + User mode: ``standard``, ``expert`` or ``research`` + + gdac: str, default: 'https://data-argo.ifremer.fr' + Default path to be used by the GDAC fetchers and Argo index stores + + erddap: str, default: 'https://erddap.ifremer.fr/erddap' + Default server address to be used by the data and index erddap fetchers + + cachedir: str, default: '~/.cache/argopy' + Absolute path to a local cache directory + + cache_expiration: int, default: 86400 + Expiration delay of cache files in seconds + + api_timeout: int, default: 60 + Time out for internet requests to web API, in seconds + + trust_env: bool, default: False + Allow for local environment variables to be used to connect to the internet. + + Argopy will get proxies information from HTTP_PROXY / HTTPS_PROXY environment variables if this option is True and it can also get proxy credentials from ~/.netrc file if this file exists + + user: str, default: None + Username to use when a simple authentication is required + + password: str, default: None + Password to use when a simple authentication is required + + argovis_api_key: str, default:'guest' + The API key to use when fetching data from the `argovis` data source + + You can get a free key at https://argovis-keygen.colorado.edu + + Other Parameters + ---------------- + server: : str, default: None + Other than expected/default server to be uses by a function/method + + This is mostly intended to be used for unit testing + + Examples + -------- You can use ``set_options`` either as a context manager for temporary setting: @@ -139,6 +160,10 @@ class set_options: >>> argopy.set_options(src='gdac') + Warns + ----- + A DeprecationWarning can be raised when a deprecated option is set + """ def __init__(self, **kwargs): self.old = {} From 37f24952dec4c23d12ced7b50331b1a9caa28e7b Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 18 Sep 2024 15:32:46 +0200 Subject: [PATCH 18/19] Update monitored_threadpool.py --- argopy/utils/monitored_threadpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argopy/utils/monitored_threadpool.py b/argopy/utils/monitored_threadpool.py index c4f0c4af..faecb894 100644 --- a/argopy/utils/monitored_threadpool.py +++ b/argopy/utils/monitored_threadpool.py @@ -1,5 +1,5 @@ """ -This sub-module provides utilities for miscellaneous computation tasks with multitheading +This submodule provides utilities for miscellaneous computation tasks with multithreading We construct the MyThreadPoolExecutor class, we create a series of classes using multiple inheritance to implement monitoring features From fa05fa78446069cf2c483f036cafc5eecf4f9066 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Wed, 25 Sep 2024 12:07:02 +0200 Subject: [PATCH 19/19] Delete test_deprecated.py --- argopy/tests/test_deprecated.py | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 argopy/tests/test_deprecated.py diff --git a/argopy/tests/test_deprecated.py b/argopy/tests/test_deprecated.py deleted file mode 100644 index 9ec4fff2..00000000 --- a/argopy/tests/test_deprecated.py +++ /dev/null @@ -1,27 +0,0 @@ -import logging -import pytest -from mocked_http import mocked_server_address -from mocked_http import mocked_httpserver as mocked_erddapserver - -import argopy -from utils import ( - requires_erddap, -) - - -log = logging.getLogger("argopy.tests.deprecated") - - -def test_deprecated_option_dataset(): - with pytest.deprecated_call(): - argopy.set_options(dataset='phy') - - -def test_deprecated_option_ftp(): - with pytest.deprecated_call(): - argopy.set_options(ftp='https://data-argo.ifremer.fr') - - -def test_deprecated_fetcher_argument_ftp(): - with pytest.deprecated_call(): - argopy.DataFetcher(src='gdac', ftp='https://data-argo.ifremer.fr')