Merge pull request #290 from euroargodev/refactor-utils

Utilities refactoring

argopy/__init__.py

@@ -26,22 +26,23 @@
 from .fetchers import ArgoIndexFetcher as IndexFetcher  # noqa: E402
 
 from .xarray import ArgoAccessor  # noqa: E402
-from . import tutorial  # noqa: E402
 
 # Other Import
-from . import utilities  # noqa: E402
+# from . import utils as utilities  # noqa: E402
+from . import utilities  # noqa: E402  # being deprecated until 0.1.15, then remove
 from . import stores  # noqa: E402
 from . import errors  # noqa: E402
 from . import plot  # noqa: E402
+from . import tutorial  # noqa: E402
 from .plot import dashboard, ArgoColors  # noqa: E402
-from .utilities import show_versions, show_options, clear_cache, lscache  # noqa: E402
-from .utilities import TopoFetcher, ArgoNVSReferenceTables, OceanOPSDeployments, ArgoDocs  # noqa: E402
-from .utilities import monitor_status as status  # noqa: E402
 from .options import set_options, reset_options  # noqa: E402
 from .data_fetchers import CTDRefDataFetcher  # noqa: E402
 from .stores import ArgoIndex  # noqa: E402
-from .related import ArgoDOI  # noqa: E402
-from .utils import compute  # noqa: E402, F401
+from .utils import show_versions, show_options  # noqa: E402
+from .utils import clear_cache, lscache  # noqa: E402
+from .utils import MonitoredThreadPoolExecutor  # noqa: E402, F401
+from .utils import monitor_status as status  # noqa: E402
+from .related import TopoFetcher, OceanOPSDeployments, ArgoNVSReferenceTables, ArgoDocs, ArgoDOI  # noqa: E402
 
 
 #
@@ -71,11 +72,10 @@
     "ArgoDOI",  # Class
 
     # Submodules:
-    "utilities",
+    "utilities",  # being deprecated until 0.1.15, then remove
     "errors",
     "plot",
     "ArgoColors",  # Class
-    # "plotters",  # Deprec, to be removed after 0.1.13
     "stores",
     "tutorial",
     # Constants

argopy/data_fetchers/argovis_data.py

@@ -9,14 +9,15 @@
 import xarray as xr
 import getpass
 import logging
-from .proto import ArgoDataFetcherProto
 from abc import abstractmethod
 import warnings
 
-from argopy.stores import httpstore
-from argopy.options import OPTIONS
-from argopy.utilities import format_oneline, Chunker
-from argopy.errors import DataNotFound
+from ..stores import httpstore
+from ..options import OPTIONS
+from ..utils.format import format_oneline
+from ..utils.chunking import Chunker
+from ..errors import DataNotFound
+from .proto import ArgoDataFetcherProto
 
 
 access_points = ["wmo", "box"]

argopy/data_fetchers/erddap_data.py

@@ -10,7 +10,6 @@
 
 """
 
-import argopy.utilities
 import xarray as xr
 import pandas as pd
 import numpy as np
@@ -21,18 +20,17 @@
 import getpass
 from typing import Union
 import fnmatch
+from aiohttp import ClientResponseError
+import logging
 
-from .proto import ArgoDataFetcherProto
-from argopy.options import OPTIONS
-from argopy.utilities import Chunker, format_oneline, to_list
-from argopy.stores import httpstore
+from ..options import OPTIONS
+from ..utils.format import format_oneline
+from ..stores import httpstore
 from ..errors import ErddapServerError, DataNotFound
-from ..stores import (
-    indexstore_pd as ArgoIndex,
-)  # make sure to work with the Pandas index store
+from ..stores import indexstore_pd as ArgoIndex  # make sure we work with the Pandas index store
+from ..utils import is_list_of_strings, to_list,Chunker
+from .proto import ArgoDataFetcherProto
 
-from aiohttp import ClientResponseError
-import logging
 
 # Load erddapy according to available version (breaking changes in v0.8.0)
 try:
@@ -201,7 +199,7 @@ def __init__(  # noqa: C901
                 raise ValueError()
             elif params[0] == "all":
                 params = self._bgc_vlist_avail
-            elif not argopy.utilities.is_list_of_strings(params):
+            elif not is_list_of_strings(params):
                 raise ValueError("'params' argument must be a list of strings")
                 # raise ValueError("'params' argument must be a list of strings (possibly with a * wildcard)")
             self._bgc_vlist_requested = [p.upper() for p in params]
@@ -222,7 +220,7 @@ def __init__(  # noqa: C901
                 measured = []
             elif self._bgc_measured[0] == "all":
                 measured = self._bgc_vlist_requested
-            elif not argopy.utilities.is_list_of_strings(self._bgc_measured):
+            elif not is_list_of_strings(self._bgc_measured):
                 raise ValueError("'measured' argument must be a list of strings")
                 # raise ValueError("'measured' argument must be a list of strings (possibly with a * wildcard)")
             self._bgc_vlist_measured = [m.upper() for m in measured]

argopy/data_fetchers/erddap_index.py

@@ -17,9 +17,10 @@
 
 from abc import ABC, abstractmethod
 
-from argopy.utilities import load_dict, mapp_dict, format_oneline
-from argopy.stores import httpstore
-from argopy.options import OPTIONS
+from ..utils.format import format_oneline
+from ..related import load_dict, mapp_dict
+from ..stores import httpstore
+from ..options import OPTIONS
 
 log = logging.getLogger("argopy.fetchers.erddap_index")
 

argopy/data_fetchers/erddap_refdata.py

@@ -2,11 +2,11 @@
 Fetcher to retrieve CTD reference data from Ifremer erddap
 """
 import xarray as xr
-from .erddap_data import ErddapArgoDataFetcher
-from argopy.options import OPTIONS
-from argopy.utilities import Chunker
-from argopy.stores import httpstore_erddap_auth
 import logging
+from ..options import OPTIONS
+from ..utils.chunking import Chunker
+from ..stores import httpstore_erddap_auth
+from .erddap_data import ErddapArgoDataFetcher
 
 # Load erddapy according to available version (breaking changes in v0.8.0)
 try:

argopy/data_fetchers/gdacftp_data.py

@@ -12,11 +12,11 @@
 import getpass
 import logging
 
-from .proto import ArgoDataFetcherProto
-from ..utilities import format_oneline, argo_split_path
+from ..utils.format import format_oneline, argo_split_path
 from ..options import OPTIONS, check_gdac_path
 from ..errors import DataNotFound
 from ..stores import ArgoIndex
+from .proto import ArgoDataFetcherProto
 
 log = logging.getLogger("argopy.gdacftp.data")
 access_points = ["wmo", "box"]

argopy/data_fetchers/gdacftp_index.py

@@ -11,7 +11,7 @@
 import logging
 import importlib
 
-from ..utilities import format_oneline
+from ..utils.format import format_oneline
 from ..options import OPTIONS, check_gdac_path
 from ..plot import dashboard
 
@@ -161,21 +161,6 @@ def clear_cache(self):
     def to_dataframe(self):
         """ Filter index file and return a pandas dataframe """
         df = self.indexfs.run().to_dataframe()
-
-        # Post-processing of the filtered index is done at the indexstore level
-        # if 'wmo' not in df:
-        #     df['wmo'] = df['file'].apply(lambda x: int(x.split('/')[1]))
-        #
-        # # institution & profiler mapping for all users
-        # # todo: may be we need to separate this for standard and expert users
-        # institution_dictionnary = load_dict('institutions')
-        # df['tmp1'] = df.institution.apply(lambda x: mapp_dict(institution_dictionnary, x))
-        # df = df.rename(columns={"institution": "institution_code", "tmp1": "institution"})
-        #
-        # profiler_dictionnary = load_dict('profilers')
-        # df['profiler'] = df.profiler_type.apply(lambda x: mapp_dict(profiler_dictionnary, int(x)))
-        # df = df.rename(columns={"profiler_type": "profiler_code"})
-
         return df
 
     def to_xarray(self):

argopy/data_fetchers/proto.py

@@ -5,7 +5,7 @@
 import hashlib
 import warnings
 from ..plot import dashboard
-from ..utilities import list_standard_variables
+from ..utils.lists import list_standard_variables
 
 
 class ArgoDataFetcherProto(ABC):

argopy/fetchers.py

@@ -15,17 +15,20 @@
 import numpy as np
 import logging
 
-from argopy.options import OPTIONS, _VALIDATORS
+from .options import OPTIONS, _VALIDATORS
 from .errors import InvalidFetcherAccessPoint, InvalidFetcher, OptionValueError
-
-from .utilities import (
-    list_available_data_src,
-    list_available_index_src,
+from .related import (
+    get_coriolis_profile_id,
+)
+from .utils.checkers import (
     is_box,
     is_indexbox,
     check_wmo,
-    check_cyc,
-    get_coriolis_profile_id,
+    check_cyc
+)
+from .utils.lists import (
+    list_available_data_src,
+    list_available_index_src,
 )
 from .plot import plot_trajectory, bar_plot, open_sat_altim_report
 

argopy/options.py

@@ -5,13 +5,13 @@
 # https://github.com/pydata/xarray/blob/cafab46aac8f7a073a32ec5aa47e213a9810ed54/xarray/core/options.py
 """
 import os
-from argopy.errors import OptionValueError, FtpPathError, ErddapPathError
 import warnings
 import logging
 import fsspec
 from fsspec.core import split_protocol
 from socket import gaierror
 from urllib.parse import urlparse
+from .errors import OptionValueError, FtpPathError, ErddapPathError
 
 
 # Define a logger

argopy/plot/argo_colors.py

@@ -1,7 +1,7 @@
 import numpy as np
 from packaging import version
 from .utils import has_mpl, has_seaborn
-from ..utilities import warnUnless
+from ..utils.loggers import warnUnless
 
 if has_mpl:
     from .utils import mpl, cm, mcolors, plt

argopy/plot/dashboards.py

@@ -11,7 +11,9 @@
 from packaging import version
 
 from .utils import has_ipython
-from ..utilities import warnUnless, check_wmo, check_cyc, get_ea_profile_page
+from ..utils.loggers import warnUnless
+from ..related.euroargo_api import get_ea_profile_page
+from ..utils import check_wmo, check_cyc
 from ..errors import InvalidDashboard
 from .. import __version__ as argopy_version
 

argopy/plot/plot.py

@@ -18,7 +18,8 @@
 from .utils import axes_style, latlongrid, land_feature
 from .argo_colors import ArgoColors
 
-from ..utilities import warnUnless, check_wmo
+from ..utils.loggers import warnUnless
+from ..utils.checkers import check_wmo
 from ..errors import InvalidDatasetStructure
 
 if has_mpl:

argopy/plot/utils.py

@@ -1,7 +1,7 @@
 import numpy as np
 from contextlib import contextmanager
 import importlib
-from ..utilities import deprecated
+from ..utils.decorators import deprecated
 
 
 def _importorskip(modname):

argopy/related/__init__.py

@@ -1,7 +1,25 @@
-from .gdac_snapshot import ArgoDOI
+from .topography import TopoFetcher
+from .ocean_ops_deployments import OceanOPSDeployments
+from .reference_tables import ArgoNVSReferenceTables
+from .argo_documentation import ArgoDocs
+from .doi_snapshot import ArgoDOI
+from .euroargo_api import get_coriolis_profile_id, get_ea_profile_page
+from .utils import load_dict, mapp_dict
 
 #
 __all__ = (
     # Classes:
+    "TopoFetcher",
+    "OceanOPSDeployments",
+    "ArgoNVSReferenceTables",
+    "ArgoDocs",
     "ArgoDOI",
+
+    # Functions:
+    "get_coriolis_profile_id",
+    "get_ea_profile_page",
+
+    # Utilities:
+    "load_dict",
+    "mapp_dict",
 )