Merge pull request #383 from euroargodev/argovis-use-mock

Refactor Argovis CI tests to use mock server

argopy/data_fetchers/argovis_data.py

@@ -256,11 +256,7 @@ def cname(self):
         return self._cname()
 
     def url_encode(self, urls):
-        """Return safely encoded list of urls
-
-        This was made to debug for fsspec caching system not working with cache of profile and region in argovis
-        Not working yet, see: https://github.com/euroargodev/argopy/issues/101
-        """
+        """Return safely encoded list of urls"""
 
         # return urls
         def safe_for_fsspec_cache(url):
@@ -269,7 +265,6 @@ def safe_for_fsspec_cache(url):
             return url
 
         return [safe_for_fsspec_cache(url) for url in urls]
-        # return [urllib.parse.quote(url, safe='/:?=[]&') for url in urls]
 
     def json2dataframe(self, profiles):
         """convert json data to Pandas DataFrame"""
@@ -372,10 +367,6 @@ def to_xarray(self, errors: str = "ignore"):
         ds.attrs = {}
         if self.dataset_id == "phy":
             ds.attrs["DATA_ID"] = "ARGO"
-        elif self.dataset_id == "ref":
-            ds.attrs["DATA_ID"] = "ARGO_Reference"
-        elif self.dataset_id == "bgc":
-            ds.attrs["DATA_ID"] = "ARGO-BGC"
         ds.attrs["DOI"] = "http://doi.org/10.17882/42182"
         ds.attrs["Fetched_from"] = self.server
         try:

argopy/errors.py

@@ -123,13 +123,13 @@ def __str__(self):
 
 
 class ErddapServerError(APIServerError):
-    """Raise this when argopy is disrupted by an error due to the Erddap, not argopy machinery."""
+    """Raise this when argopy is disrupted by an error due to the Erddap server, not argopy machinery."""
 
     pass
 
 
 class ArgovisServerError(APIServerError):
-    """Raise this when argopy is disrupted by an error due to the Erddap, not argopy machinery."""
+    """Raise this when argopy is disrupted by an error due to the Argovis server, not argopy machinery."""
 
     pass
 

argopy/fetchers.py

@@ -160,9 +160,15 @@ def __init__(self, mode: str = "", src: str = "", ds: str = "", **fetcher_kwargs
             self._mode == "standard" or self._mode == "research"
         ):
             warnings.warn(
-                "The 'bgc' dataset fetching is only available in 'expert' mode at this point."
+                "The 'bgc' dataset fetching is only available in 'expert' user modes at this point."
             )
 
+        if self._src == "argovis" and (
+            self._mode == "expert" or self._mode == "research"
+        ):
+            raise OptionValueError("The 'argovis' data source fetching is only available in 'standard' user mode")
+
+
     def __repr__(self):
         para = (
             self.fetcher_options["parallel"]

argopy/tests/helpers/mocked_http.py

@@ -4,21 +4,21 @@
 For this to work, we need to catch all the possible URI sent from unit tests and return some data (eg json or netcdf)
 we downloaded once before tests.
 
-This requires to download data using fully functional http servers: this is done by the "test_data" script located
+This requires to download data using fully functional http servers: this is done by the "citests_httpdata_manager" located
 in "argopy/cli"
 
 Servers covered by this fixture:
-    "https://github.com/euroargodev/argopy-data/raw/master",
-    "https://erddap.ifremer.fr/erddap",
-    "https://data-argo.ifremer.fr",
-    "https://api.ifremer.fr",
-    "https://coastwatch.pfeg.noaa.gov/erddap",
-    "https://www.ocean-ops.org/api/1",
-    "https://dataselection.euro-argo.eu/api",
-    "https://vocab.nerc.ac.uk/collection",
-    "https://argovisbeta02.colorado.edu",
-    "https://dx.doi.org",
-    "https://archimer.ifremer.fr",
+            "https://github.com/euroargodev/argopy-data/raw/master",
+            "https://erddap.ifremer.fr/erddap",
+            "https://data-argo.ifremer.fr",
+            "https://api.ifremer.fr",
+            "https://coastwatch.pfeg.noaa.gov/erddap",
+            "https://www.ocean-ops.org/api/1",
+            "https://dataselection.euro-argo.eu/api",
+            "https://vocab.nerc.ac.uk/collection",
+            "https://argovis-api.colorado.edu",
+            "https://dx.doi.org",
+            "https://archimer.ifremer.fr",
 
 The HTTPTestHandler class below is taken from the fsspec tests suite at:
 https://github.com/fsspec/filesystem_spec/blob/55c5d71e657445cbfbdba15049d660a5c9639ff0/fsspec/tests/conftest.py
@@ -39,7 +39,7 @@
 
 log = logging.getLogger("argopy.tests.mocked_http")
 LOG_SERVER_CONTENT = (
-    False  # Should we list files/uris available from the mocked server in the log ?
+    False  # Should we list all files/uris available from the mocked server in the log ?
 )
 
 requests = pytest.importorskip("requests")
@@ -93,7 +93,8 @@
             "https://www.ocean-ops.org/api/1",
             "https://dataselection.euro-argo.eu/api",
             "https://vocab.nerc.ac.uk/collection",
-            "https://argovisbeta02.colorado.edu",
+            "https://argovis-api.colorado.edu",
+            # "https://argovisbeta02.colorado.edu",
             "https://dx.doi.org",
             "https://archimer.ifremer.fr",
         ]
@@ -168,11 +169,11 @@ def do_GET(self):
         if "give_path" in self.headers:
             return self._respond(200, data=json.dumps({"path": self.path}).encode())
         if file_data is None:
-            # log.debug("file data empty, returning 404")
-            return self._respond(404)
-        else:
-            n = len(file_data)
+            file_data = self.files.get(self.path.rstrip("/"))  # try without unquoting
+            if file_data is None:
+                return self._respond(404)
 
+        n = len(file_data)
         status = 200
         content_range = "bytes 0-%i/%i" % (n - 1, n)
         if ("Range" in self.headers) and ("ignore_range" not in self.headers):

argopy/tests/helpers/utils.py

@@ -157,13 +157,11 @@ def _xfail(name, msg):
 has_argovis, requires_argovis = _connectskip(
     "argovis" in AVAILABLE_SOURCES, "argovis data fetcher"
 )
-
 has_connected_argovis = has_connection and has_argovis and isAPIconnected(src='argovis', data=True)
 requires_connected_argovis = pytest.mark.skipif(
     not has_connected_argovis, reason="Requires a live Argovis server"
 )
 
-
 ############
 # GDAC FTP #
 ############
@@ -374,5 +372,4 @@ def create_read_only_folder(folder_path):
     else:
         create_read_only_folder_linux(folder_path)
 
-
 log.debug("%s TESTS UTILS %s" % ("="*50, "="*50))