Adding a CMORiser for CMAP data for pr (#3766)

Co-authored-by: Max Proft <mp7041@gadi-login-05.gadi.nci.org.au>

doc/sphinx/source/input.rst

@@ -269,6 +269,8 @@ A list of the datasets for which a CMORizers is available is provided in the fol
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | CLOUDSAT-L2                  | clw, clivi, clwvi, lwp (Amon)                                                                        |   3  | NCL             |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
+| CMAP                         | pr (Amon)                                                                                            |   2  | Python          |
++------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | CowtanWay                    | tasa (Amon)                                                                                          |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | CRU                          | tas, tasmin, tasmax,  pr, clt (Amon), evspsblpot (Emon)                                              |   2  | Python          |

esmvaltool/cmorizers/data/cmor_config/CMAP.yml

@@ -0,0 +1,21 @@
+---
+# Global attributes of NetCDF file
+attributes:
+  dataset_id: CMAP
+  project_id: OBS6
+  tier: 2
+  version: "v1"
+  modeling_realm: reanaly
+  source: "https://psl.noaa.gov/data/gridded/data.cmap.html"
+  reference: "cmap"
+  comment: |
+    ''
+
+# Variables to CMORize
+variables:
+  # monthly frequency
+  pr_month:
+    short_name: pr
+    mip: Amon
+    raw: precip
+    file: "precip.mon.mean.nc"

esmvaltool/cmorizers/data/datasets.yml

@@ -264,6 +264,15 @@ datasets:
          named like the year (e.g. 2007), no subdirectories with
          days etc.
 
+  CMAP:
+    tier: 2
+    source: https://psl.noaa.gov/data/gridded/data.cmap.html
+    last_access: 2024-09-09
+    info: |
+      To facilitate the download, the links to the https server are provided.
+      https://downloads.psl.noaa.gov/Datasets/cmap/enh/
+        precip.mon.mean.nc
+
   CowtanWay:
     tier: 2
     source: https://www-users.york.ac.uk/~kdc3/papers/coverage2013/series.html

esmvaltool/cmorizers/data/downloaders/datasets/cmap.py

@@ -0,0 +1,38 @@
+"""Script to download CMAP (CPC Merged Analysis of Precipitation)."""
+
+import logging
+
+from esmvaltool.cmorizers.data.downloaders.ftp import FTPDownloader
+
+logger = logging.getLogger(__name__)
+
+
+def download_dataset(config, dataset, dataset_info, start_date, end_date,
+                     overwrite):
+    """Download dataset.
+
+    Parameters
+    ----------
+    config : dict
+        ESMValTool's user configuration
+    dataset : str
+        Name of the dataset
+    dataset_info : dict
+         Dataset information from the datasets.yml file
+    start_date : datetime
+        Start of the interval to download
+    end_date : datetime
+        End of the interval to download
+    overwrite : bool
+        Overwrite already downloaded files
+    """
+    downloader = FTPDownloader(
+        config=config,
+        server="ftp2.psl.noaa.gov",
+        dataset=dataset,
+        dataset_info=dataset_info,
+        overwrite=overwrite,
+    )
+    downloader.connect()
+
+    downloader.download_file("/Datasets/cmap/enh/precip.mon.mean.nc")

esmvaltool/cmorizers/data/formatters/datasets/cmap.py

@@ -0,0 +1,69 @@
+"""ESMValTool CMORizer for CMAP (CPC Merged Analysis of Precipitation) data.
+
+Tier
+    Tier 2: other freely-available dataset.
+
+Source
+    https://psl.noaa.gov/data/gridded/data.cmap.html
+
+Last access
+    20240909
+
+Download and processing instructions
+    To facilitate the download, the links to the ftp server are provided.
+
+    https://downloads.psl.noaa.gov/Datasets/cmap/enh/
+        precip.mon.mean.nc
+
+Caveats
+
+"""
+
+import logging
+import re
+from copy import deepcopy
+from pathlib import Path
+
+import iris
+from esmvaltool.cmorizers.data import utilities as utils
+
+
+logger = logging.getLogger(__name__)
+
+
+def _extract_variable(short_name, var, cfg, raw_filepath, out_dir):
+    cmor_info = cfg["cmor_table"].get_variable(var["mip"], short_name)
+    attributes = deepcopy(cfg["attributes"])
+    attributes["mip"] = var["mip"]
+
+    cubes = iris.load(raw_filepath)
+    for cube in cubes:
+        assert cube.units == "mm/day", f"unknown units:{cube.units}"
+        # convert data from mm/day to kg m-2 s-1
+        # mm/day ~ density_water * mm/day
+        # = 1000 kg m-3 * 1/(1000*86400) m s-1 = 1/86400 kg m-2 s-1
+        cube = cube / 86400
+        cube.units = "kg m-2 s-1"
+
+        utils.fix_var_metadata(cube, cmor_info)
+        cube = utils.fix_coords(cube)
+        utils.set_global_atts(cube, attributes)
+
+        logger.info("Saving file")
+        utils.save_variable(cube, short_name, out_dir, attributes,
+                            unlimited_dimensions=["time"])
+
+
+def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date):
+    """Cmorization func call."""
+    for short_name, var in cfg["variables"].items():
+        logger.info("CMORizing variable '%s'", short_name)
+        short_name = var["short_name"]
+        raw_filenames = Path(in_dir).rglob("*.nc")
+        filenames = []
+        for raw_filename in raw_filenames:
+            if re.search(var["file"], str(raw_filename)) is not None:
+                filenames.append(raw_filename)
+
+        for filename in sorted(filenames):
+            _extract_variable(short_name, var, cfg, filename, out_dir)

esmvaltool/recipes/examples/recipe_check_obs.yml

@@ -61,6 +61,16 @@ diagnostics:
     scripts: null
 
 
+  CMAP:
+    description: CMAP check
+    variables:
+      pr:
+    additional_datasets:
+      - {project: OBS6, dataset: CMAP, mip: Amon, tier: 2,
+        type: reanaly, version: v1}
+    scripts: null
+
+
   CRU:
     description: CRU check
     variables: