Fixed error case in gedi ancillary field processing; added pytests

SlideRuleEarth · Jan 6, 2025 · afddb22 · afddb22
1 parent 42a4fa4
commit afddb22
Show file tree

Hide file tree

Showing 5 changed files with 113 additions and 8 deletions.
diff --git a/clients/python/tests/test_ancillary.py b/clients/python/tests/test_ancillary.py
@@ -2,14 +2,14 @@
 
 import pytest
 import sliderule
-from sliderule import icesat2
+from sliderule import icesat2, gedi, earthdata
 from pathlib import Path
 import os.path
 
 TESTDIR = Path(__file__).parent
 
 @pytest.mark.network
-class TestAncillary:
+class TestIcesat2:
     def test_geo(self, init):
         region = sliderule.toregion(os.path.join(TESTDIR, "data/grandmesa.geojson"))
         parms = {
@@ -75,3 +75,92 @@ def test_atl08_phoreal(self, init):
         assert abs(gdf["rgt_y"].quantile(q=.50) - 295.0) < 0.000001
         assert abs(gdf["sigma_atlas_land%"].quantile(q=.50) - 0.24470525979995728) < 0.000001
         assert abs(gdf["cloud_flag_atm%"].quantile(q=.50) - 1.0) < 0.000001
+
+@pytest.mark.network
+class TestGedi:
+    def test_l1b(self, init):
+        region = sliderule.toregion(os.path.join(TESTDIR, "data/grandmesa.geojson"))
+        parms = {
+            "asset": "gedil1b",
+            "poly": region["poly"],
+            "beam": 0,
+            "anc_fields": ["master_frac", "rx_open"]
+        }
+        granules = earthdata.search(parms)
+        gdf = gedi.gedi01bp(parms, resources=granules[0:1])
+        assert init
+        assert len(gdf) == 22
+        assert gdf["beam"].unique()[0] == 0
+        assert gdf.master_frac.describe()["max"] < 0.3
+        assert gdf.rx_open.describe()["max"] == 2851996.0
+
+    def test_l2a(self, init):
+        region = sliderule.toregion(os.path.join(TESTDIR, "data/grandmesa.geojson"))
+        parms = {
+            "asset": "gedil2a",
+            "poly": region["poly"],
+            "beam": 0,
+            "anc_fields": ["energy_total", "geolocation/elev_highestreturn_a1"]
+        }
+        granules = earthdata.search(parms)
+        gdf = gedi.gedi02ap(parms, resources=granules[0:1])
+        assert init
+        assert len(gdf) == 22
+        assert gdf["beam"].unique()[0] == 0
+        assert abs(gdf.energy_total.describe()["max"] - 7498) < 1.0
+        assert abs(gdf["geolocation/elev_highestreturn_a1"].describe()["max"] - 2892) < 1.0
+
+    def test_l4a(self, init):
+        region = sliderule.toregion(os.path.join(TESTDIR, "data/grandmesa.geojson"))
+        parms = {
+            "asset": "gedil4a",
+            "poly": region["poly"],
+            "beam": 0,
+            "anc_fields": ["selected_algorithm", "geolocation/elev_lowestmode_a1"]
+        }
+        granules = earthdata.search(parms)
+        gdf = gedi.gedi04ap(parms, resources=granules[0:1])
+        assert init
+        assert len(gdf) == 22
+        assert gdf["beam"].unique()[0] == 0
+        assert gdf.selected_algorithm.value_counts()[2] == 14
+        assert abs(gdf["geolocation/elev_lowestmode_a1"].describe()["max"] - 2886) < 1.0
+
+    def test_l1b_failure(self, init):
+        region = sliderule.toregion(os.path.join(TESTDIR, "data/grandmesa.geojson"))
+        parms = {
+            "asset": "gedil1b",
+            "poly": region["poly"],
+            "beam": 0,
+            "anc_fields": ["non_existent_field", "rx_open"]
+        }
+        granules = earthdata.search(parms)
+        gdf = gedi.gedi01bp(parms, resources=granules[0:1])
+        assert init
+        assert len(gdf) == 0
+
+    def test_l2a_failure(self, init):
+        region = sliderule.toregion(os.path.join(TESTDIR, "data/grandmesa.geojson"))
+        parms = {
+            "asset": "gedil2a",
+            "poly": region["poly"],
+            "beam": 0,
+            "anc_fields": ["non_existent_field", "geolocation/elev_highestreturn_a1"]
+        }
+        granules = earthdata.search(parms)
+        gdf = gedi.gedi02ap(parms, resources=granules[0:1])
+        assert init
+        assert len(gdf) == 0
+
+    def test_l4a_failure(self, init):
+        region = sliderule.toregion(os.path.join(TESTDIR, "data/grandmesa.geojson"))
+        parms = {
+            "asset": "gedil4a",
+            "poly": region["poly"],
+            "beam": 0,
+            "anc_fields": ["non_existent_field", "geolocation/elev_lowestmode_a1"]
+        }
+        granules = earthdata.search(parms)
+        gdf = gedi.gedi04ap(parms, resources=granules[0:1])
+        assert init
+        assert len(gdf) == 0
diff --git a/datasets/gedi/package/FootprintReader.h b/datasets/gedi/package/FootprintReader.h
@@ -152,7 +152,7 @@ class FootprintReader: public LuaObject
                                                      const char* batch_rec_type, const char* lat_name, const char* lon_name,
                                                      subset_func_t subsetter);
                             ~FootprintReader        (void) override;
-        void                readAncillaryData       (const info_t* info, long first_footprint, long num_footprints);
+        bool                readAncillaryData       (const info_t* info, long first_footprint, long num_footprints);
         void                populateAncillaryFields (const info_t* info, long footprint, uint64_t shot_number);
         void                postRecordBatch         (stats_t* local_stats);
         static int          luaStats                (lua_State* L);
@@ -433,7 +433,7 @@ void FootprintReader<footprint_t>::Region::rasterregion (const info_t* info)
  * readAncillaryData
  *----------------------------------------------------------------------------*/
 template <class footprint_t>
-void FootprintReader<footprint_t>::readAncillaryData (const info_t* info, long first_footprint, long num_footprints)
+bool FootprintReader<footprint_t>::readAncillaryData (const info_t* info, long first_footprint, long num_footprints)
 {
     vector<H5DArray*> arrays_to_join;
 
@@ -454,10 +454,17 @@ void FootprintReader<footprint_t>::readAncillaryData (const info_t* info, long f
     }
 
     /* Join Ancillary Reads */
+    bool status = true;
     for(H5DArray* array: arrays_to_join)
     {
-        array->join(read_timeout_ms, true);
+        if(!array->join(read_timeout_ms, false))
+        {
+            status = false;
+        }
     }
+
+    /* Return Status */
+    return status;
 }
 
 /*----------------------------------------------------------------------------

diff --git a/datasets/gedi/package/Gedi01bReader.cpp b/datasets/gedi/package/Gedi01bReader.cpp
@@ -181,7 +181,10 @@ void* Gedi01bReader::subsettingThread (void* parm)
 
         /* Read GEDI Datasets */
         const Gedi01b gedi01b(info, region);
-        reader->readAncillaryData(info, region.first_footprint, region.num_footprints);
+        if(!reader->readAncillaryData(info, region.first_footprint, region.num_footprints))
+        {
+            throw RunTimeException(CRITICAL, RTE_ERROR, "failed to read ancillary data");
+        }
 
         /* Read Waveforms */
         const long tx0 = gedi01b.tx_start_index[0] - 1;

diff --git a/datasets/gedi/package/Gedi02aReader.cpp b/datasets/gedi/package/Gedi02aReader.cpp
@@ -177,7 +177,10 @@ void* Gedi02aReader::subsettingThread (void* parm)
 
         /* Read GEDI Datasets */
         const Gedi02a gedi02a(info, region);
-        reader->readAncillaryData(info, region.first_footprint, region.num_footprints);
+        if(!reader->readAncillaryData(info, region.first_footprint, region.num_footprints))
+        {
+            throw RunTimeException(CRITICAL, RTE_ERROR, "failed to read ancillary data");
+        }
 
         /* Increment Read Statistics */
         local_stats.footprints_read = region.num_footprints;

diff --git a/datasets/gedi/package/Gedi04aReader.cpp b/datasets/gedi/package/Gedi04aReader.cpp
@@ -178,7 +178,10 @@ void* Gedi04aReader::subsettingThread (void* parm)
 
         /* Read GEDI Datasets */
         const Gedi04a gedi04a(info, region);
-        reader->readAncillaryData(info, region.first_footprint, region.num_footprints);
+        if(!reader->readAncillaryData(info, region.first_footprint, region.num_footprints))
+        {
+            throw RunTimeException(CRITICAL, RTE_ERROR, "failed to read ancillary data");
+        }
 
         /* Increment Read Statistics */
         local_stats.footprints_read = region.num_footprints;