Feature/1 update 2024 08 19 (#2)

* [#1] Update scripts on libs

* [#1] Update project settings

* [#1] Update pattern settings - Rename MOD11C3 to MOD21C3

* [#1] Update execute_steps.bat

* [#1] Update step 0000

* [#1] Update step 0101

* [#1] Update step 0303

* [#1] Update source folder to include mapping/data & mapping/output/maps

* [#1] Update scripts all FinalMapOutput

* [#1] Fix dir structure for source/mapping

.gitignore

@@ -179,22 +179,22 @@ pyrightconfig.json
 .DS_Store
 
 # source files
-source/input_data/MOD11C3_LST/*
+source/input_data/MOD21C3_LST/*
 source/input_data/MOD13C2_NDVI/*
 source/input_data/CHIRPS/*
 source/input_data/soil_moisture/*
 source/map_data/*
 source/working_data/*
 source/output_data/GeoTiffs/*
-source/mapping/*
+source/mapping/data/*
 source/mapping/output/maps/*
 
-!source/input_data/MOD11C3_LST/.gitkeep
+!source/input_data/MOD21C3_LST/.gitkeep
 !source/input_data/MOD13C2_NDVI/.gitkeep
 !source/input_data/CHIRPS/.gitkeep
 !source/input_data/soil_moisture/.gitkeep
 !source/map_data/.gitkeep
 !source/working_data/.gitkeep
 !source/output_data/GeoTiffs/.gitkeep
-!source/mapping/.gitkeep
+!source/mapping/data/.gitkeep
 !source/mapping/output/maps/.gitkeep

FinalMapOutput-ESRI.py

@@ -2,11 +2,11 @@
 import datetime
 import arcpy
 ## Change these to match your directory structure. If you are missing directories you will need to create them
-map_input_dir = r'D:/ProjectData/CDI/[base directory]/mapping/data/'
-map_output_dir = r'D:/ProjectData/CDI/[base directory]/mapping/output/maps/'
-tif_dir = r'D:/ProjectData/CDI/[base directory]/output_data/GeoTiffs/'
+map_input_dir = r'./source/mapping/data/'
+map_output_dir = r'./source/mapping/output/maps/'
+tif_dir = r'./source/output_data/GeoTiffs/'
 reglyr = map_input_dir+'Percentile_colors_new.lyr'
-region = '[Country name]'
+region = 'eSwatini'
 ## Resolution can be changed here
 res = 150
 
@@ -36,6 +36,6 @@
     print 'Creating png map'
     arcpy.mapping.ExportToPNG(mxd,png_dir+region+'_'+typ+'_'+tif_year+int_month+'.png',resolution=res)
     print 'Creating pdf map'
-    arcpy.mapping.ExportToPDF(mxd,pdf_dir+region+'_'+typ+'_'+tif_year+int_month+'.png',resolution=res)
+    arcpy.mapping.ExportToPDF(mxd,pdf_dir+region+'_'+typ+'_'+tif_year+int_month+'.pdf',resolution=res)
     print 'Creating jpg map'
-    arcpy.mapping.ExportToJPEG(mxd,jpg_dir+region+'_'+typ+'_'+tif_year+int_month+'.png',resolution=res)
+    arcpy.mapping.ExportToJPEG(mxd,jpg_dir+region+'_'+typ+'_'+tif_year+int_month+'.jpg',resolution=res)

FinalMapOutput-POR-ESRI.py

@@ -2,30 +2,31 @@
 import datetime
 import arcpy
 ## Change these to match your directory structure. If you are missing directories you will need to create them
-map_input_dir = r'D:/Operations/Global_CDI/SA-Zimbabwe/mapping/data/'
-map_output_dir = r'D:/Operations/Global_CDI/SA-Zimbabwe/mapping/output/maps/'
-tif_dir = "D:\\Operations\\Global_CDI\\SA-Zimbabwe\\output_data\\GeoTiffs\\"
+map_input_dir = r'./source/mapping/data/'
+map_output_dir = r'./source/mapping/output/maps/'
+tif_dir = r'./source/output_data/GeoTiffs/'
 reglyr = map_input_dir+'Percentile_colors_new.lyr'
-region = 'Zimbabwe'
+region = 'eSwatini'
 ## Resolution can be changed here
 res = 150
+map_remote_output_dir = r'//ndmc-webp01/WebArchive/GlobalCDI/'+region+'/'
 
 for typ in ['CDI','LST','NDVI','SPI','SM']:
     print 'Mapping '+typ
     arcpy.env.overwriteOutput = True
-    tif_list = glob.glob(tif_dir+typ+'\\*.tif')
+    tif_list = glob.glob(tif_dir+typ+'/*.tif')
     #in_tif = tif_list[-1]
     for in_tif in tif_list:
         tif_year = (in_tif[-10:])[0:4]
-        int_month = ((in_tif[-10:])[0:6])[-2:]
-        monthinteger = int(int_month)
-        month = datetime.date(1900, monthinteger, 1).strftime('%b')
-        if int(tif_year) > 2019:
-            print "Inside loop"
-            print in_tif
+        if tif_year>"2023":
+
+            int_month = ((in_tif[-10:])[0:6])[-2:]
+            monthinteger = int(int_month)
+            month = datetime.date(1900, monthinteger, 1).strftime('%b')
             tif_layer = arcpy.mapping.Layer(in_tif)
             ## Change this to 10.x.mxd if not running ArcGIS 10.8 or later
             themxd = map_input_dir+region+'_'+typ+'-10.8.mxd'
+            #print themxd
             mxd = arcpy.mapping.MapDocument(themxd)
             df = arcpy.mapping.ListDataFrames(mxd, "Layers")[0]
             arcpy.mapping.AddLayer(df, tif_layer,"BOTTOM")
@@ -35,13 +36,17 @@
             mapdate = (arcpy.mapping.ListLayoutElements(mxd,"TEXT_ELEMENT","Date"))[0]
             mapdate.text = month+' '+tif_year
             png_dir = map_output_dir+typ+'/png/'
-            print png_dir
             pdf_dir = map_output_dir+typ+'/pdf/'
             jpg_dir = map_output_dir+typ+'/jpg/'
-            #print 'Creating png map'
-            print png_dir+region+'_'+typ+'_'+tif_year+int_month+'.png'
+            rpng_dir = map_remote_output_dir+typ+'/png/'
+            rpdf_dir = map_remote_output_dir+typ+'/pdf/'
+            rjpg_dir = map_remote_output_dir+typ+'/jpg/'
+            print 'Creating png map'
             arcpy.mapping.ExportToPNG(mxd,png_dir+region+'_'+typ+'_'+tif_year+int_month+'.png',resolution=res)
-            #print 'Creating pdf map'
-            arcpy.mapping.ExportToPDF(mxd,pdf_dir+region+'_'+typ+'_'+tif_year+int_month+'.png',resolution=res)
-            #$print 'Creating jpg map'
-            arcpy.mapping.ExportToJPEG(mxd,jpg_dir+region+'_'+typ+'_'+tif_year+int_month+'.png',resolution=res)
+            arcpy.mapping.ExportToPNG(mxd,rpng_dir+region+'_'+typ+'_'+tif_year+int_month+'.png',resolution=res)
+            print 'Creating pdf map'
+            arcpy.mapping.ExportToPDF(mxd,pdf_dir+region+'_'+typ+'_'+tif_year+int_month+'.pdf',resolution=res)
+            arcpy.mapping.ExportToPDF(mxd,rpdf_dir+region+'_'+typ+'_'+tif_year+int_month+'.pdf',resolution=res)
+            print 'Creating jpg map'
+            arcpy.mapping.ExportToJPEG(mxd,jpg_dir+region+'_'+typ+'_'+tif_year+int_month+'.jpg',resolution=res)
+            arcpy.mapping.ExportToJPEG(mxd,rjpg_dir+region+'_'+typ+'_'+tif_year+int_month+'.jpg',resolution=res)

STEP_0000_execute_all_steps.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from netCDF4 import Dataset
 from STEP_0101_read_hdf_create_LST_anom_netcdf import main as step_0101
 from STEP_0102_read_hdf_create_NDVI_anom_netcdf import main as step_0102
 from STEP_0103_read_chirps_create_precip_netcdf_and_spi_netcdf import main as step_0103

STEP_0101_read_hdf_create_LST_anom_netcdf.py

@@ -48,8 +48,11 @@ def __get_hdf_date(self, file_name):
         Returns:
             String of the year/month values in 'YYYYMM' format
         """
-        (year, days) = self.__raw_file_match.match(file_name).groups()
-        test_date = date(int(year), 1, 1) + timedelta(days=int(days))
+        if file_name.find('/') > -1:
+            file_name = file_name.split('/')[1]
+        match = self.__raw_file_match.match(file_name)
+        (year, month) = match.groups()
+        test_date = date(int(year), int(month), 1)
         return test_date.strftime("%Y%m")
 
     def __get_calendar_value(self, file_name):
@@ -61,6 +64,8 @@ def __get_calendar_value(self, file_name):
         Returns:
             Integer value of the number of days since Jan 1, 1900
         """
+        if file_name.find('/') > -1:
+            file_name = file_name.split('/')[1]
         origin_date = date(1900, 1, 1)
         (year, days) = self.__raw_file_match.match(file_name).groups()
         test_date = date(int(year), 1, 1) + timedelta(days=(int(days) - 1))
@@ -139,14 +144,17 @@ def get_files_to_process(self, all_hdf=False):
         """
         files = []
         try:
-            raw_files = self.__fileHandler.get_raw_file_names('lst_hdf_regex')
+            raw_files = sorted(self.__fileHandler.get_raw_file_names('lst_hdf_regex'), reverse=True)
             if all_hdf:  # include all HDF files
                 files = raw_files
             else:  # determine which HDF files have not been converted to NetCDF
                 working_files = self.__fileHandler.get_working_file_names('lst_netcdf_regex')
                 # parse out the year/days from the file name #
                 for f in raw_files:
-                    file_date = self.__get_hdf_date(f)
+                    name = f
+                    if name.find('/') > -1:
+                        name = name.split('/')[1]
+                    file_date = self.__get_hdf_date(name)
                     test_file = "STEP_0101_LST_{}_{}.nc".format(self.__region, file_date)
                     if test_file not in working_files:
                         files.append(f)
@@ -174,21 +182,16 @@ def create_lst_netcdf_file(self, file_name):
 
             # extract SubGrids of the required parameters #
             with HDFSubGrid(self.__bounds, raw_file_path, self.__hdf_group) as sg:
-                lst_day = sg.create_sub_grid('LST_Day_CMG') * 0.02  # data is scaled in the HDF file
-                lst_night = sg.create_sub_grid('LST_Night_CMG') * 0.02  # data is scaled in the HDF file
+                lst_day = sg.create_sub_grid('LST_Day').astype(float) * 0.02  # data is scaled in the HDF file
+                lst_night = sg.create_sub_grid('LST_Night').astype(float) * 0.02  # data is scaled in the HDF file
                 qc_day = sg.create_sub_grid('QC_Day')
                 qc_night = sg.create_sub_grid('QC_Night')
 
             # compute the LST delta #
-            qc_day_filter = np.logical_or(np.logical_and(qc_day > 16, qc_day < 64), qc_day > 79)
-            filtered_lst_day = ma.masked_where(qc_day_filter, lst_day)
-
-            qc_night_filter = np.logical_or(np.logical_and(qc_night > 16, qc_night < 64), qc_night > 95)
-            filtered_lst_night = ma.masked_where(qc_night_filter, lst_night)
-
-            delta = filtered_lst_day - filtered_lst_night
-            delta_filter = np.logical_or(filtered_lst_day == 0, filtered_lst_night == 0, delta <= 0)
-            lst_delta = np.round(ma.masked_where(delta_filter, delta).filled(self.__missing), 3)
+            filtered_lst_day = ma.masked_where(np.logical_or(qc_day < 16, lst_day == 0), lst_day)
+            filtered_lst_night = ma.masked_where(np.logical_or(qc_night < 16, lst_night == 0), lst_night)
+            delta = np.ma.clip(np.ma.subtract(filtered_lst_day, filtered_lst_night), -40.0, 40.0)
+            lst_delta = np.round(delta.filled(self.__missing), 3)
 
             # create the output file #
             out_properties = {
@@ -253,7 +256,7 @@ def update_lst_anomaly_file(self):
                 index = idx
                 # loop thru the years and add the data to the NetCDF file #
                 for y in month_anomalies:
-                    lst_var[index] = y
+                    lst_var[index] = y  # np.where(self.__missing, y, np.multiply(y, -1))  # reverse so positive anomalies are dry
                     index += 12  # increment 1 year
         except IOError:
             raise