Compound flooding scripts: fixed a bug in depth loading. Added BlueTo…

…po in a few regions into the DEM database.

src/Utility/Pre-Processing/SECOFS/Bathy_loading/DEM_info.json

@@ -0,0 +1,122 @@
+{
+    "gebco": {
+        "Annotation": "General Bathymetric Chart of the Oceans",
+        "Note": "",
+        "File": "/sciclone/schism10/lcui01/schism20/DEM/GEBCO/gebco_2020_n90.0_s0.0_w-90.0_e0.0.asc"
+    },
+    "etopo1": {
+        "Annotation": "ETOPO1 1 Arc-Minute Global Relief Model",
+        "Note": "",
+        "File": "/sciclone/data10/whuang07/NWM/DEM/DEM/etopo1.asc"
+    },
+    "crm": {
+        "Annotation": "CRM (Coastal Relief Model) 3 arc seconds",
+        "Note": "",
+        "File": "/sciclone/data10/whuang07/NWM/NOAA/3arcs/central_gom_crm_v[1-5].asc"
+    },
+    "cdem": {
+        "Annotation": "Coastal 1/3 arc seconds DEM from NOAA NGDC",
+        "Note": "https://docs.google.com/document/d/1IsLhZchWqg6F03vapwfjqbd1Ho_c5nrZcz2Gp3dOzu8/edit?usp=sharing",
+        "File": "/sciclone/schism10/Hgrid_projects/DEMs/CDEM/; /ches/data10/whuang07/NWM/NOAA/13arcs/"
+    },
+    "CoNED_North_Carolina": {
+        "Annotation": "Coastal National Elevation Database: North Carolina region, coarsened to 3m, projected to lon/lat",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/OneNote.aspx?id=%2Fpersonal%2Ffeiye_vims_edu%2FDocuments%2FNotebooks%2FNOAA%20TWL%20project%E2%80%8B&wd=target%28DEM.one%7CAE181E84-4618-4FAF-8AF2-32FE3A6C3D29%2FCoNED%7CC017A206-7C6F-4EED-8414-6244AB1BCFB8%2F%29",
+        "File": "/sciclone/data10/whuang07/NWM/CONED/NorthCarolina_USGS_CoNED_3m/North_Carolina_USGS_3m_*"
+    },
+    "CuDEM_NCEI_al": {
+        "Annotation": "CuDEM (Continously updated DEM): Alabama",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FCuDEM%7C7ed05443-5395-4dee-81b2-a8f7b0d530f2%2F%29&wdorigin=703&wdpreservelink=1",
+        "File": "/sciclone/schism10/Hgrid_projects/DEMs/CuDEM2/CuDEM_NCEI_al_ll*.asc"
+    },
+    "CuDEM_NCEI_nc": {
+        "Annotation": "CuDEM (Continously updated DEM): North Carolina",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FCuDEM%7C7ed05443-5395-4dee-81b2-a8f7b0d530f2%2F%29&wdorigin=703&wdpreservelink=1",
+        "File": "/sciclone/schism10/Hgrid_projects/DEMs/CuDEM2/CuDEM_NCEI_nc_ll*.asc"
+    },
+    "CuDEM_NCEI_fl": {
+        "Annotation": "CuDEM (Continously updated DEM): Florida",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FCuDEM%7C7ed05443-5395-4dee-81b2-a8f7b0d530f2%2F%29&wdorigin=703&wdpreservelink=1",
+        "File": "/sciclone/schism10/Hgrid_projects/DEMs/CuDEM2/CuDEM_NCEI_fl_ll*.asc"
+    },
+    "CoNED_gulf_1": {
+        "Annotation": "Coastal National Elevation Database: DEM for Gulf region 1, coarsened to 3m",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/OneNote.aspx?id=%2Fpersonal%2Ffeiye_vims_edu%2FDocuments%2FNotebooks%2FNOAA%20TWL%20project%E2%80%8B&wd=target%28DEM.one%7CAE181E84-4618-4FAF-8AF2-32FE3A6C3D29%2FCoNED%7CC017A206-7C6F-4EED-8414-6244AB1BCFB8%2F%29",
+        "File": "/sciclone/schism10/Hgrid_projects/DEMs/CoNED/CoNED_gulf1*; /sciclone/data10/whuang07/NWM/CONED/Northern_Gulf_of_Mexico/Gulf_1"
+    },
+    "CoNED_gulf_3": {
+        "Annotation": "Coastal National Elevation Database: DEM for Gulf region 3, coarsened to 3m",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FDelaware%20Bay%7Cb4b569d1-6f88-459e-9acd-f381f7d897a9%2F%29&wdorigin=703",
+        "File": "/sciclone/schism10/Hgrid_projects/DEMs/CoNED/CoNED_gulf3*; /sciclone/data10/whuang07/NWM/CONED/Northern_Gulf_of_Mexico/Gulf_3"
+    },
+    "CuDEM_NCEI_ge": {
+        "Annotation": "CuDEM (Continously updated DEM): Georgia",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FCuDEM%7C7ed05443-5395-4dee-81b2-a8f7b0d530f2%2F%29&wdorigin=703&wdpreservelink=1",
+        "File": "/sciclone/schism10/Hgrid_projects/DEMs/CuDEM2/CuDEM_NCEI_ge_ll*.asc"
+    },
+    "CuDEM_NCEI_sc": {
+        "Annotation": "CuDEM (Continously updated DEM): South Carolina",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FCuDEM%7C7ed05443-5395-4dee-81b2-a8f7b0d530f2%2F%29&wdorigin=703&wdpreservelink=1",
+        "File": "/sciclone/schism10/Hgrid_projects/DEMs/CuDEM2/CuDEM_NCEI_sc_ll*.asc"
+    },
+    "CoNED_cb_ll": {
+        "Annotation": "Coastal National Elevation Database: Chesapeake Bay region, coarsened to 3m",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/OneNote.aspx?id=%2Fpersonal%2Ffeiye_vims_edu%2FDocuments%2FNotebooks%2FNOAA%20TWL%20project%E2%80%8B&wd=target%28DEM.one%7CAE181E84-4618-4FAF-8AF2-32FE3A6C3D29%2FCoNED%7CC017A206-7C6F-4EED-8414-6244AB1BCFB8%2F%29",
+        "File": "/sciclone/data10/whuang07/NWM/DEM/DEM/cb_ll*"
+    },
+    "CoNED_db_ll": {
+        "Annotation": "Coastal National Elevation Database: DEM for Delaware Bay, coarsened to 3m",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/OneNote.aspx?id=%2Fpersonal%2Ffeiye_vims_edu%2FDocuments%2FNotebooks%2FNOAA%20TWL%20project%E2%80%8B&wd=target%28DEM.one%7CAE181E84-4618-4FAF-8AF2-32FE3A6C3D29%2FCoNED%7CC017A206-7C6F-4EED-8414-6244AB1BCFB8%2F%29",
+        "File": "/sciclone/data10/whuang07/NWM/DEM/DEM/db_*"
+    },
+    "CoNED_new_england": {
+        "Annotation": "Coastal National Elevation Database: New England region, coarsened to 3m",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/OneNote.aspx?id=%2Fpersonal%2Ffeiye_vims_edu%2FDocuments%2FNotebooks%2FNOAA%20TWL%20project%E2%80%8B&wd=target%28DEM.one%7CAE181E84-4618-4FAF-8AF2-32FE3A6C3D29%2FCoNED%7CC017A206-7C6F-4EED-8414-6244AB1BCFB8%2F%29",
+        "File": "/sciclone/schism10/Hgrid_projects/DEMs/CoNED/CoNED_new_england_topobathy_dem_3m_dd*; /sciclone/data10/whuang07/NWM/CONED/New_England_USGS_CoNED_1m"
+    },
+    "FEMA": {
+        "Annotation": "FEMA Region III DEM",
+        "Note": "",
+        "File": "/sciclone/data10/whuang07/NWM/CUDEM/FEMA-10m/Tile3_R3_DEMv2_3.asc"
+    },
+    "chesbay_v3.1": {
+        "Annotation": "DEM v3.1 for Chesapeake Bay from the CB database",
+        "Note": "https://docs.google.com/document/d/1NhGplKh7OyEXmo1EwN6pElrP-seZbWzKuzjvKJC9ew8/edit?usp=sharing",
+        "File": "/sciclone/data10/feiye/data20/ChesBay/DEM_version3/bay_dem_v3.1.grd; /sciclone/data10/wangzg/DEM/ChesBay/bay_dem_v3.1.npz"
+    },
+    "CuDEM_NCEI_pr": {
+        "Annotation": "CuDEM (Continously updated DEM): Puerto Rico",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FCuDEM%7C7ed05443-5395-4dee-81b2-a8f7b0d530f2%2F%29&wdorigin=703&wdpreservelink=1",
+        "File": "/sciclone/data10/wangzg/DEM/Puerto_Rico/tif/"
+    },
+    "CuDEM_NCEI_tx": {
+        "Annotation": "CuDEM (Continously updated DEM): Texas",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FCuDEM%7C7ed05443-5395-4dee-81b2-a8f7b0d530f2%2F%29&wdorigin=703&wdpreservelink=1",
+        "File": "/sciclone/data10/whuang07/NWM/CUDEM/TX_all/"
+    },
+    "CuDEM_NCEI_ms_la": {
+        "Annotation": "CuDEM (Continously updated DEM): Mississippi and Louisiana",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FCuDEM%7C7ed05443-5395-4dee-81b2-a8f7b0d530f2%2F%29&wdorigin=703&wdpreservelink=1",
+        "File": "/sciclone/data10/whuang07/NWM/CUDEM/MS_LA/"
+    },
+    "CuDEM_NCEI_ma_nh_ne": {
+        "Annotation": "CuDEM (Continously updated DEM): MA, NH, and NE",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FCuDEM%7C7ed05443-5395-4dee-81b2-a8f7b0d530f2%2F%29&wdorigin=703&wdpreservelink=1",
+        "File": "/sciclone/schism10/lcui01/schism20/DEM/CUDEM/MA_NH_ME/"
+    },
+    "CuDEM_NCEI_al_nwfl": {
+        "Annotation": "CuDEM (Continously updated DEM): Alabama and Northwest Florida",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FCuDEM%7C7ed05443-5395-4dee-81b2-a8f7b0d530f2%2F%29&wdorigin=703&wdpreservelink=1",
+        "File": "/sciclone/schism10/lcui01/schism20/DEM/CUDEM/AL_nwFL/"
+    },
+    "CuDEM_NCEI19_FL": {
+        "Annotation": "CuDEM (Continously updated DEM): Updated Florida",
+        "Note": "https://vims0-my.sharepoint.com/personal/feiye_vims_edu/_layouts/15/Doc.aspx?sourcedoc={d6aa1259-6a28-4b12-9caf-58f05600b3b5}&action=edit&wd=target%28DEM.one%7Cae181e84-4618-4faf-8af2-32fe3a6c3d29%2FCuDEM%7C7ed05443-5395-4dee-81b2-a8f7b0d530f2%2F%29&wdorigin=703&wdpreservelink=1",
+        "File": "/sciclone/schism10/lcui01/schism20/DEM/CUDEM/FL/"
+    },
+    "BlueTopo": {
+        "Annotation": "BlueTopo",
+        "Note": "",
+        "File": "/sciclone/schism10/Hgrid_projects/DEMs/BlueTopo/"
+    }
+}

src/Utility/Pre-Processing/SECOFS/Bathy_loading/__init__.py

@@ -0,0 +1 @@
+../../STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_loading/__init__.py

src/Utility/Pre-Processing/SECOFS/Bathy_loading/gen_dem_id_map.py

@@ -0,0 +1 @@
+../../STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_loading/gen_dem_id_map.py

src/Utility/Pre-Processing/SECOFS/Bathy_loading/pload_depth.py

@@ -0,0 +1 @@
+../../STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_loading/pload_depth.py

src/Utility/Pre-Processing/SECOFS/Bathy_loading/regions

@@ -0,0 +1 @@
+../../STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_loading/regions/

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_loading/pload_depth.py

@@ -17,22 +17,24 @@
 grdout='hgrid.ll.new' #grid name with depth loaded
 
 #parameter
-regions=['/sciclone/schism10/Hgrid_projects/DEMs/regions/'+i for i in ("min_5m_ll_noPR.reg","SabinePass.reg","BergenPoint.reg","Washington_3.reg",
-         "Elk_river.reg","Hudson_river.reg","James_river.reg","NorthEast_river.reg",
-         "Rappahannock_river.reg","Susquehanna_river.reg","York_river.reg",
-         "Androscoggin_Kennebec_rivers.reg","Merrimack_river.reg","Patuxent_river.reg",
-         "Penobscot_river.reg","Saco_river.reg","StCroix_river.reg","Oyster_landing.reg",
-         "st_lawrence1.reg", "st_lawrence2.reg", "st_lawrence3.reg",
-         )] #regions for modifying depth
-rvalues=(5,7,5,15,
-         2,16,14,5,
-         6,10,10,
-         3,3,5,
-         5,3,5,1,
-         10,10,10
-         ) #minimum depth in regions
-
-dem_info_dict = json.load(open('/sciclone/schism10/Hgrid_projects/DEMs/DEM_info.json'))  # existing json containing the DEM info
+regions=[
+    '/sciclone/schism10/Hgrid_projects/DEMs/regions/'+i for i in ("min_5m_ll_noPR.reg","SabinePass.reg","BergenPoint.reg","Washington_3.reg",
+    "Elk_river.reg","Hudson_river.reg","James_river.reg","NorthEast_river.reg",
+    "Rappahannock_river.reg","Susquehanna_river.reg","York_river.reg",
+    "Androscoggin_Kennebec_rivers.reg","Merrimack_river.reg","Patuxent_river.reg",
+    "Penobscot_river.reg","Saco_river.reg","StCroix_river.reg","Oyster_landing.reg",
+    "st_lawrence1.reg", "st_lawrence2.reg", "st_lawrence3.reg",
+)] #regions for modifying depth
+rvalues=(
+    5,7,5,15,
+    2,16,14,5,
+    6,10,10,
+    3,3,5,
+    5,3,5,1,
+    10,10,10
+) #minimum depth in regions
+
+dem_info_dict = json.load(open('DEM_info.json'))  # existing json containing the DEM info
 headers=tuple(dem_info_dict.keys()) #DEM headers used for glob'ing the DEM files; python3.6+ only, which preserves the order of the keys
 
 sdir=r'/sciclone/schism10/Hgrid_projects/DEMs/npz2/'  #directory of DEM data
@@ -94,7 +96,7 @@
 #filter with headers, and sort by id numbers
 fnames_sort=[]
 for header in headers:
-    fnames_sub=array([i for i in fnames0 if i.startswith(header)])
+    fnames_sub=array([fname for fname in fnames0 if fname.startswith(header)])
     if len(fnames_sub)==1: fnames_sort.extend(fnames_sub); continue
 
     #get id number, which is the last number before the suffix ".*"
@@ -131,19 +133,23 @@
 #load bathymetry on each core
 S=zdata(); S.dp=dict(); S.sind=dict()
 for m,fname in enumerate(fnames):
-    bname=fname.split('.')[0]
-
-    #interpolate depth
-    while(True):
-        try:
-           dpi,sindi=load_bathymetry(gd.x,gd.y,'{}/{}'.format(sdir,fname),fmt=1)
-           break
-        except:
-            time.sleep(15)
-
-    #save results
-    S.dp[bname]=dpi; S.sind[bname]=sindi
-    print('finished reading {},: {}, myrank={}'.format(fname,inum[m],myrank)); sys.stdout.flush()
+   bname=fname.split('.')[0]
+
+   #interpolate depth
+   while(True):
+       try:
+          dpi,sindi=load_bathymetry(gd.x,gd.y,'{}/{}'.format(sdir,fname),fmt=1)
+          break
+       except:
+           time.sleep(15)
+
+   # reverse depth sign
+   if reverse_sign==1:
+      dpi=-dpi
+
+   #save results
+   S.dp[bname]=dpi; S.sind[bname]=sindi
+   print('finished reading {},: {}, myrank={}'.format(fname,inum[m],myrank)); sys.stdout.flush()
 #savez('S_{}'.format(myrank),S)
 
 #gather results