Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add LastObs reading and writing for command-line run of main_V04 with HYfeature hydrofabric #701

Merged
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
9730555
add LastObs reading and writing by command-line run main_V04 with HYf…
kumdonoaa Nov 17, 2023
c41000f
add lastobs folder and files
kumdonoaa Nov 17, 2023
3fc084a
add wrf_hydro_lastobs_file parameter in test_AnA_V4_NHD.yaml and modi…
kumdonoaa Nov 18, 2023
b036795
Delete src/troute-routing/troute/routing/compute.py
kumdonoaa Nov 21, 2023
3cc07d2
mistakenly removed compute.py
kumdonoaa Nov 21, 2023
ffdf35a
refreshed compute.py
kumdonoaa Nov 21, 2023
45da97b
no change for __main.py__
kumdonoaa Nov 21, 2023
dfb5924
add lite_restart_output_directory
kumdonoaa Nov 21, 2023
0af5507
remove space in __main__.py
kumdonoaa Nov 21, 2023
14a886b
deleted one lastobs file
kumdonoaa Nov 21, 2023
36da201
remove one lastobs file again
kumdonoaa Nov 21, 2023
4ece1c5
remove lastobos_output_folder in compute_parameters and update yaml file
kumdonoaa Nov 22, 2023
8790a42
remove commented lines
kumdonoaa Nov 22, 2023
89f2371
remove unused lastobs file
kumdonoaa Nov 22, 2023
f2c08b1
remove lastobs_output_folder parameter in config
kumdonoaa Dec 4, 2023
a4b5b17
Merge branch 'master' into add_lastobs_read_write_HYfeature_commandli…
kumdonoaa Dec 4, 2023
5e6dd67
remove wrf_hydro_last_obs file in both config and DataAssimilation.py
kumdonoaa Dec 4, 2023
58be2ed
Merge branch 'add_lastobs_read_write_HYfeature_commandlinerun' of htt…
kumdonoaa Dec 4, 2023
a35810e
remove wrf_hydro_lastobs_file in compute_parameters.py and undo chang…
kumdonoaa Dec 5, 2023
a148280
Merge branch 'master' into add_lastobs_read_write_HYfeature_commandli…
kumdonoaa Dec 5, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add wrf_hydro_lastobs_file parameter in test_AnA_V4_NHD.yaml and modi…
…fied related files
  • Loading branch information
kumdonoaa committed Nov 18, 2023
commit 3fc084a713112d817e65c39abd1eab78c803295f
1 change: 1 addition & 0 deletions src/troute-config/troute/config/compute_parameters.py
Original file line number Diff line number Diff line change
@@ -119,6 +119,7 @@ class StreamflowDA(BaseModel, extra='forbid'):
# TODO: changed the name of this parameter from "wrf_hydro_lastobs_file" to "lastobs_file"
# Need to update this in t-route as well.
lastobs_file: Optional[FilePath] = None
wrf_hydro_lastobs_file: Optional[FilePath] = None
shorvath-noaa marked this conversation as resolved.
Show resolved Hide resolved

# NOTE: required if lastobs are to be written out during and after simulations
lastobs_output_folder: Optional[DirectoryPath] = None
60 changes: 31 additions & 29 deletions src/troute-network/troute/DataAssimilation.py
Original file line number Diff line number Diff line change
@@ -98,36 +98,38 @@ def __init__(self, network, from_files, value_dict, da_run=[]):
self._last_obs_df = _reindex_link_to_lake_id(lastobs_df, network.link_lake_crosswalk)

else:
import pdb; pdb.set_trace()
# lastobs Dataframe for NHD Hydrofabric
lastobs_file = streamflow_da_parameters.get("wrf_hydro_lastobs_file", None)
#TODO: Is it a sustainable to figure out if using NHD or HYfeature based on lastobs_crosswalk_file?
lastobs_crosswalk_file = streamflow_da_parameters.get("gage_segID_crosswalk_file", None)
lastobs_start = streamflow_da_parameters.get("wrf_hydro_lastobs_lead_time_relative_to_simulation_start_time", 0)

if lastobs_file:
self._last_obs_df = build_lastobs_df(
lastobs_file,
lastobs_crosswalk_file,
lastobs_start,
)

# lastobs Dataframe forHYfeature HYdrofabric
lastobs_file = data_assimilation_parameters.get('streamflow_da', {}).get('lastobs_file', False)
if lastobs_file:
lastobs_df = _read_lastobs_file(lastobs_file)
lastobs_df = lastobs_df.set_index('gages')
link_gage_df = network.link_gage_df.reset_index().set_index('gages')
col_name = link_gage_df.columns[0]
link_gage_df[col_name] = link_gage_df[col_name].astype(int)
gages_dict = link_gage_df.to_dict().get(col_name)
lastobs_df = lastobs_df.rename(index=gages_dict)
# remove 'nan' values from index
temp_df = lastobs_df.reset_index()
temp_df = temp_df[temp_df['gages']!='nan']
temp_df['gages'] = temp_df['gages'].astype(int)
lastobs_df = temp_df.set_index('gages').dropna()
self._last_obs_df = lastobs_df

if lastobs_crosswalk_file:
shorvath-noaa marked this conversation as resolved.
Show resolved Hide resolved
# lastobs Dataframe for NHD Hydrofabric
lastobs_file = streamflow_da_parameters.get("wrf_hydro_lastobs_file", None)
lastobs_crosswalk_file = streamflow_da_parameters.get("gage_segID_crosswalk_file", None)
lastobs_start = streamflow_da_parameters.get("wrf_hydro_lastobs_lead_time_relative_to_simulation_start_time", 0)

if lastobs_file:
self._last_obs_df = build_lastobs_df(
lastobs_file,
lastobs_crosswalk_file,
lastobs_start,
)
else:
# lastobs Dataframe forHYfeature HYdrofabric
lastobs_file = data_assimilation_parameters.get('streamflow_da', {}).get('lastobs_file', False)
if lastobs_file:
lastobs_df = _read_lastobs_file(lastobs_file)
lastobs_df = lastobs_df.set_index('gages')
link_gage_df = network.link_gage_df.reset_index().set_index('gages')
col_name = link_gage_df.columns[0]
link_gage_df[col_name] = link_gage_df[col_name].astype(int)
gages_dict = link_gage_df.to_dict().get(col_name)
lastobs_df = lastobs_df.rename(index=gages_dict)
# remove 'nan' values from index
temp_df = lastobs_df.reset_index()
temp_df = temp_df[temp_df['gages']!='nan']
temp_df['gages'] = temp_df['gages'].astype(int)
lastobs_df = temp_df.set_index('gages').dropna()
self._last_obs_df = lastobs_df

# replace link ids with lake ids, for gages at waterbody outlets,
# otherwise, gage data will not be assimilated at waterbody outlet
# segments because connections dic has replaced all link ids within
2 changes: 0 additions & 2 deletions src/troute-nwm/src/nwm_routing/output.py
shorvath-noaa marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -390,7 +390,6 @@ def nwm_output_generator(
LOG.debug("writing lastobs files took %s seconds." % (time.time() - start))

if lastobso:
#if 1==1:
# Write out LastObs as netcdf when using main_v04 or troute_model with HYfeature.
# This is only needed if 1) streamflow nudging is ON and 2) a lastobs output
# folder is provided by the user.
@@ -420,7 +419,6 @@ def nwm_output_generator(
LOG.debug("writing lastobs files took %s seconds." % (time.time() - start))



if 'flowveldepth' in locals():
LOG.debug(flowveldepth)

2 changes: 1 addition & 1 deletion test/LowerColorado_TX/test_AnA_V4_NHD.yaml
Original file line number Diff line number Diff line change
@@ -89,7 +89,7 @@ compute_parameters:
gage_segID_crosswalk_file : domain/RouteLink.nc
crosswalk_gage_field : 'gages'
crosswalk_segID_field : 'link'
lastobs_file : lastobs/nudgingLastObs.2021-08-23_12:00:00.nc
wrf_hydro_lastobs_file : lastobs/nudgingLastObs.2021-08-23_12:00:00.nc
lastobs_output_folder : lastobs/
shorvath-noaa marked this conversation as resolved.
Show resolved Hide resolved
reservoir_da:
#----------