Ac/mizuroute migration

.gitignore

@@ -2,9 +2,13 @@
 *.pdf
 *.csv
 *.nc
+*.json
 *~
 *.vscode*
 *.DS_Store
+*.dbf
+*.shp
+*.txt
 
 # Files generated by invoking Julia with --code-coverage
 *.jl.cov

Project.toml

@@ -3,5 +3,28 @@ uuid = "9363829b-25e6-4e43-a03e-985c67d112e3"
 authors = ["Mauricio Lima, Andrew Chiang, Katherine Deck, Ollie Dunbar"]
 version = "0.1.0"
 
+[deps]
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
+ColorSchemes = "35d6a980-a343-548e-a6ea-1d62b119f2f4"
+DSP = "717857b8-e6f2-59f4-9121-6e50c889abd2"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+GeometryBasics = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
+Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
+NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9"
+ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Shapefile = "8e980c4a-a4fe-5da2-b3a7-4b4b0353a2f4"
+SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
+
 [compat]
-julia = "1.10"
+julia = "1.10"

examples/routing_models/README.md

@@ -0,0 +1,13 @@
+## 1 - Engineerining
+
+Data setup for the model in `engineering/`.
+
+## 2 - Routing
+
+For physics-based routing, see `gamma-IRF/`.
+
+For a hybrid approach, see `LSTM-IRF/`.
+
+## 3 - Analysis
+
+A simple check of the behaviour of the models can be found in `analysis/`.

examples/routing_models/analysis/check_plot.jl

@@ -0,0 +1,42 @@
+using CSV
+using DataFrames
+using Dates
+using CairoMakie
+
+function main()
+    base = "/groups/esm/achiang/Rivers/data_from_sampo"
+    # 6050344660 -> source basin (ok)
+    # 7050675400 -> rout_lv = 05 (?)
+    # 3050639250 -> rout_lv = 05 (ok)
+    basin_id = 3050639250
+
+    # Read DataFrames
+    obs_df = CSV.read(joinpath(base, "routing/timeseries/timeseries_lv05/basin_$basin_id.csv"), DataFrame)
+    mizu_df = CSV.read(joinpath(base, "routing/simulations/simulations_lv05/simulation_gamma-IRF/basin_$basin_id.csv"), DataFrame)
+
+    # Define date limits
+    start_date = Date(1996, 01, 01)
+    end_date = Date(1997, 12, 31)
+
+    # Filter dates
+    obs_df = filter(row -> start_date <= row[:date] <= end_date, obs_df)
+
+    # Select date indexes
+    mizu_df = filter(row -> start_date <= row[:date] <= end_date, mizu_df)
+
+    fig = Figure()
+    ax = Axis(
+        fig[1,1], 
+        xlabel="Number of days", 
+        ylabel="Streamflow (m³/s)", 
+        title="Descendent basin"
+    )
+    lines!(obs_df[:,:streamflow], label="Observed")
+    lines!(mizu_df[:,:streamflow], label="gamma-IRF")
+    axislegend(ax)
+
+    # save("../analysis/png_files/source_basin.png", fig)
+    save("examples/routing_models/analysis/png_files/descendent_basin.png", fig)
+end
+
+main()

examples/routing_models/engineering/Config.toml

@@ -0,0 +1,25 @@
+# midway data is assumed to be already present
+
+[base]
+base = "/groups/esm/achiang/ClimaRivers.jl/data"
+
+[level]
+hydro_lv = "05"
+
+[source]
+hydroatlas_shp_file = "source_data/BasinATLAS_v10_shp/BasinATLAS_v10_lev05.shp"
+
+[midway]
+basin_gauge_dict_file = "midway_data/mapping_dicts/gauge_to_basin_dict_lv05_grdc.json"
+graph_dict_file = "midway_data/graph_dicts/graph_lv05.json"
+grdc_nc_file = "midway_data/GRDC-Globe/grdc-merged.nc"
+xd_dir = "midway_data/xd_lv05"
+
+[simulation]
+attributes_dir = "routing/attributes/attributes_lv05"
+timeseries_dir = "routing/timeseries/timeseries_lv05"
+routing_levels_dir = "routing/routing_lvs/routing_lvs_lv05"
+
+[dates]
+start_date = "1990-01-01"
+end_date = "2014-12-31"

examples/routing_models/engineering/engineer_lv05.jl

@@ -0,0 +1,57 @@
+using ClimaRivers
+using Dates
+using TOML
+
+function main()
+    # 0. Unpack configuration file
+    config = TOML.parsefile("examples/routing_models/engineering/Config.toml")
+    base = config["base"]["base"]
+    hydro_lv = config["level"]["hydro_lv"]
+    hydroatlas_shp_file = joinpath(base, config["source"]["hydroatlas_shp_file"])
+    basin_gauge_dict_file = joinpath(base, config["midway"]["basin_gauge_dict_file"])
+    graph_dict_file = joinpath(base, config["midway"]["graph_dict_file"])
+    grdc_nc_file = joinpath(base, config["midway"]["grdc_nc_file"])
+    xd_dir = joinpath(base, config["midway"]["xd_dir"])
+    attributes_dir = joinpath(base, config["simulation"]["attributes_dir"])
+    timeseries_dir = joinpath(base, config["simulation"]["timeseries_dir"])
+    routing_levels_dir = joinpath(base, config["simulation"]["routing_levels_dir"])
+    start_date = Date(config["dates"]["start_date"], "yyyy-mm-dd")
+    end_date = Date(config["dates"]["end_date"], "yyyy-mm-dd")
+
+
+    # 1. Ensure the graph is written under the simulation directory
+    cp(graph_dict_file, joinpath(base, "routing/graphs/graph_lv$hydro_lv.json"), force=true)
+    graph_dict_file = joinpath(base, "routing/graphs/graph_lv$hydro_lv.json")
+
+
+    # 2. Write routing levels
+    write_routing_levels(graph_dict_file, hydroatlas_shp_file, routing_levels_dir)
+
+
+    # 3. Write the proper basin -> dict file
+    gauges_to_basins(
+        grdc_nc_file, 
+        hydroatlas_shp_file, 
+        basin_gauge_dict_file, 
+        true,
+        "grdc",
+        graph_dict_file,
+    )
+
+
+    # 4. Write timeseries
+    write_routing_timeseries(    
+        xd_dir, 
+        start_date, 
+        end_date,
+        grdc_nc_file, 
+        basin_gauge_dict_file,
+        timeseries_dir,
+    )
+
+
+    # 5. Write attributes
+    write_routing_attributes(hydroatlas_shp_file, attributes_dir)
+end
+
+main()

examples/routing_models/routing/LSTM-IRF/Config.toml

@@ -0,0 +1,32 @@
+[base]
+base = "/groups/esm/achiang/ClimaRivers.jl/data"
+
+[level]
+hydro_lv = "05"
+
+[source]
+hydroatlas_shp_file = "source_data/BasinATLAS_v10_shp/BasinATLAS_v10_lev05.shp"
+
+[midway]
+basin_gauge_dict_file = "midway_data/mapping_dicts/gauge_to_basin_dict_lv05_grdc.json"
+grdc_nc_file = "midway_data/GRDC-Globe/grdc-merged.nc"
+routing_levels_dir = "routing/routing_lvs/routing_lvs_lv05"
+xd_dir = "midway_data/xd_lv05"
+
+[simulation]
+attributes_dir = "routing/attributes/attributes_lv05"
+graph_dict_file = "routing/graphs/graph_lv05.json"
+timeseries_dir = "routing/timeseries/timeseries_lv05"
+simulation_dir = "routing/simulations/simulations_lv05/simulation_LSTM-IRF"
+
+[dates]
+start_date = "1996-01-01"
+end_date = "1996-01-31"
+
+[training]
+hillslope_method = "LSTM"
+hillslope_learning_rate = 0.001
+hillslope_epochs = 10
+river_channel_method = "IRF"
+river_channel_learning_rate = 0.001
+river_channel_epochs = 2

examples/routing_models/routing/LSTM-IRF/route_lv05.jl

@@ -0,0 +1,53 @@
+# using Rivers
+
+# TODO: put these includes in the Rivers module
+include("../../../src/routing/routing.jl")
+
+let
+    # Unpack config
+    config = TOML.parsefile(filename)
+
+    base = config["base"]["base"]
+    hydro_lv = config["level"]["hydro_lv"]
+
+    hydroatlas_shp_file = config["source"]["hydroatlas_shp_file"]
+
+    basin_gauge_dict_file = config["midway"]["basin_gauge_dict_file"]
+    grdc_nc_file = config["midway"]["grdc_nc_file"]
+    routing_levels_dir = config["midway"]["routing_levels_dir"]
+    xd_dir = config["midway"]["xd_dir"]
+
+    attributes_dir = config["simulation"]["attributes_dir"]
+    graph_dict_file = config["simulation"]["graph_dict_file"]
+    timeseries_dir = config["simulation"]["timeseries_dir"]
+    simulation_dir = config["simulation"]["simulation_dir"]
+
+    start_date = Date(config["dates"]["start_date"], "yyyy-mm-dd")
+    end_date = Date(config["dates"]["end_date"], "yyyy-mm-dd")
+
+    hillslope_method = config["training"]["hillslope_method"]
+    hillslope_learning_rate = config["training"]["hillslope_learning_rate"]
+    hillslope_epochs = config["training"]["hillslope_epochs"]
+    river_channel_method = config["training"]["river_channel_method"]
+    river_channel_learning_rate = config["training"]["river_channel_learning_rate"]
+    river_channel_epochs = config["training"]["river_channel_epochs"]
+
+    # Route
+    route(
+        timeseries_dir,
+        attributes_dir, 
+        graph_dict_file, 
+        routing_levels_dir,
+        hillslope_method,
+        true,
+        river_channel_method,
+        false,
+        start_date,
+        end_date,
+        simulation_dir,
+        hillslope_learning_rate,
+        hillslope_epochs,
+        river_channel_learning_rate,
+        river_channel_epochs,
+    )
+end

examples/routing_models/routing/Project.toml

@@ -0,0 +1,16 @@
+[deps]
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+CUDNN_jll = "62b44479-cb7b-5706-934f-f13b2eb2e645"
+CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
+DSP = "717857b8-e6f2-59f4-9121-6e50c889abd2"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+NNlibCUDA = "a00861dc-f156-4864-bf3c-e6376f28a68d"
+ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
+SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"

examples/routing_models/routing/gamma-IRF/Config.toml

@@ -0,0 +1,30 @@
+[base]
+base = "/groups/esm/achiang/ClimaRivers.jl/data"
+
+[level]
+hydro_lv = "05"
+
+[source]
+hydroatlas_shp_file = "source_data/BasinATLAS_v10_shp/BasinATLAS_v10_lev05.shp"
+
+[midway]
+basin_gauge_dict_file = "midway_data/mapping_dicts/gauge_to_basin_dict_lv05_grdc.json"
+grdc_nc_file = "midway_data/GRDC-Globe/grdc-merged.nc"
+routing_levels_dir = "routing/routing_lvs/routing_lvs_lv05"
+xd_dir = "midway_data/xd_lv05"
+
+[simulation]
+attributes_dir = "routing/attributes/attributes_lv05"
+graph_dict_file = "routing/graphs/graph_lv05.json"
+timeseries_dir = "routing/timeseries/timeseries_lv05"
+simulation_dir = "routing/simulations/simulations_lv05/simulation_gamma-IRF"
+
+[dates]
+start_date = "1996-01-01"
+end_date = "2014-12-31"
+
+[training]
+hillslope_method = "gamma"
+is_hillslope_training = false
+river_channel_method = "IRF"
+is_river_channel_training = false

examples/routing_models/routing/gamma-IRF/route_lv05.jl

@@ -0,0 +1,47 @@
+using Dates
+using TOML
+include("../../../../src/routing/routing.jl")
+
+function main()
+    # Unpack config
+    config = TOML.parsefile("examples/routing_models/routing/gamma-IRF/Config.toml")
+    base = config["base"]["base"]
+    hydro_lv = config["level"]["hydro_lv"]
+    hydroatlas_shp_file = joinpath(base, config["source"]["hydroatlas_shp_file"])
+    routing_levels_dir = joinpath(base, config["midway"]["routing_levels_dir"])
+    basin_gauge_dict_file = joinpath(base, config["midway"]["basin_gauge_dict_file"])
+    grdc_nc_file = joinpath(base, config["midway"]["grdc_nc_file"])
+    routing_levels_dir = joinpath(base, config["midway"]["routing_levels_dir"])
+    xd_dir = joinpath(base, config["midway"]["xd_dir"])
+    attributes_dir = joinpath(base, config["simulation"]["attributes_dir"])
+    graph_dict_file = joinpath(base, config["simulation"]["graph_dict_file"])
+    timeseries_dir = joinpath(base, config["simulation"]["timeseries_dir"])
+    simulation_dir = joinpath(base, config["simulation"]["simulation_dir"])
+    start_date = Date(config["dates"]["start_date"], "yyyy-mm-dd")
+    end_date = Date(config["dates"]["end_date"], "yyyy-mm-dd")
+    hillslope_method = config["training"]["hillslope_method"]
+    is_hillslope_training = config["training"]["is_hillslope_training"]
+    river_channel_method = config["training"]["river_channel_method"]
+    is_river_channel_training = config["training"]["is_river_channel_training"]
+
+    # Route route route
+    route(
+        timeseries_dir,
+        attributes_dir, 
+        graph_dict_file, 
+        routing_levels_dir,
+        hillslope_method,
+        is_hillslope_training,
+        river_channel_method,
+        is_river_channel_training,
+        start_date,
+        end_date,
+        simulation_dir,
+        1,
+        1.,
+        1,
+        1.,
+    )
+end
+
+main()