Add other models to leaderboard

experiments/ClimaEarth/Artifacts.toml

@@ -12,3 +12,10 @@ git-tree-sha1 = "9969543acd3c9084b8238b7858d0c76aa0bf98a7"
 
 [test]
 git-tree-sha1 = "b377e7c997804b47720b8e7243361d399d108441"
+
+[cmip_model_rmse]
+git-tree-sha1 = "aa2d5178a656e9ce7a37698d978d8ed92aa46d99"
+
+    [[cmip_model_rmse.download]]
+    sha256 = "b019d40510cebc678899a74d2cf14e33aa60452805478b2bf7176b891012aa97"
+    url = "https://caltech.box.com/shared/static/lmzvnjkxctlb0fanieeqxll0vdrbei9e.gz"

experiments/ClimaEarth/Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.3"
 manifest_format = "2.0"
-project_hash = "c2b41c62ef471acee30499943ca8e1f5eb29f796"
+project_hash = "1a2614b593c01321f60aa5713efe4d6d7b0cf5ba"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "daf26bbdec60d9ca1c0003b70f389d821ddb4224"

experiments/ClimaEarth/Project.toml

@@ -2,6 +2,7 @@
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
 ArtifactWrappers = "a14bc488-3040-4b00-9dc1-f6467924858a"
+Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 AtmosphericProfilesLibrary = "86bc3604-9858-485a-bdbe-831ec50de11d"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"

experiments/ClimaEarth/user_io/leaderboard/Leaderboard.jl

@@ -8,6 +8,7 @@ import CairoMakie
 import GeoMakie
 import ClimaCoupler
 import Statistics: mean
+import Artifacts
 
 include(joinpath(pkgdir(ClimaCoupler), "artifacts", "artifact_funcs.jl"))
 include("data_sources.jl")

experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl

@@ -0,0 +1,74 @@
+import Statistics: median
+
+pr_file_path = joinpath(artifact"cmip_model_rmse", "pr_rmse_amippr_amip.csv")
+
+open(pr_file_path, "r") do io
+    # Skip the header
+    header = readline(io)
+
+    # Process each line
+    for line in eachline(io)
+        # Split the line by comma
+        fields = split(line, ',')
+        model_name = fields[1]
+        DJF, MAM, JJA, SON, ANN = map(x -> parse(Float64, x), fields[2:end])
+
+        push!(OTHER_MODELS_RMSEs["pr"], RMSEs(; model_name, DJF, MAM, JJA, SON, ANN))
+    end
+end
+
+"""
+    best_single_model(RMSEs)
+
+Return the one model that has the overall smallest error.
+"""
+function best_single_model(RMSEs)
+    _, index = findmin(r -> abs.(values(r)), RMSEs)
+    return RMSEs[index]
+end
+
+"""
+    RSME_stats(RMSEs)
+
+Return:
+- best single model
+- "model" with all the medians
+- "model" with all the best values
+- "model" with all the worst values
+"""
+function RSME_stats(vecRMSEs)
+    # Collect into vectors that we can process independently
+    all_values = stack(values.(vecRMSEs))
+    ANN, DJF, JJA, MAM, SON = ntuple(i -> all_values[i, :], 5)
+
+    median_model = RMSEs(;
+        model_name = "Median",
+        ANN = median(ANN),
+        DJF = median(DJF),
+        JJA = median(JJA),
+        MAM = median(MAM),
+        SON = median(SON),
+    )
+
+    worst_model = RMSEs(;
+        model_name = "Worst",
+        ANN = maximum(abs.(ANN)),
+        DJF = maximum(abs.(DJF)),
+        JJA = maximum(abs.(JJA)),
+        MAM = maximum(abs.(MAM)),
+        SON = maximum(abs.(SON)),
+    )
+
+    best_model = RMSEs(;
+        model_name = "Best",
+        ANN = minimum(abs.(ANN)),
+        DJF = minimum(abs.(DJF)),
+        JJA = minimum(abs.(JJA)),
+        MAM = minimum(abs.(MAM)),
+        SON = minimum(abs.(SON)),
+    )
+
+    (; best_single_model = best_single_model(vecRMSEs), median_model, worst_model, best_model)
+end
+
+COMPARISON_RMSEs["pr"] = RSME_stats(OTHER_MODELS_RMSEs["pr"])

experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl

@@ -1,6 +1,7 @@
 const OBS_DS = Dict()
 const SIM_DS_KWARGS = Dict()
 const OTHER_MODELS_RMSEs = Dict()
+const COMPARISON_RMSEs = Dict()
 
 function preprocess_pr_fn(data)
     # -1 kg/m/s2 -> 1 mm/day
@@ -28,8 +29,9 @@ OBS_DS["pr"] =
 
 SIM_DS_KWARGS["pr"] = (; preprocess_data_fn = preprocess_pr_fn, new_units = "mm / day")
 
-# TODO: These numbers are eyeballed and should not be really used. Use instead real values from the various models
-OTHER_MODELS_RMSEs["pr"] = [RMSEs(; model_name = "AM4.0", ANN = 0.5, DJF = 1.0, JJA = 1.5, MAM = 0.5, SON = 1.0)]
+OTHER_MODELS_RMSEs["pr"] = []
+
+include("cmip_rmse.jl")
 
 # OBS_DS["rsut"] = ObsDataSource(;
 #                              path = "OBS/CERES_EBAF-TOA_Ed4.2_Subset_200003-202303.g025.nc",
@@ -74,12 +76,33 @@ function plot_leaderboard(rmses; output_path)
             xticks = (1:5, ["Ann", "DJF", "JJA", "MAM", "SON"]),
             title = "Global RMSE",
         )
-        CairoMakie.scatter!(ax, 1:5, values(rmse), label = rmse.model_name)
-        for other_model_rmse in OTHER_MODELS_RMSEs[short_name]
-            CairoMakie.scatter!(ax, 1:5, values(other_model_rmse), label = other_model_rmse.model_name)
-        end
+
+        # Against other models
+        (; best_single_model, median_model, worst_model, best_model) = COMPARISON_RMSEs[short_name]
+
+        CairoMakie.errorbars!(
+            ax,
+            1:5,
+            values(median_model),
+            values(best_model),
+            values(worst_model),
+            whiskerwidth = 10,
+            color = :black,
+            linewidth = 0.5,
+        )
+        CairoMakie.scatter!(
+            ax,
+            1:5,
+            values(median_model),
+            label = median_model.model_name,
+            color = :black,
+            marker = :hline,
+        )
+        CairoMakie.scatter!(ax, 1:5, values(best_single_model), label = best_single_model.model_name)
+        CairoMakie.scatter!(ax, 1:5, values(rmse), label = rmse.model_name, marker = :star5)
+
         # Add a fake extra point to center the legend a little better
-        CairoMakie.scatter!(ax, [6], [0.1], markersize = 0.01)
+        CairoMakie.scatter!(ax, [6.5], [0.1], markersize = 0.01)
         CairoMakie.axislegend()
         loc = loc + 1
     end