Merge pull request #845 from CliMA/gb/leaderboard

Add boxes in bias_leaderboard
CliMA · Jun 12, 2024 · 416cee5 · 416cee5
2 parents 99fe211 + 4e8ffb6
commit 416cee5
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 16 deletions.
diff --git a/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl b/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl
@@ -1,4 +1,4 @@
-import Statistics: median
+import Statistics: median, quantile
 
 const RMSE_FILE_PATHS = Dict()
 
@@ -75,7 +75,25 @@ function RSME_stats(vecRMSEs)
         SON = minimum(abs.(SON)),
     )
 
-    (; best_single_model = best_single_model(vecRMSEs), median_model, worst_model, best_model)
+    quantile25 = RMSEs(;
+        model_name = "Quantile 0.25",
+        ANN = quantile(ANN, 0.25),
+        DJF = quantile(DJF, 0.25),
+        JJA = quantile(JJA, 0.25),
+        MAM = quantile(MAM, 0.25),
+        SON = quantile(SON, 0.25),
+    )
+
+    quantile75 = RMSEs(;
+        model_name = "Quantile 0.75",
+        ANN = quantile(ANN, 0.75),
+        DJF = quantile(DJF, 0.75),
+        JJA = quantile(JJA, 0.75),
+        MAM = quantile(MAM, 0.75),
+        SON = quantile(SON, 0.75),
+    )
+
+    (; best_single_model = best_single_model(vecRMSEs), median_model, worst_model, best_model, quantile25, quantile75)
 end
 
 for short_name in short_names

diff --git a/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl b/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl
@@ -86,7 +86,7 @@ function plot_leaderboard(rmses; output_path)
     loc_squares = length(rmses) + 1
     ax_squares = CairoMakie.Axis(
         fig[loc_squares, 1],
-        yticks = (1:num_variables, var_names),
+        yticks = (1:num_variables, reverse(var_names)),
         xticks = ([3, NUM_BOXES + 3], ["CliMA", "Best model"]),
         aspect = NUM_BOXES * NUM_MODELS,
     )
@@ -116,29 +116,46 @@ function plot_leaderboard(rmses; output_path)
         # Against other models
         (; best_single_model, median_model, worst_model, best_model) = COMPARISON_RMSEs[short_name]
 
-        squares[begin:NUM_BOXES, var_num] .= values(rmse) ./ values(median_model)
-        squares[(NUM_BOXES + 1):end, var_num] .= values(best_single_model) ./ values(median_model)
+        squares[begin:NUM_BOXES, end - var_num + 1] .= values(rmse) ./ values(median_model)
+        squares[(NUM_BOXES + 1):end, end - var_num + 1] .= values(best_single_model) ./ values(median_model)
 
-        CairoMakie.errorbars!(
+        CairoMakie.scatter!(
             ax,
             1:5,
             values(median_model),
-            values(best_model),
-            values(worst_model),
-            whiskerwidth = 10,
+            label = median_model.model_name,
             color = :black,
-            linewidth = 0.5,
+            marker = :hline,
+            markersize = 15,
+        )
+
+        categories = vcat(map(_ -> collect(1:5), 1:length(OTHER_MODELS_RMSEs[short_name]))...)
+
+        CairoMakie.boxplot!(
+            ax,
+            categories,
+            vcat(values.(OTHER_MODELS_RMSEs[short_name])...);
+            whiskerwidth = 1,
+            width = 0.35,
+            mediancolor = :black,
+            color = :gray,
+            whiskerlinewidth = 1,
         )
+
+        # If we want to plot other models
+        # for model in OTHER_MODELS_RMSEs[short_name]
+        #     CairoMakie.scatter!(ax, 1:5, values(model), marker = :hline)
+        # end
+
         CairoMakie.scatter!(
             ax,
             1:5,
-            values(median_model),
-            label = median_model.model_name,
-            color = :black,
-            marker = :hline,
+            values(rmse),
+            label = rmse.model_name,
+            marker = :star5,
+            markersize = 20,
+            color = :orange,
         )
-        CairoMakie.scatter!(ax, 1:5, values(best_single_model), label = best_single_model.model_name)
-        CairoMakie.scatter!(ax, 1:5, values(rmse), label = rmse.model_name, marker = :star5)
 
         # Add a fake extra point to center the legend a little better
         CairoMakie.scatter!(ax, [6.5], [0.1], markersize = 0.01)