From c357da36eb08f7dc9f8c710d5846858f776febfa Mon Sep 17 00:00:00 2001
From: Julia Sloan <jsloan@caltech.edu>
Date: Fri, 8 Mar 2024 15:37:37 -0800
Subject: [PATCH] 2 gpu ss sum before max [skip ci]

---
 .buildkite/gpu/pipeline.yml | 60 ++++++++++++++++++-------------------
 src/Regridder.jl            | 25 ++++++++++------
 2 files changed, 46 insertions(+), 39 deletions(-)

diff --git a/.buildkite/gpu/pipeline.yml b/.buildkite/gpu/pipeline.yml
index ee78539191..5007acbaa6 100644
--- a/.buildkite/gpu/pipeline.yml
+++ b/.buildkite/gpu/pipeline.yml
@@ -39,8 +39,8 @@ steps:
 
   - wait
 
-  # - group: "CHAP GPU strong scaling"
-  #   steps:
+  - group: "CHAP GPU strong scaling"
+    steps:
 
   #     - label: "GPU AMIP CHAP - strong scaling - 1 GPU"
   #       key: "gpu_amip_chap"
@@ -56,20 +56,20 @@ steps:
   #         slurm_mem: 32G
   #         slurm_exclusive:
 
-      # - label: "GPU AMIP CHAP - strong scaling - 2 GPUs"
-      #   key: "gpu_amip_chap_2process"
-      #   command:
-      #     - >
-      #       srun --cpu-bind=threads --cpus-per-task=4
-      #       julia --threads=3 --color=yes --project=experiments/AMIP experiments/AMIP/coupler_driver.jl
-      #       --config_file $GPU_CONFIG_PATH/gpu_amip_chap_2process.yml
-      #   artifact_paths: "gpu_amip_chap_2process/*"
-      #   agents:
-      #     slurm_gpus_per_task: 1
-      #     slurm_cpus_per_task: 4
-      #     slurm_ntasks: 2
-      #     slurm_mem: 32G
-      #     slurm_exclusive:
+      - label: "GPU AMIP CHAP - strong scaling - 2 GPUs"
+        key: "gpu_amip_chap_2process"
+        command:
+          - >
+            srun --cpu-bind=threads --cpus-per-task=4
+            julia --threads=3 --color=yes --project=experiments/AMIP experiments/AMIP/coupler_driver.jl
+            --config_file $GPU_CONFIG_PATH/gpu_amip_chap_2process.yml
+        artifact_paths: "gpu_amip_chap_2process/*"
+        agents:
+          slurm_gpus_per_task: 1
+          slurm_cpus_per_task: 4
+          slurm_ntasks: 2
+          slurm_mem: 32G
+          slurm_exclusive:
 
   #     - label: "GPU AMIP CHAP - strong scaling - 4 GPUs"
   #       key: "gpu_amip_chap_4process"
@@ -86,21 +86,21 @@ steps:
   #         slurm_mem: 32G
   #         slurm_exclusive:
 
-  - group: "CHAP GPU weak scaling"
-    steps:
+  # - group: "CHAP GPU weak scaling"
+  #   steps:
 
-      - label: "GPU AMIP CHAP - weak scaling - 1 GPU"
-        key: "gpu_amip_chap_ws"
-        command:
-          - >
-            julia --threads=3 --color=yes --project=experiments/AMIP experiments/AMIP/coupler_driver.jl --config_file $GPU_CONFIG_PATH/gpu_amip_chap_ws.yml
-        artifact_paths: "gpu_amip_chap_ws/*"
-        agents:
-          slurm_gpus_per_task: 1
-          slurm_cpus_per_task: 4
-          slurm_ntasks: 1
-          slurm_mem: 32G
-          slurm_exclusive:
+  #     - label: "GPU AMIP CHAP - weak scaling - 1 GPU"
+  #       key: "gpu_amip_chap_ws"
+  #       command:
+  #         - >
+  #           julia --threads=3 --color=yes --project=experiments/AMIP experiments/AMIP/coupler_driver.jl --config_file $GPU_CONFIG_PATH/gpu_amip_chap_ws.yml
+  #       artifact_paths: "gpu_amip_chap_ws/*"
+  #       agents:
+  #         slurm_gpus_per_task: 1
+  #         slurm_cpus_per_task: 4
+  #         slurm_ntasks: 1
+  #         slurm_mem: 32G
+  #         slurm_exclusive:
 
       # - label: "GPU AMIP CHAP - weak scaling - 2 GPUs"
       #   key: "gpu_amip_chap_ws_2process"
diff --git a/src/Regridder.jl b/src/Regridder.jl
index 1737e7c789..c636825950 100644
--- a/src/Regridder.jl
+++ b/src/Regridder.jl
@@ -507,17 +507,24 @@ function update_surface_fractions!(cs::CoupledSimulation)
     cs.surface_fractions.ice .= max.(min.(ice_d, FT(1) .- land_s), FT(0))
     cs.surface_fractions.ocean .= max.(FT(1) .- (cs.surface_fractions.ice .+ land_s), FT(0))
 
-    # sf_sum = cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean
-    # if abs(minimum(sf_sum) - FT(1)) > eps(FT) || abs(maximum(sf_sum) - FT(1)) > eps(FT)
-    #     @show cs.surface_fractions.ice
-    #     @show cs.surface_fractions.land
-    #     @show cs.surface_fractions.ocean
-    #     @show cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean
-    #     @show minimum(cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean)
-    #     @show maximum(cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean)
-    # end
+    sf_sum = cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean
+    if abs(minimum(sf_sum) - FT(1)) > eps(FT) || abs(maximum(sf_sum) - FT(1)) > eps(FT)
+        @show minimum(FT(1) .- (cs.surface_fractions.ice .+ land_s))
+        @show maximum(FT(1) .- (cs.surface_fractions.ice .+ land_s))
+        # @show cs.surface_fractions.ice
+        # @show cs.surface_fractions.land
+        # @show cs.surface_fractions.ocean
+        # @show cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean
+        # @show minimum(cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean)
+        # @show maximum(cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean)
+    end
     comms_ctx = axes(land_s).grid.topology.context
     ClimaComms.barrier(comms_ctx)
+
+    @show minimum(cs.surface_masks.ice) >= FT(0)
+    @show minimum(cs.surface_masks.land) >= FT(0)
+    @show minimum(cs.surface_masks.ocean) >= FT(0)
+
     @assert minimum(cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean) ≈ FT(1)
     @assert maximum(cs.surface_fractions.ice .+ cs.surface_fractions.land .+ cs.surface_fractions.ocean) ≈ FT(1)