diff --git a/.buildkite/gpu/pipeline.yml b/.buildkite/benchmarks/pipeline.yml similarity index 85% rename from .buildkite/gpu/pipeline.yml rename to .buildkite/benchmarks/pipeline.yml index f8aa2768c8..8f70dd07e8 100644 --- a/.buildkite/gpu/pipeline.yml +++ b/.buildkite/benchmarks/pipeline.yml @@ -90,3 +90,14 @@ steps: agents: slurm_gpus: 1 slurm_mem: 16GB + + - group: "Generate output table" + steps: + - label: "Compare AMIP/aquaplanet with diagnostic EDMF" + key: "compare_slabplanet_albedo_function" + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/user_io/benchmarks.jl --cpu_run_name_coupled amip_diagedmf --cpu_run_name_atmos aquaplanet_diagedmf --gpu_run_name_coupled gpu_amip_diagedmf --gpu_run_name_atmos gpu_aquaplanet_diagedmf --mode_name amip" + depends_on: + - "aquaplanet_diagedmf" + - "amip_diagedmf" + - "gpu_aquaplanet_diagedmf" + - "gpu_amip_diagedmf" diff --git a/experiments/AMIP/user_io/benchmarks.jl b/experiments/AMIP/user_io/benchmarks.jl new file mode 100644 index 0000000000..d20407ba98 --- /dev/null +++ b/experiments/AMIP/user_io/benchmarks.jl @@ -0,0 +1,135 @@ +using PrettyTables + +#= +Our goal here is to output a table displaying some results from benchmark runs +in the coupler. We want to be able to compare between CPU and GPU runs, as well +as between coupled and atmos-only runs. The metrics we want to compare are +SYPD, allocations, and the maximum, median, and mean differences between the +CPU and GPU states. + +The table should look something like this (note that the last 3 columns will be +added in a future PR): +------------------------------------------------------------------------------------ +| | CPU Run | GPU Run | Max Diff | Median Diff | Mean Diff | +------------------------------------------------------------------------------------ +| | $run_name | $run_name | | | | +| Coupled run | $SYPD | $SYPD | $max_diff | $median_diff | $mean_diff | +| | $cpu_allocs | $gpu_allocs | | | | +------------------------------------------------------------------------------------ +| | $run_name | $run_name | | | | +| Atmos-only | $SYPD | $SYPD | $max_diff | $median_diff | $mean_diff | +| | $cpu_allocs | $gpu_allocs | | | | +------------------------------------------------------------------------------------ + +=# + +import ArgParse +import PrettyTables +import ClimaCoupler + +function argparse_settings() + s = ArgParse.ArgParseSettings() + ArgParse.@add_arg_table! s begin + "--cpu_run_name_coupled" + help = "The name of the CPU coupled run we want to compare. User must specify CPU and/or GPU coupled run name." + arg_type = String + default = nothing + "--gpu_run_name_coupled" + help = "The name of the GPU coupled run we want to compare." + arg_type = String + default = nothing + "--cpu_run_name_atmos" + help = "The name of the CPU atmos-only run we want to compare. User must specify CPU and/or GPU atmos-only run name." + arg_type = String + default = nothing + "--gpu_run_name_atmos" + help = "The name of the GPU atmos-only run we want to compare." + arg_type = String + default = nothing + "--mode_name" + help = "The mode of the simulations being compared (`slabplanet` or `AMIP`)." + arg_type = String + default = nothing + "--coupler_output_dir" + help = "Directory to save output files." + arg_type = String + default = "experiments/AMIP/output" + end + return s +end + +# Read in CPU and GPU run name info from command line +parsed_args = ArgParse.parse_args(ARGS, argparse_settings()) + +# Coupled runs +cpu_run_name_coupled = parsed_args["cpu_run_name_coupled"] +gpu_run_name_coupled = parsed_args["gpu_run_name_coupled"] +if isnothing(cpu_run_name_coupled) && isnothing(gpu_run_name_coupled) + error("Must pass CPU and/or GPU coupled run name to compare them.") +elseif isnothing(gpu_run_name_coupled) + gpu_run_name_coupled = "gpu_" * cpu_run_name_coupled +elseif isnothing(cpu_run_name_coupled) + cpu_run_name_coupled = gpu_run_name_coupled[5:end] +end + +# Atmos-only runs +cpu_run_name_atmos = parsed_args["cpu_run_name_atmos"] +gpu_run_name_atmos = parsed_args["gpu_run_name_atmos"] +if isnothing(cpu_run_name_atmos) && isnothing(gpu_run_name_atmos) + error("Must pass CPU and/or GPU coupled run name to compare them.") +elseif isnothing(gpu_run_name_atmos) + gpu_run_name_atmos = "gpu_" * cpu_run_name_atmos +elseif isnothing(cpu_run_name_atmos) + cpu_run_name_atmos = gpu_run_name_atmos[5:end] +end + +# Read in mode name from command line (or retrieve from run name). +#Note that we expect this to be the same for all 4 simulations being compared. +mode_name = parsed_args["mode_name"] +if isnothing(mode_name) + mode_name = + occursin("amip", cpu_run_name) ? "amip" : + (occursin("slabplanet", cpu_run_name) ? "slabplanet" : error("Please provide a valid `mode_name`.")) +end + +# Construct CPU and GPU artifacts directories +output_dir = parsed_args["coupler_output_dir"] + +# Read in SYPD and allocations info from artifacts directories +function get_sypd_allocs(output_dir, cpu_run_name, gpu_run_name) + cpu_artifacts_dir = joinpath(output_dir, joinpath(mode_name, cpu_run_name)) * "_artifacts" + gpu_artifacts_dir = joinpath(output_dir, joinpath(mode_name, gpu_run_name)) * "_artifacts" + + # Read in SYPD info + cpu_sypd_file = open(joinpath(cpu_artifacts_dir, "sypd.txt"), "r"); + cpu_sypd = read(cpu_sypd_file, String) + + gpu_sypd_file = open(joinpath(gpu_artifacts_dir, "sypd.txt"), "r"); + gpu_sypd = read(gpu_sypd_file, String) + + # Read in allocations info + cpu_allocs_file = open(joinpath(cpu_artifacts_dir, "allocations.txt"), "r"); + cpu_allocs = read(cpu_allocs_file, String) + + gpu_allocs_file = open(joinpath(gpu_artifacts_dir, "allocations.txt"), "r"); + gpu_allocs = read(gpu_allocs_file, String) + + return (cpu_sypd, gpu_sypd, cpu_allocs, gpu_allocs) +end + +cpu_sypd_coupled, gpu_sypd_coupled, cpu_allocs_coupled, gpu_allocs_coupled = get_sypd_allocs(output_dir, cpu_run_name_coupled, gpu_run_name_coupled) +cpu_sypd_atmos, gpu_sypd_atmos, cpu_allocs_atmos, gpu_allocs_atmos = get_sypd_allocs(output_dir, cpu_run_name_atmos, gpu_run_name_atmos) + +# Set up info for PrettyTables.jl +headers = ["", "CPU Run", "GPU Run"] +data = [ + ["", cpu_run_name_coupled, gpu_run_name_coupled], + ["Coupled run", cpu_sypd_coupled, gpu_sypd_coupled], + ["", cpu_allocs_coupled, gpu_allocs_coupled], + ["", cpu_run_name_atmos, gpu_run_name_atmos], + ["Atmos-only", cpu_sypd_atmos, gpu_sypd_atmos], + ["", cpu_allocs_atmos, gpu_allocs_atmos] +] + +# Output the table, including lines before and after the header, and after the 4th row +PrettyTables.pretty_table(data, headers, hlines=[0, 1, 4])