From 81d822a449b00de55e3791bf797a25f0b64caef9 Mon Sep 17 00:00:00 2001
From: Gabriele Bozzola <gbozzola@caltech.edu>
Date: Mon, 18 Dec 2023 12:13:53 -0800
Subject: [PATCH] Tar up files for easier read in buildkite

---
 .buildkite/pipeline.yml   | 10 ++++++++++
 examples/Manifest.toml    |  2 +-
 examples/Project.toml     |  1 +
 examples/hybrid/driver.jl | 20 ++++++++++++++++++++
 4 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 902e0372ba1..0a69875c4af 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -386,6 +386,8 @@ steps:
         key: "remap_pipeline"
         depends_on: "prep_remap"
         command: >
+          tar xvf prep_remap/hdf5_files.tar
+
           julia --color=yes --project=examples post_processing/remap/remap_pipeline.jl
           --data_dir prep_remap/ --out_dir remap_pipeline_output
         artifact_paths: "remap_pipeline_output/*"
@@ -597,6 +599,10 @@ steps:
 
       - label: "GPU: compare BW with CPU"
         command: >
+          tar xvf sphere_baroclinic_wave_rhoe/hdf5_files.tar
+
+          tar xvf gpu_baroclinic_wave_rhoe/hdf5_files.tar
+
           julia --color=yes --project=examples post_processing/compare_outputs.jl
           --output_folder_1 sphere_baroclinic_wave_rhoe/
           --output_folder_2 gpu_baroclinic_wave_rhoe/ --t_end 10days
@@ -616,6 +622,10 @@ steps:
 
       - label: "GPU: compare HS (ρe) hightop with CPU"
         command: >
+          tar xvf sphere_held_suarez_rhoe_hightop/hdf5_files.tar
+
+          tar xvf gpu_held_suarez_rhoe_hightop/hdf5_files.tar
+
           julia --color=yes --project=examples post_processing/compare_outputs.jl
           --output_folder_1 sphere_held_suarez_rhoe_hightop/
           --output_folder_2 gpu_held_suarez_rhoe_hightop/
diff --git a/examples/Manifest.toml b/examples/Manifest.toml
index 399fe4b0722..73bbc12762e 100644
--- a/examples/Manifest.toml
+++ b/examples/Manifest.toml
@@ -2,7 +2,7 @@
 
 julia_version = "1.9.4"
 manifest_format = "2.0"
-project_hash = "8b5e7db06aabbfc02784320a5a0b46105a9852dd"
+project_hash = "1efea2db8dbdf443c1c46cf2bc43d6ed80120b5a"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "332e5d7baeff8497b923b730b994fa480601efc7"
diff --git a/examples/Project.toml b/examples/Project.toml
index 0616e8e57f8..2ea69890f91 100644
--- a/examples/Project.toml
+++ b/examples/Project.toml
@@ -59,6 +59,7 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 SurfaceFluxes = "49b00bb7-8bd4-4f2b-b78c-51cd0450215f"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c"
 YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
diff --git a/examples/hybrid/driver.jl b/examples/hybrid/driver.jl
index c1199ad415c..5b3fd7f5246 100644
--- a/examples/hybrid/driver.jl
+++ b/examples/hybrid/driver.jl
@@ -27,6 +27,8 @@ using NCDatasets
 using ClimaTimeSteppers
 import JSON
 using Test
+import Tar
+import Base.Filesystem: rm
 import OrderedCollections
 using ClimaCoreTempestRemap
 using ClimaCorePlots, Plots
@@ -51,6 +53,24 @@ CA.verify_callbacks(sol.t)
 make_plots(Val(Symbol(reference_job_id)), simulation.output_dir)
 @info "Plotting done"
 
+@info "Creating tarballs"
+Tar.create(
+    f -> endswith(f, ".nc"),
+    simulation.output_dir,
+    joinpath(simulation.output_dir, "nc_files.tar"),
+)
+Tar.create(
+    f -> endswith(f, r"hdf5|h5"),
+    simulation.output_dir,
+    joinpath(simulation.output_dir, "hdf5_files.tar"),
+)
+
+foreach(readdir(simulation.output_dir)) do f
+    endswith(f, r"nc|hdf5|h5") && rm(joinpath(simulation.output_dir, f))
+end
+
+@info "Tarballs created"
+
 if CA.is_distributed(config.comms_ctx)
     nprocs = ClimaComms.nprocs(config.comms_ctx)
     comms_ctx = config.comms_ctx