Tar up files for easier read in buildkite

CliMA · Dec 18, 2023 · e7c741f · e7c741f
1 parent 8169a37
commit e7c741f
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 1 deletion.
diff --git a/examples/Manifest.toml b/examples/Manifest.toml
@@ -2,7 +2,7 @@
 
 julia_version = "1.9.4"
 manifest_format = "2.0"
-project_hash = "8b5e7db06aabbfc02784320a5a0b46105a9852dd"
+project_hash = "1efea2db8dbdf443c1c46cf2bc43d6ed80120b5a"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "332e5d7baeff8497b923b730b994fa480601efc7"

diff --git a/examples/Project.toml b/examples/Project.toml
@@ -59,6 +59,7 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 SurfaceFluxes = "49b00bb7-8bd4-4f2b-b78c-51cd0450215f"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c"
 YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"

diff --git a/examples/hybrid/driver.jl b/examples/hybrid/driver.jl
@@ -27,6 +27,8 @@ using NCDatasets
 using ClimaTimeSteppers
 import JSON
 using Test
+import Tar
+import Base.Filesystem: rm
 import OrderedCollections
 using ClimaCoreTempestRemap
 using ClimaCorePlots, Plots
@@ -51,6 +53,19 @@ CA.verify_callbacks(sol.t)
 make_plots(Val(Symbol(reference_job_id)), simulation.output_dir)
 @info "Plotting done"
 
+@info "Creating tarballs"
+Tar.create(f -> endswith(f, ".nc"), simulation.output_dir,
+           joinpath(simulation.output_dir, "nc_files.tar"))
+Tar.create(f -> endswith(f, r"hdf5|h5"),
+           simulation.output_dir,
+           joinpath(simulation.output_dir, "hdf5_files.tar"))
+
+foreach(readdir(simulation.output_dir)) do f
+    endswith(f, r"nc|hdf5|h5") && rm(joinpath(simulation.output_dir, f))
+end
+
+@info "Tarballs created"
+
 if CA.is_distributed(config.comms_ctx)
     nprocs = ClimaComms.nprocs(config.comms_ctx)
     comms_ctx = config.comms_ctx