Batch logstash uploads to speed up CI jobs

pika-org · Nov 29, 2024 · 1344f11 · 1344f11
1 parent f182d67
commit 1344f11
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 12 deletions.
diff --git a/.gitlab/scripts/collect_ctest_metrics.sh b/.gitlab/scripts/collect_ctest_metrics.sh
@@ -17,6 +17,7 @@ metadata_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json)
 create_metadata_file "${metadata_file}"
 
 # Submit individual test data
+result_file_all=$(mktemp --tmpdir "ctest.XXXXXXXXXX.json")
 num_tests=$(xq . "${ctest_xml}" | jq '.testsuite.testcase | length')
 for i in $(seq 1 ${num_tests}); do
     result_file=$(mktemp --tmpdir "ctest_${i}.XXXXXXXXXX.json")
@@ -27,8 +28,9 @@ for i in $(seq 1 ${num_tests}); do
     json_add_value_json "${result_file}" "ctest.testcase" "${ctest_object}"
 
     json_merge "${metadata_file}" "${result_file}" "${result_file}"
-    submit_logstash "${result_file}"
+    cat "${result_file}" >>"${result_file_all}"
 done
+submit_logstash "${result_file_all}"
 
 result_file=$(mktemp --tmpdir "ctest.XXXXXXXXXX.json")
 echo '{}' >"${result_file}"

diff --git a/.gitlab/scripts/collect_file_sizes.sh b/.gitlab/scripts/collect_file_sizes.sh
@@ -20,6 +20,7 @@ function submit_filesizes {
     directory="${1}"
     IFS='
 '
+    result_file_all=$(mktemp --tmpdir "filesizes.XXXXXXXXXX.json")
     for line in $(ls --time-style long-iso -l $(find "${directory}" -maxdepth 1 -type f)); do
         filename=$(echo "${line}" | awk '{ print $8 }')
         filesize=$(echo "${line}" | awk '{ print $5 }')
@@ -31,8 +32,9 @@ function submit_filesizes {
         json_add_value_number "${result_file}" "files.size" "${filesize}"
 
         json_merge "${metadata_file}" "${result_file}" "${result_file}"
-        submit_logstash "${result_file}"
-done
+        cat "${result_file}" >>"${result_file_all}"
+    done
+    submit_logstash "${result_file_all}"
 }
 
 # Submit file name and size for files under lib and bin in the build directory

diff --git a/.gitlab/scripts/run_performance_benchmarks.sh b/.gitlab/scripts/run_performance_benchmarks.sh
@@ -111,6 +111,7 @@ pika_test_options=(
 
 index=0
 failures=0
+result_file_all=$(mktemp --tmpdir "benchmarks.XXXXXXXXXX.json")
 for executable in "${pika_targets[@]}"; do
     test_opts=${pika_test_options[$index]}
     raw_result_file=$(mktemp --tmpdir "${executable}_raw.XXXXXXXXXX.json")
@@ -140,11 +141,12 @@ for executable in "${pika_targets[@]}"; do
         json_add_value_json "${result_file}" "metric.benchmark.series" "${benchmark_series}"
 
         json_merge "${metadata_file}" "${result_file}" "${result_file}"
-        submit_logstash "${result_file}"
+        cat "${result_file}" >> "${result_file_all}
     fi
 
     index=$((index + 1))
 done
+submit_logstash "${result_file_all}"
 
 if ((failures > 0)); then
     exit 1

diff --git a/.gitlab/scripts/utilities.sh b/.gitlab/scripts/utilities.sh
@@ -10,14 +10,18 @@ function submit_logstash {
         jq . "${1}"
     fi
 
-    curl \
-        --silent \
-        --show-error \
-        --request POST \
-        --header "Content-Type: application/json" \
-        --data "@${1}" \
-        "${CSCS_LOGSTASH_URL}" \
-        || true
+    array_of_objects_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json)
+    jq '[inputs]' <"${1}" >"${array_of_objects_file}"
+    cat "${array_of_objects_file}"
+    # curl \
+    #     --silent \
+    #     --output /dev/null \
+    #     --show-error \
+    #     --request POST \
+    #     --header "Content-Type: application/json" \
+    #     --data "@${array_of_objects_file}" \
+    #     "${CSCS_LOGSTASH_URL}" \
+    #     || true
 }
 
 function json_merge {