Fixed aggregate, improved logging

.github/workflows/sycl-benchmark-aggregate.yml

@@ -57,7 +57,7 @@ jobs:
       with:
         # TODO populate default values before loading configuration
         path: ${{ env.PERF_RES_PATH }}
-        repository: ${{ env.PERF_RES_GIT_REPO }}
+        repository: intel-sandbox/llvm-ci-perf-results #${{ env.PERF_RES_GIT_REPO }}
         branch: ${{ env.PERF_RES_BRANCH }}
     - name: Run aggregator on cloned data
       run: |

.github/workflows/sycl-linux-benchmark.yml

@@ -51,7 +51,7 @@ on:
       cache_results:
         type: boolean
         default: false
-        required: False
+        required: false
 
   workflow_dispatch:
     inputs:
@@ -109,7 +109,7 @@ on:
       cache_results:
         type: boolean
         default: false
-        required: False
+        required: false
 
 permissions:
   contents: write
@@ -242,8 +242,7 @@ jobs:
         clang++ --version
         ls
         export CMPLR_ROOT=$PWD/toolchain
-        ./devops/scripts/benchmarking/benchmark.sh
-        exit $?
+        ./devops/scripts/benchmarking/benchmark.sh ${{ inputs.cache_results == true && '-s' }}
     - name: debug -- delete after
       env:
         SSH_KEY: ${{secrets.ACTIONS_DEPLOY_KEY}}

devops/scripts/benchmarking/aggregate.py

@@ -40,13 +40,13 @@ def csv_samples() -> list[str]:
 		with Path(f"{common.PERF_RES_PATH}/{benchmark}") as cache_dir:
 			# TODO check for time range; What time range do I want?
 			return filter(lambda f: f.is_file() and
-						  common.valid_timestamp(str(f)[-13:]) and str(f)[-13:] > cutoff,
+						  common.valid_timestamp(str(f)[-17:-4]) and str(f)[-17:-4] > cutoff,
 						  cache_dir.glob(f"{benchmark}-*_*.csv"))
 
 	# Calculate median of every desired metric:
 	aggregate_s = dict()
 	for sample_path in csv_samples():
-		with open(sample_path, mode='r') as sample_file:
+		with open(sample_path, 'r') as sample_file:
 			for s in csv.DictReader(sample_file):
 				if s["TestCase"] not in aggregate_s:
 					aggregate_s[s["TestCase"]] = \
@@ -71,5 +71,5 @@ def csv_samples() -> list[str]:
 	if not common.valid_timestamp(sys.argv[2]):
 		print(f"Bad cutoff timestamp, please use YYMMDD_HHMMSS.")
 		exit(1)
-
+	common.load_configs()
 	aggregate_median(sys.argv[1], sys.argv[2])

devops/scripts/benchmarking/benchmark-ci.conf

@@ -1,5 +1,5 @@
 # Git branch settings for llvm-ci-perf-results
-PERF_RES_GIT_REPO="https://github.com/ianayl/llvm-ci-perf-results"
+PERF_RES_GIT_REPO="https://github.com/intel-sandbox/llvm-ci-perf-results"
 PERF_RES_BRANCH="test-compute-bench"
 # Path where llvm-ci-perf-results are cloned
 PERF_RES_PATH="./llvm-ci-perf-res"
@@ -17,17 +17,22 @@ COMPUTE_BENCH_COMPILE_FLAGS="-j2"
 OUTPUT_PATH="."
 
 # Metrics to benchmark, and their allowed variance as a Python dictionary 
-METRICS_VARIANCE='{"Median": 0.5}'
+METRICS_VARIANCE='{"Median": 0.15}'
 #METRICS_VARIANCE='{"Median": 0.5, "StdDev": 4.0}'
 
 # Metrics to record using aggregate.py
 METRICS_RECORDED='["Median", "StdDev"]'
 
 # Threshold to store benchmark files before benchmarking
 # TODO reconsider this
-AVERAGE_THRESHOLD=7
+AVERAGE_THRESHOLD=3
 # Default period of time to aggregate for the average
 AVERAGE_CUTOFF_RANGE="7 days ago"
 
 # Format of timestamps used (linux `date` format string)
-TIMESTAMP_FORMAT='%Y%m%d_%H%M%S'
+TIMESTAMP_FORMAT='%Y%m%d_%H%M%S'
+
+# Log file for test cases that perform over the allowed variance
+BENCHMARK_SLOW_LOG="./benchmarks-over_tolerance.log"
+# Log file for test cases that errored / failed to build
+BENCHMARK_ERROR_LOG="./benchmarks-errored.log"

devops/scripts/benchmarking/benchmark.sh

@@ -73,7 +73,7 @@ build_compute_bench() {
 
 ###
 STATUS_SUCCESS=0
-STATUS_FAILED=1
+STATUS_ERROR=1
 ###
 
 samples_under_threshold () {
@@ -89,7 +89,6 @@ check_regression() {
     fi
     BENCHMARKING_ROOT="$BENCHMARKING_ROOT" python "$BENCHMARKING_ROOT/compare.py" "$1" "$2"
     return $?
-    # return $STATUS_FAILED
 }
 
 cache() {
@@ -100,10 +99,14 @@ cache() {
 check_and_cache() {
     echo "Checking $testcase..."
     if check_regression $1 $2; then
-        echo "Caching $testcase..."
-        cache $1 $2
+        if [ "$CACHE_RESULTS" -eq "1" ]; then
+            echo "Caching $testcase..."
+            cache $1 $2
+        fi
     else
-        echo "Not caching!"
+        if [ "$CACHE_RESULTS" -eq "1" ]; then
+            echo "Not caching!"
+        fi
     fi
 }
 
@@ -113,8 +116,9 @@ process_benchmarks() {
     echo "### Running and processing selected benchmarks ###"
     if [ -z "$TESTS_CONFIG" ]; then
         echo "Setting tests to run via cli is not currently supported."
-        exit $STATUS_FAILED
+        exit $STATUS_ERROR
     else
+        rm "$BENCHMARK_ERROR_LOG" "$BENCHMARK_SLOW_LOG" 2> /dev/null
         # Ignore lines in the test config starting with #'s
         grep "^[^#]" "$TESTS_CONFIG" | while read -r testcase; do
             echo "# Running $testcase..."
@@ -124,16 +128,32 @@ process_benchmarks() {
             if [ "$?" -eq 0 ] && [ -s "$test_csv_output" ]; then 
                 check_and_cache $testcase $test_csv_output
             else
+                # TODO consider capturing error for logging
                 echo "ERROR @ $test_case"
+                echo "-- $testcase: error $?" >> "$BENCHMARK_ERROR_LOG"
             fi
         done
     fi
 }
 
+process_results() {
+    if [ -s "$BENCHMARK_SLOW_LOG" ]; then
+        printf "\n### Tests performing over acceptable range of average: ###\n"
+        cat "$BENCHMARK_SLOW_LOG"
+        echo ""
+    fi
+    if [ -s "$BENCHMARK_ERROR_LOG" ]; then
+        printf "\n### Tests that failed to run: ###\n"
+        cat "$BENCHMARK_ERROR_LOG"
+        echo ""
+    fi
+    [ ! -s "$BENCHMARKING_SLOW_LOG" ] && [ ! -s "$BENCHMARK_ERROR_LOG" ]
+}
+
 cleanup() {
     echo "### Cleaning up compute-benchmark builds from prior runs ###"
     rm -rf $COMPUTE_BENCH_PATH
-    #rm -rf $PERF_RES_PATH
+    rm -rf $PERF_RES_PATH
     [ ! -z "$_exit_after_cleanup" ] && exit
 }
 
@@ -163,10 +183,11 @@ load_configs() {
 load_configs
 
 COMPUTE_BENCH_COMPILE_FLAGS=""
+CACHE_RESULTS="0"
 TIMESTAMP="$(date +"$TIMESTAMP_FORMAT")"
 
 # CLI overrides to configuration options
-while getopts "p:b:r:f:cC" opt; do
+while getopts "p:b:r:f:cCs" opt; do
     case $opt in
         p) COMPUTE_BENCH_PATH=$OPTARG ;;
         r) COMPUTE_BENCH_GIT_REPO=$OPTARG ;;
@@ -176,6 +197,7 @@ while getopts "p:b:r:f:cC" opt; do
         # performing cleanup
         c) _cleanup=1 ;;
         C) _cleanup=1 && _exit_after_cleanup=1 ;;
+        s) CACHE_RESULTS="1";;
         \?) usage ;;
     esac
 done
@@ -189,4 +211,5 @@ fi
 [ ! -d "$PERF_RES_PATH"            ] && clone_perf_res
 [ ! -d "$COMPUTE_BENCH_PATH"       ] && clone_compute_bench
 [ ! -d "$COMPUTE_BENCH_PATH/build" ] && build_compute_bench
-process_benchmarks
+process_benchmarks
+process_results

devops/scripts/benchmarking/common.py

@@ -3,6 +3,7 @@
 import ast
 
 PERF_RES_PATH, metrics_variance, metrics_recorded = None, None, None
+BENCHMARK_SLOW_LOG, BENCHMARK_ERROR_LOG = None, None
 
 def sanitize(stat: str) -> float:
 	# Get rid of %
@@ -22,9 +23,12 @@ def load_configs():
         raise Exception(f"Please provide path to a valid BENCHMARKING_ROOT.")
 
     global PERF_RES_PATH, metrics_variance, metrics_recorded
+    global BENCHMARK_ERROR_LOG, BENCHMARK_SLOW_LOG
     perf_res_re   = re.compile(r'^PERF_RES_PATH=(.*)$', re.M)
     m_variance_re = re.compile(r'^METRICS_VARIANCE=(.*)$', re.M)
     m_recorded_re = re.compile(r'^METRICS_RECORDED=(.*)$', re.M)
+    b_slow_re     = re.compile(r'^BENCHMARK_SLOW_LOG=(.*)$', re.M)
+    b_error_re    = re.compile(r'^BENCHMARK_ERROR_LOG=(.*)$', re.M)
 
     with open(benchmarking_ci_conf_path, 'r') as configs_file:
         configs_str = configs_file.read()
@@ -41,6 +45,12 @@ def load_configs():
 
         for perf_res in perf_res_re.findall(configs_str):
             PERF_RES_PATH = str(perf_res[1:-1])
+
+        for b_slow_log in b_slow_re.findall(configs_str):
+            BENCHMARK_SLOW_LOG = str(b_slow_log[1:-1])
+
+        for b_error_log in b_error_re.findall(configs_str):
+            BENCHMARK_ERROR_LOG = str(b_error_log[1:-1])
 
 
 def valid_timestamp(timestamp: str) -> bool:

devops/scripts/benchmarking/compare.py

@@ -18,15 +18,15 @@ def compare_to_median(test_name: str, test_csv_path: str):
 		exit(-1)
 
 	median = dict()
-	with open(median_path, mode='r') as median_csv:
+	with open(median_path, 'r') as median_csv:
 		for stat in csv.DictReader(median_csv):
 			median[stat["TestCase"]] = \
 					{ metric: float(stat[metric]) for metric in common.metrics_variance }
 
 	# TODO read status codes from a config file
 	status = 0
 	failure_counts = { metric: 0 for metric in common.metrics_variance }
-	with open(test_csv_path, mode='r') as sample_csv:
+	with open(test_csv_path, 'r') as sample_csv:
 		for sample in csv.DictReader(sample_csv):
 			# Ignore test cases we haven't profiled before
 			if sample["TestCase"] not in median:
@@ -37,8 +37,13 @@ def compare_to_median(test_name: str, test_csv_path: str):
 				if common.sanitize(sample[metric]) >  max_tolerated:
 					print("vvv FAILED vvv")
 					print(sample['TestCase'])
-					print(f"{metric}: {metric} {common.sanitize(sample[metric])} -- Historic avg. {test_median[metric]} (max tolerance {threshold*100}% -- {max_tolerated})")
+					print(f"{metric}: {common.sanitize(sample[metric])} -- Historic avg. {test_median[metric]} (max tolerance {threshold*100}%: {max_tolerated})")
 					print("^^^^^^^^^^^^^^")
+					with open(common.BENCHMARK_SLOW_LOG, 'a') as slow_log:
+						slow_log.write(
+							f"-- {test_name}::{sample['TestCase']}\n"
+							f"   {metric}: {common.sanitize(sample[metric])} -- Historic avg. {test_median[metric]} (max tol. {threshold*100}%: {max_tolerated})\n"
+						)
 					status = 1
 					failure_counts[metric] += 1
 	if status != 0: