chore: lower time for fuzz/race tests (#15141)

* chore: lower time for fuzz/race tests

* fix: use fuzz timeout directly

* fix: greater gap for fuzz test seconds vs timeout

* fix: greater gap for fuzz test seconds vs timeout

* fix: lower fuzz test duration

* fix: fuzzing timeout calculation

* adjust runtimes

* Adjust fuzzing runtime

* adjust race time

* chore: further lower race/fuzz times

* remove -json for non-debug runs

* test: enable build cache for race test

* lower fuzz test timeout further

.github/workflows/ci-core.yml

@@ -166,7 +166,7 @@ jobs:
         uses: ./.github/actions/setup-go
         with:
           # race/fuzz tests don't benefit repeated caching, so restore from develop's build cache
-          restore-build-cache-only: ${{ matrix.type.cmd == 'go_core_race_tests' || matrix.type.cmd == 'go_core_fuzz' }}
+          restore-build-cache-only: ${{ matrix.type.cmd == 'go_core_fuzz' }}
           build-cache-version: ${{ matrix.type.cmd }}
 
       - name: Replace chainlink-evm deps
@@ -220,12 +220,13 @@ jobs:
           go install ./pkg/chainlink/cmd/chainlink-starknet
           popd
 
-      - name: Increase Race Timeout
-        # Increase race timeout for scheduled runs only
+      - name: Increase Timeouts for Fuzz/Race
+        # Increase timeouts for scheduled runs only
         if: ${{ github.event.schedule != '' && needs.filter.outputs.should-run-ci-core == 'true' }}
         run: |
           echo "TIMEOUT=10m" >> $GITHUB_ENV
           echo "COUNT=50" >> $GITHUB_ENV
+          echo "FUZZ_TIMEOUT_MINUTES=10">> $GITHUB_ENV
 
       - name: Install gotestloghelper
         if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }}

fuzz/fuzz_all_native.py

@@ -6,6 +6,7 @@
 import re
 import subprocess
 import sys
+import time
 
 def main():
     parser = argparse.ArgumentParser(
@@ -22,35 +23,51 @@ def main():
 
     # use float for remaining_seconds so we can represent infinity
     if args.seconds:
-        remaining_seconds = float(args.seconds)
+        total_time = float(args.seconds)
     else:
-        remaining_seconds = float("inf")
+        total_time = float("inf")
+
+    start_time = time.time()    
+    remaining_seconds = total_time
 
     fuzzers = discover_fuzzers(args.go_module_root)
-    print(f"🐝 Discovered fuzzers:", file=sys.stderr)
+    num_fuzzers = len(fuzzers)
+    print(f"🐝 Discovered {num_fuzzers} fuzzers:", file=sys.stderr)
     for fuzzfn, path in fuzzers.items():
         print(f"{fuzzfn} in {path}", file=sys.stderr)
 
+    if num_fuzzers == 0:
+        print(f"No fuzzers found, this is likely an error. Exiting.")
+        exit(1)
+
+    # run forever or until --seconds, with increasingly longer durations per fuzz run
+    durations_seconds = itertools.chain([5, 10, 30, 90, 270], itertools.repeat(600))
     if args.ci:
-        # only run each fuzzer once for 60 seconds in CI
-        durations_seconds = [60]
-    else:
-        # run forever or until --seconds, with increasingly longer durations per fuzz run
-        durations_seconds = itertools.chain([5, 10, 30, 90, 270], itertools.repeat(600))
+        # In CI - default to 60s fuzzes for scheduled runs, and 45 seconds for everything else
+        durations_seconds = [60] if os.getenv('GITHUB_EVENT_NAME') == 'scheduled' else [45]
+        if args.seconds:
+            # However, if seconds was specified, evenly divide total time among all fuzzers
+            # leaving a 10 second buffer for processing/building time between fuzz runs
+            actual_fuzz_time = total_time - (num_fuzzers * 10)
+            if actual_fuzz_time <= 5 * num_fuzzers:
+                print(f"Seconds (--seconds {arg.seconds}) is too low to properly run fuzzers for 5sec each. Exiting.")
+                exit(1)
+            durations_seconds = [ actual_fuzz_time / num_fuzzers ]
 
     for duration_seconds in durations_seconds:
         print(f"🐝 Running each fuzzer for {duration_seconds}s before switching to next fuzzer", file=sys.stderr)
         for fuzzfn, path in fuzzers.items():
+            elapsed_time = time.time() - start_time
+            remaining_seconds = total_time - elapsed_time 
+
             if remaining_seconds <= 0:
                 print(f"🐝 Time budget of {args.seconds}s is exhausted. Exiting.", file=sys.stderr)
                 return
 
             next_duration_seconds = min(remaining_seconds, duration_seconds)
-            remaining_seconds -= next_duration_seconds
-
-            print(f"🐝 Running {fuzzfn} in {path} for {next_duration_seconds}s before switching to next fuzzer", file=sys.stderr)
+            print(f"🐝 Running {fuzzfn} in {path} for {next_duration_seconds}s (Elapsed: {elapsed_time:.2f}s, Remaining: {remaining_seconds:.2f}s)", file=sys.stderr)
             run_fuzzer(fuzzfn, path, next_duration_seconds, args.go_module_root)
-            print(f"🐝 Completed running {fuzzfn} in {path} for {next_duration_seconds}s. Total remaining time is {remaining_seconds}s", file=sys.stderr)
+            print(f"🐝 Completed running {fuzzfn} in {path} for {next_duration_seconds}s.", file=sys.stderr)
 
 def discover_fuzzers(go_module_root):
     fuzzers = {}

tools/bin/go_core_fuzz

@@ -4,14 +4,22 @@ set +e
 
 SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"`
 OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"}
-FUZZ_TIMEOUT=${FUZZ_TIMEOUT:-10m}
+FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"3"}
 
+TOTAL_SECONDS=$((FUZZ_TIMEOUT_MINUTES * 60))
+if (( TOTAL_SECONDS >= 120 )); then
+    # Allow for a 30 second buffer between the timeout, and fuzz test runtime
+    FUZZ_SECONDS=$((TOTAL_SECONDS - 30))
+else
+    echo "Increase FUZZ_TIMEOUT_MINUTES to >=2, received $FUZZ_TIMEOUT_MINUTES"
+    exit 1
+fi
+
+echo "timeout minutes: $FUZZ_TIMEOUT_MINUTES"
+echo "fuzz seconds: $FUZZ_SECONDS"
 echo "Failed fuzz tests and panics: ---------------------"
 echo ""
-# the amount of --seconds here is subject to change based on how long the CI job takes in the future
-# as we add more fuzz tests, we should take into consideration increasing this timelapse, so we can have enough coverage.
-# We are timing out after ~10mins in case the tests hang. (Current CI duration is ~8m, modify if needed)
-timeout "${FUZZ_TIMEOUT}" ./fuzz/fuzz_all_native.py --ci --seconds 420 --go_module_root ./ | tee $OUTPUT_FILE
+timeout "${FUZZ_TIMEOUT_MINUTES}"m ./fuzz/fuzz_all_native.py --ci --seconds "$FUZZ_SECONDS" --go_module_root ./ | tee $OUTPUT_FILE
 EXITCODE=${PIPESTATUS[0]}
 
 # Assert no known sensitive strings present in test logger output

tools/bin/go_core_race_tests

@@ -1,22 +1,22 @@
 #!/usr/bin/env bash
 set -ex
 OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"}
-TIMEOUT="${TIMEOUT:-30s}"
-COUNT="${COUNT:-10}"
+TIMEOUT="${TIMEOUT:-10s}"
+COUNT="${COUNT:-5}"
 
 echo "Failed tests and panics: ---------------------"
 echo ""
 if [[ $GITHUB_EVENT_NAME == "schedule" ]]; then
   if [[  $DEBUG == "true" ]]; then 
     GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | tee $OUTPUT_FILE
   else
-    GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE
+    GORACE="log_path=$PWD/race" go test -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE
   fi
 else
   if [[ $DEBUG == "true" ]]; then 
         GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | tee $OUTPUT_FILE
   else
-    GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE
+    GORACE="log_path=$PWD/race" go test -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE
   fi
 fi
 EXITCODE=${PIPESTATUS[0]}