Add nightlyfuzz CI

.github/workflows/nightlyfuzz.yml

@@ -0,0 +1,53 @@
+name: 'nightly/tag fuzz'
+on:
+  schedule:
+    # Note: The schedule event can be delayed during periods of high
+    # loads of GitHub Actions workflow runs. High load times include
+    # the start of every hour. To decrease the chance of delay,
+    # schedule your workflow to run at a different time of the hour.
+    - cron: "25 0 * * *" # at 25 past midnight every day
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: null
+jobs:
+  fuzzrun:
+    name: "run native fuzzers"
+    runs-on: "ubuntu20.04-4cores-16GB"
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11" # v4.1.1
+      - name: "Setup go"
+        uses: "actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491" # v5.0.0
+        with:
+          go-version-file: 'go.mod'
+          cache: true
+          cache-dependency-path: 'go.sum'
+      - name: "Get corpus directory"
+        id: "get-corpus-dir"
+        run: echo "corpus_dir=$(go env GOCACHE)/fuzz" >> $GITHUB_OUTPUT
+        shell: "bash"
+      - name: "Restore corpus"
+        uses: "actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2" # v4.0.0
+        id: "restore-corpus"
+        with:
+          path: "${{ steps.get-corpus-dir.outputs.corpus_dir }}"
+          # We need to ensure uniqueness of the key, as saving to a key more than once will fail (see Save corpus step).
+          # We never expect a cache hit with the key but we do expect a hit with the restore-keys prefix that is going
+          # to match the latest cache that has that prefix.
+          key: "nightlyfuzz-corpus-${{ github.run_id }}-${{ github.run_attempt }}"
+          restore-keys: "nightlyfuzz-corpus-"
+      - name: "Run native fuzzers"
+        # Fuzz for 1 hour
+        run: "cd fuzz && ./fuzz_all_native.py --seconds 3600"
+      - name: "Print failing testcases"
+        if: failure()
+        run: find . -type f|fgrep '/testdata/fuzz/'|while read f; do echo $f; cat $f; done
+      - name: "Save corpus"
+        uses: "actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2" # v4.0.0
+        # We save also on failure, so that we can keep the valuable corpus generated that led to finding a crash.
+        # If the corpus gets clobbered for any reason, we can remove the offending cache from the Github UI.
+        if: always()
+        with:
+          path: "${{ steps.get-corpus-dir.outputs.corpus_dir }}"
+          key: "${{ steps.restore-corpus.outputs.cache-primary-key }}"

fuzz/.gitignore

@@ -0,0 +1 @@
+*/fuzzer

fuzz/fuzz_all_native.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+import argparse
+import itertools
+import os
+import re
+import subprocess
+import sys
+
+LIBROOT = "../"
+
+def main():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="\n".join([
+            "Fuzz helper to run all native go fuzzers in chainlink",
+            "",
+        ]),
+    )
+    parser.add_argument("--ci", required=False, help="In CI mode we run each parser only briefly once", action="store_true")
+    parser.add_argument("--seconds", required=False, help="Run for this many seconds of total fuzz time before exiting")
+    args = parser.parse_args()
+
+    # use float for remaining_seconds so we can represent infinity
+    if args.seconds:
+        remaining_seconds = float(args.seconds)
+    else:
+        remaining_seconds = float("inf")
+
+    fuzzers = discover_fuzzers()
+    print(f"🐝 Discovered fuzzers:", file=sys.stderr)
+    for fuzzfn, path in fuzzers.items():
+        print(f"{fuzzfn} in {path}", file=sys.stderr)
+
+    if args.ci:
+        # only run each fuzzer once for 60 seconds in CI
+        durations_seconds = [60]
+    else:
+        # run forever or until --seconds, with increasingly longer durations per fuzz run
+        durations_seconds = itertools.chain([5, 10, 30, 90, 270], itertools.repeat(600))
+
+    for duration_seconds in durations_seconds:
+        print(f"🐝 Running each fuzzer for {duration_seconds}s before switching to next fuzzer", file=sys.stderr)
+        for fuzzfn, path in fuzzers.items():
+            if remaining_seconds <= 0:
+                print(f"🐝 Time budget of {args.seconds}s is exhausted. Exiting.", file=sys.stderr)
+                return
+
+            next_duration_seconds = min(remaining_seconds, duration_seconds)
+            remaining_seconds -= next_duration_seconds
+
+            print(f"🐝 Running {fuzzfn} in {path} for {next_duration_seconds}s before switching to next fuzzer", file=sys.stderr)
+            run_fuzzer(fuzzfn, path, next_duration_seconds)
+            print(f"🐝 Completed running {fuzzfn} in {path} for {next_duration_seconds}s. Total remaining time is {remaining_seconds}s", file=sys.stderr)
+
+def discover_fuzzers():
+    fuzzers = {}
+    for root, dirs, files in os.walk(LIBROOT):
+        for file in files:
+            if not file.endswith("test.go"): continue
+            with open(os.path.join(root, file), "r") as f:
+                text = f.read()
+            # ignore multiline comments
+            text = re.sub(r"(?s)/[*].*?[*]/", "", text)
+            # ignore single line comments *except* build tags
+            text = re.sub(r"//.*", "", text)
+            # Find every function with a name like FuzzXXX
+            for fuzzfn in re.findall(r"func\s+(Fuzz\w+)", text):
+                if fuzzfn in fuzzers:
+                    raise Exception(f"Duplicate fuzz function: {fuzzfn}")
+                fuzzers[fuzzfn] = os.path.relpath(root, LIBROOT)
+    return fuzzers
+
+def run_fuzzer(fuzzfn, dir, duration_seconds):
+    subprocess.check_call(["go", "test", "-run=^$", f"-fuzz=^{fuzzfn}$", f"-fuzztime={duration_seconds}s", f"./{dir}"], cwd=LIBROOT)
+
+if __name__ == "__main__":
+    main()

sonar-project.properties

@@ -3,7 +3,7 @@ sonar.sources=.
 sonar.python.version=3.8
 
 # Full exclusions from the static analysis
-sonar.exclusions=**/node_modules/**/*,**/mocks/**/*, **/testdata/**/*, **/contracts/typechain/**/*, **/contracts/artifacts/**/*, **/contracts/cache/**/*, **/contracts/scripts/**/*, **/generated/**/*, **/fixtures/**/*, **/docs/**/*, **/tools/**/*, **/*.pb.go, **/*report.xml, **/*.config.ts, **/*.txt, **/*.abi, **/*.bin, **/*_codecgen.go, core/services/relay/evm/types/*_gen.go, core/services/relay/evm/types/gen/main.go, core/services/relay/evm/testfiles/*, **/core/web/assets**, core/scripts/chaincli/handler/debug.go
+sonar.exclusions=**/node_modules/**/*,**/mocks/**/*, **/testdata/**/*, **/contracts/typechain/**/*, **/contracts/artifacts/**/*, **/contracts/cache/**/*, **/contracts/scripts/**/*, **/generated/**/*, **/fixtures/**/*, **/docs/**/*, **/tools/**/*, **/*.pb.go, **/*report.xml, **/*.config.ts, **/*.txt, **/*.abi, **/*.bin, **/*_codecgen.go, core/services/relay/evm/types/*_gen.go, core/services/relay/evm/types/gen/main.go, core/services/relay/evm/testfiles/*, **/core/web/assets**, core/scripts/chaincli/handler/debug.go, **/fuzz/**/*
 # Coverage exclusions
 sonar.coverage.exclusions=**/*.test.ts, **/*_test.go, **/contracts/test/**/*, **/contracts/**/tests/**/*, **/core/**/testutils/**/*, **/core/**/mocks/**/*, **/core/**/cltest/**/*, **/integration-tests/**/*, **/generated/**/*, **/core/scripts**/* , **/*.pb.go, ./plugins/**/*, **/main.go, **/0195_add_not_null_to_evm_chain_id_in_job_specs.go, **/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/mercury/streams/streams.go
 # Duplication exclusions

tools/bin/go_core_fuzz

@@ -6,17 +6,19 @@ SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"`
 OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"}
 USE_TEE="${USE_TEE:-true}"
 
-echo "Failed tests and panics: ---------------------"
+echo "Failed fuzz tests and panics: ---------------------"
 echo ""
-GO_LDFLAGS=$(bash tools/bin/ldflags)
 use_tee() {
   if [ "$USE_TEE" = "true" ]; then
     tee "$@"
   else
     cat > "$@"
   fi
 }
-go test -json -ldflags "$GO_LDFLAGS" github.com/smartcontractkit/chainlink/v2/core/services/relay/evm -fuzz . -fuzztime 12s | use_tee $OUTPUT_FILE
+
+# the amount of --seconds here is subject to change based on how long the CI job takes in the future
+# as we add more fuzz tests, we should take into consideration increasing this timelapse, so we can have enough coverage.
+cd ./fuzz && ./fuzz_all_native.py --ci --seconds 420 | use_tee $OUTPUT_FILE
 EXITCODE=${PIPESTATUS[0]}
 
 # Assert no known sensitive strings present in test logger output
@@ -29,8 +31,9 @@ fi
 
 echo "Exit code: $EXITCODE"
 if [[ $EXITCODE != 0 ]]; then
-  echo "Encountered test failures."
+  echo "Encountered fuzz test failures. Logging all failing fuzz inputs:"
+  find . -type f|fgrep '/testdata/fuzz/'|while read f; do echo $f; cat $f; done
 else
-  echo "All tests passed!"
+  echo "All fuzz tests passed!"
 fi
 exit $EXITCODE