From 1347424c6372d8bd00ac38426369009cfd510605 Mon Sep 17 00:00:00 2001 From: Cedric Date: Fri, 8 Sep 2023 11:14:02 +0100 Subject: [PATCH] WIP (#10542) --- .github/workflows/ci-core.yml | 2 +- tools/flakeytests/runner.go | 65 ++++++---- tools/flakeytests/runner_test.go | 197 ++++--------------------------- 3 files changed, 69 insertions(+), 195 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 7b0b5e36ad7..8dc29d5db77 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -289,7 +289,7 @@ jobs: -gh_sha=$GITHUB_SHA \ -gh_event_path=$GITHUB_EVENT_PATH \ -command=./tools/bin/go_core_tests \ - `ls -R ./artifacts/go_core_tests*/output-short.txt` + `ls -R ./artifacts/go_core_tests*/output.txt` - name: Store logs artifacts if: always() uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 diff --git a/tools/flakeytests/runner.go b/tools/flakeytests/runner.go index 592b673a23b..87777e85109 100644 --- a/tools/flakeytests/runner.go +++ b/tools/flakeytests/runner.go @@ -3,6 +3,7 @@ package flakeytests import ( "bufio" "bytes" + "encoding/json" "fmt" "io" "log" @@ -10,13 +11,11 @@ import ( "os/exec" "regexp" "strings" + "time" ) var ( - failedTestRe = regexp.MustCompile(`^--- FAIL: (Test\w+)`) - logPanicRe = regexp.MustCompile(`^panic: Log in goroutine after (Test\w+)`) - - failedPkgRe = regexp.MustCompile(`^FAIL\s+github\.com\/smartcontractkit\/chainlink\/v2\/(\S+)`) + panicRe = regexp.MustCompile(`^panic:`) ) type Runner struct { @@ -53,29 +52,55 @@ func runGoTest(pkg string, tests []string, numReruns int, w io.Writer) error { return cmd.Run() } +type TestEvent struct { + Time time.Time + Action string + Package string + Test string + Elapsed float64 // seconds + Output string +} + +func newEvent(b []byte) (*TestEvent, error) { + e := &TestEvent{} + err := json.Unmarshal(b, e) + if err != nil { + return nil, err + } + + e.Package = strings.Replace(e.Package, "github.com/smartcontractkit/chainlink/v2/", "", -1) + return e, nil + +} + func parseOutput(readers ...io.Reader) (map[string]map[string]int, error) { - testsWithoutPackage := []string{} tests := map[string]map[string]int{} for _, r := range readers { s := bufio.NewScanner(r) for s.Scan() { - t := s.Text() - switch { - case failedTestRe.MatchString(t): - m := failedTestRe.FindStringSubmatch(t) - testsWithoutPackage = append(testsWithoutPackage, m[1]) - case logPanicRe.MatchString(t): - m := logPanicRe.FindStringSubmatch(t) - testsWithoutPackage = append(testsWithoutPackage, m[1]) - case failedPkgRe.MatchString(t): - p := failedPkgRe.FindStringSubmatch(t) - for _, t := range testsWithoutPackage { - if tests[p[1]] == nil { - tests[p[1]] = map[string]int{} + t := s.Bytes() + if len(t) == 0 { + continue + } + + e, err := newEvent(t) + if err != nil { + return nil, err + } + + switch e.Action { + case "fail": + if tests[e.Package] == nil { + tests[e.Package] = map[string]int{} + } + tests[e.Package][e.Test]++ + case "output": + if panicRe.MatchString(e.Output) { + if tests[e.Package] == nil { + tests[e.Package] = map[string]int{} } - tests[p[1]][t]++ + tests[e.Package][e.Test]++ } - testsWithoutPackage = []string{} } } diff --git a/tools/flakeytests/runner_test.go b/tools/flakeytests/runner_test.go index 6560d43e2f9..63c7e7e3a68 100644 --- a/tools/flakeytests/runner_test.go +++ b/tools/flakeytests/runner_test.go @@ -23,25 +23,7 @@ func newMockReporter() *mockReporter { } func TestParser(t *testing.T) { - output := ` ---- FAIL: TestLink (0.00s) - --- FAIL: TestLink/1.1_link#01 (0.00s) - currencies_test.go:325: - Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325 - Error: Not equal: - expected: "1.2 link" - actual : "1.1 link" - - Diff: - --- Expected - +++ Actual - @@ -1 +1 @@ - -1.2 link - +1.1 link - Test: TestLink/1.1_link#01 -FAIL -FAIL github.com/smartcontractkit/chainlink/v2/core/assets 0.338s -FAIL + output := `{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0} ` r := strings.NewReader(output) @@ -53,65 +35,9 @@ FAIL assert.Equal(t, ts["core/assets"]["TestLink"], 1) } -func TestParser_PanicInTest(t *testing.T) { - output := ` -? github.com/smartcontractkit/chainlink/v2/tools/flakeytests/cmd/runner [no test files] ---- FAIL: TestParser (0.00s) -panic: foo [recovered] - panic: foo - -goroutine 21 [running]: -testing.tRunner.func1.2({0x1009953c0, 0x1009d1e40}) - /opt/homebrew/Cellar/go/1.20.3/libexec/src/testing/testing.go:1526 +0x1c8 -testing.tRunner.func1() - /opt/homebrew/Cellar/go/1.20.3/libexec/src/testing/testing.go:1529 +0x384 -panic({0x1009953c0, 0x1009d1e40}) - /opt/homebrew/Cellar/go/1.20.3/libexec/src/runtime/panic.go:884 +0x204 -github.com/smartcontractkit/chainlink/v2/tools/flakeytests.TestParser(0x0?) - /Users/ccordenier/Development/chainlink/tools/flakeytests/runner_test.go:50 +0xa4 -testing.tRunner(0x14000083520, 0x1009d1588) - /opt/homebrew/Cellar/go/1.20.3/libexec/src/testing/testing.go:1576 +0x10c -created by testing.(*T).Run - /opt/homebrew/Cellar/go/1.20.3/libexec/src/testing/testing.go:1629 +0x368 -FAIL github.com/smartcontractkit/chainlink/v2/tools/flakeytests 0.197s -FAIL` - - r := strings.NewReader(output) - ts, err := parseOutput(r) - require.NoError(t, err) - - assert.Len(t, ts, 1) - assert.Len(t, ts["tools/flakeytests"], 1) - assert.Equal(t, ts["tools/flakeytests"]["TestParser"], 1) -} - func TestParser_PanicDueToLogging(t *testing.T) { output := ` -panic: Log in goroutine after TestIntegration_LogEventProvider_Backfill has completed: 2023-07-19T10:10:45.925Z WARN KeepersRegistry.LogEventProvider logprovider/provider.go:218 failed to read logs {"version": "2.3.0@d898528", "where": "reader", "err": "fetched logs with errors: context canceled"} - -goroutine 4999 [running]: -testing.(*common).logDepth(0xc0051f6000, {0xc003011960, 0xd3}, 0x3) - /opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1003 +0x4e7 -testing.(*common).log(...) - /opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:985 -testing.(*common).Logf(0xc0051f6000, {0x21ba777?, 0x41ac8a?}, {0xc00217c330?, 0x1e530c0?, 0x1?}) - /opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1036 +0x5a -go.uber.org/zap/zaptest.testingWriter.Write({{0x7f4c5c94f018?, 0xc0051f6000?}, 0xa8?}, {0xc003017000?, 0xd4, 0xc00217c320?}) - /home/runner/go/pkg/mod/go.uber.org/zap@v1.24.0/zaptest/logger.go:130 +0xe6 -go.uber.org/zap/zapcore.(*ioCore).Write(0xc0022f60f0, {0x1, {0xc1260b897723e1ca, 0x4bdc75e54, 0x3d56e40}, {0x22e96a1, 0x20}, {0x22c3204, 0x13}, {0x1, ...}, ...}, ...) - /home/runner/go/pkg/mod/go.uber.org/zap@v1.24.0/zapcore/core.go:99 +0xb5 -go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc001265ba0, {0xc0023ca100, 0x1, 0x2}) - /home/runner/go/pkg/mod/go.uber.org/zap@v1.24.0/zapcore/entry.go:255 +0x1d9 -go.uber.org/zap.(*SugaredLogger).log(0xc00363a008, 0x1, {0x22c3204?, 0x13?}, {0x0?, 0x0?, 0x0?}, {0xc004ea3f80, 0x2, 0x2}) - /home/runner/go/pkg/mod/go.uber.org/zap@v1.24.0/sugar.go:295 +0xee -go.uber.org/zap.(*SugaredLogger).Warnw(...) - /home/runner/go/pkg/mod/go.uber.org/zap@v1.24.0/sugar.go:216 -github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/logprovider.(*logEventProvider).startReader(0xc00076d730, {0x2917018?, 0xc0043c4000?}, 0xc003b2a000) - /home/runner/work/chainlink/chainlink/core/services/ocr2/plugins/ocr2keeper/evm21/logprovider/provider.go:218 +0x29f -created by github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/logprovider.(*logEventProvider).Start - /home/runner/work/chainlink/chainlink/core/services/ocr2/plugins/ocr2keeper/evm21/logprovider/provider.go:108 +0x133 -FAIL github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/logprovider 20.380s -FAIL +{"Time":"2023-09-07T16:01:40.649849+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_LinkScanValue","Output":"panic: foo\n"} ` r := strings.NewReader(output) @@ -119,14 +45,24 @@ FAIL require.NoError(t, err) assert.Len(t, ts, 1) - assert.Len(t, ts["core/services/ocr2/plugins/ocr2keeper/evm21/logprovider"], 1) - assert.Equal(t, ts["core/services/ocr2/plugins/ocr2keeper/evm21/logprovider"]["TestIntegration_LogEventProvider_Backfill"], 1) + assert.Len(t, ts["core/assets"], 1) + assert.Equal(t, ts["core/assets"]["TestAssets_LinkScanValue"], 1) } func TestParser_SuccessfulOutput(t *testing.T) { output := ` -? github.com/smartcontractkit/chainlink/v2/tools/flakeytests/cmd/runner [no test files] -ok github.com/smartcontractkit/chainlink/v2/tools/flakeytests 0.320s +{"Time":"2023-09-07T16:22:52.556853+01:00","Action":"start","Package":"github.com/smartcontractkit/chainlink/v2/core/assets"} +{"Time":"2023-09-07T16:22:52.762353+01:00","Action":"run","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString"} +{"Time":"2023-09-07T16:22:52.762456+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString","Output":"=== RUN TestAssets_NewLinkAndString\n"} +{"Time":"2023-09-07T16:22:52.76249+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString","Output":"=== PAUSE TestAssets_NewLinkAndString\n"} +{"Time":"2023-09-07T16:22:52.7625+01:00","Action":"pause","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString"} +{"Time":"2023-09-07T16:22:52.762511+01:00","Action":"cont","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString"} +{"Time":"2023-09-07T16:22:52.762528+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString","Output":"=== CONT TestAssets_NewLinkAndString\n"} +{"Time":"2023-09-07T16:22:52.762546+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString","Output":"--- PASS: TestAssets_NewLinkAndString (0.00s)\n"} +{"Time":"2023-09-07T16:22:52.762557+01:00","Action":"pass","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString","Elapsed":0} +{"Time":"2023-09-07T16:22:52.762566+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Output":"PASS\n"} +{"Time":"2023-09-07T16:22:52.762955+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Output":"ok \tgithub.com/smartcontractkit/chainlink/v2/core/assets\t0.206s\n"} +{"Time":"2023-09-07T16:22:52.765598+01:00","Action":"pass","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Elapsed":0.209} ` r := strings.NewReader(output) @@ -136,26 +72,7 @@ ok github.com/smartcontractkit/chainlink/v2/tools/flakeytests 0.320s } func TestRunner_WithFlake(t *testing.T) { - output := ` ---- FAIL: TestLink (0.00s) - --- FAIL: TestLink/1.1_link#01 (0.00s) - currencies_test.go:325: - Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325 - Error: Not equal: - expected: "1.2 link" - actual : "1.1 link" - - Diff: - --- Expected - +++ Actual - @@ -1 +1 @@ - -1.2 link - +1.1 link - Test: TestLink/1.1_link#01 -FAIL -FAIL github.com/smartcontractkit/chainlink/v2/core/assets 0.338s -FAIL -` + output := `{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0}` m := newMockReporter() r := &Runner{ numReruns: 2, @@ -177,61 +94,11 @@ FAIL } func TestRunner_AllFailures(t *testing.T) { - output := ` ---- FAIL: TestLink (0.00s) - --- FAIL: TestLink/1.1_link#01 (0.00s) - currencies_test.go:325: - Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325 - Error: Not equal: - expected: "1.2 link" - actual : "1.1 link" - - Diff: - --- Expected - +++ Actual - @@ -1 +1 @@ - -1.2 link - +1.1 link - Test: TestLink/1.1_link#01 -FAIL -FAIL github.com/smartcontractkit/chainlink/v2/core/assets 0.338s -FAIL -` + output := `{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0}` rerunOutput := ` ---- FAIL: TestLink (0.00s) - --- FAIL: TestLink/1.1_link#01 (0.00s) - currencies_test.go:325: - Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325 - Error: Not equal: - expected: "1.2 link" - actual : "1.1 link" - - Diff: - --- Expected - +++ Actual - @@ -1 +1 @@ - -1.2 link - +1.1 link - Test: TestLink/1.1_link#01 ---- FAIL: TestLink (0.00s) - --- FAIL: TestLink/1.1_link#01 (0.00s) - currencies_test.go:325: - Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325 - Error: Not equal: - expected: "1.2 link" - actual : "1.1 link" - - Diff: - --- Expected - +++ Actual - @@ -1 +1 @@ - -1.2 link - +1.1 link - Test: TestLink/1.1_link#01 -FAIL -FAIL github.com/smartcontractkit/chainlink/v2/core/assets 0.315s -FAIL +{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0} +{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0} ` m := newMockReporter() r := &Runner{ @@ -251,29 +118,11 @@ FAIL } func TestRunner_RerunSuccessful(t *testing.T) { - output := ` ---- FAIL: TestLink (0.00s) - --- FAIL: TestLink/1.1_link#01 (0.00s) - currencies_test.go:325: - Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325 - Error: Not equal: - expected: "1.2 link" - actual : "1.1 link" - - Diff: - --- Expected - +++ Actual - @@ -1 +1 @@ - -1.2 link - +1.1 link - Test: TestLink/1.1_link#01 -FAIL -FAIL github.com/smartcontractkit/chainlink/v2/core/assets 0.338s -FAIL -` + output := `{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0}` rerunOutput := ` -ok github.com/smartcontractkit/chainlink/v2/core/assets 0.320s +{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0} +{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"pass","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0} ` m := newMockReporter() r := &Runner{