From 17a99efa9ca91c7aed053e27fb1706a4e4131252 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Mon, 22 Jul 2024 15:04:21 -0600
Subject: [PATCH 01/18] test(systest): improve unavailable err handling

---
 systest/tests/common.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/systest/tests/common.go b/systest/tests/common.go
index d4df0db710..e812239182 100644
--- a/systest/tests/common.go
+++ b/systest/tests/common.go
@@ -28,6 +28,8 @@ const (
 	attempts = 3
 )
 
+var retryBackoff = 10 * time.Second
+
 func sendTransactions(
 	ctx context.Context,
 	eg *errgroup.Group,
@@ -135,6 +137,8 @@ func stateHashStream(
 	logger *zap.Logger,
 	collector func(*pb.GlobalStateStreamResponse) (bool, error),
 ) error {
+	retries := 0
+BACKOFF:
 	stateapi := pb.NewGlobalStateServiceClient(node.PubConn())
 	states, err := stateapi.GlobalStateStream(ctx,
 		&pb.GlobalStateStreamRequest{
@@ -153,7 +157,12 @@ func stateHashStream(
 				zap.Any("status", s),
 			)
 			if s.Code() == codes.Unavailable {
-				return nil
+				if retries == attempts {
+					return errors.New("state stream unavailable")
+				}
+				retries++
+				time.Sleep(retryBackoff)
+				goto BACKOFF
 			}
 		}
 		if err != nil {

From ea3db7731a7b2f3184167f7db5adef7e8f5d142c Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Mon, 22 Jul 2024 17:54:02 -0600
Subject: [PATCH 02/18] chore: try to improve streaming checks

---
 systest/tests/common.go         | 33 ++++++++++++++-----------
 systest/tests/partition_test.go | 44 ++++++++++++++++++---------------
 2 files changed, 43 insertions(+), 34 deletions(-)

diff --git a/systest/tests/common.go b/systest/tests/common.go
index e812239182..ab0dc30d73 100644
--- a/systest/tests/common.go
+++ b/systest/tests/common.go
@@ -119,18 +119,6 @@ func submitTransaction(ctx context.Context, tx []byte, node *cluster.NodeClient)
 	return response.Txstate.Id.Id, nil
 }
 
-func watchStateHashes(
-	ctx context.Context,
-	eg *errgroup.Group,
-	node *cluster.NodeClient,
-	logger *zap.Logger,
-	collector func(*pb.GlobalStateStreamResponse) (bool, error),
-) {
-	eg.Go(func() error {
-		return stateHashStream(ctx, node, logger, collector)
-	})
-}
-
 func stateHashStream(
 	ctx context.Context,
 	node *cluster.NodeClient,
@@ -194,6 +182,9 @@ func layersStream(
 	logger *zap.Logger,
 	collector layerCollector,
 ) error {
+	retries := 0
+BACKOFF:
+
 	meshapi := pb.NewMeshServiceClient(node.PubConn())
 	layers, err := meshapi.LayerStream(ctx, &pb.LayerStreamRequest{})
 	if err != nil {
@@ -205,7 +196,12 @@ func layersStream(
 		if ok && s.Code() != codes.OK {
 			logger.Warn("layers stream error", zap.String("client", node.Name), zap.Error(err), zap.Any("status", s))
 			if s.Code() == codes.Unavailable {
-				return nil
+				if retries == attempts {
+					return errors.New("layer stream unavailable")
+				}
+				retries++
+				time.Sleep(retryBackoff)
+				goto BACKOFF
 			}
 		}
 		if err != nil {
@@ -223,6 +219,9 @@ func malfeasanceStream(
 	logger *zap.Logger,
 	collector func(*pb.MalfeasanceStreamResponse) (bool, error),
 ) error {
+	retries := 0
+BACKOFF:
+
 	meshapi := pb.NewMeshServiceClient(node.PubConn())
 	layers, err := meshapi.MalfeasanceStream(ctx, &pb.MalfeasanceStreamRequest{IncludeProof: true})
 	if err != nil {
@@ -238,7 +237,13 @@ func malfeasanceStream(
 				zap.Any("status", s),
 			)
 			if s.Code() == codes.Unavailable {
-				return nil
+				if retries == attempts {
+					return errors.New("layer stream unavailable")
+				}
+				retries++
+				time.Sleep(retryBackoff)
+				goto BACKOFF
+
 			}
 		}
 		if err != nil {
diff --git a/systest/tests/partition_test.go b/systest/tests/partition_test.go
index 89431bfdfe..596dcda3e6 100644
--- a/systest/tests/partition_test.go
+++ b/systest/tests/partition_test.go
@@ -81,28 +81,32 @@ func testPartition(t *testing.T, tctx *testcontext.Context, cl *cluster.Cluster,
 	tctx.Log.Debug("listening to state hashes...")
 	for i := range cl.Total() {
 		client := cl.Client(i)
-		watchStateHashes(ctx, eg, client, tctx.Log.Desugar(), func(state *pb.GlobalStateStreamResponse) (bool, error) {
-			data := state.Datum.Datum
-			require.IsType(t, &pb.GlobalStateData_GlobalState{}, data)
-
-			resp := data.(*pb.GlobalStateData_GlobalState)
-			layer := resp.GlobalState.Layer.Number
-			if layer > stop {
-				return false, nil
-			}
 
-			stateHash := types.BytesToHash(resp.GlobalState.RootHash)
-			tctx.Log.Debugw("state hash collected",
-				"client", client.Name,
-				"layer", layer,
-				"state", stateHash.ShortString())
-			stateCh <- &stateUpdate{
-				layer:  layer,
-				hash:   stateHash,
-				client: client.Name,
-			}
-			return true, nil
+		eg.Go(func() error {
+			return stateHashStream(ctx, node, logger, func(state *pb.GlobalStateStreamResponse) (bool, error) {
+				data := state.Datum.Datum
+				require.IsType(t, &pb.GlobalStateData_GlobalState{}, data)
+
+				resp := data.(*pb.GlobalStateData_GlobalState)
+				layer := resp.GlobalState.Layer.Number
+				if layer > stop {
+					return false, nil
+				}
+
+				stateHash := types.BytesToHash(resp.GlobalState.RootHash)
+				tctx.Log.Debugw("state hash collected",
+					"client", client.Name,
+					"layer", layer,
+					"state", stateHash.ShortString())
+				stateCh <- &stateUpdate{
+					layer:  layer,
+					hash:   stateHash,
+					client: client.Name,
+				}
+				return true, nil
+			})
 		})
+
 	}
 
 	finalErr := eg.Wait()

From caacee9a9025e833d808ea9fc21dd095c5d80732 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Mon, 22 Jul 2024 18:23:39 -0600
Subject: [PATCH 03/18] chore: lint

---
 systest/tests/partition_test.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/systest/tests/partition_test.go b/systest/tests/partition_test.go
index 596dcda3e6..4247f25401 100644
--- a/systest/tests/partition_test.go
+++ b/systest/tests/partition_test.go
@@ -80,10 +80,10 @@ func testPartition(t *testing.T, tctx *testcontext.Context, cl *cluster.Cluster,
 	stateCh := make(chan *stateUpdate, uint32(cl.Total())*numLayers*10)
 	tctx.Log.Debug("listening to state hashes...")
 	for i := range cl.Total() {
-		client := cl.Client(i)
+		node := cl.Client(i)
 
 		eg.Go(func() error {
-			return stateHashStream(ctx, node, logger, func(state *pb.GlobalStateStreamResponse) (bool, error) {
+			return stateHashStream(ctx, node, tctx.Log.Desugar(), func(state *pb.GlobalStateStreamResponse) (bool, error) {
 				data := state.Datum.Datum
 				require.IsType(t, &pb.GlobalStateData_GlobalState{}, data)
 
@@ -106,7 +106,6 @@ func testPartition(t *testing.T, tctx *testcontext.Context, cl *cluster.Cluster,
 				return true, nil
 			})
 		})
-
 	}
 
 	finalErr := eg.Wait()

From 41a3efa00c966c973e295dacf1ffbd9c87d39fd5 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Mon, 22 Jul 2024 18:43:15 -0600
Subject: [PATCH 04/18] chore: lint

---
 systest/tests/partition_test.go | 45 +++++++++++++++++----------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/systest/tests/partition_test.go b/systest/tests/partition_test.go
index 4247f25401..ba65b833cc 100644
--- a/systest/tests/partition_test.go
+++ b/systest/tests/partition_test.go
@@ -83,28 +83,29 @@ func testPartition(t *testing.T, tctx *testcontext.Context, cl *cluster.Cluster,
 		node := cl.Client(i)
 
 		eg.Go(func() error {
-			return stateHashStream(ctx, node, tctx.Log.Desugar(), func(state *pb.GlobalStateStreamResponse) (bool, error) {
-				data := state.Datum.Datum
-				require.IsType(t, &pb.GlobalStateData_GlobalState{}, data)
-
-				resp := data.(*pb.GlobalStateData_GlobalState)
-				layer := resp.GlobalState.Layer.Number
-				if layer > stop {
-					return false, nil
-				}
-
-				stateHash := types.BytesToHash(resp.GlobalState.RootHash)
-				tctx.Log.Debugw("state hash collected",
-					"client", client.Name,
-					"layer", layer,
-					"state", stateHash.ShortString())
-				stateCh <- &stateUpdate{
-					layer:  layer,
-					hash:   stateHash,
-					client: client.Name,
-				}
-				return true, nil
-			})
+			return stateHashStream(ctx, node, tctx.Log.Desugar(),
+				func(state *pb.GlobalStateStreamResponse) (bool, error) {
+					data := state.Datum.Datum
+					require.IsType(t, &pb.GlobalStateData_GlobalState{}, data)
+
+					resp := data.(*pb.GlobalStateData_GlobalState)
+					layer := resp.GlobalState.Layer.Number
+					if layer > stop {
+						return false, nil
+					}
+
+					stateHash := types.BytesToHash(resp.GlobalState.RootHash)
+					tctx.Log.Debugw("state hash collected",
+						"client", node.Name,
+						"layer", layer,
+						"state", stateHash.ShortString())
+					stateCh <- &stateUpdate{
+						layer:  layer,
+						hash:   stateHash,
+						client: node.Name,
+					}
+					return true, nil
+				})
 		})
 	}
 

From 5a7b9c168a19349579bbf26ebc2a426bf06ab01b Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Tue, 23 Jul 2024 09:37:54 -0600
Subject: [PATCH 05/18] chore: control parallelism according to nr of clusters

---
 systest/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/systest/Makefile b/systest/Makefile
index 9a2cb71dac..343bbc2d5d 100644
--- a/systest/Makefile
+++ b/systest/Makefile
@@ -36,8 +36,8 @@ ifeq ($(configname),$(test_job_name))
 	run_deps = config
 endif
 
-command := tests -test.v -test.count=$(count) -test.timeout=0 -test.run=$(test_name) -clusters=$(clusters) \
--level=$(level) -configname=$(configname)
+command := tests -test.v -test.count=$(count) -test.timeout=0 -test.run=$(test_name) -test.p=$(clusters) \
+					 -clusters=$(clusters) -level=$(level) -configname=$(configname)
 
 ifeq ($(failfast),true)
 	command := $(command) -test.failfast

From c7c5fa4898da00f3ac0218b040f1d71c01234c38 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Tue, 23 Jul 2024 09:40:54 -0600
Subject: [PATCH 06/18] chore: fix context on ensure conn, change api limits

---
 systest/cluster/nodes.go       | 2 +-
 systest/testcontext/context.go | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/systest/cluster/nodes.go b/systest/cluster/nodes.go
index aca5764a1e..cba322d326 100644
--- a/systest/cluster/nodes.go
+++ b/systest/cluster/nodes.go
@@ -186,7 +186,7 @@ func (n *NodeClient) ensurePubConn(ctx context.Context) (*grpc.ClientConn, error
 	if err != nil {
 		return nil, err
 	}
-	if err := n.waitForConnectionReady(context.Background(), conn); err != nil {
+	if err := n.waitForConnectionReady(ctx, conn); err != nil {
 		return nil, err
 	}
 	n.pubConn = conn
diff --git a/systest/testcontext/context.go b/systest/testcontext/context.go
index e06d99cb57..9ff5acff53 100644
--- a/systest/testcontext/context.go
+++ b/systest/testcontext/context.go
@@ -293,6 +293,8 @@ func New(t *testing.T, opts ...Opt) *Context {
 	// The default rate limiter is too slow 5qps and 10 burst, This will prevent the client from being throttled
 	config.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(20, 50)
 	require.NoError(t, err)
+	config.Burst = 20
+	config.QPS = 10.0
 
 	clientset, err := kubernetes.NewForConfig(config)
 	require.NoError(t, err)

From 6239c019fd976c5c2908074803aa1aac4971dee9 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Tue, 23 Jul 2024 09:57:37 -0600
Subject: [PATCH 07/18] chore: change to -parallel

---
 systest/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/systest/Makefile b/systest/Makefile
index 343bbc2d5d..26fd5b63b2 100644
--- a/systest/Makefile
+++ b/systest/Makefile
@@ -36,7 +36,7 @@ ifeq ($(configname),$(test_job_name))
 	run_deps = config
 endif
 
-command := tests -test.v -test.count=$(count) -test.timeout=0 -test.run=$(test_name) -test.p=$(clusters) \
+command := tests -test.v -test.count=$(count) -test.timeout=0 -test.run=$(test_name) -test.parallel=$(clusters) \
 					 -clusters=$(clusters) -level=$(level) -configname=$(configname)
 
 ifeq ($(failfast),true)

From 0497e3433e84cebb62da37a7238d18bcbc9d5f2f Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Tue, 23 Jul 2024 11:51:03 -0600
Subject: [PATCH 08/18] chore: add test.timeout to see which tests are still
 executing when failing

---
 .github/workflows/systest.yml | 1 -
 systest/Makefile              | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/systest.yml b/.github/workflows/systest.yml
index dfd15b59e5..144c365946 100644
--- a/.github/workflows/systest.yml
+++ b/.github/workflows/systest.yml
@@ -130,7 +130,6 @@ jobs:
           go-version-file: "go.mod"
 
       - name: Run tests
-        timeout-minutes: 60
         env:
           test_id: systest-${{ steps.vars.outputs.sha_short }}
           storage: premium-rwo=10Gi
diff --git a/systest/Makefile b/systest/Makefile
index 26fd5b63b2..221aa7533a 100644
--- a/systest/Makefile
+++ b/systest/Makefile
@@ -36,7 +36,7 @@ ifeq ($(configname),$(test_job_name))
 	run_deps = config
 endif
 
-command := tests -test.v -test.count=$(count) -test.timeout=0 -test.run=$(test_name) -test.parallel=$(clusters) \
+command := tests -test.v -test.count=$(count) -test.timeout=60m -test.run=$(test_name) -test.parallel=$(clusters) \
 					 -clusters=$(clusters) -level=$(level) -configname=$(configname)
 
 ifeq ($(failfast),true)

From 32c51341ac9ee678cdc168331d749c632a5c9ea4 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Tue, 23 Jul 2024 13:37:02 -0600
Subject: [PATCH 09/18] chore: more grpc err handling

---
 systest/tests/common.go | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/systest/tests/common.go b/systest/tests/common.go
index ab0dc30d73..9b31754e47 100644
--- a/systest/tests/common.go
+++ b/systest/tests/common.go
@@ -184,7 +184,6 @@ func layersStream(
 ) error {
 	retries := 0
 BACKOFF:
-
 	meshapi := pb.NewMeshServiceClient(node.PubConn())
 	layers, err := meshapi.LayerStream(ctx, &pb.LayerStreamRequest{})
 	if err != nil {
@@ -323,6 +322,9 @@ func watchTransactionResults(ctx context.Context,
 	collector func(*pb.TransactionResult) (bool, error),
 ) {
 	eg.Go(func() error {
+		retries := 0
+	BACKOFF:
+
 		api := pb.NewTransactionServiceClient(client.PubConn())
 		rsts, err := api.StreamResults(ctx, &pb.TransactionResultsRequest{Watch: true})
 		if err != nil {
@@ -338,7 +340,12 @@ func watchTransactionResults(ctx context.Context,
 					zap.Any("status", s),
 				)
 				if s.Code() == codes.Unavailable {
-					return nil
+					if retries == attempts {
+						return errors.New("transaction results unavailable")
+					}
+					retries++
+					time.Sleep(retryBackoff)
+					goto BACKOFF
 				}
 			}
 			if err != nil {
@@ -359,6 +366,8 @@ func watchProposals(
 	collector func(*pb.Proposal) (bool, error),
 ) {
 	eg.Go(func() error {
+		retries := 0
+	BACKOFF:
 		dbg := pb.NewDebugServiceClient(client.PrivConn())
 		proposals, err := dbg.ProposalsStream(ctx, &emptypb.Empty{})
 		if err != nil {
@@ -374,7 +383,12 @@ func watchProposals(
 					zap.Any("status", s),
 				)
 				if s.Code() == codes.Unavailable {
-					return nil
+					if retries == attempts {
+						return errors.New("watch proposals unavailable")
+					}
+					retries++
+					time.Sleep(retryBackoff)
+					goto BACKOFF
 				}
 			}
 			if err != nil {

From 4e4784a9a6d6986b930386b1298eed2849117e45 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Tue, 23 Jul 2024 14:29:22 -0600
Subject: [PATCH 10/18] echo the important line

---
 systest/Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/systest/Makefile b/systest/Makefile
index 221aa7533a..b0b87a56fa 100644
--- a/systest/Makefile
+++ b/systest/Makefile
@@ -43,6 +43,8 @@ ifeq ($(failfast),true)
 	command := $(command) -test.failfast
 endif
 
+
+
 .PHONY: docker
 docker:
 	@DOCKER_BUILDKIT=1 docker build \
@@ -89,7 +91,7 @@ gomplate:
 .PHONY: run
 run: gomplate $(run_deps)
 	@echo "launching test job with name=$(test_job_name) and testid=$(test_id)"
-	@ns=$$(kubectl config view --minify -o jsonpath='{..namespace}' 2>/dev/null); \
+	ns=$$(kubectl config view --minify -o jsonpath='{..namespace}' 2>/dev/null); \
 	export ns="$${ns:-default}"; \
 	if [ -z "$${norbac}" ]; then \
 	  testid=$(test_id) job_name=$(test_job_name) image=$(image_name) command="$(command)" rbac=1 \

From a832d8e177782fe88740212e46b11ff22ccf6c73 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Tue, 23 Jul 2024 14:30:01 -0600
Subject: [PATCH 11/18] print the command

---
 systest/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/systest/Makefile b/systest/Makefile
index b0b87a56fa..2c8dda1eb7 100644
--- a/systest/Makefile
+++ b/systest/Makefile
@@ -90,8 +90,8 @@ gomplate:
 # where /bin/bash is an old bash
 .PHONY: run
 run: gomplate $(run_deps)
-	@echo "launching test job with name=$(test_job_name) and testid=$(test_id)"
-	ns=$$(kubectl config view --minify -o jsonpath='{..namespace}' 2>/dev/null); \
+	@echo "launching test job with name=$(test_job_name) and testid=$(test_id), command=$(command)"
+	@ns=$$(kubectl config view --minify -o jsonpath='{..namespace}' 2>/dev/null); \
 	export ns="$${ns:-default}"; \
 	if [ -z "$${norbac}" ]; then \
 	  testid=$(test_id) job_name=$(test_job_name) image=$(image_name) command="$(command)" rbac=1 \

From 5aa1703338512fd65149f81842525efbcd5c6cf3 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Thu, 25 Jul 2024 11:20:26 -0600
Subject: [PATCH 12/18] chore: allow parallel tests to bail out earlier

---
 systest/cluster/cluster.go     | 6 ++++++
 systest/testcontext/context.go | 9 +++++++++
 2 files changed, 15 insertions(+)

diff --git a/systest/cluster/cluster.go b/systest/cluster/cluster.go
index 54f54f58c6..a5135b3993 100644
--- a/systest/cluster/cluster.go
+++ b/systest/cluster/cluster.go
@@ -150,6 +150,12 @@ func ReuseWait(cctx *testcontext.Context, opts ...Opt) (*Cluster, error) {
 	if err := cl.WaitAllTimeout(cctx.BootstrapDuration); err != nil {
 		return nil, err
 	}
+
+	select {
+	case <-failed:
+		return nil, errors.New("some test failed. cancelling test execution")
+	default:
+	}
 	return cl, nil
 }
 
diff --git a/systest/testcontext/context.go b/systest/testcontext/context.go
index 9ff5acff53..9a289b75b3 100644
--- a/systest/testcontext/context.go
+++ b/systest/testcontext/context.go
@@ -53,6 +53,9 @@ var (
 
 	tokens     chan struct{}
 	initTokens sync.Once
+
+	failed   = make(chan struct{})
+	failOnce sync.Once
 )
 
 var (
@@ -288,6 +291,12 @@ func New(t *testing.T, opts ...Opt) *Context {
 		tokens <- struct{}{}
 		t.Cleanup(func() { <-tokens })
 	}
+
+	t.Cleanup(func() {
+		if t.Failed() {
+			failOnce.Do(func() { close(failed) }())
+		}
+	}())
 	config, err := rest.InClusterConfig()
 
 	// The default rate limiter is too slow 5qps and 10 burst, This will prevent the client from being throttled

From f84e8278ff328a0adbbe31a19f6881ae8cd9df3a Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Thu, 25 Jul 2024 13:10:09 -0600
Subject: [PATCH 13/18] lint

---
 systest/testcontext/context.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/systest/testcontext/context.go b/systest/testcontext/context.go
index 9a289b75b3..ea78e0179e 100644
--- a/systest/testcontext/context.go
+++ b/systest/testcontext/context.go
@@ -294,9 +294,9 @@ func New(t *testing.T, opts ...Opt) *Context {
 
 	t.Cleanup(func() {
 		if t.Failed() {
-			failOnce.Do(func() { close(failed) }())
+			failOnce.Do(func() { close(failed) })
 		}
-	}())
+	})
 	config, err := rest.InClusterConfig()
 
 	// The default rate limiter is too slow 5qps and 10 burst, This will prevent the client from being throttled

From e3b8ce3391c3a44cec3ce1e03052e1444d04d084 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Thu, 25 Jul 2024 13:39:53 -0600
Subject: [PATCH 14/18] move around

---
 systest/cluster/cluster.go     | 7 ++-----
 systest/testcontext/context.go | 9 +++++++++
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/systest/cluster/cluster.go b/systest/cluster/cluster.go
index a5135b3993..36e495d77f 100644
--- a/systest/cluster/cluster.go
+++ b/systest/cluster/cluster.go
@@ -150,11 +150,8 @@ func ReuseWait(cctx *testcontext.Context, opts ...Opt) (*Cluster, error) {
 	if err := cl.WaitAllTimeout(cctx.BootstrapDuration); err != nil {
 		return nil, err
 	}
-
-	select {
-	case <-failed:
-		return nil, errors.New("some test failed. cancelling test execution")
-	default:
+	if err = cctx.CheckFail(); err != nil {
+		return nil, err
 	}
 	return cl, nil
 }
diff --git a/systest/testcontext/context.go b/systest/testcontext/context.go
index ea78e0179e..a21a9de024 100644
--- a/systest/testcontext/context.go
+++ b/systest/testcontext/context.go
@@ -376,3 +376,12 @@ func New(t *testing.T, opts ...Opt) *Context {
 	cctx.Log.Infow("using", "namespace", cctx.Namespace)
 	return cctx
 }
+
+func (c *Context) CheckFail() error {
+	select {
+	case <-failed:
+		return errors.New("test suite failed. aborting test execution")
+	default:
+	}
+	return nil
+}

From 78edf637d3bd6d5ef107d77578f993ff3fa65293 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Thu, 25 Jul 2024 14:03:46 -0600
Subject: [PATCH 15/18] lint 4real now

---
 systest/testcontext/context.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/systest/testcontext/context.go b/systest/testcontext/context.go
index a21a9de024..8eef0016bb 100644
--- a/systest/testcontext/context.go
+++ b/systest/testcontext/context.go
@@ -2,6 +2,7 @@ package testcontext
 
 import (
 	"context"
+	goerr "errors"
 	"flag"
 	"fmt"
 	"math/rand/v2"
@@ -380,7 +381,7 @@ func New(t *testing.T, opts ...Opt) *Context {
 func (c *Context) CheckFail() error {
 	select {
 	case <-failed:
-		return errors.New("test suite failed. aborting test execution")
+		return goerr.New("test suite failed. aborting test execution")
 	default:
 	}
 	return nil

From 41ef6e87b9b5f6a8e4c2d0b719cb7c0c57ec068a Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Thu, 25 Jul 2024 16:11:00 -0600
Subject: [PATCH 16/18] clean

---
 systest/testcontext/context.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/systest/testcontext/context.go b/systest/testcontext/context.go
index 8eef0016bb..ce3ff47cea 100644
--- a/systest/testcontext/context.go
+++ b/systest/testcontext/context.go
@@ -303,8 +303,6 @@ func New(t *testing.T, opts ...Opt) *Context {
 	// The default rate limiter is too slow 5qps and 10 burst, This will prevent the client from being throttled
 	config.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(20, 50)
 	require.NoError(t, err)
-	config.Burst = 20
-	config.QPS = 10.0
 
 	clientset, err := kubernetes.NewForConfig(config)
 	require.NoError(t, err)

From e48889492434d66d388435320888f7e1a6b2a3a7 Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Fri, 26 Jul 2024 13:35:19 -0600
Subject: [PATCH 17/18] Update systest/Makefile

Co-authored-by: Matthias Fasching <5011972+fasmat@users.noreply.github.com>
---
 systest/Makefile | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/systest/Makefile b/systest/Makefile
index 2c8dda1eb7..08829590b3 100644
--- a/systest/Makefile
+++ b/systest/Makefile
@@ -37,11 +37,7 @@ ifeq ($(configname),$(test_job_name))
 endif
 
 command := tests -test.v -test.count=$(count) -test.timeout=60m -test.run=$(test_name) -test.parallel=$(clusters) \
-					 -clusters=$(clusters) -level=$(level) -configname=$(configname)
-
-ifeq ($(failfast),true)
-	command := $(command) -test.failfast
-endif
+	-test.failfast=$(failfast) -clusters=$(clusters) -level=$(level) -configname=$(configname)
 
 
 

From 0787ce2509ad98a75a69f0cc846250b95cbd19bb Mon Sep 17 00:00:00 2001
From: acud <12988138+acud@users.noreply.github.com>
Date: Fri, 26 Jul 2024 14:01:15 -0600
Subject: [PATCH 18/18] pr comments

---
 systest/testcontext/context.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/systest/testcontext/context.go b/systest/testcontext/context.go
index ce3ff47cea..a7712fdaa0 100644
--- a/systest/testcontext/context.go
+++ b/systest/testcontext/context.go
@@ -2,7 +2,7 @@ package testcontext
 
 import (
 	"context"
-	goerr "errors"
+	"errors"
 	"flag"
 	"fmt"
 	"math/rand/v2"
@@ -19,7 +19,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zaptest"
-	"k8s.io/apimachinery/pkg/api/errors"
+	k8serr "k8s.io/apimachinery/pkg/api/errors"
 	apimetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	corev1 "k8s.io/client-go/applyconfigurations/core/v1"
@@ -231,7 +231,7 @@ func updateContext(ctx *Context) error {
 	ns, err := ctx.Client.CoreV1().Namespaces().Get(ctx, ctx.Namespace,
 		apimetav1.GetOptions{})
 	if err != nil || ns == nil {
-		if errors.IsNotFound(err) {
+		if k8serr.IsNotFound(err) {
 			return nil
 		}
 		return err
@@ -379,7 +379,7 @@ func New(t *testing.T, opts ...Opt) *Context {
 func (c *Context) CheckFail() error {
 	select {
 	case <-failed:
-		return goerr.New("test suite failed. aborting test execution")
+		return errors.New("test suite failed. aborting test execution")
 	default:
 	}
 	return nil