From 472f05977acd1587dd6ab8194668c90d04210c2b Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Thu, 5 Dec 2024 01:28:16 +0000
Subject: [PATCH] test/e2e: dump cluster status when SyncedToRemote doesn't
 become True

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
---
 test/e2e/multik8s/suite_test.go | 38 +++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
index 4796eea..3e67286 100644
--- a/test/e2e/multik8s/suite_test.go
+++ b/test/e2e/multik8s/suite_test.go
@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/cybozu-go/mantle/test/util"
+	"github.com/go-logr/logr"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -80,12 +81,17 @@ func replicationTestSuite() {
 			}).Should(Succeed())
 
 			By("checking MantleBackup's SyncedToRemote status")
+			repeatCount := 0
 			Eventually(func() error {
+				repeatCount++
 				mb, err := getMB(primaryK8sCluster, namespace, backupName)
 				if err != nil {
 					return err
 				}
 				if !meta.IsStatusConditionTrue(mb.Status.Conditions, mantlev1.BackupConditionSyncedToRemote) {
+					if repeatCount == 500 {
+						_ = dumpClusterStatus(GinkgoLogr)
+					}
 					return errors.New("status of SyncedToRemote condition is not True")
 				}
 				return nil
@@ -181,3 +187,35 @@ func replicationTestSuite() {
 		})
 	})
 }
+
+func dumpClusterStatus(logger logr.Logger) error {
+	primaryPods, _, err := kubectl(primaryK8sCluster, nil, "get", "pod", "-A")
+	if err != nil {
+		return err
+	}
+	logger.Info("CLUSTER STATUS: PRIMARY: kubectl get pod -A\n" + string(primaryPods))
+
+	primaryControllerLog, _, err := kubectl(
+		primaryK8sCluster, nil, "logs", "-n", "rook-ceph", "deploy/mantle-controller")
+	if err != nil {
+		return err
+	}
+	logger.Info("CLUSTER STATUS: PRIMARY: kubectl logs -n rook-ceph deploy/mantle-controller\n" +
+		string(primaryControllerLog))
+
+	secondaryPods, _, err := kubectl(secondaryK8sCluster, nil, "get pod -A")
+	if err != nil {
+		return err
+	}
+	logger.Info("CLUSTER STATUS: SECONDARY: kubectl get pod -A\n" + string(secondaryPods))
+
+	secondaryControllerLog, _, err := kubectl(
+		secondaryK8sCluster, nil, "logs", "-n", "rook-ceph", "deploy/mantle-controller")
+	if err != nil {
+		return err
+	}
+	logger.Info("CLUSTER STATUS: SECONDARY: kubectl logs -n rook-ceph deploy/mantle-controller" +
+		string(secondaryControllerLog))
+
+	return nil
+}