From 81c3676d20c6367830e167e2015622a6acfbd71f Mon Sep 17 00:00:00 2001 From: Pankti Shah <58618433+panktishah26@users.noreply.github.com> Date: Tue, 3 Oct 2023 10:56:35 -0700 Subject: [PATCH] Add extra logs and increase verbosity of upgrade command for e2e tests (#6756) --- pkg/cluster/wait.go | 10 +++++++--- pkg/cluster/wait_test.go | 4 ++-- pkg/clustermanager/applier.go | 12 ++++++------ test/framework/cluster.go | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pkg/cluster/wait.go b/pkg/cluster/wait.go index 1bec3167c0fc..7b35ab342f8b 100644 --- a/pkg/cluster/wait.go +++ b/pkg/cluster/wait.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/cluster-api/util/conditions" @@ -16,8 +17,8 @@ import ( // WaitForCondition blocks until either the cluster has this condition as True // or the retrier timeouts. If observedGeneration is not equal to generation, // the condition is considered false regardless of the status value. -func WaitForCondition(ctx context.Context, client kubernetes.Reader, cluster *anywherev1.Cluster, retrier *retrier.Retrier, conditionType anywherev1.ConditionType) error { - return WaitFor(ctx, client, cluster, retrier, func(c *anywherev1.Cluster) error { +func WaitForCondition(ctx context.Context, log logr.Logger, client kubernetes.Reader, cluster *anywherev1.Cluster, retrier *retrier.Retrier, conditionType anywherev1.ConditionType) error { + return WaitFor(ctx, log, client, cluster, retrier, func(c *anywherev1.Cluster) error { condition := conditions.Get(c, conditionType) if condition == nil { return fmt.Errorf("cluster doesn't yet have condition %s", conditionType) @@ -36,7 +37,7 @@ type Matcher func(*anywherev1.Cluster) error // WaitFor gets the cluster object from the client // checks for generation and observedGeneration condition // matches condition and returns error if the condition is not met. -func WaitFor(ctx context.Context, client kubernetes.Reader, cluster *anywherev1.Cluster, retrier *retrier.Retrier, matcher Matcher) error { +func WaitFor(ctx context.Context, log logr.Logger, client kubernetes.Reader, cluster *anywherev1.Cluster, retrier *retrier.Retrier, matcher Matcher) error { return retrier.Retry(func() error { c := &anywherev1.Cluster{} @@ -51,6 +52,9 @@ func WaitFor(ctx context.Context, client kubernetes.Reader, cluster *anywherev1. observedGeneration := c.Status.ObservedGeneration generation := c.Generation + + log.V(9).Info("Cluster generation and observedGeneration", "Generation", generation, "ObservedGeneration", observedGeneration) + if observedGeneration != generation { return fmt.Errorf("cluster generation (%d) and observedGeneration (%d) differ", generation, observedGeneration) } diff --git a/pkg/cluster/wait_test.go b/pkg/cluster/wait_test.go index fa1532f9767c..927468621136 100644 --- a/pkg/cluster/wait_test.go +++ b/pkg/cluster/wait_test.go @@ -145,7 +145,7 @@ func TestWaitForCondition(t *testing.T) { g := NewWithT(t) client := test.NewFakeKubeClient(tt.currentCluster) - gotErr := cluster.WaitForCondition(ctx, client, tt.clusterInput, tt.retrier, tt.condition) + gotErr := cluster.WaitForCondition(ctx, test.NewNullLogger(), client, tt.clusterInput, tt.retrier, tt.condition) if tt.wantErr != "" { g.Expect(gotErr).To(MatchError(tt.wantErr)) } else { @@ -254,7 +254,7 @@ func TestWaitFor(t *testing.T) { g := NewWithT(t) client := test.NewFakeKubeClient(tt.currentCluster) - gotErr := cluster.WaitFor(ctx, client, tt.clusterInput, tt.retrier, tt.matcher) + gotErr := cluster.WaitFor(ctx, test.NewNullLogger(), client, tt.clusterInput, tt.retrier, tt.matcher) if tt.wantErr != "" { g.Expect(gotErr).To(MatchError(tt.wantErr)) } else { diff --git a/pkg/clustermanager/applier.go b/pkg/clustermanager/applier.go index 28864a933b0b..0dccb257b9ca 100644 --- a/pkg/clustermanager/applier.go +++ b/pkg/clustermanager/applier.go @@ -95,7 +95,7 @@ func WithApplierRetryBackOff(backOff time.Duration) ApplierOpt { // until the changes are fully reconciled. func (a Applier) Run(ctx context.Context, spec *cluster.Spec, managementCluster types.Cluster) error { var client kubernetes.Client - + a.log.V(9).Info("Cluster generation before applying specs", "generation", spec.Cluster.Generation) a.log.V(3).Info("Applying cluster spec") err := retrier.New( a.applyClusterTimeout, @@ -133,7 +133,7 @@ func (a Applier) Run(ctx context.Context, spec *cluster.Spec, managementCluster waitStartTime := time.Now() retry := a.retrierForWait(waitStartTime) - if err := cluster.WaitFor(ctx, client, spec.Cluster, a.retrierForFailureMessage(), func(c *anywherev1.Cluster) error { + if err := cluster.WaitFor(ctx, a.log, client, spec.Cluster, a.retrierForFailureMessage(), func(c *anywherev1.Cluster) error { if c.Status.FailureMessage != nil && *c.Status.FailureMessage != "" { return fmt.Errorf("cluster has an error: %s", *c.Status.FailureMessage) } @@ -143,27 +143,27 @@ func (a Applier) Run(ctx context.Context, spec *cluster.Spec, managementCluster } a.log.V(3).Info("Waiting for control plane to be ready") - if err := cluster.WaitForCondition(ctx, client, spec.Cluster, retry, anywherev1.ControlPlaneReadyCondition); err != nil { + if err := cluster.WaitForCondition(ctx, a.log, client, spec.Cluster, retry, anywherev1.ControlPlaneReadyCondition); err != nil { return errors.Wrapf(err, "waiting for cluster's control plane to be ready") } if spec.Cluster.Spec.ClusterNetwork.CNIConfig.IsManaged() { a.log.V(3).Info("Waiting for default CNI to be updated") retry = a.retrierForWait(waitStartTime) - if err := cluster.WaitForCondition(ctx, client, spec.Cluster, retry, anywherev1.DefaultCNIConfiguredCondition); err != nil { + if err := cluster.WaitForCondition(ctx, a.log, client, spec.Cluster, retry, anywherev1.DefaultCNIConfiguredCondition); err != nil { return errors.Wrapf(err, "waiting for cluster's CNI to be configured") } } a.log.V(3).Info("Waiting for worker nodes to be ready after upgrade") retry = a.retrierForWait(waitStartTime) - if err := cluster.WaitForCondition(ctx, client, spec.Cluster, retry, anywherev1.WorkersReadyCondition); err != nil { + if err := cluster.WaitForCondition(ctx, a.log, client, spec.Cluster, retry, anywherev1.WorkersReadyCondition); err != nil { return errors.Wrapf(err, "waiting for cluster's workers to be ready") } a.log.V(3).Info("Waiting for cluster upgrade to be completed") retry = a.retrierForWait(waitStartTime) - if err := cluster.WaitForCondition(ctx, client, spec.Cluster, retry, anywherev1.ReadyCondition); err != nil { + if err := cluster.WaitForCondition(ctx, a.log, client, spec.Cluster, retry, anywherev1.ReadyCondition); err != nil { return errors.Wrapf(err, "waiting for cluster to be ready") } diff --git a/test/framework/cluster.go b/test/framework/cluster.go index 7f50181cf14f..d9f66a09030a 100644 --- a/test/framework/cluster.go +++ b/test/framework/cluster.go @@ -844,7 +844,7 @@ func (e *ClusterE2ETest) upgradeCluster(clusterOpts []ClusterE2ETestOpt, command // UpgradeCluster runs the CLI upgrade command. func (e *ClusterE2ETest) UpgradeCluster(commandOpts ...CommandOpt) { - upgradeClusterArgs := []string{"upgrade", "cluster", "-f", e.ClusterConfigLocation, "-v", "4"} + upgradeClusterArgs := []string{"upgrade", "cluster", "-f", e.ClusterConfigLocation, "-v", "9"} if getBundlesOverride() == "true" { upgradeClusterArgs = append(upgradeClusterArgs, "--bundles-override", defaultBundleReleaseManifestFile) }