Wait for clusters to be marked as ready

Previously, the remote assemblage controller was made to skip (and
come back to) a cluster when its kubeconfig secret is missing. This
accounts broadly for races in provisioning clusters, since ultimately
its the secret that matters.

However, with Cluster API providers its possible for a kubeconfig
secret to exist before the cluster is ready to be used. This situation
_also_ leads to long backoffs and apparent lack of responsiveness. To
mitigate that, this commit adds a similar mechanism to the above, at
another layer: clusters that are not marked as ready (in
`.status.controlPlaneReady`) are skipped. This is at the module (and
bootstrapmodule) level, because it's modules that react to clusters --
assemblages just refer to kubeconfig secrets.

Signed-off-by: Michael Bridgen <mikeb@squaremobius.net>

module/config/rbac/role.yaml

@@ -21,6 +21,7 @@ rules:
   verbs:
   - get
   - list
+  - watch
 - apiGroups:
   - fleet.squaremo.dev
   resources:

module/controllers/bootstrap_test.go

@@ -178,6 +178,8 @@ var _ = Describe("bootstrap module controller", func() {
 			cluster.Namespace = namespace.Name
 			clusters[cluster.Name] = cluster
 			Expect(k8sClient.Create(context.TODO(), cluster)).To(Succeed())
+			cluster.Status.ControlPlaneReady = true
+			Expect(k8sClient.Status().Update(context.Background(), cluster)).To(Succeed())
 		}
 
 	})

module/controllers/bootstrapmodule_controller.go

@@ -109,6 +109,13 @@ func (r *BootstrapModuleReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 
 clusters:
 	for _, cluster := range clusters.Items {
+		// Don't bother if the cluster isn't marked as ready yet. Creating an assemblage targeting
+		// an unready cluster means lots of failures and back-off.
+		if !cluster.Status.ControlPlaneReady {
+			// TODO: should this be a separate field in the summary, e.g., "waiting"?
+			log.Info("waiting for cluster to be ready", "name", cluster.GetName())
+			continue
+		}
 		summary.Total++
 		requiredAsm[cluster.GetName()] = struct{}{}
 

module/controllers/module_controller.go

@@ -82,6 +82,13 @@ func (r *ModuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 
 clusters:
 	for _, cluster := range clusters.Items {
+		// Don't bother if the cluster isn't marked as ready yet. Creating an assemblage targeting
+		// an unready cluster means lots of failures and back-off.
+		if !cluster.Status.ControlPlaneReady {
+			// TODO: should this be a separate field in the summary, e.g., "waiting"?
+			log.Info("waiting for cluster to be ready", "name", cluster.GetName())
+			continue
+		}
 		summary.Total++
 		// This loop makes sure every cluster that matches the
 		// selector has a remote assemblage with the latest definition

module/controllers/module_test.go

@@ -119,6 +119,8 @@ var _ = Describe("modules", func() {
 					"environment": "production",
 				})
 				Expect(k8sClient.Create(context.Background(), cluster)).To(Succeed())
+				cluster.Status.ControlPlaneReady = true
+				Expect(k8sClient.Status().Update(context.Background(), cluster)).To(Succeed())
 			}
 		})
 
@@ -177,6 +179,8 @@ var _ = Describe("modules", func() {
 				newCluster.Name = "newcluster"
 				newCluster.Namespace = namespace.Name
 				Expect(k8sClient.Create(context.TODO(), &newCluster)).To(Succeed())
+				newCluster.Status.ControlPlaneReady = true
+				Expect(k8sClient.Status().Update(context.Background(), &newCluster)).To(Succeed())
 
 				var newAsm fleetv1.ProxyAssemblage
 				Eventually(func() bool {
@@ -326,6 +330,8 @@ var _ = Describe("modules", func() {
 			cluster.Namespace = namespace.Name
 			cluster.Name = "clus-" + randString(5)
 			Expect(k8sClient.Create(context.TODO(), &cluster)).To(Succeed())
+			cluster.Status.ControlPlaneReady = true
+			Expect(k8sClient.Status().Update(context.Background(), &cluster)).To(Succeed())
 
 			module := fleetv1.Module{
 				Spec: fleetv1.ModuleSpec{