Rollout automation

assemblage/controllers/assemblage_controller.go

@@ -9,10 +9,7 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/fluxcd/pkg/apis/meta"
 	"github.com/go-logr/logr"
-	apimeta "k8s.io/apimachinery/pkg/api/meta"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -122,7 +119,7 @@ func (r *AssemblageReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			if syncStatus.State == "" {
 				switch op {
 				case controllerutil.OperationResultNone:
-					syncStatus.State = readyState(&kustom)
+					syncStatus.State = syncapi.KustomizeReadyState(&kustom)
 				default:
 					syncStatus.State = syncapi.StateUpdating
 				}
@@ -141,25 +138,6 @@ func (r *AssemblageReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	return ctrl.Result{}, nil
 }
 
-func readyState(obj meta.ObjectWithStatusConditions) syncapi.SyncState {
-	conditions := obj.GetStatusConditions()
-	c := apimeta.FindStatusCondition(*conditions, meta.ReadyCondition)
-	switch {
-	case c == nil:
-		return syncapi.StateUpdating
-	case c.Status == metav1.ConditionTrue:
-		return syncapi.StateSucceeded
-	case c.Status == metav1.ConditionFalse:
-		if c.Reason == meta.ReconciliationFailedReason {
-			return syncapi.StateFailed
-		} else {
-			return syncapi.StateUpdating
-		}
-	default: // FIXME possibly StateUnknown?
-		return syncapi.StateUpdating
-	}
-}
-
 func makeBindingFunc(ctx context.Context, log logr.Logger, namespacedClient client.Client, bindings []syncapi.Binding, stack []string) func(string) string {
 	memo := map[string]string{}
 	return func(name string) string {

assemblage/go.mod

@@ -4,12 +4,11 @@ go 1.15
 
 require (
 	github.com/fluxcd/kustomize-controller/api v0.12.0
-	github.com/fluxcd/pkg/apis/meta v0.9.0
 	github.com/fluxcd/source-controller/api v0.12.2
 	github.com/go-logr/logr v0.4.0
 	github.com/onsi/ginkgo v1.14.1
 	github.com/onsi/gomega v1.10.2
-	github.com/squaremo/fleeet/pkg v0.0.2
+	github.com/squaremo/fleeet/pkg v0.0.3-rc1
 	k8s.io/api v0.20.4
 	k8s.io/apimachinery v0.21.0
 	k8s.io/client-go v0.20.4

assemblage/go.sum

@@ -373,8 +373,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/squaremo/fleeet/pkg v0.0.2 h1:yMfRYspuDb4S/qdXJ+J1rZjyBLpeNqglRbtXfOseeoM=
-github.com/squaremo/fleeet/pkg v0.0.2/go.mod h1:zmOneM0CcQv4RdOgsOCMVzVIlwt0yEGykPvpQqfSG8s=
+github.com/squaremo/fleeet/pkg v0.0.3-rc1 h1:IVlqw29VOBly03IyRiuUxvNBreFToxlYN0iYbiZwtUc=
+github.com/squaremo/fleeet/pkg v0.0.3-rc1/go.mod h1:zmOneM0CcQv4RdOgsOCMVzVIlwt0yEGykPvpQqfSG8s=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

demo/delete-cluster.sh

@@ -10,10 +10,11 @@ name="$1"
 
 echo "--> removing Cluster object and secret"
 kubectl delete --ignore-not-found secret "$name-kubeconfig"
-kubectl delete --ignore-not-found -f "$name.yaml"
+if [ -f "$name.yaml" ]; then kubectl delete --ignore-not-found -f "$name.yaml"; fi
 
 echo "--> deleting kind cluster $name"
 kind delete cluster --name "$name"
 
-echo "--> cleaning up kubeconfig"
-rm "$name.kubeconfig"
+echo "--> cleaning up kubeconfig and YAML"
+rm -f "$name.kubeconfig"
+rm -f "$name.yaml"

demo/enrol-kind-cluster.sh

@@ -17,7 +17,7 @@ echo "--> create secret $name-kubeconfig"
 kubectl create secret generic "$name-kubeconfig" --from-file="value=$kubeconfig"
 
 host=$(yq eval '.networking.apiServerAddress' "kind.config")
-port=$(yq eval '.clusters[0].cluster.server'"$kubeconfig" | sed 's#https://.*:\([0-9]\{4,5\}\)#\1#')
+port=$(yq eval '.clusters[0].cluster.server' "$kubeconfig" | sed 's#https://.*:\([0-9]\{4,5\}\)#\1#')
 
 echo "<!> Using host $host from kind.config apiServerAddress, this is assumed to be"
 echo "    an IP address accessible from the control cluster. For example, the IP address"

demo/kind.config

@@ -1,4 +1,4 @@
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 networking:
-  apiServerAddress: 192.168.86.33
+  apiServerAddress: 192.168.86.84 # needs to be the IP of a local interface

demo/transcript.md

@@ -136,12 +136,22 @@ This ties another knot: how does a downstream cluster start syncing anything? Th
 a `BootstrapModule`, which will install the required GitOps Toolkit and Fleeet machinery on each
 downstream cluster.
 
-Now create bootstrap modules referring to these bits of repository:
+This goes in another directory, with an indirection in the form of a Kustomization to sync it. That
+is to avoid a chicken-and-egg problem of having a BootstrapModule object in an "earlier" sync (the
+one created by `flux bootstrap`) than the one that defines its type (the `Kustomization` object
+`fleeet-control`, created above).
+
+Create bootstrap modules referring to these bits of repository:
 
 ```bash
+# A directory for the fleet objects
+mkdir fleet
+# Sync the fleet directory
+flux create kustomization fleet-objects --source=flux-system --path=fleet --prune=true --depends-on=fleeet-control --export > upstream/fleet-objects-sync.yaml
+#
 # This bootstrap module will be applied to all downstream clusters that show up in the namespace. The module must be given a particular revision or tag (but not a branch -- that would be the same as using image:latest).
 CONFIG_VERSION=v0.1
-cat > upstream/bootstrap-worker.yaml <<EOF
+cat > fleet/bootstrap-worker.yaml <<EOF
 ---
 apiVersion: fleet.squaremo.dev/v1alpha1
 kind: BootstrapModule
@@ -179,8 +189,8 @@ spec:
 EOF
 #
 # Add it to git, to be synced to the management cluster
-git add upstream/bootstrap-worker.yaml
-git commit -s -m "Add bootstrap modules for downstream"
+git add fleet/bootstrap-worker.yaml upstream/fleet-objects-sync.yaml
+git commit -s -m "Add bootstrap modules and a sync for them"
 git push
 ```
 
@@ -193,18 +203,37 @@ git push
 sh create-cluster.sh cluster-1
 ```
 
+This creates a `kind` cluster and saves a kubeconfig for it to the current directory (e.g., as
+`cluster-1.kubeconfig`). So that you can create a bunch of clusters, which takes a while, and make
+it look like they are coming online later quickly, there's a separate step to make the cluster
+appear in the control plane:
+
+```bash
+sh enrol-cluster.sh cluster-1.kubeconfig
+```
+
+(NB you supply the kubeconfig file; the script will work out the cluster name from that.)
+
 See what happened in the downstream cluster:
 
 ```bash
 kubectl --kubeconfig ./cluster-1.kubeconfig get namespace
 # and explore from there ...
 ```
 
+If something doesn't seem like it's working, you can have a look at the cluster nodes:
+
+```bash
+kubectl --kubeconfig ./cluster-1.kubeconfig describe nodes
+```
+
+You may see problems with the CNI plugin not working. Upgrading `kind` and trying again might help.
+
 ### Create a module
 
 ```bash
 # Create a module and add that to syncing
-cat > upstream/podinfo-module.yaml <<EOF
+cat > fleet/podinfo-module.yaml <<EOF
 apiVersion: fleet.squaremo.dev/v1alpha1
 kind: Module
 metadata:
@@ -222,7 +251,7 @@ spec:
       kustomize:
         path: ./kustomize
 EOF
-git add upstream/podinfo-module.yaml
+git add fleet/podinfo-module.yaml
 git commit -s -m "Add module for podinfo app"
 git push
 ```
@@ -247,3 +276,22 @@ kubectl --kubeconfig ./cluster-a.kubeconfig get deploy
 NAME      READY   UP-TO-DATE   AVAILABLE   AGE
 podinfo   2/2     2            2           5m17s
 ```
+
+# Cleaning up
+
+The script `delete-cluster.sh` will remove the Kubernetes resources associated with a cluster, the
+files created by scripts, and delete the `kind` cluster itself.
+
+If you need to clean things up manually, this is what might need to be deleted for, say,
+`cluster-1`:
+
+In the directory in which you ran scripts:
+ - a file `cluster-1.kubeconfig`
+ - a file `cluster-1.yaml`
+
+In the cluster:
+ - a `Cluster` object named `cluster-1`
+ - a `Secret` named `cluster-1-kubeconfig`
+ - a `RemoteAssemblage` named `cluster-1`
+ - Kustomization objects with names including `cluster-1`
+ - a `ProxyAssemblage` named `cluster-1`