Use Cluster Mover in create and delete workflows

aws · Mar 28, 2024 · 167de78 · 167de78
1 parent 689f338
commit 167de78
Show file tree

Hide file tree

Showing 17 changed files with 191 additions and 148 deletions.
diff --git a/Makefile b/Makefile
@@ -568,7 +568,7 @@ mocks: ## Generate mocks
 	${MOCKGEN} -destination=pkg/bootstrapper/mocks/bootstrapper.go -package=mocks "github.com/aws/eks-anywhere/pkg/bootstrapper" ClusterClient
 	${MOCKGEN} -destination=pkg/git/providers/github/mocks/github.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/providers/github" GithubClient
 	${MOCKGEN} -destination=pkg/git/mocks/git.go -package=mocks "github.com/aws/eks-anywhere/pkg/git" Client,ProviderClient
-	${MOCKGEN} -destination=pkg/workflows/interfaces/mocks/clients.go -package=mocks "github.com/aws/eks-anywhere/pkg/workflows/interfaces" Bootstrapper,ClusterManager,GitOpsManager,Validator,CAPIManager,EksdInstaller,EksdUpgrader,PackageInstaller,ClusterUpgrader,ClusterCreator,ClientFactory,EksaInstaller,ClusterDeleter
+	${MOCKGEN} -destination=pkg/workflows/interfaces/mocks/clients.go -package=mocks "github.com/aws/eks-anywhere/pkg/workflows/interfaces" Bootstrapper,ClusterManager,GitOpsManager,Validator,CAPIManager,EksdInstaller,EksdUpgrader,PackageInstaller,ClusterUpgrader,ClusterCreator,ClientFactory,EksaInstaller,ClusterDeleter,ClusterMover
 	${MOCKGEN} -destination=pkg/git/gogithub/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/gogithub" Client
 	${MOCKGEN} -destination=pkg/git/gitclient/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/gitclient" GoGit
 	${MOCKGEN} -destination=pkg/validations/mocks/docker.go -package=mocks "github.com/aws/eks-anywhere/pkg/validations" DockerExecutable

diff --git a/cmd/eksctl-anywhere/cmd/createcluster.go b/cmd/eksctl-anywhere/cmd/createcluster.go
@@ -190,7 +190,8 @@ func (cc *createClusterOptions) createCluster(cmd *cobra.Command, _ []string) er
 		WithCreateClusterDefaulter(createCLIConfig).
 		WithClusterApplier().
 		WithKubeconfigWriter(clusterSpec.Cluster).
-		WithClusterCreator(clusterSpec.Cluster)
+		WithClusterCreator(clusterSpec.Cluster).
+		WithClusterMover()
 
 	if cc.timeoutOptions.noTimeouts {
 		factory.WithNoTimeouts()
@@ -274,6 +275,7 @@ func (cc *createClusterOptions) createCluster(cmd *cobra.Command, _ []string) er
 			deps.PackageInstaller,
 			deps.ClusterCreator,
 			deps.EksaInstaller,
+			deps.ClusterMover,
 		)
 
 		err = createMgmtCluster.Run(ctx, clusterSpec, createValidations)

diff --git a/cmd/eksctl-anywhere/cmd/deletecluster.go b/cmd/eksctl-anywhere/cmd/deletecluster.go
@@ -126,6 +126,7 @@ func (dc *deleteClusterOptions) deleteCluster(ctx context.Context) error {
 		WithEksdInstaller().
 		WithEKSAInstaller().
 		WithUnAuthKubeClient().
+		WithClusterMover().
 		Build(ctx)
 	if err != nil {
 		return err
@@ -154,7 +155,7 @@ func (dc *deleteClusterOptions) deleteCluster(ctx context.Context) error {
 		deleteWorkload := workload.NewDelete(deps.Provider, deps.Writer, deps.ClusterManager, deps.ClusterDeleter, deps.GitOpsFlux)
 		err = deleteWorkload.Run(ctx, cluster, clusterSpec)
 	} else {
-		deleteManagement := management.NewDelete(deps.Bootstrapper, deps.Provider, deps.Writer, deps.ClusterManager, deps.GitOpsFlux, deps.ClusterDeleter, deps.EksdInstaller, deps.EksaInstaller, deps.UnAuthKubeClient)
+		deleteManagement := management.NewDelete(deps.Bootstrapper, deps.Provider, deps.Writer, deps.ClusterManager, deps.GitOpsFlux, deps.ClusterDeleter, deps.EksdInstaller, deps.EksaInstaller, deps.UnAuthKubeClient, deps.ClusterMover)
 		err = deleteManagement.Run(ctx, cluster, clusterSpec)
 	}
 	cleanup(deps, &err)

diff --git a/pkg/clustermanager/cluster_manager.go b/pkg/clustermanager/cluster_manager.go
@@ -693,6 +693,20 @@ func (c *ClusterManager) ResumeCAPIWorkloadClusters(ctx context.Context, managem
 	return nil
 }
 
+func (c *ClusterManager) AllowDeleteWhilePaused(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec) error {
+	return c.allowDeleteWhilePaused(ctx, cluster, clusterSpec.Cluster)
+}
+
+func (c *ClusterManager) allowDeleteWhilePaused(ctx context.Context, clusterCreds *types.Cluster, cluster *v1alpha1.Cluster) error {
+	allowDelete := map[string]string{v1alpha1.AllowDeleteWhenPausedAnnotation: "true"}
+
+	if err := c.clusterClient.UpdateAnnotationInNamespace(ctx, cluster.ResourceType(), cluster.Name, allowDelete, clusterCreds, cluster.Namespace); err != nil {
+		return fmt.Errorf("updating paused annotation in cluster reconciliation: %v", err)
+	}
+
+	return nil
+}
+
 func (c *ClusterManager) PauseEKSAControllerReconcile(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec, provider providers.Provider) error {
 	if clusterSpec.Cluster.IsSelfManaged() {
 		return c.pauseEksaReconcileForManagementAndWorkloadClusters(ctx, cluster, clusterSpec, provider)

diff --git a/pkg/task/task.go b/pkg/task/task.go
@@ -52,6 +52,7 @@ type CommandContext struct {
 	OriginalError         error
 	BackupClusterStateDir string
 	ForceCleanup          bool
+	ClusterMover          interfaces.ClusterMover
 }
 
 func (c *CommandContext) SetError(err error) {

diff --git a/pkg/workflows/create_prep.go b/pkg/workflows/create_prep.go
@@ -7,22 +7,11 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	"github.com/aws/eks-anywhere/pkg/workflows/interfaces"
+	"github.com/aws/eks-anywhere/pkg/clients/kubernetes"
 )
 
 // CreateNamespaceIfNotPresent creates the namespace on the cluster if it does not already exist.
-func CreateNamespaceIfNotPresent(ctx context.Context, namespace, kubeconfig string, clientFactory interfaces.ClientFactory) error {
-	client, err := clientFactory.BuildClientFromKubeconfig(kubeconfig)
-	if err != nil {
-		return err
-	}
-
-	if err := client.Get(ctx, namespace, "", &corev1.Namespace{}); err != nil && !errors.IsNotFound(err) {
-		return err
-	} else if err == nil {
-		return nil
-	}
-
+func CreateNamespaceIfNotPresent(ctx context.Context, namespace string, client kubernetes.Client) error {
 	ns := &corev1.Namespace{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "v1",
@@ -33,7 +22,7 @@ func CreateNamespaceIfNotPresent(ctx context.Context, namespace, kubeconfig stri
 		},
 	}
 
-	if err = client.Create(ctx, ns); err != nil {
+	if err := client.Create(ctx, ns); err != nil && !errors.IsAlreadyExists(err) {
 		return err
 	}
 

diff --git a/pkg/workflows/create_prep_test.go b/pkg/workflows/create_prep_test.go
@@ -50,72 +50,35 @@ func newNamespace(name string) *corev1.Namespace {
 
 func TestCreateNamespaceNotExistsSuccess(t *testing.T) {
 	test := newCreatePrepTest(t)
-	kubeconfig := "testpath"
 	namespace := "test-ns"
 
-	test.clientFactory.EXPECT().BuildClientFromKubeconfig(kubeconfig).Return(test.client, nil)
-	test.client.EXPECT().Get(test.ctx, namespace, "", &corev1.Namespace{}).Return(apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: ""}, ""))
 	test.client.EXPECT().Create(test.ctx, newNamespace(namespace)).Return(nil)
 
-	err := workflows.CreateNamespaceIfNotPresent(test.ctx, namespace, kubeconfig, test.clientFactory)
+	err := workflows.CreateNamespaceIfNotPresent(test.ctx, namespace, test.client)
 	if err != nil {
 		t.Fatalf("Expected nil, but got %v", err)
 	}
 }
 
 func TestCreateNamespaceAlreadyExistsSuccess(t *testing.T) {
 	test := newCreatePrepTest(t)
-	kubeconfig := "testpath"
 	namespace := "default"
 
-	test.clientFactory.EXPECT().BuildClientFromKubeconfig(kubeconfig).Return(test.client, nil)
-	test.client.EXPECT().Get(test.ctx, namespace, "", &corev1.Namespace{}).Return(nil)
+	test.client.EXPECT().Create(test.ctx, newNamespace(namespace)).Return(apierrors.NewAlreadyExists(schema.GroupResource{Group: "", Resource: ""}, ""))
 
-	err := workflows.CreateNamespaceIfNotPresent(test.ctx, namespace, kubeconfig, test.clientFactory)
+	err := workflows.CreateNamespaceIfNotPresent(test.ctx, namespace, test.client)
 	if err != nil {
 		t.Fatalf("Expected nil, but got %v", err)
 	}
 }
 
-func TestCreateNamespaceBuildClientFail(t *testing.T) {
-	test := newCreatePrepTest(t)
-	kubeconfig := "testpath"
-	namespace := "test-ns"
-
-	test.clientFactory.EXPECT().BuildClientFromKubeconfig(kubeconfig).Return(test.client, fmt.Errorf(""))
-
-	err := workflows.CreateNamespaceIfNotPresent(test.ctx, namespace, kubeconfig, test.clientFactory)
-
-	if err == nil {
-		t.Fatalf("Expected error, but got nil")
-	}
-}
-
-func TestCreateNamespaceGetNamespaceFail(t *testing.T) {
-	test := newCreatePrepTest(t)
-	kubeconfig := "testpath"
-	namespace := "test-ns"
-
-	test.clientFactory.EXPECT().BuildClientFromKubeconfig(kubeconfig).Return(test.client, nil)
-	test.client.EXPECT().Get(test.ctx, namespace, "", &corev1.Namespace{}).Return(fmt.Errorf(""))
-
-	err := workflows.CreateNamespaceIfNotPresent(test.ctx, namespace, kubeconfig, test.clientFactory)
-
-	if err == nil {
-		t.Fatalf("Expected error, but got nil")
-	}
-}
-
 func TestCreateNamespaceFail(t *testing.T) {
 	test := newCreatePrepTest(t)
-	kubeconfig := "testpath"
 	namespace := "test-ns"
 
-	test.clientFactory.EXPECT().BuildClientFromKubeconfig(kubeconfig).Return(test.client, nil)
-	test.client.EXPECT().Get(test.ctx, namespace, "", &corev1.Namespace{}).Return(apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: ""}, ""))
 	test.client.EXPECT().Create(test.ctx, newNamespace(namespace)).Return(fmt.Errorf(""))
 
-	err := workflows.CreateNamespaceIfNotPresent(test.ctx, namespace, kubeconfig, test.clientFactory)
+	err := workflows.CreateNamespaceIfNotPresent(test.ctx, namespace, test.client)
 
 	if err == nil {
 		t.Fatalf("Expected error, but got nil")

diff --git a/pkg/workflows/interfaces/interfaces.go b/pkg/workflows/interfaces/interfaces.go
@@ -98,3 +98,8 @@ type EksaInstaller interface {
 type ClusterDeleter interface {
 	Run(ctx context.Context, spec *cluster.Spec, managementCluster types.Cluster) error
 }
+
+// ClusterMover moves the EKS-A cluster.
+type ClusterMover interface {
+	Move(ctx context.Context, spec *cluster.Spec, srcClient, dstClient kubernetes.Client) error
+}
diff --git a/pkg/workflows/interfaces/mocks/clients.go b/pkg/workflows/interfaces/mocks/clients.go
diff --git a/pkg/workflows/management/create.go b/pkg/workflows/management/create.go
@@ -22,6 +22,7 @@ type Create struct {
 	packageInstaller interfaces.PackageInstaller
 	clusterCreator   interfaces.ClusterCreator
 	eksaInstaller    interfaces.EksaInstaller
+	clusterMover     interfaces.ClusterMover
 }
 
 // NewCreate builds a new create construct.
@@ -32,6 +33,7 @@ func NewCreate(bootstrapper interfaces.Bootstrapper,
 	packageInstaller interfaces.PackageInstaller,
 	clusterCreator interfaces.ClusterCreator,
 	eksaInstaller interfaces.EksaInstaller,
+	mover interfaces.ClusterMover,
 ) *Create {
 	return &Create{
 		bootstrapper:     bootstrapper,
@@ -44,6 +46,7 @@ func NewCreate(bootstrapper interfaces.Bootstrapper,
 		packageInstaller: packageInstaller,
 		clusterCreator:   clusterCreator,
 		eksaInstaller:    eksaInstaller,
+		clusterMover:     mover,
 	}
 }
 
@@ -62,6 +65,7 @@ func (c *Create) Run(ctx context.Context, clusterSpec *cluster.Spec, validator i
 		PackageInstaller: c.packageInstaller,
 		ClusterCreator:   c.clusterCreator,
 		EksaInstaller:    c.eksaInstaller,
+		ClusterMover:     c.clusterMover,
 	}
 
 	return task.NewTaskRunner(&setupAndValidateCreate{}, c.writer).RunTask(ctx, commandContext)

diff --git a/pkg/workflows/management/create_install_eksa.go b/pkg/workflows/management/create_install_eksa.go
@@ -58,15 +58,32 @@ func (s *installEksaComponentsOnWorkloadTask) Run(ctx context.Context, commandCo
 	commandContext.ClusterSpec.Cluster.AddManagedByCLIAnnotation()
 	commandContext.ClusterSpec.Cluster.SetManagementComponentsVersion(commandContext.ClusterSpec.EKSARelease.Spec.Version)
 
+	srcClient, err := commandContext.ClientFactory.BuildClientFromKubeconfig(commandContext.BootstrapCluster.KubeconfigFile)
+	if err != nil {
+		commandContext.SetError(err)
+		return &workflows.CollectMgmtClusterDiagnosticsTask{}
+	}
+
+	dstClient, err := commandContext.ClientFactory.BuildClientFromKubeconfig(commandContext.WorkloadCluster.KubeconfigFile)
+	if err != nil {
+		commandContext.SetError(err)
+		return &workflows.CollectMgmtClusterDiagnosticsTask{}
+	}
+
 	if commandContext.ClusterSpec.Cluster.Namespace != "" {
-		if err := workflows.CreateNamespaceIfNotPresent(ctx, commandContext.ClusterSpec.Cluster.Namespace, commandContext.WorkloadCluster.KubeconfigFile, commandContext.ClientFactory); err != nil {
+		if err := workflows.CreateNamespaceIfNotPresent(ctx, commandContext.ClusterSpec.Cluster.Namespace, dstClient); err != nil {
 			commandContext.SetError(err)
 			return &workflows.CollectMgmtClusterDiagnosticsTask{}
 		}
 	}
 
-	logger.Info("Applying cluster spec to workload cluster")
-	if err = commandContext.ClusterCreator.Run(ctx, commandContext.ClusterSpec, *commandContext.WorkloadCluster); err != nil {
+	logger.Info("Moving cluster spec to workload cluster")
+	if err = commandContext.ClusterMover.Move(ctx, commandContext.ClusterSpec, srcClient, dstClient); err != nil {
+		commandContext.SetError(err)
+		return &workflows.CollectMgmtClusterDiagnosticsTask{}
+	}
+
+	if err = commandContext.ClusterManager.ResumeEKSAControllerReconcile(ctx, commandContext.WorkloadCluster, commandContext.ClusterSpec, commandContext.Provider); err != nil {
 		commandContext.SetError(err)
 		return &workflows.CollectMgmtClusterDiagnosticsTask{}
 	}

diff --git a/pkg/workflows/management/create_test.go b/pkg/workflows/management/create_test.go
@@ -49,6 +49,7 @@ type createTestSetup struct {
 	workflow             *management.Create
 	client               *clientmocks.MockClient
 	clientFactory        *mocks.MockClientFactory
+	mover                *mocks.MockClusterMover
 }
 
 func newCreateTest(t *testing.T) *createTestSetup {
@@ -71,6 +72,7 @@ func newCreateTest(t *testing.T) *createTestSetup {
 	validator := mocks.NewMockValidator(mockCtrl)
 	client := clientmocks.NewMockClient(mockCtrl)
 	clientFactory := mocks.NewMockClientFactory(mockCtrl)
+	mover := mocks.NewMockClusterMover(mockCtrl)
 
 	workflow := management.NewCreate(
 		bootstrapper,
@@ -83,6 +85,7 @@ func newCreateTest(t *testing.T) *createTestSetup {
 		packageInstaller,
 		clusterCreator,
 		eksaInstaller,
+		mover,
 	)
 
 	for _, e := range featureEnvVars {
@@ -119,6 +122,7 @@ func newCreateTest(t *testing.T) *createTestSetup {
 		managementComponents: managementComponents,
 		clusterSpec:          clusterSpec,
 		client:               client,
+		mover:                mover,
 	}
 }
 
@@ -231,9 +235,13 @@ func (c *createTestSetup) expectInstallEksaComponentsWorkload(err1, err2, err3 e
 		c.eksdInstaller.EXPECT().InstallEksdManifest(
 			c.ctx, c.clusterSpec, c.workloadCluster),
 
-		c.clientFactory.EXPECT().BuildClientFromKubeconfig(c.workloadCluster.KubeconfigFile).Return(c.client, err3),
+		c.clientFactory.EXPECT().BuildClientFromKubeconfig(c.bootstrapCluster.KubeconfigFile).Return(c.client, nil),
 
-		c.clusterCreator.EXPECT().Run(c.ctx, c.clusterSpec, *c.workloadCluster).Return(err2),
+		c.clientFactory.EXPECT().BuildClientFromKubeconfig(c.workloadCluster.KubeconfigFile).Return(c.client, nil),
+
+		c.mover.EXPECT().Move(c.ctx, c.clusterSpec, c.client, c.client).Return(err2),
+
+		c.clusterManager.EXPECT().ResumeEKSAControllerReconcile(c.ctx, c.workloadCluster, c.clusterSpec, c.provider).Return(err3).MaxTimes(1),
 	)
 }
 
@@ -747,33 +755,19 @@ func TestCreateEKSAWorkloadFailure(t *testing.T) {
 	}
 }
 
-func TestCreateEKSAWorkloadNamespaceFailure(t *testing.T) {
+func TestCreateEKSAResumeWorkloadFailure(t *testing.T) {
 	test := newCreateTest(t)
 	test.expectSetup()
 	test.expectPreflightValidationsToPass()
 	test.expectCreateBootstrap()
 	test.expectCAPIInstall(nil, nil, nil)
 	test.expectInstallEksaComponentsBootstrap(nil, nil, nil, nil)
 	test.expectCreateWorkload(nil, nil, nil, nil, nil, nil)
-	test.expectCreateNamespace()
 	test.expectInstallResourcesOnManagementTask(nil)
 	test.expectPauseReconcile(nil)
 	test.expectMoveManagement(nil)
-	gomock.InOrder(
-
-		test.eksdInstaller.EXPECT().InstallEksdCRDs(test.ctx, test.clusterSpec, test.workloadCluster),
-
-		test.eksaInstaller.EXPECT().Install(
-			test.ctx, logger.Get(), test.workloadCluster, test.managementComponents, test.clusterSpec),
-
-		test.provider.EXPECT().InstallCustomProviderComponents(
-			test.ctx, test.workloadCluster.KubeconfigFile),
-
-		test.eksdInstaller.EXPECT().InstallEksdManifest(
-			test.ctx, test.clusterSpec, test.workloadCluster),
-
-		test.clientFactory.EXPECT().BuildClientFromKubeconfig(test.workloadCluster.KubeconfigFile).Return(test.client, fmt.Errorf("")),
-	)
+	test.expectInstallEksaComponentsWorkload(nil, nil, fmt.Errorf("test"))
+	test.expectCreateNamespace()
 
 	test.clusterManager.EXPECT().SaveLogsManagementCluster(test.ctx, test.clusterSpec, test.bootstrapCluster)