Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use Cluster Mover in create and delete workflows #7909

Merged
merged 3 commits into from
Apr 4, 2024
Merged

Use Cluster Mover in create and delete workflows #7909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
167de78
Use Cluster Mover in create and delete workflows
tatlat Mar 28, 2024
2aef03b
update tests and mocks
tatlat Apr 1, 2024
006453e
additional test coverage
tatlat Apr 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -568,7 +568,7 @@ mocks: ## Generate mocks
${MOCKGEN} -destination=pkg/bootstrapper/mocks/bootstrapper.go -package=mocks "github.com/aws/eks-anywhere/pkg/bootstrapper" ClusterClient
${MOCKGEN} -destination=pkg/git/providers/github/mocks/github.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/providers/github" GithubClient
${MOCKGEN} -destination=pkg/git/mocks/git.go -package=mocks "github.com/aws/eks-anywhere/pkg/git" Client,ProviderClient
${MOCKGEN} -destination=pkg/workflows/interfaces/mocks/clients.go -package=mocks "github.com/aws/eks-anywhere/pkg/workflows/interfaces" Bootstrapper,ClusterManager,GitOpsManager,Validator,CAPIManager,EksdInstaller,EksdUpgrader,PackageInstaller,ClusterUpgrader,ClusterCreator,ClientFactory,EksaInstaller,ClusterDeleter
${MOCKGEN} -destination=pkg/workflows/interfaces/mocks/clients.go -package=mocks "github.com/aws/eks-anywhere/pkg/workflows/interfaces" Bootstrapper,ClusterManager,GitOpsManager,Validator,CAPIManager,EksdInstaller,EksdUpgrader,PackageInstaller,ClusterUpgrader,ClusterCreator,ClientFactory,EksaInstaller,ClusterDeleter,ClusterMover
${MOCKGEN} -destination=pkg/git/gogithub/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/gogithub" Client
${MOCKGEN} -destination=pkg/git/gitclient/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/gitclient" GoGit
${MOCKGEN} -destination=pkg/validations/mocks/docker.go -package=mocks "github.com/aws/eks-anywhere/pkg/validations" DockerExecutable
Expand Down
4 changes: 3 additions & 1 deletion cmd/eksctl-anywhere/cmd/createcluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,8 @@ func (cc *createClusterOptions) createCluster(cmd *cobra.Command, _ []string) er
WithCreateClusterDefaulter(createCLIConfig).
WithClusterApplier().
WithKubeconfigWriter(clusterSpec.Cluster).
WithClusterCreator(clusterSpec.Cluster)
WithClusterCreator(clusterSpec.Cluster).
WithClusterMover()

if cc.timeoutOptions.noTimeouts {
factory.WithNoTimeouts()
Expand Down Expand Up @@ -274,6 +275,7 @@ func (cc *createClusterOptions) createCluster(cmd *cobra.Command, _ []string) er
deps.PackageInstaller,
deps.ClusterCreator,
deps.EksaInstaller,
deps.ClusterMover,
)

err = createMgmtCluster.Run(ctx, clusterSpec, createValidations)
Expand Down
3 changes: 2 additions & 1 deletion cmd/eksctl-anywhere/cmd/deletecluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ func (dc *deleteClusterOptions) deleteCluster(ctx context.Context) error {
WithEksdInstaller().
WithEKSAInstaller().
WithUnAuthKubeClient().
WithClusterMover().
Build(ctx)
if err != nil {
return err
Expand Down Expand Up @@ -154,7 +155,7 @@ func (dc *deleteClusterOptions) deleteCluster(ctx context.Context) error {
deleteWorkload := workload.NewDelete(deps.Provider, deps.Writer, deps.ClusterManager, deps.ClusterDeleter, deps.GitOpsFlux)
err = deleteWorkload.Run(ctx, cluster, clusterSpec)
} else {
deleteManagement := management.NewDelete(deps.Bootstrapper, deps.Provider, deps.Writer, deps.ClusterManager, deps.GitOpsFlux, deps.ClusterDeleter, deps.EksdInstaller, deps.EksaInstaller, deps.UnAuthKubeClient)
deleteManagement := management.NewDelete(deps.Bootstrapper, deps.Provider, deps.Writer, deps.ClusterManager, deps.GitOpsFlux, deps.ClusterDeleter, deps.EksdInstaller, deps.EksaInstaller, deps.UnAuthKubeClient, deps.ClusterMover)
err = deleteManagement.Run(ctx, cluster, clusterSpec)
}
cleanup(deps, &err)
Expand Down
82 changes: 82 additions & 0 deletions pkg/clustermanager/cluster_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -674,6 +674,73 @@ func (c *ClusterManager) PauseCAPIWorkloadClusters(ctx context.Context, manageme
return nil
}

func (c *ClusterManager) resumeEksaReconcileForManagementAndWorkloadClusters(ctx context.Context, managementCluster *types.Cluster, clusterSpec *cluster.Spec, provider providers.Provider) error {
clusters := &v1alpha1.ClusterList{}
err := c.clusterClient.ListObjects(ctx, eksaClusterResourceType, clusterSpec.Cluster.Namespace, managementCluster.KubeconfigFile, clusters)
if err != nil {
return err
}

for _, w := range clusters.Items {
if w.ManagedBy() != clusterSpec.Cluster.Name {
continue
}

if err := c.resumeReconcileForCluster(ctx, managementCluster, &w, provider); err != nil {
return err
}
}

return nil
}

// ResumeEKSAControllerReconcile resumes a paused EKS-Anywhere cluster.
func (c *ClusterManager) ResumeEKSAControllerReconcile(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec, provider providers.Provider) error {
// clear pause annotation
clusterSpec.Cluster.ClearPauseAnnotation()
provider.DatacenterConfig(clusterSpec).ClearPauseAnnotation()

if clusterSpec.Cluster.IsSelfManaged() {
return c.resumeEksaReconcileForManagementAndWorkloadClusters(ctx, cluster, clusterSpec, provider)
}

return c.resumeReconcileForCluster(ctx, cluster, clusterSpec.Cluster, provider)
}

func (c *ClusterManager) resumeReconcileForCluster(ctx context.Context, clusterCreds *types.Cluster, cluster *v1alpha1.Cluster, provider providers.Provider) error {
pausedAnnotation := cluster.PausedAnnotation()
err := c.clusterClient.RemoveAnnotationInNamespace(ctx, provider.DatacenterResourceType(), cluster.Spec.DatacenterRef.Name, pausedAnnotation, clusterCreds, cluster.Namespace)
if err != nil {
return fmt.Errorf("removing paused annotation when resuming datacenterconfig reconciliation: %v", err)
}

if provider.MachineResourceType() != "" {
for _, machineConfigRef := range cluster.MachineConfigRefs() {
err = c.clusterClient.RemoveAnnotationInNamespace(ctx, provider.MachineResourceType(), machineConfigRef.Name, pausedAnnotation, clusterCreds, cluster.Namespace)
if err != nil {
return fmt.Errorf("removing paused annotation when resuming reconciliation for machine config %s: %v", machineConfigRef.Name, err)
}
}
}

err = c.clusterClient.RemoveAnnotationInNamespace(ctx, cluster.ResourceType(), cluster.Name, pausedAnnotation, clusterCreds, cluster.Namespace)
if err != nil {
return fmt.Errorf("removing paused annotation when resuming cluster reconciliation: %v", err)
}

if err = c.clusterClient.RemoveAnnotationInNamespace(ctx,
cluster.ResourceType(),
cluster.Name,
v1alpha1.ManagedByCLIAnnotation,
clusterCreds,
cluster.Namespace,
); err != nil {
return fmt.Errorf("removing managed by CLI annotation when resuming cluster reconciliation: %v", err)
}

return nil
}

// ResumeCAPIWorkloadClusters resumes all workload CAPI clusters except the management cluster.
func (c *ClusterManager) ResumeCAPIWorkloadClusters(ctx context.Context, managementCluster *types.Cluster) error {
clusters, err := c.clusterClient.GetClusters(ctx, managementCluster)
Expand All @@ -693,6 +760,21 @@ func (c *ClusterManager) ResumeCAPIWorkloadClusters(ctx context.Context, managem
return nil
}

// AllowDeleteWhilePaused allows the deletion of paused clusters.
func (c *ClusterManager) AllowDeleteWhilePaused(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec) error {
return c.allowDeleteWhilePaused(ctx, cluster, clusterSpec.Cluster)
}

func (c *ClusterManager) allowDeleteWhilePaused(ctx context.Context, clusterCreds *types.Cluster, cluster *v1alpha1.Cluster) error {
allowDelete := map[string]string{v1alpha1.AllowDeleteWhenPausedAnnotation: "true"}

if err := c.clusterClient.UpdateAnnotationInNamespace(ctx, cluster.ResourceType(), cluster.Name, allowDelete, clusterCreds, cluster.Namespace); err != nil {
return fmt.Errorf("updating paused annotation in cluster reconciliation: %v", err)
}

return nil
}

func (c *ClusterManager) PauseEKSAControllerReconcile(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec, provider providers.Provider) error {
if clusterSpec.Cluster.IsSelfManaged() {
return c.pauseEksaReconcileForManagementAndWorkloadClusters(ctx, cluster, clusterSpec, provider)
Expand Down
97 changes: 78 additions & 19 deletions pkg/clustermanager/cluster_manager_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -771,7 +771,7 @@ func TestPauseEKSAControllerReconcileWorkloadCluster(t *testing.T) {
tt.Expect(tt.clusterManager.PauseEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).To(Succeed())
}

func TestPauseEKSAControllerReconcileWorkloadClusterUpdateAnnotationError(t *testing.T) {
func TestResumeEKSAControllerReconcileWorkloadClusterUpdateAnnotationError(t *testing.T) {
tt := newTest(t, clustermanager.WithRetrier(retrier.NewWithMaxRetries(1, 0)))
tt.clusterSpec.Cluster = &v1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -788,15 +788,26 @@ func TestPauseEKSAControllerReconcileWorkloadClusterUpdateAnnotationError(t *tes
},
}

datacenterConfig := &v1alpha1.VSphereDatacenterConfig{
ObjectMeta: metav1.ObjectMeta{
Name: tt.clusterName,
},
Spec: v1alpha1.VSphereDatacenterConfigSpec{
Insecure: true,
},
}
pauseAnnotation := "anywhere.eks.amazonaws.com/paused"

tt.mocks.provider.EXPECT().DatacenterResourceType().Return(eksaVSphereDatacenterResourceType)
tt.mocks.provider.EXPECT().MachineResourceType().Return("")
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, eksaVSphereDatacenterResourceType, tt.clusterSpec.Cluster.Spec.DatacenterRef.Name, expectedPauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, eksaClusterResourceType, tt.clusterSpec.Cluster.Name, expectedPauseAnnotation, tt.cluster, "").Return(errors.New("pause eksa cluster error"))
tt.mocks.provider.EXPECT().DatacenterConfig(tt.clusterSpec).Return(datacenterConfig)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(tt.ctx, eksaVSphereDatacenterResourceType, tt.clusterSpec.Cluster.Spec.DatacenterRef.Name, pauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(tt.ctx, eksaClusterResourceType, tt.clusterSpec.Cluster.Name, pauseAnnotation, tt.cluster, "").Return(errors.New("pause eksa cluster error"))

tt.Expect(tt.clusterManager.PauseEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).NotTo(Succeed())
tt.Expect(tt.clusterManager.ResumeEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).NotTo(Succeed())
}

func TestPauseEKSAControllerReconcileManagementCluster(t *testing.T) {
func TestResumeEKSAControllerReconcileManagementCluster(t *testing.T) {
tt := newTest(t)
tt.clusterSpec.Cluster = &v1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -813,6 +824,18 @@ func TestPauseEKSAControllerReconcileManagementCluster(t *testing.T) {
},
}

tt.clusterSpec.Cluster.PauseReconcile()

datacenterConfig := &v1alpha1.VSphereDatacenterConfig{
ObjectMeta: metav1.ObjectMeta{
Name: tt.clusterName,
},
Spec: v1alpha1.VSphereDatacenterConfigSpec{
Insecure: true,
},
}
pauseAnnotation := "anywhere.eks.amazonaws.com/paused"

tt.mocks.client.EXPECT().
ListObjects(tt.ctx, eksaClusterResourceType, "", "", &v1alpha1.ClusterList{}).
DoAndReturn(func(_ context.Context, _, _, _ string, obj *v1alpha1.ClusterList) error {
Expand Down Expand Up @@ -851,34 +874,31 @@ func TestPauseEKSAControllerReconcileManagementCluster(t *testing.T) {
})
tt.mocks.provider.EXPECT().DatacenterResourceType().Return(eksaVSphereDatacenterResourceType).Times(2)
tt.mocks.provider.EXPECT().MachineResourceType().Return("").Times(2)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, eksaVSphereDatacenterResourceType, tt.clusterSpec.Cluster.Spec.DatacenterRef.Name, expectedPauseAnnotation, tt.cluster, "").Return(nil).Times(2)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, eksaClusterResourceType, tt.clusterSpec.Cluster.Name, expectedPauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(
tt.mocks.provider.EXPECT().DatacenterConfig(tt.clusterSpec).Return(datacenterConfig)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(tt.ctx, eksaVSphereDatacenterResourceType, tt.clusterSpec.Cluster.Spec.DatacenterRef.Name, pauseAnnotation, tt.cluster, "").Return(nil).Times(2)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(tt.ctx, eksaClusterResourceType, tt.clusterSpec.Cluster.Name, pauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(
tt.ctx,
eksaClusterResourceType,
tt.clusterSpec.Cluster.Name,
map[string]string{
v1alpha1.ManagedByCLIAnnotation: "true",
},
v1alpha1.ManagedByCLIAnnotation,
tt.cluster,
"",
).Return(nil)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, eksaClusterResourceType, "workload-cluster-1", expectedPauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(tt.ctx, eksaClusterResourceType, "workload-cluster-1", pauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(
tt.ctx,
eksaClusterResourceType,
"workload-cluster-1",
map[string]string{
v1alpha1.ManagedByCLIAnnotation: "true",
},
v1alpha1.ManagedByCLIAnnotation,
tt.cluster,
"",
).Return(nil)

tt.Expect(tt.clusterManager.PauseEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).To(Succeed())
tt.Expect(tt.clusterManager.ResumeEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).To(Succeed())
}

func TestPauseEKSAControllerReconcileManagementClusterListObjectsError(t *testing.T) {
func TestResumeEKSAControllerReconcileManagementClusterListObjectsError(t *testing.T) {
tt := newTest(t, clustermanager.WithRetrier(retrier.NewWithMaxRetries(1, 0)))
tt.clusterSpec.Cluster = &v1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -891,9 +911,20 @@ func TestPauseEKSAControllerReconcileManagementClusterListObjectsError(t *testin
},
}

datacenterConfig := &v1alpha1.VSphereDatacenterConfig{
ObjectMeta: metav1.ObjectMeta{
Name: tt.clusterName,
},
Spec: v1alpha1.VSphereDatacenterConfigSpec{
Insecure: true,
},
}

tt.mocks.provider.EXPECT().DatacenterConfig(tt.clusterSpec).Return(datacenterConfig)

tt.mocks.client.EXPECT().ListObjects(tt.ctx, eksaClusterResourceType, "", "", &v1alpha1.ClusterList{}).Return(errors.New("list error"))

tt.Expect(tt.clusterManager.PauseEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).NotTo(Succeed())
tt.Expect(tt.clusterManager.ResumeEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).NotTo(Succeed())
}

func TestPauseEKSAControllerReconcileWorkloadClusterWithMachineConfig(t *testing.T) {
Expand Down Expand Up @@ -1084,3 +1115,31 @@ func TestCreateRegistryCredSecretSuccess(t *testing.T) {
err := tt.clusterManager.CreateRegistryCredSecret(tt.ctx, tt.cluster)
tt.Expect(err).To(BeNil())
}

func TestAllowDeleteWhilePaused(t *testing.T) {
tests := []struct {
name string
err error
}{
{
name: "success allow delete while paused",
err: nil,
},
{
name: "fail allow delete while paused",
err: fmt.Errorf("failure"),
},
}
allowDelete := map[string]string{v1alpha1.AllowDeleteWhenPausedAnnotation: "true"}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
tt := newTest(t)
cluster := tt.clusterSpec.Cluster
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, cluster.ResourceType(), cluster.Name, allowDelete, tt.cluster, cluster.Namespace).Return(test.err)
err := tt.clusterManager.AllowDeleteWhilePaused(tt.ctx, tt.cluster, tt.clusterSpec)
expectedErr := fmt.Errorf("updating paused annotation in cluster reconciliation: %v", test.err)
tt.Expect(err).To(Or(BeNil(), MatchError(expectedErr)))
})
}
}
1 change: 1 addition & 0 deletions pkg/task/task.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ type CommandContext struct {
OriginalError error
BackupClusterStateDir string
ForceCleanup bool
ClusterMover interfaces.ClusterMover
}

func (c *CommandContext) SetError(err error) {
Expand Down
17 changes: 3 additions & 14 deletions pkg/workflows/create_prep.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,11 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/aws/eks-anywhere/pkg/workflows/interfaces"
"github.com/aws/eks-anywhere/pkg/clients/kubernetes"
)

// CreateNamespaceIfNotPresent creates the namespace on the cluster if it does not already exist.
func CreateNamespaceIfNotPresent(ctx context.Context, namespace, kubeconfig string, clientFactory interfaces.ClientFactory) error {
client, err := clientFactory.BuildClientFromKubeconfig(kubeconfig)
if err != nil {
return err
}

if err := client.Get(ctx, namespace, "", &corev1.Namespace{}); err != nil && !errors.IsNotFound(err) {
return err
} else if err == nil {
return nil
}

func CreateNamespaceIfNotPresent(ctx context.Context, namespace string, client kubernetes.Client) error {
ns := &corev1.Namespace{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Expand All @@ -33,7 +22,7 @@ func CreateNamespaceIfNotPresent(ctx context.Context, namespace, kubeconfig stri
},
}

if err = client.Create(ctx, ns); err != nil {
if err := client.Create(ctx, ns); err != nil && !errors.IsAlreadyExists(err) {
return err
}

Expand Down
Loading