Upgrade workload using controller behind flag (#7228)

* Upgrade workload using controller behind flag

* change workflow success message based on command

* add tests

* fix success message between create and upgrade

* remove logs mentioning POC

cmd/eksctl-anywhere/cmd/upgradecluster.go

@@ -11,6 +11,7 @@ import (
 	"github.com/aws/eks-anywhere/cmd/eksctl-anywhere/cmd/aflag"
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
 	"github.com/aws/eks-anywhere/pkg/dependencies"
+	"github.com/aws/eks-anywhere/pkg/features"
 	"github.com/aws/eks-anywhere/pkg/kubeconfig"
 	"github.com/aws/eks-anywhere/pkg/logger"
 	"github.com/aws/eks-anywhere/pkg/providers/tinkerbell/hardware"
@@ -19,6 +20,7 @@ import (
 	"github.com/aws/eks-anywhere/pkg/validations/upgradevalidations"
 	"github.com/aws/eks-anywhere/pkg/workflows"
 	"github.com/aws/eks-anywhere/pkg/workflows/management"
+	"github.com/aws/eks-anywhere/pkg/workflows/workload"
 )
 
 type upgradeClusterOptions struct {
@@ -212,6 +214,17 @@ func (uc *upgradeClusterOptions) upgradeCluster(cmd *cobra.Command, args []strin
 
 		err = upgrade.Run(ctx, clusterSpec, managementCluster, upgradeValidations)
 
+	} else if features.UseControllerViaCLIWorkflow().IsActive() {
+		upgradeWorkloadCluster := workload.NewUpgrade(
+			deps.Provider,
+			deps.ClusterManager,
+			deps.GitOpsFlux,
+			deps.Writer,
+			deps.ClusterApplier,
+			deps.EksdInstaller,
+			deps.PackageInstaller,
+		)
+		err = upgradeWorkloadCluster.Run(ctx, workloadCluster, clusterSpec, upgradeValidations)
 	} else {
 		upgrade := workflows.NewUpgrade(
 			deps.Bootstrapper,

pkg/workflows/workload/upgrade.go

@@ -0,0 +1,59 @@
+package workload
+
+import (
+	"context"
+
+	"github.com/aws/eks-anywhere/pkg/cluster"
+	"github.com/aws/eks-anywhere/pkg/filewriter"
+	"github.com/aws/eks-anywhere/pkg/providers"
+	"github.com/aws/eks-anywhere/pkg/task"
+	"github.com/aws/eks-anywhere/pkg/types"
+	"github.com/aws/eks-anywhere/pkg/workflows/interfaces"
+)
+
+// Upgrade is a schema for upgrade cluster.
+type Upgrade struct {
+	provider         providers.Provider
+	clusterManager   interfaces.ClusterManager
+	gitOpsManager    interfaces.GitOpsManager
+	writer           filewriter.FileWriter
+	eksdInstaller    interfaces.EksdInstaller
+	ClusterUpgrader  interfaces.ClusterUpgrader
+	packageInstaller interfaces.PackageInstaller
+}
+
+// NewUpgrade builds a new upgrade construct.
+func NewUpgrade(provider providers.Provider,
+	clusterManager interfaces.ClusterManager, gitOpsManager interfaces.GitOpsManager,
+	writer filewriter.FileWriter,
+	clusterUpgrade interfaces.ClusterUpgrader,
+	eksdInstaller interfaces.EksdInstaller,
+	packageInstaller interfaces.PackageInstaller,
+) *Upgrade {
+	return &Upgrade{
+		provider:         provider,
+		clusterManager:   clusterManager,
+		gitOpsManager:    gitOpsManager,
+		writer:           writer,
+		eksdInstaller:    eksdInstaller,
+		ClusterUpgrader:  clusterUpgrade,
+		packageInstaller: packageInstaller,
+	}
+}
+
+// Run Upgrade implements upgrade functionality for workload cluster's upgrade operation.
+func (c *Upgrade) Run(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec, validator interfaces.Validator) error {
+	commandContext := &task.CommandContext{
+		Provider:          c.provider,
+		ClusterManager:    c.clusterManager,
+		GitOpsManager:     c.gitOpsManager,
+		ClusterSpec:       clusterSpec,
+		Writer:            c.writer,
+		Validations:       validator,
+		ManagementCluster: clusterSpec.ManagementCluster,
+		WorkloadCluster:   cluster,
+		ClusterUpgrader:   c.ClusterUpgrader,
+	}
+
+	return task.NewTaskRunner(&setAndValidateWorkloadTask{}, c.writer).RunTask(ctx, commandContext)
+}

pkg/workflows/workload/upgrade_test.go

@@ -0,0 +1,229 @@
+package workload_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+
+	"github.com/aws/eks-anywhere/internal/test"
+	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	"github.com/aws/eks-anywhere/pkg/cluster"
+	"github.com/aws/eks-anywhere/pkg/features"
+	writermocks "github.com/aws/eks-anywhere/pkg/filewriter/mocks"
+	"github.com/aws/eks-anywhere/pkg/providers"
+	providermocks "github.com/aws/eks-anywhere/pkg/providers/mocks"
+	"github.com/aws/eks-anywhere/pkg/types"
+	"github.com/aws/eks-anywhere/pkg/workflows/interfaces/mocks"
+	"github.com/aws/eks-anywhere/pkg/workflows/workload"
+)
+
+type upgradeTestSetup struct {
+	t                  *testing.T
+	clusterManager     *mocks.MockClusterManager
+	gitOpsManager      *mocks.MockGitOpsManager
+	provider           *providermocks.MockProvider
+	writer             *writermocks.MockFileWriter
+	validator          *mocks.MockValidator
+	eksd               *mocks.MockEksdInstaller
+	packageInstaller   *mocks.MockPackageInstaller
+	clusterUpgrader    *mocks.MockClusterUpgrader
+	datacenterConfig   providers.DatacenterConfig
+	machineConfigs     []providers.MachineConfig
+	ctx                context.Context
+	currentClusterSpec *cluster.Spec
+	clusterSpec        *cluster.Spec
+	workloadCluster    *types.Cluster
+	workload           *workload.Upgrade
+}
+
+func newUpgradeTest(t *testing.T) *upgradeTestSetup {
+	featureEnvVars := []string{}
+	mockCtrl := gomock.NewController(t)
+	clusterManager := mocks.NewMockClusterManager(mockCtrl)
+	gitOpsManager := mocks.NewMockGitOpsManager(mockCtrl)
+	provider := providermocks.NewMockProvider(mockCtrl)
+	writer := writermocks.NewMockFileWriter(mockCtrl)
+	eksd := mocks.NewMockEksdInstaller(mockCtrl)
+	packageInstaller := mocks.NewMockPackageInstaller(mockCtrl)
+	eksdInstaller := mocks.NewMockEksdInstaller(mockCtrl)
+
+	datacenterConfig := &v1alpha1.VSphereDatacenterConfig{}
+	machineConfigs := []providers.MachineConfig{&v1alpha1.VSphereMachineConfig{}}
+	clusterUpgrader := mocks.NewMockClusterUpgrader(mockCtrl)
+
+	validator := mocks.NewMockValidator(mockCtrl)
+
+	workload := workload.NewUpgrade(
+		provider,
+		clusterManager,
+		gitOpsManager,
+		writer,
+		clusterUpgrader,
+		eksdInstaller,
+		packageInstaller,
+	)
+
+	for _, e := range featureEnvVars {
+		t.Setenv(e, "true")
+	}
+
+	return &upgradeTestSetup{
+		t:                t,
+		clusterManager:   clusterManager,
+		gitOpsManager:    gitOpsManager,
+		provider:         provider,
+		writer:           writer,
+		validator:        validator,
+		eksd:             eksd,
+		packageInstaller: packageInstaller,
+		datacenterConfig: datacenterConfig,
+		machineConfigs:   machineConfigs,
+		workload:         workload,
+		ctx:              context.Background(),
+		clusterUpgrader:  clusterUpgrader,
+		currentClusterSpec: test.NewClusterSpec(func(s *cluster.Spec) {
+			s.Cluster.Name = "workload"
+			s.Cluster.Spec.DatacenterRef.Kind = v1alpha1.VSphereDatacenterKind
+			s.ManagementCluster = &types.Cluster{Name: "management"}
+			s.Cluster.Spec.KubernetesVersion = v1alpha1.Kube127
+		}),
+		clusterSpec: test.NewClusterSpec(func(s *cluster.Spec) {
+			s.Cluster.Name = "workload"
+			s.Cluster.Spec.DatacenterRef.Kind = v1alpha1.VSphereDatacenterKind
+			s.ManagementCluster = &types.Cluster{Name: "management"}
+			s.Cluster.Spec.KubernetesVersion = v1alpha1.Kube128
+		}),
+		workloadCluster: &types.Cluster{Name: "workload"},
+	}
+}
+
+func (c *upgradeTestSetup) expectSetup() {
+	c.clusterManager.EXPECT().GetCurrentClusterSpec(c.ctx, c.clusterSpec.ManagementCluster, c.clusterSpec.Cluster.Name).AnyTimes().Return(c.currentClusterSpec, nil)
+	c.provider.EXPECT().SetupAndValidateUpgradeCluster(c.ctx, c.workloadCluster, c.clusterSpec, c.currentClusterSpec).AnyTimes()
+	c.provider.EXPECT().Name().AnyTimes()
+	c.gitOpsManager.EXPECT().Validations(c.ctx, c.clusterSpec).AnyTimes()
+}
+
+func (c *upgradeTestSetup) expectUpgradeWorkloadCluster(err error) {
+	c.clusterUpgrader.EXPECT().Run(c.ctx, c.clusterSpec, *c.clusterSpec.ManagementCluster).Return(err)
+}
+
+func (c *upgradeTestSetup) expectWriteWorkloadClusterConfig(err error) {
+	gomock.InOrder(
+		c.writer.EXPECT().Write("workload-eks-a-cluster.yaml", gomock.Any(), gomock.Any()).Return("workload-eks-a-cluster.yaml", err),
+	)
+}
+
+func (c *upgradeTestSetup) expectDatacenterConfig() {
+	gomock.InOrder(
+		c.provider.EXPECT().DatacenterConfig(c.clusterSpec).Return(c.datacenterConfig).AnyTimes(),
+	)
+}
+
+func (c *upgradeTestSetup) expectMachineConfigs() {
+	gomock.InOrder(
+		c.provider.EXPECT().MachineConfigs(c.clusterSpec).Return(c.machineConfigs).AnyTimes(),
+	)
+}
+
+func (c *upgradeTestSetup) run() error {
+	return c.workload.Run(c.ctx, c.workloadCluster, c.clusterSpec, c.validator)
+}
+
+func (c *upgradeTestSetup) expectPreflightValidationsToPass() {
+	c.validator.EXPECT().PreflightValidations(c.ctx).Return(nil)
+}
+
+func (c *upgradeTestSetup) expectSaveLogsManagement() {
+	c.clusterManager.EXPECT().SaveLogsManagementCluster(c.ctx, c.clusterSpec, c.clusterSpec.ManagementCluster)
+	c.expectWrite()
+}
+
+func (c *upgradeTestSetup) expectWrite() {
+	c.writer.EXPECT().Write(gomock.Any(), gomock.Any(), gomock.Any()).Return("", nil)
+}
+
+func TestUpgradeRunSuccess(t *testing.T) {
+	features.ClearCache()
+	os.Setenv(features.UseControllerForCli, "true")
+	test := newUpgradeTest(t)
+	test.expectSetup()
+	test.expectPreflightValidationsToPass()
+	test.expectDatacenterConfig()
+	test.expectMachineConfigs()
+	test.expectUpgradeWorkloadCluster(nil)
+	test.expectWriteWorkloadClusterConfig(nil)
+
+	err := test.run()
+	if err != nil {
+		t.Fatalf("Upgrade.Run() err = %v, want err = nil", err)
+	}
+}
+
+func TestUpgradeRunUpgradeFail(t *testing.T) {
+	features.ClearCache()
+	os.Setenv(features.UseControllerForCli, "true")
+	test := newUpgradeTest(t)
+	test.expectSetup()
+	test.expectPreflightValidationsToPass()
+	test.expectDatacenterConfig()
+	test.expectMachineConfigs()
+	test.expectUpgradeWorkloadCluster(fmt.Errorf("boom"))
+	test.expectSaveLogsManagement()
+
+	err := test.run()
+	if err == nil {
+		t.Fatalf("Upgrade.Run() err = %v, want err = nil", err)
+	}
+}
+
+func TestUpgradeRunGetCurrentClusterSpecFail(t *testing.T) {
+	features.ClearCache()
+	os.Setenv(features.UseControllerForCli, "true")
+	test := newUpgradeTest(t)
+	test.clusterManager.EXPECT().GetCurrentClusterSpec(test.ctx, test.clusterSpec.ManagementCluster, test.clusterSpec.Cluster.Name).Return(nil, fmt.Errorf("boom"))
+	test.expectWrite()
+
+	err := test.run()
+	if err == nil {
+		t.Fatalf("Upgrade.Run() err = %v, want err = nil", err)
+	}
+}
+
+func TestUpgradeRunValidateFail(t *testing.T) {
+	features.ClearCache()
+	os.Setenv(features.UseControllerForCli, "true")
+	test := newUpgradeTest(t)
+	test.clusterManager.EXPECT().GetCurrentClusterSpec(test.ctx, test.clusterSpec.ManagementCluster, test.clusterSpec.Cluster.Name).AnyTimes().Return(test.currentClusterSpec, nil)
+	test.provider.EXPECT().Name().AnyTimes()
+	test.gitOpsManager.EXPECT().Validations(test.ctx, test.clusterSpec).AnyTimes()
+	test.provider.EXPECT().SetupAndValidateUpgradeCluster(test.ctx, test.workloadCluster, test.clusterSpec, test.currentClusterSpec).Return(fmt.Errorf("boom"))
+	test.expectPreflightValidationsToPass()
+	test.expectWrite()
+
+	err := test.run()
+	if err == nil {
+		t.Fatalf("Upgrade.Run() err = %v, want err = nil", err)
+	}
+}
+
+func TestUpgradeRunWriteClusterConfigFail(t *testing.T) {
+	features.ClearCache()
+	os.Setenv(features.UseControllerForCli, "true")
+	test := newUpgradeTest(t)
+	test.expectSetup()
+	test.expectPreflightValidationsToPass()
+	test.expectDatacenterConfig()
+	test.expectMachineConfigs()
+	test.expectUpgradeWorkloadCluster(nil)
+	test.expectWriteWorkloadClusterConfig(fmt.Errorf("boom"))
+	test.expectWrite()
+
+	err := test.run()
+	if err == nil {
+		t.Fatalf("Upgrade.Run() err = %v, want err = nil", err)
+	}
+}

pkg/workflows/workload/upgradecluster.go

@@ -0,0 +1,36 @@
+package workload
+
+import (
+	"context"
+
+	"github.com/aws/eks-anywhere/pkg/logger"
+	"github.com/aws/eks-anywhere/pkg/task"
+	"github.com/aws/eks-anywhere/pkg/workflows"
+)
+
+type upgradeCluster struct{}
+
+// Run upgradeCluster performs actions needed to upgrade the workload cluster.
+func (s *upgradeCluster) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
+	logger.Info("Upgrading workload cluster components")
+	if err := commandContext.ClusterUpgrader.Run(ctx, commandContext.ClusterSpec, *commandContext.ManagementCluster); err != nil {
+		commandContext.SetError(err)
+		return &workflows.CollectMgmtClusterDiagnosticsTask{}
+	}
+
+	return &writeClusterConfig{}
+}
+
+func (s *upgradeCluster) Name() string {
+	return "upgrade-workload-cluster"
+}
+
+func (s *upgradeCluster) Checkpoint() *task.CompletedTask {
+	return &task.CompletedTask{
+		Checkpoint: nil,
+	}
+}
+
+func (s *upgradeCluster) Restore(ctx context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) {
+	return &writeClusterConfig{}, nil
+}