Upgrade registry certificate in packages (#7881) (#8256)

* upgrade packages workflow

* Change PackageInstaller to PackageManager

Makefile

@@ -568,7 +568,7 @@ mocks: ## Generate mocks
 	${MOCKGEN} -destination=pkg/bootstrapper/mocks/bootstrapper.go -package=mocks "github.com/aws/eks-anywhere/pkg/bootstrapper" ClusterClient
 	${MOCKGEN} -destination=pkg/git/providers/github/mocks/github.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/providers/github" GithubClient
 	${MOCKGEN} -destination=pkg/git/mocks/git.go -package=mocks "github.com/aws/eks-anywhere/pkg/git" Client,ProviderClient
-	${MOCKGEN} -destination=pkg/workflows/interfaces/mocks/clients.go -package=mocks "github.com/aws/eks-anywhere/pkg/workflows/interfaces" Bootstrapper,ClusterManager,GitOpsManager,Validator,CAPIManager,EksdInstaller,EksdUpgrader,PackageInstaller,ClusterUpgrader,ClusterCreator,ClientFactory,EksaInstaller,ClusterDeleter
+	${MOCKGEN} -destination=pkg/workflows/interfaces/mocks/clients.go -package=mocks "github.com/aws/eks-anywhere/pkg/workflows/interfaces" Bootstrapper,ClusterManager,GitOpsManager,Validator,CAPIManager,EksdInstaller,EksdUpgrader,PackageManager,ClusterUpgrader,ClusterCreator,ClientFactory,EksaInstaller,ClusterDeleter
 	${MOCKGEN} -destination=pkg/git/gogithub/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/gogithub" Client
 	${MOCKGEN} -destination=pkg/git/gitclient/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/gitclient" GoGit
 	${MOCKGEN} -destination=pkg/validations/mocks/docker.go -package=mocks "github.com/aws/eks-anywhere/pkg/validations" DockerExecutable

cmd/eksctl-anywhere/cmd/createcluster.go

@@ -185,7 +185,7 @@ func (cc *createClusterOptions) createCluster(cmd *cobra.Command, _ []string) er
 		WithGitOpsFlux(clusterSpec.Cluster, clusterSpec.FluxConfig, cliConfig).
 		WithWriter().
 		WithEksdInstaller().
-		WithPackageInstaller(clusterSpec, cc.installPackages, cc.managementKubeconfig).
+		WithPackageManager(clusterSpec, cc.installPackages, cc.managementKubeconfig).
 		WithValidatorClients().
 		WithCreateClusterDefaulter(createCLIConfig).
 		WithClusterApplier().
@@ -256,7 +256,7 @@ func (cc *createClusterOptions) createCluster(cmd *cobra.Command, _ []string) er
 			deps.GitOpsFlux,
 			deps.Writer,
 			deps.EksdInstaller,
-			deps.PackageInstaller,
+			deps.PackageManager,
 			deps.ClusterCreator,
 			deps.UnAuthKubectlClient,
 		)
@@ -273,7 +273,7 @@ func (cc *createClusterOptions) createCluster(cmd *cobra.Command, _ []string) er
 			deps.GitOpsFlux,
 			deps.Writer,
 			deps.EksdInstaller,
-			deps.PackageInstaller,
+			deps.PackageManager,
 			deps.ClusterCreator,
 			deps.EksaInstaller,
 		)

cmd/eksctl-anywhere/cmd/upgradecluster.go

@@ -159,6 +159,7 @@ func (uc *upgradeClusterOptions) upgradeCluster(cmd *cobra.Command, args []strin
 		WithEksdInstaller().
 		WithKubectl().
 		WithValidatorClients().
+		WithPackageManagerWithoutWait(clusterSpec, "", uc.managementKubeconfig).
 		WithUpgradeClusterDefaulter(upgradeCLIConfig)
 
 	if uc.timeoutOptions.noTimeouts {
@@ -212,6 +213,7 @@ func (uc *upgradeClusterOptions) upgradeCluster(cmd *cobra.Command, args []strin
 			deps.EksdUpgrader,
 			deps.EksdInstaller,
 			deps.ClusterApplier,
+			deps.PackageManager,
 		)
 
 		err = upgrade.Run(ctx, clusterSpec, managementCluster, upgradeValidations)
@@ -225,7 +227,7 @@ func (uc *upgradeClusterOptions) upgradeCluster(cmd *cobra.Command, args []strin
 			deps.Writer,
 			deps.ClusterApplier,
 			deps.EksdInstaller,
-			deps.PackageInstaller,
+			deps.PackageManager,
 		)
 		err = upgradeWorkloadCluster.Run(ctx, workloadCluster, clusterSpec, upgradeValidations)
 	}

pkg/curatedpackages/packagecontrollerclient.go

@@ -645,3 +645,10 @@ func WithRegistryAccessTester(registryTester RegistryAccessTester) func(client *
 		config.registryAccessTester = registryTester
 	}
 }
+
+// WithSkipWait sets skipWaitForPackageBundle.
+func WithSkipWait() func(client *PackageControllerClient) {
+	return func(config *PackageControllerClient) {
+		config.skipWaitForPackageBundle = true
+	}
+}

pkg/curatedpackages/packagecontrollerclient_test.go

@@ -524,6 +524,55 @@ func TestEnableWithEmptyProxy(t *testing.T) {
 	}
 }
 
+func TestEnableWithSkipWait(t *testing.T) {
+	for _, tt := range newPackageControllerTests(t) {
+		tt.command = curatedpackages.NewPackageControllerClient(
+			tt.chartManager, tt.kubectl, "billy", tt.kubeConfig, tt.chart,
+			tt.registryMirror,
+			curatedpackages.WithEksaSecretAccessKey(tt.eksaAccessKey),
+			curatedpackages.WithEksaRegion(tt.eksaRegion),
+			curatedpackages.WithEksaAccessKeyId(tt.eksaAccessID),
+			curatedpackages.WithSkipWait(),
+			curatedpackages.WithManagementClusterName(tt.clusterName),
+			curatedpackages.WithValuesFileWriter(tt.writer),
+		)
+		clusterName := fmt.Sprintf("clusterName=%s", "billy")
+		valueFilePath := filepath.Join("billy", filewriter.DefaultTmpFolder, valueFileName)
+		ociURI := fmt.Sprintf("%s%s", "oci://", tt.registryMirror.ReplaceRegistry(tt.chart.Image()))
+		sourceRegistry, defaultRegistry, defaultImageRegistry := tt.command.GetCuratedPackagesRegistries(context.Background())
+		sourceRegistry = fmt.Sprintf("sourceRegistry=%s", sourceRegistry)
+		defaultRegistry = fmt.Sprintf("defaultRegistry=%s", defaultRegistry)
+		defaultImageRegistry = fmt.Sprintf("defaultImageRegistry=%s", defaultImageRegistry)
+		if tt.registryMirror != nil {
+			t.Setenv("REGISTRY_USERNAME", "username")
+			t.Setenv("REGISTRY_PASSWORD", "password")
+		} else {
+			if tt.eksaRegion == "" {
+				tt.eksaRegion = "us-west-2"
+			}
+			defaultImageRegistry = strings.ReplaceAll(defaultImageRegistry, "us-west-2", tt.eksaRegion)
+		}
+		values := []string{sourceRegistry, defaultRegistry, defaultImageRegistry, clusterName}
+		if (tt.eksaAccessID == "" || tt.eksaAccessKey == "") && tt.registryMirror == nil {
+			values = append(values, "cronjob.suspend=true")
+		}
+		tt.chartManager.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, false, values).Return(nil)
+		tt.kubectl.EXPECT().
+			GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
+			DoAndReturn(getPBCSuccess(t)).
+			AnyTimes()
+		tt.kubectl.EXPECT().
+			HasResource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
+			DoAndReturn(func(_, _, _, _, _ interface{}) (bool, error) { return true, nil }).
+			AnyTimes()
+
+		err := tt.command.Enable(tt.ctx)
+		if err != nil {
+			t.Errorf("Install Controller Should succeed when installation passes")
+		}
+	}
+}
+
 func TestEnableFail(t *testing.T) {
 	for _, tt := range newPackageControllerTests(t) {
 		clusterName := fmt.Sprintf("clusterName=%s", "billy")

pkg/curatedpackages/packageinstaller.go

@@ -67,6 +67,23 @@ func (pi *Installer) InstallCuratedPackages(ctx context.Context) {
 	}
 }
 
+// UpgradeCuratedPackages upgrades curated packages as part of the cluster upgrade.
+func (pi *Installer) UpgradeCuratedPackages(ctx context.Context) {
+	if IsPackageControllerDisabled(pi.spec.Cluster) {
+		logger.Info("Package controller disabled")
+		return
+	}
+	PrintLicense()
+	if err := pi.installPackagesController(ctx); err != nil {
+		logger.MarkWarning("Failed to upgrade the optional EKS-A Curated Package Controller.", "warning", err)
+		return
+	}
+
+	if err := pi.installPackages(ctx); err != nil {
+		logger.MarkWarning("Failed upgrading curated packages on the cluster.", "error", err)
+	}
+}
+
 func (pi *Installer) installPackagesController(ctx context.Context) error {
 	logger.Info("Enabling curated packages on the cluster")
 	err := pi.packageController.Enable(ctx)

pkg/dependencies/factory.go

@@ -97,7 +97,7 @@ type Dependencies struct {
 	closers                     []types.Closer
 	CliConfig                   *cliconfig.CliConfig
 	CreateCliConfig             *cliconfig.CreateClusterCLIConfig
-	PackageInstaller            interfaces.PackageInstaller
+	PackageManager              interfaces.PackageManager
 	BundleRegistry              curatedpackages.BundleRegistry
 	PackageControllerClient     *curatedpackages.PackageControllerClient
 	PackageClient               curatedpackages.PackageHandler
@@ -1384,16 +1384,17 @@ func (f *Factory) WithGitOpsFlux(clusterConfig *v1alpha1.Cluster, fluxConfig *v1
 	return f
 }
 
-func (f *Factory) WithPackageInstaller(spec *cluster.Spec, packagesLocation, kubeConfig string) *Factory {
+// WithPackageManager builds a package manager.
+func (f *Factory) WithPackageManager(spec *cluster.Spec, packagesLocation, kubeConfig string) *Factory {
 	f.WithKubectl().WithPackageControllerClient(spec, kubeConfig).WithPackageClient()
-	f.buildSteps = append(f.buildSteps, func(ctx context.Context) error {
-		if f.dependencies.PackageInstaller != nil {
+	f.buildSteps = append(f.buildSteps, func(_ context.Context) error {
+		if f.dependencies.PackageManager != nil {
 			return nil
 		}
 		managementClusterName := getManagementClusterName(spec)
 		mgmtKubeConfig := kubeconfig.ResolveFilename(kubeConfig, managementClusterName)
 
-		f.dependencies.PackageInstaller = curatedpackages.NewInstaller(
+		f.dependencies.PackageManager = curatedpackages.NewInstaller(
 			f.dependencies.Kubectl,
 			f.dependencies.PackageClient,
 			f.dependencies.PackageControllerClient,
@@ -1406,10 +1407,18 @@ func (f *Factory) WithPackageInstaller(spec *cluster.Spec, packagesLocation, kub
 	return f
 }
 
-func (f *Factory) WithPackageControllerClient(spec *cluster.Spec, kubeConfig string) *Factory {
+// WithPackageManagerWithoutWait builds a package manager that doesn't wait for active bundles.
+func (f *Factory) WithPackageManagerWithoutWait(spec *cluster.Spec, packagesLocation, kubeConfig string) *Factory {
+	f.WithPackageControllerClient(spec, kubeConfig, curatedpackages.WithSkipWait()).
+		WithPackageManager(spec, packagesLocation, kubeConfig)
+	return f
+}
+
+// WithPackageControllerClient builds a client for package controller.
+func (f *Factory) WithPackageControllerClient(spec *cluster.Spec, kubeConfig string, opts ...curatedpackages.PackageControllerClientOpt) *Factory {
 	f.WithHelm(helm.WithInsecure()).WithKubectl()
 
-	f.buildSteps = append(f.buildSteps, func(ctx context.Context) error {
+	f.buildSteps = append(f.buildSteps, func(_ context.Context) error {
 		if f.dependencies.PackageControllerClient != nil || spec == nil {
 			return nil
 		}
@@ -1436,13 +1445,8 @@ func (f *Factory) WithPackageControllerClient(spec *cluster.Spec, kubeConfig str
 		if bundle == nil {
 			return fmt.Errorf("could not find VersionsBundle")
 		}
-		f.dependencies.PackageControllerClient = curatedpackages.NewPackageControllerClient(
-			f.dependencies.Helm,
-			f.dependencies.Kubectl,
-			spec.Cluster.Name,
-			mgmtKubeConfig,
-			&bundle.PackageController.HelmChart,
-			f.registryMirror,
+
+		options := []curatedpackages.PackageControllerClientOpt{
 			curatedpackages.WithEksaAccessKeyId(eksaAccessKeyID),
 			curatedpackages.WithEksaSecretAccessKey(eksaSecretKey),
 			curatedpackages.WithEksaRegion(eksaRegion),
@@ -1453,6 +1457,18 @@ func (f *Factory) WithPackageControllerClient(spec *cluster.Spec, kubeConfig str
 			curatedpackages.WithManagementClusterName(managementClusterName),
 			curatedpackages.WithValuesFileWriter(writer),
 			curatedpackages.WithClusterSpec(spec),
+		}
+
+		options = append(options, opts...)
+
+		f.dependencies.PackageControllerClient = curatedpackages.NewPackageControllerClient(
+			f.dependencies.Helm,
+			f.dependencies.Kubectl,
+			spec.Cluster.Name,
+			mgmtKubeConfig,
+			&bundle.PackageController.HelmChart,
+			f.registryMirror,
+			options...,
 		)
 		return nil
 	})

pkg/dependencies/factory_test.go

@@ -433,10 +433,46 @@ func TestFactoryBuildWithPackageInstaller(t *testing.T) {
 		WithLocalExecutables().
 		WithHelm(helm.WithInsecure()).
 		WithKubectl().
-		WithPackageInstaller(spec, "/test/packages.yaml", "kubeconfig.kubeconfig").
+		WithPackageManager(spec, "/test/packages.yaml", "kubeconfig.kubeconfig").
 		Build(context.Background())
 	tt.Expect(err).To(BeNil())
-	tt.Expect(deps.PackageInstaller).NotTo(BeNil())
+	tt.Expect(deps.PackageManager).NotTo(BeNil())
+}
+
+func TestFactoryBuildWithPackageInstallerWithoutWait(t *testing.T) {
+	spec := &cluster.Spec{
+		Config: &cluster.Config{
+			Cluster: &anywherev1.Cluster{
+				ObjectMeta: v1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				Spec: anywherev1.ClusterSpec{
+					KubernetesVersion: "1.19",
+				},
+			},
+		},
+		VersionsBundles: map[anywherev1.KubernetesVersion]*cluster.VersionsBundle{
+			"1.19": {
+				VersionsBundle: &v1alpha1.VersionsBundle{
+					PackageController: v1alpha1.PackageBundle{
+						HelmChart: v1alpha1.Image{
+							URI:  "test_registry/test/eks-anywhere-packages:v1",
+							Name: "test_chart",
+						},
+					},
+				},
+			},
+		},
+	}
+	tt := newTest(t, vsphere)
+	deps, err := dependencies.NewFactory().
+		WithLocalExecutables().
+		WithHelm(helm.WithInsecure()).
+		WithKubectl().
+		WithPackageManagerWithoutWait(spec, "/test/packages.yaml", "kubeconfig.kubeconfig").
+		Build(context.Background())
+	tt.Expect(err).To(BeNil())
+	tt.Expect(deps.PackageManager).NotTo(BeNil())
 }
 
 func TestFactoryBuildWithCuratedPackagesCustomRegistry(t *testing.T) {

pkg/task/task.go

@@ -36,7 +36,7 @@ type CommandContext struct {
 	Writer                filewriter.FileWriter
 	EksdInstaller         interfaces.EksdInstaller
 	EksaInstaller         interfaces.EksaInstaller
-	PackageInstaller      interfaces.PackageInstaller
+	PackageManager        interfaces.PackageManager
 	EksdUpgrader          interfaces.EksdUpgrader
 	ClusterUpgrader       interfaces.ClusterUpgrader
 	ClusterCreator        interfaces.ClusterCreator

pkg/workflows/create.go

@@ -25,14 +25,14 @@ type Create struct {
 	gitOpsManager    interfaces.GitOpsManager
 	writer           filewriter.FileWriter
 	eksdInstaller    interfaces.EksdInstaller
-	packageInstaller interfaces.PackageInstaller
+	packageInstaller interfaces.PackageManager
 }
 
 // NewCreate returns a Create instance.
 func NewCreate(clientFactory interfaces.ClientFactory, bootstrapper interfaces.Bootstrapper, provider providers.Provider,
 	clusterManager interfaces.ClusterManager, gitOpsManager interfaces.GitOpsManager,
 	writer filewriter.FileWriter, eksdInstaller interfaces.EksdInstaller,
-	packageInstaller interfaces.PackageInstaller,
+	packageInstaller interfaces.PackageManager,
 ) *Create {
 	return &Create{
 		clientFactory:    clientFactory,
@@ -55,16 +55,16 @@ func (c *Create) Run(ctx context.Context, clusterSpec *cluster.Spec, validator i
 		}
 	}
 	commandContext := &task.CommandContext{
-		ClientFactory:    c.clientFactory,
-		Bootstrapper:     c.bootstrapper,
-		Provider:         c.provider,
-		ClusterManager:   c.clusterManager,
-		GitOpsManager:    c.gitOpsManager,
-		ClusterSpec:      clusterSpec,
-		Writer:           c.writer,
-		Validations:      validator,
-		EksdInstaller:    c.eksdInstaller,
-		PackageInstaller: c.packageInstaller,
+		ClientFactory:  c.clientFactory,
+		Bootstrapper:   c.bootstrapper,
+		Provider:       c.provider,
+		ClusterManager: c.clusterManager,
+		GitOpsManager:  c.gitOpsManager,
+		ClusterSpec:    clusterSpec,
+		Writer:         c.writer,
+		Validations:    validator,
+		EksdInstaller:  c.eksdInstaller,
+		PackageManager: c.packageInstaller,
 	}
 
 	if clusterSpec.ManagementCluster != nil {
@@ -486,7 +486,7 @@ func (s *DeleteBootstrapClusterTask) Name() string {
 }
 
 func (cp *InstallCuratedPackagesTask) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
-	commandContext.PackageInstaller.InstallCuratedPackages(ctx)
+	commandContext.PackageManager.InstallCuratedPackages(ctx)
 	return nil
 }
 

pkg/workflows/create_test.go

@@ -28,7 +28,7 @@ type createTestSetup struct {
 	t                    *testing.T
 	client               *clientmocks.MockClient
 	clientFactory        *mocks.MockClientFactory
-	packageInstaller     *mocks.MockPackageInstaller
+	packageInstaller     *mocks.MockPackageManager
 	bootstrapper         *mocks.MockBootstrapper
 	clusterManager       *mocks.MockClusterManager
 	gitOpsManager        *mocks.MockGitOpsManager
@@ -57,7 +57,7 @@ func newCreateTest(t *testing.T) *createTestSetup {
 	provider := providermocks.NewMockProvider(mockCtrl)
 	writer := writermocks.NewMockFileWriter(mockCtrl)
 	eksd := mocks.NewMockEksdInstaller(mockCtrl)
-	packageInstaller := mocks.NewMockPackageInstaller(mockCtrl)
+	packageInstaller := mocks.NewMockPackageManager(mockCtrl)
 
 	datacenterConfig := &v1alpha1.VSphereDatacenterConfig{}
 	machineConfigs := []providers.MachineConfig{&v1alpha1.VSphereMachineConfig{}}

pkg/workflows/interfaces/interfaces.go

@@ -90,8 +90,10 @@ type EksdUpgrader interface {
 	Upgrade(ctx context.Context, cluster *types.Cluster, currentSpec, newSpec *cluster.Spec) error
 }
 
-type PackageInstaller interface {
+// PackageManager handles installation and upgrades of curated packages.
+type PackageManager interface {
 	InstallCuratedPackages(ctx context.Context)
+	UpgradeCuratedPackages(ctx context.Context)
 }
 
 // ClusterUpgrader upgrades the cluster and waits until it's ready.