Skip to content

Commit

Permalink
Add workflow for management kindless upgrades (#6622)
Browse files Browse the repository at this point in the history
This branchs out the code at the command level and uses a different
workflow if kindless upgrades are enabled. This facilitates the
transition once we are ready to flip the switch and avoids having to
maintain multiple codepaths in the existing workflow, which from
experience, it has proven to be difficult and prone to errors. The
tradeoff is we need to mainetain the same functionality in two workflows
until we decide to deprecate the old one.
  • Loading branch information
panktishah26 authored Sep 7, 2023
1 parent 770bfdd commit 5349a0b
Show file tree
Hide file tree
Showing 20 changed files with 1,359 additions and 23 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -544,7 +544,7 @@ mocks: ## Generate mocks
${MOCKGEN} -destination=pkg/bootstrapper/mocks/bootstrapper.go -package=mocks "github.com/aws/eks-anywhere/pkg/bootstrapper" ClusterClient
${MOCKGEN} -destination=pkg/git/providers/github/mocks/github.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/providers/github" GithubClient
${MOCKGEN} -destination=pkg/git/mocks/git.go -package=mocks "github.com/aws/eks-anywhere/pkg/git" Client,ProviderClient
${MOCKGEN} -destination=pkg/workflows/interfaces/mocks/clients.go -package=mocks "github.com/aws/eks-anywhere/pkg/workflows/interfaces" Bootstrapper,ClusterManager,GitOpsManager,Validator,CAPIManager,EksdInstaller,EksdUpgrader,PackageInstaller
${MOCKGEN} -destination=pkg/workflows/interfaces/mocks/clients.go -package=mocks "github.com/aws/eks-anywhere/pkg/workflows/interfaces" Bootstrapper,ClusterManager,GitOpsManager,Validator,CAPIManager,EksdInstaller,EksdUpgrader,PackageInstaller,ClusterUpgrader
${MOCKGEN} -destination=pkg/git/gogithub/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/gogithub" Client
${MOCKGEN} -destination=pkg/git/gitclient/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/gitclient" GoGit
${MOCKGEN} -destination=pkg/validations/mocks/docker.go -package=mocks "github.com/aws/eks-anywhere/pkg/validations" DockerExecutable
Expand Down
45 changes: 33 additions & 12 deletions cmd/eksctl-anywhere/cmd/upgradecluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ import (
"github.com/aws/eks-anywhere/cmd/eksctl-anywhere/cmd/flags"
"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
"github.com/aws/eks-anywhere/pkg/dependencies"
"github.com/aws/eks-anywhere/pkg/features"
"github.com/aws/eks-anywhere/pkg/kubeconfig"
"github.com/aws/eks-anywhere/pkg/logger"
"github.com/aws/eks-anywhere/pkg/types"
"github.com/aws/eks-anywhere/pkg/validations"
"github.com/aws/eks-anywhere/pkg/validations/upgradevalidations"
"github.com/aws/eks-anywhere/pkg/workflows"
"github.com/aws/eks-anywhere/pkg/workflows/management"
)

type upgradeClusterOptions struct {
Expand Down Expand Up @@ -131,6 +133,7 @@ func (uc *upgradeClusterOptions) upgradeCluster(cmd *cobra.Command) error {
WithBootstrapper().
WithCliConfig(cliConfig).
WithClusterManager(clusterSpec.Cluster, clusterManagerTimeoutOpts).
WithClusterApplier().
WithProvider(uc.fileName, clusterSpec.Cluster, cc.skipIpCheck, uc.hardwareCSVPath, uc.forceClean, uc.tinkerbellBootstrapIP, skippedValidations).
WithGitOpsFlux(clusterSpec.Cluster, clusterSpec.FluxConfig, cliConfig).
WithWriter().
Expand All @@ -156,17 +159,6 @@ func (uc *upgradeClusterOptions) upgradeCluster(cmd *cobra.Command) error {
return err
}

upgradeCluster := workflows.NewUpgrade(
deps.Bootstrapper,
deps.Provider,
deps.CAPIManager,
deps.ClusterManager,
deps.GitOpsFlux,
deps.Writer,
deps.EksdUpgrader,
deps.EksdInstaller,
)

workloadCluster := &types.Cluster{
Name: clusterSpec.Cluster.Name,
KubeconfigFile: getKubeconfigPath(clusterSpec.Cluster.Name, uc.clusterKubeconfig),
Expand All @@ -191,7 +183,36 @@ func (uc *upgradeClusterOptions) upgradeCluster(cmd *cobra.Command) error {

upgradeValidations := upgradevalidations.New(validationOpts)

err = upgradeCluster.Run(ctx, clusterSpec, managementCluster, workloadCluster, upgradeValidations, uc.forceClean)
if features.ExperimentalSelfManagedClusterUpgrade().IsActive() && clusterConfig.IsSelfManaged() {
logger.Info("Management kindless upgrade")
upgrade := management.NewUpgrade(
deps.Provider,
deps.CAPIManager,
deps.ClusterManager,
deps.GitOpsFlux,
deps.Writer,
deps.EksdUpgrader,
deps.EksdInstaller,
deps.ClusterApplier,
)

err = upgrade.Run(ctx, clusterSpec, managementCluster, upgradeValidations)

} else {
upgrade := workflows.NewUpgrade(
deps.Bootstrapper,
deps.Provider,
deps.CAPIManager,
deps.ClusterManager,
deps.GitOpsFlux,
deps.Writer,
deps.EksdUpgrader,
deps.EksdInstaller,
)

err = upgrade.Run(ctx, clusterSpec, managementCluster, workloadCluster, upgradeValidations, uc.forceClean)
}

cleanup(deps, &err)
return err
}
Expand Down
6 changes: 6 additions & 0 deletions pkg/api/v1alpha1/vspheremachineconfig_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/webhook"

"github.com/aws/eks-anywhere/pkg/features"
)

// log is for logging in this package.
Expand Down Expand Up @@ -129,6 +131,10 @@ func validateImmutableFieldsVSphereMachineConfig(new, old *VSphereMachineConfig)
return allErrs
}

if features.IsActive(features.ExperimentalSelfManagedClusterUpgrade()) {
return allErrs
}

vspheremachineconfiglog.Info("Machine config is associated with management cluster's control plane or etcd", "name", old.Name)

if !reflect.DeepEqual(old.Spec.Users, new.Spec.Users) {
Expand Down
17 changes: 12 additions & 5 deletions pkg/cluster/wait.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@ package cluster

import (
"context"
"fmt"

"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/cluster-api/util/conditions"

anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1"
"github.com/aws/eks-anywhere/pkg/clients/kubernetes"
"github.com/aws/eks-anywhere/pkg/constants"
"github.com/aws/eks-anywhere/pkg/retrier"
)

Expand All @@ -18,23 +19,29 @@ import (
func WaitForCondition(ctx context.Context, client kubernetes.Reader, cluster *anywherev1.Cluster, retrier *retrier.Retrier, conditionType anywherev1.ConditionType) error {
return retrier.Retry(func() error {
c := &anywherev1.Cluster{}
if err := client.Get(ctx, cluster.Name, cluster.Namespace, c); err != nil {

namespace := cluster.Namespace
if namespace == "" {
namespace = constants.DefaultNamespace
}

if err := client.Get(ctx, cluster.Name, namespace, c); err != nil {
return err
}

observedGeneration := c.Status.ObservedGeneration
generation := c.Generation
if observedGeneration != generation {
return errors.Errorf("cluster generation (%d) and observedGeneration (%d) differ", generation, observedGeneration)
return fmt.Errorf("cluster generation (%d) and observedGeneration (%d) differ", generation, observedGeneration)
}

condition := conditions.Get(c, conditionType)
if condition == nil {
return errors.Errorf("cluster doesn't yet have condition %s", conditionType)
return fmt.Errorf("cluster doesn't yet have condition %s", conditionType)
}

if condition.Status != corev1.ConditionTrue {
return errors.Errorf("cluster condition %s is %s: %s", conditionType, condition.Status, condition.Message)
return fmt.Errorf("cluster condition %s is %s: %s", conditionType, condition.Status, condition.Message)
}

return nil
Expand Down
5 changes: 2 additions & 3 deletions pkg/cluster/wait_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,14 +114,13 @@ func TestWaitForCondition(t *testing.T) {
name: "condition is True",
clusterInput: &anywherev1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "my-c",
Namespace: "my-n",
Name: "my-c",
},
},
currentCluster: &anywherev1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "my-c",
Namespace: "my-n",
Namespace: "default",
Generation: 5,
},
Status: anywherev1.ClusterStatus{
Expand Down
1 change: 1 addition & 0 deletions pkg/task/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ type CommandContext struct {
EksdInstaller interfaces.EksdInstaller
PackageInstaller interfaces.PackageInstaller
EksdUpgrader interfaces.EksdUpgrader
ClusterUpgrader interfaces.ClusterUpgrader
CAPIManager interfaces.CAPIManager
ClusterSpec *cluster.Spec
CurrentClusterSpec *cluster.Spec
Expand Down
8 changes: 7 additions & 1 deletion pkg/workflows/diagnostics.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package workflows
import (
"context"

"github.com/aws/eks-anywhere/pkg/features"
"github.com/aws/eks-anywhere/pkg/logger"
"github.com/aws/eks-anywhere/pkg/task"
)
Expand Down Expand Up @@ -53,7 +54,12 @@ func (s *CollectWorkloadClusterDiagnosticsTask) Name() string {

func (s *CollectMgmtClusterDiagnosticsTask) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
logger.Info("collecting management cluster diagnostics")
_ = commandContext.ClusterManager.SaveLogsManagementCluster(ctx, commandContext.ClusterSpec, commandContext.BootstrapCluster)
mgmt := commandContext.BootstrapCluster
if features.IsActive(features.ExperimentalSelfManagedClusterUpgrade()) {
mgmt = commandContext.ManagementCluster
}

_ = commandContext.ClusterManager.SaveLogsManagementCluster(ctx, commandContext.ClusterSpec, mgmt)
return nil
}

Expand Down
5 changes: 5 additions & 0 deletions pkg/workflows/interfaces/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,3 +80,8 @@ type EksdUpgrader interface {
type PackageInstaller interface {
InstallCuratedPackages(ctx context.Context)
}

// ClusterUpgrader upgrades the cluster and waits until it's ready.
type ClusterUpgrader interface {
Run(ctx context.Context, spec *cluster.Spec, managementCluster types.Cluster) error
}
39 changes: 38 additions & 1 deletion pkg/workflows/interfaces/mocks/clients.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

110 changes: 110 additions & 0 deletions pkg/workflows/management/core_components.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package management

import (
"context"

"github.com/aws/eks-anywhere/pkg/logger"
"github.com/aws/eks-anywhere/pkg/task"
"github.com/aws/eks-anywhere/pkg/types"
"github.com/aws/eks-anywhere/pkg/workflows"
)

type ensureEtcdCAPIComponentsExist struct{}

// Run ensureEtcdCAPIComponentsExist ensures ETCD CAPI providers on the management cluster.
func (s *ensureEtcdCAPIComponentsExist) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
logger.Info("Ensuring etcd CAPI providers exist on management cluster before upgrade")
if err := commandContext.CAPIManager.EnsureEtcdProvidersInstallation(ctx, commandContext.ManagementCluster, commandContext.Provider, commandContext.CurrentClusterSpec); err != nil {
commandContext.SetError(err)
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}
return &pauseGitOpsReconcile{}
}

func (s *ensureEtcdCAPIComponentsExist) Name() string {
return "ensure-etcd-capi-components-exist"
}

func (s *ensureEtcdCAPIComponentsExist) Checkpoint() *task.CompletedTask {
return &task.CompletedTask{
Checkpoint: nil,
}
}

func (s *ensureEtcdCAPIComponentsExist) Restore(ctx context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) {
return &pauseGitOpsReconcile{}, nil
}

type upgradeCoreComponents struct {
UpgradeChangeDiff *types.ChangeDiff
}

// Run upgradeCoreComponents upgrades pre cluster upgrade components.
func (s *upgradeCoreComponents) Run(ctx context.Context, commandContext *task.CommandContext) task.Task {
logger.Info("Upgrading core components")

err := commandContext.Provider.PreCoreComponentsUpgrade(
ctx,
commandContext.ManagementCluster,
commandContext.ClusterSpec,
)
if err != nil {
commandContext.SetError(err)
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}

changeDiff, err := commandContext.CAPIManager.Upgrade(ctx, commandContext.ManagementCluster, commandContext.Provider, commandContext.CurrentClusterSpec, commandContext.ClusterSpec)
if err != nil {
commandContext.SetError(err)
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}
commandContext.UpgradeChangeDiff.Append(changeDiff)

if err = commandContext.GitOpsManager.Install(ctx, commandContext.ManagementCluster, commandContext.CurrentClusterSpec, commandContext.ClusterSpec); err != nil {
commandContext.SetError(err)
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}

changeDiff, err = commandContext.GitOpsManager.Upgrade(ctx, commandContext.ManagementCluster, commandContext.CurrentClusterSpec, commandContext.ClusterSpec)
if err != nil {
commandContext.SetError(err)
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}
commandContext.UpgradeChangeDiff.Append(changeDiff)

changeDiff, err = commandContext.ClusterManager.Upgrade(ctx, commandContext.ManagementCluster, commandContext.CurrentClusterSpec, commandContext.ClusterSpec)
if err != nil {
commandContext.SetError(err)
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}
commandContext.UpgradeChangeDiff.Append(changeDiff)

changeDiff, err = commandContext.EksdUpgrader.Upgrade(ctx, commandContext.ManagementCluster, commandContext.CurrentClusterSpec, commandContext.ClusterSpec)
if err != nil {
commandContext.SetError(err)
return &workflows.CollectMgmtClusterDiagnosticsTask{}
}
commandContext.UpgradeChangeDiff.Append(changeDiff)
s.UpgradeChangeDiff = commandContext.UpgradeChangeDiff

return &preClusterUpgrade{}
}

func (s *upgradeCoreComponents) Name() string {
return "upgrade-core-components"
}

func (s *upgradeCoreComponents) Checkpoint() *task.CompletedTask {
return &task.CompletedTask{
Checkpoint: s.UpgradeChangeDiff,
}
}

func (s *upgradeCoreComponents) Restore(ctx context.Context, commandContext *task.CommandContext, completedTask *task.CompletedTask) (task.Task, error) {
s.UpgradeChangeDiff = &types.ChangeDiff{}
if err := task.UnmarshalTaskCheckpoint(completedTask.Checkpoint, s.UpgradeChangeDiff); err != nil {
return nil, err
}
commandContext.UpgradeChangeDiff = s.UpgradeChangeDiff
return &preClusterUpgrade{}, nil
}
Loading

0 comments on commit 5349a0b

Please sign in to comment.