Merge remote-tracking branch 'origin/main' into kim-setup-seed-region

hack/runtime-migrator/cmd/main.go

@@ -5,16 +5,17 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/go-playground/validator/v10"
-	"github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs"
 	"io"
-	v12 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/types"
 	"log/slog"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/go-playground/validator/v10"
+	"github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs"
+	v12 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+
 	"github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	gardener_types "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1"
 	"github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/config"
@@ -28,7 +29,7 @@ import (
 )
 
 const (
-	contextTimeout      = 5 * time.Minute
+	timeoutK8sOperation = 5 * time.Second
 	expirationTime      = 60 * time.Minute
 	runtimeIDAnnotation = "kcp.provisioner.kyma-project.io/runtime-id"
 )
@@ -89,7 +90,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	err = migrator.Do(runtimeIds)
+	err = migrator.Do(context.Background(), runtimeIds)
 	if err != nil {
 		slog.Error(fmt.Sprintf("Failed to migrate runtimes: %v", slog.Any("error", err)))
 		os.Exit(1)

hack/runtime-migrator/cmd/migration.go

@@ -3,6 +3,9 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/pkg/errors"
+	"log/slog"
+
 	"github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	gardener_types "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1"
 	runtimev1 "github.com/kyma-project/infrastructure-manager/api/v1"
@@ -13,7 +16,6 @@ import (
 	"github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig"
 	"github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"log/slog"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -43,29 +45,26 @@ func NewMigration(migratorConfig config2.Config, converterConfig config.Converte
 	}, nil
 }
 
-func (m Migration) Do(runtimeIDs []string) error {
+func (m Migration) Do(ctx context.Context, runtimeIDs []string) error {
 
-	migratorContext, cancel := context.WithTimeout(context.Background(), contextTimeout)
+	listCtx, cancel := context.WithTimeout(ctx, timeoutK8sOperation)
 	defer cancel()
 
-	shootList, err := m.shootClient.List(migratorContext, v1.ListOptions{})
+	shootList, err := m.shootClient.List(listCtx, v1.ListOptions{})
 	if err != nil {
 		return err
 	}
 
 	results := migration.NewMigratorResults(m.outputWriter.NewResultsDir)
 
 	reportError := func(runtimeID, shootName string, msg string, err error) {
-		var errorMsg string
 
 		if err != nil {
-			errorMsg = fmt.Sprintf("%s: %v", msg, err)
-		} else {
-			errorMsg = fmt.Sprintf(msg)
+			msg = fmt.Sprintf("%s: %v", msg, err)
 		}
 
-		results.ErrorOccurred(runtimeID, shootName, errorMsg)
-		slog.Error(errorMsg, "runtimeID", runtimeID)
+		results.ErrorOccurred(runtimeID, shootName, msg)
+		slog.Error(msg, "runtimeID", runtimeID)
 	}
 
 	reportValidationError := func(runtimeID, shootName string, msg string, err error) {
@@ -84,58 +83,70 @@ func (m Migration) Do(runtimeIDs []string) error {
 		slog.Info(msg, "runtimeID", runtimeID)
 	}
 
-	for _, runtimeID := range runtimeIDs {
+	run := func(runtimeID string) {
 		shoot := findShoot(runtimeID, shootList)
 		if shoot == nil {
-			reportError(runtimeID, "", "Failed to find shoot", nil)
-
-			continue
+			reportError(runtimeID, "", "Failed to find shoot", errors.New("no shoot with given runtimeID found"))
+			return
 		}
 
-		runtime, err := m.runtimeMigrator.Do(migratorContext, *shoot)
+		migrationCtx, cancel := context.WithTimeout(ctx, timeoutK8sOperation)
+		defer cancel()
+
+		runtime, err := m.runtimeMigrator.Do(migrationCtx, *shoot)
+
 		if err != nil {
 			reportError(runtimeID, shoot.Name, "Failed to migrate runtime", err)
-
-			continue
+			return
 		}
 
 		err = m.outputWriter.SaveRuntimeCR(runtime)
 		if err != nil {
 			reportError(runtimeID, shoot.Name, "Failed to save runtime CR", err)
-
-			continue
+			return
 		}
 
 		shootComparisonResult, err := m.runtimeVerifier.Do(runtime, *shoot)
 		if err != nil {
 			reportValidationError(runtimeID, shoot.Name, "Failed to verify runtime", err)
-
-			continue
+			return
 		}
 
 		if !shootComparisonResult.IsEqual() {
 			err = m.outputWriter.SaveComparisonResult(shootComparisonResult)
 			if err != nil {
 				reportError(runtimeID, shoot.Name, "Failed to save comparison result", err)
-			} else {
-				reportUnwantedUpdateDetected(runtimeID, shoot.Name, "Runtime CR can cause unwanted update in Gardener. Please verify the runtime CR.")
+				return
 			}
 
-			continue
+			reportUnwantedUpdateDetected(runtimeID, shoot.Name, "Runtime CR can cause unwanted update in Gardener. Please verify the runtime CR.")
+			return
 		}
 
 		if !m.isDryRun {
 			err = m.applyRuntimeCR(runtime)
 			if err != nil {
 				reportError(runtimeID, shoot.Name, "Failed to apply Runtime CR", err)
 			}
-
-			continue
+			return
 		}
 
 		reportSuccess(runtimeID, shoot.Name, "Runtime processed successfully")
 	}
 
+main:
+	for _, runtimeID := range runtimeIDs {
+		select {
+		case <-ctx.Done():
+			// application context was canceled
+			reportError(runtimeID, "", "Failed to find shoot", nil)
+			break main
+
+		default:
+			run(runtimeID)
+		}
+	}
+
 	resultsFile, err := m.outputWriter.SaveMigrationResults(results)
 	if err != nil {
 		return err

hack/runtime-migrator/go.mod

@@ -40,6 +40,8 @@ require (
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect

hack/runtime-migrator/go.sum

@@ -60,6 +60,11 @@ github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSF
 github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -106,6 +111,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=

hack/runtime-migrator/internal/runtime/verifier.go

@@ -87,7 +87,7 @@ func getZones(workers []v1beta1.Worker) []string {
 }
 
 func compare(originalShoot, convertedShoot v1beta1.Shoot) (*Difference, error) {
-	matcher := shoot.NewMatcher(originalShoot)
+	matcher := shoot.NewMatcherForPatch(originalShoot)
 	equal, err := matcher.Match(convertedShoot)
 	if err != nil {
 		return nil, err

hack/shoot-comparator/internal/files/comparator.go

@@ -20,7 +20,7 @@ func CompareFiles(leftFile, rightFile string) (bool, string, error) {
 		return false, "", err
 	}
 
-	matcher := shoot.NewMatcher(leftObject)
+	matcher := shoot.NewMatcherForCreate(leftObject)
 
 	success, err := matcher.Match(rightObject)
 	if err != nil {

hack/shoot-comparator/pkg/runtime/raw_extension_matcher.go

@@ -2,6 +2,7 @@ package runtime
 
 import (
 	"sort"
+	"strings"
 
 	"github.com/kyma-project/infrastructure-manager/hack/shoot-comparator/pkg/utilz"
 	"github.com/onsi/gomega"
@@ -43,7 +44,9 @@ func (m *RawExtensionMatcher) Match(actual interface{}) (bool, error) {
 	sort.Sort(sortBytes(rawActual))
 	sort.Sort(sortBytes(rawToMatch))
 
-	return gomega.BeComparableTo(rawActual).Match(rawToMatch)
+	return gomega.
+		BeComparableTo(strings.TrimSpace(string(rawActual))).
+		Match(strings.TrimSpace(string(rawToMatch)))
 }
 
 func (m *RawExtensionMatcher) NegatedFailureMessage(_ interface{}) string {

hack/shoot-comparator/pkg/shoot/matcher.go

@@ -14,16 +14,24 @@ import (
 )
 
 type Matcher struct {
-	toMatch interface{}
-	fails   []string
+	toMatch    interface{}
+	fails      []string
+	DNSMatcher types.GomegaMatcher
 }
 
-func NewMatcher(i interface{}) types.GomegaMatcher {
+func NewMatcherForCreate(i interface{}) types.GomegaMatcher {
 	return &Matcher{
 		toMatch: i,
 	}
 }
 
+func NewMatcherForPatch(i interface{}) types.GomegaMatcher {
+	return &Matcher{
+		toMatch:    i,
+		DNSMatcher: gstruct.Ignore(),
+	}
+}
+
 func getShoot(i interface{}) (shoot v1beta1.Shoot, err error) {
 	if i == nil {
 		return v1beta1.Shoot{}, fmt.Errorf("invalid value nil")
@@ -56,6 +64,10 @@ func (m *Matcher) Match(actual interface{}) (success bool, err error) {
 		return false, err
 	}
 
+	if m.DNSMatcher == nil {
+		m.DNSMatcher = newDNSMatcher(shootToMatch.Spec.DNS)
+	}
+
 	matchers := []propertyMatcher{
 		{
 			GomegaMatcher: gomega.BeComparableTo(shootToMatch.Name),
@@ -118,7 +130,7 @@ func (m *Matcher) Match(actual interface{}) (success bool, err error) {
 			path:          "spec/secretBindingName",
 		},
 		{
-			GomegaMatcher: newDNSMatcher(shootToMatch.Spec.DNS),
+			GomegaMatcher: m.DNSMatcher,
 			path:          "spec/dns",
 			actual:        shootActual.Spec.DNS,
 		},

hack/shoot-comparator/pkg/shoot/matcher_test.go

@@ -45,13 +45,13 @@ func deepCp(s v1beta1.Shoot, opts ...deepCpOpts) v1beta1.Shoot {
 }
 
 func testInvalidArgs(actual, expected interface{}) {
-	matcher := NewMatcher(expected)
+	matcher := NewMatcherForCreate(expected)
 	_, err := matcher.Match(actual)
 	Expect(err).To(HaveOccurred())
 }
 
 func testResults(actual, expected interface{}, expectedMatch bool) {
-	matcher := NewMatcher(expected)
+	matcher := NewMatcherForCreate(expected)
 	actualMatch, err := matcher.Match(actual)
 	Expect(err).ShouldNot(HaveOccurred(), err)
 	Expect(actualMatch).Should(Equal(expectedMatch), matcher.FailureMessage(actual))

internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go

@@ -35,7 +35,8 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s
 	var crbList rbacv1.ClusterRoleBindingList
 	if err := shootAdminClient.List(ctx, &crbList); err != nil {
 		updateCRBApplyFailed(&s.instance)
-		return updateStatusAndStopWithError(err)
+		m.log.Info("Cannot list Cluster Role Bindings on shoot, scheduling for retry", "RuntimeCR", s.instance.Name, "shoot", s.shoot.Name)
+		return requeue()
 	}
 
 	removed := getRemoved(crbList.Items, s.instance.Spec.Security.Administrators)
@@ -47,7 +48,8 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s
 	} {
 		if err := fn(); err != nil {
 			updateCRBApplyFailed(&s.instance)
-			return updateStatusAndStopWithError(err)
+			m.log.Info("Cannot setup Cluster Role Bindings on shoot, scheduling for retry", "RuntimeCR", s.instance.Name, "shoot", s.shoot.Name)
+			return requeue()
 		}
 	}
 

internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing.go

@@ -10,8 +10,6 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
-type ErrReason string
-
 func sFnSelectShootProcessing(_ context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) {
 	m.log.Info("Select shoot processing state")
 

internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go

@@ -6,6 +6,7 @@ import (
 
 	gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+	imgardenerhandler "github.com/kyma-project/infrastructure-manager/pkg/gardener"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
@@ -25,10 +26,17 @@ func sFnWaitForShootReconcile(_ context.Context, m *fsm, s *systemState) (stateF
 		return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration)
 
 	case gardener.LastOperationStateFailed:
-		var reason ErrReason
+		lastErrors := s.shoot.Status.LastErrors
+		reason := imgardenerhandler.ToErrReason(lastErrors...)
 
-		if len(s.shoot.Status.LastErrors) > 0 {
-			reason = gardenerErrCodesToErrReason(s.shoot.Status.LastErrors...)
+		if imgardenerhandler.IsRetryable(lastErrors) {
+			m.log.Info(fmt.Sprintf("Retryable gardener errors during cluster provisioning for Shoot %s, reason: %s, scheduling for retry", s.shoot.Name, reason))
+			s.instance.UpdateStatePending(
+				imv1.ConditionTypeRuntimeProvisioned,
+				imv1.ConditionReasonShootCreationPending,
+				"Unknown",
+				"Retryable gardener errors during cluster reconcile")
+			return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration)
 		}
 
 		msg := fmt.Sprintf("error during cluster processing: reconcilation failed for shoot %s, reason: %s, exiting with no retry", s.shoot.Name, reason)