From e32b121ec78b6f3db23f8ab221c1cb988787ce09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Thu, 7 Nov 2024 07:46:24 +0100 Subject: [PATCH 01/10] gardener retryable errors handling --- .../runtime_fsm_select_shoot_processing.go | 2 - ...runtime_fsm_waiting_for_shoot_reconcile.go | 10 +- .../fsm/runtime_fsm_waiting_shoot_creation.go | 33 +--- pkg/gardener/gardener_error_handler.go | 44 +++++ pkg/gardener/gardener_error_handler_test.go | 151 ++++++++++++++++++ 5 files changed, 209 insertions(+), 31 deletions(-) create mode 100644 pkg/gardener/gardener_error_handler.go create mode 100644 pkg/gardener/gardener_error_handler_test.go diff --git a/internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing.go b/internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing.go index 9d6788be..0de3f10b 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing.go +++ b/internal/controller/runtime/fsm/runtime_fsm_select_shoot_processing.go @@ -10,8 +10,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ) -type ErrReason string - func sFnSelectShootProcessing(_ context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) { m.log.Info("Select shoot processing state") diff --git a/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go b/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go index 10b47e40..66db8534 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go +++ b/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go @@ -6,6 +6,7 @@ import ( gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" + imgardenerhandler "github.com/kyma-project/infrastructure-manager/pkg/gardener" ctrl "sigs.k8s.io/controller-runtime" ) @@ -25,10 +26,13 @@ func sFnWaitForShootReconcile(_ context.Context, m *fsm, s *systemState) (stateF return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) case gardener.LastOperationStateFailed: - var reason ErrReason + lastErrors := s.shoot.Status.LastErrors + reason := imgardenerhandler.ToErrReason(lastErrors...) - if len(s.shoot.Status.LastErrors) > 0 { - reason = gardenerErrCodesToErrReason(s.shoot.Status.LastErrors...) + if imgardenerhandler.IsRetryable(lastErrors) { + m.log.Info(fmt.Sprintf("Retryable gardener errors during cluster provisioning for Shoot %s, reason: %s, scheduling for retry", s.shoot.Name, reason)) + // TODO: should update status? + return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) } msg := fmt.Sprintf("error during cluster processing: reconcilation failed for shoot %s, reason: %s, exiting with no retry", s.shoot.Name, reason) diff --git a/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go b/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go index 1acd3e1b..ae98ab4a 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go +++ b/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go @@ -3,11 +3,9 @@ package fsm import ( "context" "fmt" - "strings" - gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" - gardenerhelper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" + imgardenerhandler "github.com/kyma-project/infrastructure-manager/pkg/gardener" ctrl "sigs.k8s.io/controller-runtime" ) @@ -43,15 +41,15 @@ func sFnWaitForShootCreation(_ context.Context, m *fsm, s *systemState) (stateFn return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) case gardener.LastOperationStateFailed: - if gardenerhelper.HasErrorCode(s.shoot.Status.LastErrors, gardener.ErrorInfraRateLimitsExceeded) { - m.log.Info(fmt.Sprintf("Error during cluster provisioning: Rate limits exceeded for Shoot %s, scheduling for retry", s.shoot.Name)) + lastErrors := s.shoot.Status.LastErrors + reason := imgardenerhandler.ToErrReason(lastErrors...) + + if imgardenerhandler.IsRetryable(lastErrors) { + m.log.Info(fmt.Sprintf("Retryable gardener errors during cluster provisioning for Shoot %s, reason: %s, scheduling for retry", s.shoot.Name, reason)) + //TODO: should update status? return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) } - // also handle other retryable errors here - // ErrorRetryableConfigurationProblem - // ErrorRetryableInfraDependencies - msg := fmt.Sprintf("Provisioning failed for shoot: %s ! Last state: %s, Description: %s", s.shoot.Name, s.shoot.Status.LastOperation.State, s.shoot.Status.LastOperation.Description) m.log.Info(msg) @@ -78,20 +76,3 @@ func sFnWaitForShootCreation(_ context.Context, m *fsm, s *systemState) (stateFn return stopWithMetrics() } } - -func gardenerErrCodesToErrReason(lastErrors ...gardener.LastError) ErrReason { - var codes []gardener.ErrorCode - var vals []string - - for _, e := range lastErrors { - if len(e.Codes) > 0 { - codes = append(codes, e.Codes...) - } - } - - for _, code := range codes { - vals = append(vals, string(code)) - } - - return ErrReason(strings.Join(vals, ", ")) -} diff --git a/pkg/gardener/gardener_error_handler.go b/pkg/gardener/gardener_error_handler.go new file mode 100644 index 00000000..a4f2e8f7 --- /dev/null +++ b/pkg/gardener/gardener_error_handler.go @@ -0,0 +1,44 @@ +package gardener + +import ( + "fmt" + gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" + gardenerhelper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" + "strconv" + "strings" +) + +type ErrReason string + +func IsRetryable(lastErrors []gardener.LastError) bool { + if len(lastErrors) > 0 && + !gardenerhelper.HasNonRetryableErrorCode(lastErrors...) { + return true + } + return false +} + +func ToErrReason(lastErrors ...gardener.LastError) ErrReason { + var codes []gardener.ErrorCode + var vals []string + + for _, e := range lastErrors { + if len(e.Codes) > 0 { + codes = append(codes, e.Codes...) + } + } + + for _, code := range codes { + vals = append(vals, string(code)) + } + + return ErrReason(strings.Join(vals, ", ")) +} + +func CombineErrorDescriptions(lastErrors []gardener.LastError) string { + var descriptions string + for i, lastError := range lastErrors { + descriptions += fmt.Sprint(strconv.Itoa(i+1), ") ", lastError.Description, " ") + } + return descriptions +} diff --git a/pkg/gardener/gardener_error_handler_test.go b/pkg/gardener/gardener_error_handler_test.go new file mode 100644 index 00000000..01928ffd --- /dev/null +++ b/pkg/gardener/gardener_error_handler_test.go @@ -0,0 +1,151 @@ +package gardener + +import ( + gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGardenerErrorHandler(t *testing.T) { + t.Run("Should combine error descriptions", func(t *testing.T) { + //given + lastErrors := []gardener.LastError{ + { + Description: "First description", + }, + { + Description: "Second description", + }, + } + + //when + combinedDescritpion := CombineErrorDescriptions(lastErrors) + + //then + assert.Equal(t, "1) First description 2) Second description ", combinedDescritpion) + }) + + for _, testCase := range []struct { + name string + lastErrors []gardener.LastError + expectedRetryable bool + }{ + { + name: "Should return true for retryable gardener errors", + lastErrors: fixRetryableErrors(), + expectedRetryable: true, + }, + { + name: "Should return false for retryable gardener errors", + lastErrors: fixNonRetryableErrors(), + expectedRetryable: false, + }, + { + name: "Should return false for mixture of retryable and non-retryable gardener errors", + lastErrors: fixMixtureOfErrors(), + expectedRetryable: false, + }, + { + name: "Should return false empty list", + lastErrors: []gardener.LastError{}, + expectedRetryable: false, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + // given + + // when + retryable := IsRetryable(testCase.lastErrors) + + // then + assert.Equal(t, testCase.expectedRetryable, retryable) + }) + } +} + +func fixRetryableErrors() []gardener.LastError { + return []gardener.LastError{ + { + Description: "First description - retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorRetryableConfigurationProblem, + }, + }, + { + Description: "Second description - retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorRetryableInfraDependencies, + }, + }, + } +} + +func fixNonRetryableErrors() []gardener.LastError { + return []gardener.LastError{ + { + Description: "First description - non-retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorInfraDependencies, + }, + }, + { + Description: "Second description - non-retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorInfraQuotaExceeded, + }, + }, + { + Description: "Third description - non-retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorInfraUnauthenticated, + }, + }, + { + Description: "Fourth description - non-retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorInfraUnauthorized, + }, + }, + { + Description: "Fifth description - non-retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorInfraRateLimitsExceeded, + }, + }, + { + Description: "Sixth description - non-retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorConfigurationProblem, + }, + }, + { + Description: "Seventh description - non-retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorProblematicWebhook, + }, + }, + } +} + +func fixMixtureOfErrors() []gardener.LastError { + return []gardener.LastError{ + { + Description: "First description - non-retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorInfraDependencies, + }, + }, + { + Description: "Second description - retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorRetryableConfigurationProblem, + }, + }, + { + Description: "Third description - non-retryable", + Codes: []gardener.ErrorCode{ + gardener.ErrorInfraQuotaExceeded, + }, + }, + } +} From e41c07cdf0a2d16fb087fc320773a44ee50371ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Thu, 7 Nov 2024 07:48:44 +0100 Subject: [PATCH 02/10] cluster rolebinding requeues when listing or update fails --- .../runtime/fsm/runtime_fsm_apply_clusterrolebindings.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go index 7e7737ed..d5539aaa 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go +++ b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go @@ -35,7 +35,7 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s var crbList rbacv1.ClusterRoleBindingList if err := shootAdminClient.List(ctx, &crbList); err != nil { updateCRBApplyFailed(&s.instance) - return updateStatusAndStopWithError(err) + return requeue() } removed := getRemoved(crbList.Items, s.instance.Spec.Security.Administrators) @@ -47,7 +47,7 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s } { if err := fn(); err != nil { updateCRBApplyFailed(&s.instance) - return updateStatusAndStopWithError(err) + return requeue() } } From 3c1a33e94eb51a864fc48a7f121c257ef920bb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Thu, 7 Nov 2024 08:18:03 +0100 Subject: [PATCH 03/10] go mod tidy in migrator --- ...ager.kyma-project.io_gardenerclusters.yaml | 21 +++++++++++--- ...cturemanager.kyma-project.io_runtimes.yaml | 29 +++++++++++++++---- hack/runtime-migrator/go.mod | 3 ++ hack/runtime-migrator/go.sum | 9 ++++++ 4 files changed, 52 insertions(+), 10 deletions(-) diff --git a/config/crd/bases/infrastructuremanager.kyma-project.io_gardenerclusters.yaml b/config/crd/bases/infrastructuremanager.kyma-project.io_gardenerclusters.yaml index 2a26b5e2..48ee34fa 100644 --- a/config/crd/bases/infrastructuremanager.kyma-project.io_gardenerclusters.yaml +++ b/config/crd/bases/infrastructuremanager.kyma-project.io_gardenerclusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.15.0 name: gardenerclusters.infrastructuremanager.kyma-project.io spec: group: infrastructuremanager.kyma-project.io @@ -92,8 +92,16 @@ spec: description: List of status conditions to indicate the status of a ServiceInstance. items: - description: Condition contains details for one aspect of the current - state of this API Resource. + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" properties: lastTransitionTime: description: |- @@ -134,7 +142,12 @@ spec: - Unknown type: string type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string diff --git a/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml b/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml index cbdbefb3..37ed9d50 100644 --- a/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml +++ b/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.4 + controller-gen.kubebuilder.io/version: v0.15.0 name: runtimes.infrastructuremanager.kyma-project.io spec: group: infrastructuremanager.kyma-project.io @@ -134,8 +134,10 @@ spec: description: |- ClientAuthentication can optionally contain client configuration used for kubeconfig generation. + Deprecated: This field has no implemented use and will be forbidden starting from Kubernetes 1.31. It's use was planned for genereting OIDC kubeconfig https://github.com/gardener/gardener/issues/1433 + TODO(AleksandarSavchev): Drop this field after support for Kubernetes 1.30 is dropped. properties: extraConfig: additionalProperties: @@ -218,8 +220,10 @@ spec: description: |- ClientAuthentication can optionally contain client configuration used for kubeconfig generation. + Deprecated: This field has no implemented use and will be forbidden starting from Kubernetes 1.31. It's use was planned for genereting OIDC kubeconfig https://github.com/gardener/gardener/issues/1433 + TODO(AleksandarSavchev): Drop this field after support for Kubernetes 1.30 is dropped. properties: extraConfig: additionalProperties: @@ -757,8 +761,10 @@ spec: SystemReserved is the configuration for resources reserved for system processes not managed by kubernetes (e.g. journald). When updating these values, be aware that cgroup resizes may not succeed on active worker nodes. Look for the NodeAllocatableEnforced event to determine if the configuration was applied. + Deprecated: Separately configuring resource reservations for system processes is deprecated in Gardener and will be forbidden starting from Kubernetes 1.31. Please merge existing resource reservations into the kubeReserved field. + TODO(MichaelEischer): Drop this field after support for Kubernetes 1.30 is dropped. properties: cpu: anyOf: @@ -1025,8 +1031,16 @@ spec: description: List of status conditions to indicate the status of a ServiceInstance. items: - description: Condition contains details for one aspect of the current - state of this API Resource. + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" properties: lastTransitionTime: description: |- @@ -1067,7 +1081,12 @@ spec: - Unknown type: string type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -1087,8 +1106,6 @@ spec: - Terminating - Failed type: string - required: - - state type: object type: object served: true diff --git a/hack/runtime-migrator/go.mod b/hack/runtime-migrator/go.mod index 9c515c85..b98321f3 100644 --- a/hack/runtime-migrator/go.mod +++ b/hack/runtime-migrator/go.mod @@ -16,6 +16,7 @@ require ( ) require ( + github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect @@ -35,6 +36,8 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/hack/runtime-migrator/go.sum b/hack/runtime-migrator/go.sum index 691e95af..d685b9a9 100644 --- a/hack/runtime-migrator/go.sum +++ b/hack/runtime-migrator/go.sum @@ -1,3 +1,5 @@ +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -56,6 +58,11 @@ github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSF github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -102,6 +109,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= From b642bad8f982223e158ef12db2646b66cfe65263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Thu, 7 Nov 2024 08:25:42 +0100 Subject: [PATCH 04/10] Revert "cluster rolebinding requeues when listing or update fails" This reverts commit e41c07cdf0a2d16fb087fc320773a44ee50371ac. --- .../runtime/fsm/runtime_fsm_apply_clusterrolebindings.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go index d5539aaa..7e7737ed 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go +++ b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go @@ -35,7 +35,7 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s var crbList rbacv1.ClusterRoleBindingList if err := shootAdminClient.List(ctx, &crbList); err != nil { updateCRBApplyFailed(&s.instance) - return requeue() + return updateStatusAndStopWithError(err) } removed := getRemoved(crbList.Items, s.instance.Spec.Security.Administrators) @@ -47,7 +47,7 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s } { if err := fn(); err != nil { updateCRBApplyFailed(&s.instance) - return requeue() + return updateStatusAndStopWithError(err) } } From 95784f633d35fbb84515716d0b37709c80322350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Thu, 7 Nov 2024 08:27:47 +0100 Subject: [PATCH 05/10] Revert "Revert "cluster rolebinding requeues when listing or update fails"" This reverts commit b642bad8f982223e158ef12db2646b66cfe65263. --- .../runtime/fsm/runtime_fsm_apply_clusterrolebindings.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go index 7e7737ed..d5539aaa 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go +++ b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go @@ -35,7 +35,7 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s var crbList rbacv1.ClusterRoleBindingList if err := shootAdminClient.List(ctx, &crbList); err != nil { updateCRBApplyFailed(&s.instance) - return updateStatusAndStopWithError(err) + return requeue() } removed := getRemoved(crbList.Items, s.instance.Spec.Security.Administrators) @@ -47,7 +47,7 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s } { if err := fn(); err != nil { updateCRBApplyFailed(&s.instance) - return updateStatusAndStopWithError(err) + return requeue() } } From a273cce3470d0d51b704a8b4af5bb89d05f24b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Thu, 7 Nov 2024 08:28:01 +0100 Subject: [PATCH 06/10] Revert "go mod tidy in migrator" This reverts commit 3c1a33e94eb51a864fc48a7f121c257ef920bb99. --- ...ager.kyma-project.io_gardenerclusters.yaml | 21 +++----------- ...cturemanager.kyma-project.io_runtimes.yaml | 29 ++++--------------- hack/runtime-migrator/go.mod | 3 -- hack/runtime-migrator/go.sum | 9 ------ 4 files changed, 10 insertions(+), 52 deletions(-) diff --git a/config/crd/bases/infrastructuremanager.kyma-project.io_gardenerclusters.yaml b/config/crd/bases/infrastructuremanager.kyma-project.io_gardenerclusters.yaml index 48ee34fa..2a26b5e2 100644 --- a/config/crd/bases/infrastructuremanager.kyma-project.io_gardenerclusters.yaml +++ b/config/crd/bases/infrastructuremanager.kyma-project.io_gardenerclusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.4 name: gardenerclusters.infrastructuremanager.kyma-project.io spec: group: infrastructuremanager.kyma-project.io @@ -92,16 +92,8 @@ spec: description: List of status conditions to indicate the status of a ServiceInstance. items: - description: "Condition contains details for one aspect of the current - state of this API Resource.\n---\nThis struct is intended for - direct use as an array at the field path .status.conditions. For - example,\n\n\n\ttype FooStatus struct{\n\t // Represents the - observations of a foo's current state.\n\t // Known .status.conditions.type - are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // - +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t - \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t - \ // other fields\n\t}" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: description: |- @@ -142,12 +134,7 @@ spec: - Unknown type: string type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + description: type of condition in CamelCase or in foo.example.com/CamelCase. maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string diff --git a/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml b/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml index 37ed9d50..cbdbefb3 100644 --- a/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml +++ b/config/crd/bases/infrastructuremanager.kyma-project.io_runtimes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.4 name: runtimes.infrastructuremanager.kyma-project.io spec: group: infrastructuremanager.kyma-project.io @@ -134,10 +134,8 @@ spec: description: |- ClientAuthentication can optionally contain client configuration used for kubeconfig generation. - Deprecated: This field has no implemented use and will be forbidden starting from Kubernetes 1.31. It's use was planned for genereting OIDC kubeconfig https://github.com/gardener/gardener/issues/1433 - TODO(AleksandarSavchev): Drop this field after support for Kubernetes 1.30 is dropped. properties: extraConfig: additionalProperties: @@ -220,10 +218,8 @@ spec: description: |- ClientAuthentication can optionally contain client configuration used for kubeconfig generation. - Deprecated: This field has no implemented use and will be forbidden starting from Kubernetes 1.31. It's use was planned for genereting OIDC kubeconfig https://github.com/gardener/gardener/issues/1433 - TODO(AleksandarSavchev): Drop this field after support for Kubernetes 1.30 is dropped. properties: extraConfig: additionalProperties: @@ -761,10 +757,8 @@ spec: SystemReserved is the configuration for resources reserved for system processes not managed by kubernetes (e.g. journald). When updating these values, be aware that cgroup resizes may not succeed on active worker nodes. Look for the NodeAllocatableEnforced event to determine if the configuration was applied. - Deprecated: Separately configuring resource reservations for system processes is deprecated in Gardener and will be forbidden starting from Kubernetes 1.31. Please merge existing resource reservations into the kubeReserved field. - TODO(MichaelEischer): Drop this field after support for Kubernetes 1.30 is dropped. properties: cpu: anyOf: @@ -1031,16 +1025,8 @@ spec: description: List of status conditions to indicate the status of a ServiceInstance. items: - description: "Condition contains details for one aspect of the current - state of this API Resource.\n---\nThis struct is intended for - direct use as an array at the field path .status.conditions. For - example,\n\n\n\ttype FooStatus struct{\n\t // Represents the - observations of a foo's current state.\n\t // Known .status.conditions.type - are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // - +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t - \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t - \ // other fields\n\t}" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: description: |- @@ -1081,12 +1067,7 @@ spec: - Unknown type: string type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + description: type of condition in CamelCase or in foo.example.com/CamelCase. maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -1106,6 +1087,8 @@ spec: - Terminating - Failed type: string + required: + - state type: object type: object served: true diff --git a/hack/runtime-migrator/go.mod b/hack/runtime-migrator/go.mod index b98321f3..9c515c85 100644 --- a/hack/runtime-migrator/go.mod +++ b/hack/runtime-migrator/go.mod @@ -16,7 +16,6 @@ require ( ) require ( - github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect @@ -36,8 +35,6 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/hack/runtime-migrator/go.sum b/hack/runtime-migrator/go.sum index d685b9a9..691e95af 100644 --- a/hack/runtime-migrator/go.sum +++ b/hack/runtime-migrator/go.sum @@ -1,5 +1,3 @@ -github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= -github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -58,11 +56,6 @@ github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSF github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -109,8 +102,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= -go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= From 20c4fe60ebf6e159e02c36fef7b1067825bfe64f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Thu, 7 Nov 2024 08:28:28 +0100 Subject: [PATCH 07/10] go mod tidy in migrator --- hack/runtime-migrator/go.mod | 3 +++ hack/runtime-migrator/go.sum | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/hack/runtime-migrator/go.mod b/hack/runtime-migrator/go.mod index 9c515c85..b98321f3 100644 --- a/hack/runtime-migrator/go.mod +++ b/hack/runtime-migrator/go.mod @@ -16,6 +16,7 @@ require ( ) require ( + github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect @@ -35,6 +36,8 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/hack/runtime-migrator/go.sum b/hack/runtime-migrator/go.sum index 691e95af..d685b9a9 100644 --- a/hack/runtime-migrator/go.sum +++ b/hack/runtime-migrator/go.sum @@ -1,3 +1,5 @@ +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -56,6 +58,11 @@ github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSF github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -102,6 +109,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= From d579b8cac30d5b5b1eb9cca030abd8b8658b7d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Thu, 7 Nov 2024 08:30:33 +0100 Subject: [PATCH 08/10] linter fixes --- .../runtime/fsm/runtime_fsm_waiting_shoot_creation.go | 1 + internal/controller/runtime/suite_test.go | 1 - pkg/gardener/gardener_error_handler.go | 5 +++-- pkg/gardener/gardener_error_handler_test.go | 9 +++++---- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go b/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go index ae98ab4a..db2a0fa3 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go +++ b/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go @@ -3,6 +3,7 @@ package fsm import ( "context" "fmt" + gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" imgardenerhandler "github.com/kyma-project/infrastructure-manager/pkg/gardener" diff --git a/internal/controller/runtime/suite_test.go b/internal/controller/runtime/suite_test.go index 41d969fe..169c8174 100644 --- a/internal/controller/runtime/suite_test.go +++ b/internal/controller/runtime/suite_test.go @@ -39,7 +39,6 @@ import ( v12 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - //nolint:revive "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" diff --git a/pkg/gardener/gardener_error_handler.go b/pkg/gardener/gardener_error_handler.go index a4f2e8f7..0ae41da9 100644 --- a/pkg/gardener/gardener_error_handler.go +++ b/pkg/gardener/gardener_error_handler.go @@ -2,10 +2,11 @@ package gardener import ( "fmt" - gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" - gardenerhelper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" "strconv" "strings" + + gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" + gardenerhelper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" ) type ErrReason string diff --git a/pkg/gardener/gardener_error_handler_test.go b/pkg/gardener/gardener_error_handler_test.go index 01928ffd..ac83ab22 100644 --- a/pkg/gardener/gardener_error_handler_test.go +++ b/pkg/gardener/gardener_error_handler_test.go @@ -1,14 +1,15 @@ package gardener import ( + "testing" + gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1" "github.com/stretchr/testify/assert" - "testing" ) func TestGardenerErrorHandler(t *testing.T) { t.Run("Should combine error descriptions", func(t *testing.T) { - //given + // given lastErrors := []gardener.LastError{ { Description: "First description", @@ -18,10 +19,10 @@ func TestGardenerErrorHandler(t *testing.T) { }, } - //when + // when combinedDescritpion := CombineErrorDescriptions(lastErrors) - //then + // then assert.Equal(t, "1) First description 2) Second description ", combinedDescritpion) }) From bad87c78d17a18ee3c92c64466bb0b139f40c8a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Thu, 7 Nov 2024 09:27:34 +0100 Subject: [PATCH 09/10] adds additional logs on unexpected rolebindings operations --- .../runtime/fsm/runtime_fsm_apply_clusterrolebindings.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go index d5539aaa..0a6b9c17 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go +++ b/internal/controller/runtime/fsm/runtime_fsm_apply_clusterrolebindings.go @@ -35,6 +35,7 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s var crbList rbacv1.ClusterRoleBindingList if err := shootAdminClient.List(ctx, &crbList); err != nil { updateCRBApplyFailed(&s.instance) + m.log.Info("Cannot list Cluster Role Bindings on shoot, scheduling for retry", "RuntimeCR", s.instance.Name, "shoot", s.shoot.Name) return requeue() } @@ -47,6 +48,7 @@ func sFnApplyClusterRoleBindings(ctx context.Context, m *fsm, s *systemState) (s } { if err := fn(); err != nil { updateCRBApplyFailed(&s.instance) + m.log.Info("Cannot setup Cluster Role Bindings on shoot, scheduling for retry", "RuntimeCR", s.instance.Name, "shoot", s.shoot.Name) return requeue() } } From cbd6dbfd432f93730a2a1c51ea3dabf889be282b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Mon, 18 Nov 2024 07:07:12 +0100 Subject: [PATCH 10/10] updates to pending when retryable error occured --- .../runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go | 6 +++++- .../runtime/fsm/runtime_fsm_waiting_shoot_creation.go | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go b/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go index 66db8534..788e60d1 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go +++ b/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go @@ -31,7 +31,11 @@ func sFnWaitForShootReconcile(_ context.Context, m *fsm, s *systemState) (stateF if imgardenerhandler.IsRetryable(lastErrors) { m.log.Info(fmt.Sprintf("Retryable gardener errors during cluster provisioning for Shoot %s, reason: %s, scheduling for retry", s.shoot.Name, reason)) - // TODO: should update status? + s.instance.UpdateStatePending( + imv1.ConditionTypeRuntimeProvisioned, + imv1.ConditionReasonShootCreationPending, + "Unknown", + "Retryable gardener errors during cluster reconcile") return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) } diff --git a/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go b/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go index db2a0fa3..2c0036fc 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go +++ b/internal/controller/runtime/fsm/runtime_fsm_waiting_shoot_creation.go @@ -47,7 +47,11 @@ func sFnWaitForShootCreation(_ context.Context, m *fsm, s *systemState) (stateFn if imgardenerhandler.IsRetryable(lastErrors) { m.log.Info(fmt.Sprintf("Retryable gardener errors during cluster provisioning for Shoot %s, reason: %s, scheduling for retry", s.shoot.Name, reason)) - //TODO: should update status? + s.instance.UpdateStatePending( + imv1.ConditionTypeRuntimeProvisioned, + imv1.ConditionReasonShootCreationPending, + "Unknown", + "Retryable gardener errors during cluster provisioning") return updateStatusAndRequeueAfter(m.RCCfg.GardenerRequeueDuration) }