From 686088b0bdbb2eb064db48288077814e915beb45 Mon Sep 17 00:00:00 2001 From: m00g3n Date: Fri, 15 Nov 2024 12:38:17 +0100 Subject: [PATCH] fix issue with migrator's timeout --- hack/runtime-migrator/cmd/main.go | 13 +++--- hack/runtime-migrator/cmd/migration.go | 60 +++++++++++++++----------- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/hack/runtime-migrator/cmd/main.go b/hack/runtime-migrator/cmd/main.go index 4e98dc87..34e15cd8 100644 --- a/hack/runtime-migrator/cmd/main.go +++ b/hack/runtime-migrator/cmd/main.go @@ -5,16 +5,17 @@ import ( "context" "encoding/json" "fmt" - "github.com/go-playground/validator/v10" - "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs" "io" - v12 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" "log/slog" "os" "strings" "time" + "github.com/go-playground/validator/v10" + "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs" + v12 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "github.com/gardener/gardener/pkg/apis/core/v1beta1" gardener_types "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/config" @@ -28,7 +29,7 @@ import ( ) const ( - contextTimeout = 5 * time.Minute + timeoutK8sOperation = 5 * time.Second expirationTime = 60 * time.Minute runtimeIDAnnotation = "kcp.provisioner.kyma-project.io/runtime-id" ) @@ -89,7 +90,7 @@ func main() { os.Exit(1) } - err = migrator.Do(runtimeIds) + err = migrator.Do(context.Background(), runtimeIds) if err != nil { slog.Error(fmt.Sprintf("Failed to migrate runtimes: %v", slog.Any("error", err))) os.Exit(1) diff --git a/hack/runtime-migrator/cmd/migration.go b/hack/runtime-migrator/cmd/migration.go index 7d23a283..d19499fb 100644 --- a/hack/runtime-migrator/cmd/migration.go +++ b/hack/runtime-migrator/cmd/migration.go @@ -3,6 +3,8 @@ package main import ( "context" "fmt" + "log/slog" + "github.com/gardener/gardener/pkg/apis/core/v1beta1" gardener_types "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1" runtimev1 "github.com/kyma-project/infrastructure-manager/api/v1" @@ -13,7 +15,6 @@ import ( "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "log/slog" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -43,12 +44,12 @@ func NewMigration(migratorConfig config2.Config, converterConfig config.Converte }, nil } -func (m Migration) Do(runtimeIDs []string) error { +func (m Migration) Do(ctx context.Context, runtimeIDs []string) error { - migratorContext, cancel := context.WithTimeout(context.Background(), contextTimeout) + listCtx, cancel := context.WithTimeout(ctx, timeoutK8sOperation) defer cancel() - shootList, err := m.shootClient.List(migratorContext, v1.ListOptions{}) + shootList, err := m.shootClient.List(listCtx, v1.ListOptions{}) if err != nil { return err } @@ -56,16 +57,13 @@ func (m Migration) Do(runtimeIDs []string) error { results := migration.NewMigratorResults(m.outputWriter.NewResultsDir) reportError := func(runtimeID, shootName string, msg string, err error) { - var errorMsg string if err != nil { - errorMsg = fmt.Sprintf("%s: %v", msg, err) - } else { - errorMsg = fmt.Sprintf(msg) + msg = fmt.Sprintf("%s: %v", msg, err) } - results.ErrorOccurred(runtimeID, shootName, errorMsg) - slog.Error(errorMsg, "runtimeID", runtimeID) + results.ErrorOccurred(runtimeID, shootName, msg) + slog.Error(msg, "runtimeID", runtimeID) } reportValidationError := func(runtimeID, shootName string, msg string, err error) { @@ -84,44 +82,44 @@ func (m Migration) Do(runtimeIDs []string) error { slog.Info(msg, "runtimeID", runtimeID) } - for _, runtimeID := range runtimeIDs { + run := func(runtimeID string) { shoot := findShoot(runtimeID, shootList) if shoot == nil { reportError(runtimeID, "", "Failed to find shoot", nil) - - continue + return } - runtime, err := m.runtimeMigrator.Do(migratorContext, *shoot) + migrationCtx, cancel := context.WithTimeout(ctx, timeoutK8sOperation) + defer cancel() + + runtime, err := m.runtimeMigrator.Do(migrationCtx, *shoot) + if err != nil { reportError(runtimeID, shoot.Name, "Failed to migrate runtime", err) - - continue + return } err = m.outputWriter.SaveRuntimeCR(runtime) if err != nil { reportError(runtimeID, shoot.Name, "Failed to save runtime CR", err) - - continue + return } shootComparisonResult, err := m.runtimeVerifier.Do(runtime, *shoot) if err != nil { reportValidationError(runtimeID, shoot.Name, "Failed to verify runtime", err) - - continue + return } if !shootComparisonResult.IsEqual() { err = m.outputWriter.SaveComparisonResult(shootComparisonResult) if err != nil { reportError(runtimeID, shoot.Name, "Failed to save comparison result", err) - } else { - reportUnwantedUpdateDetected(runtimeID, shoot.Name, "Runtime CR can cause unwanted update in Gardener. Please verify the runtime CR.") + return } - continue + reportUnwantedUpdateDetected(runtimeID, shoot.Name, "Runtime CR can cause unwanted update in Gardener. Please verify the runtime CR.") + return } if !m.isDryRun { @@ -129,13 +127,25 @@ func (m Migration) Do(runtimeIDs []string) error { if err != nil { reportError(runtimeID, shoot.Name, "Failed to apply Runtime CR", err) } - - continue + return } reportSuccess(runtimeID, shoot.Name, "Runtime processed successfully") } +main: + for _, runtimeID := range runtimeIDs { + select { + case <-ctx.Done(): + // application context was canceled + reportError(runtimeID, "", "Failed to find shoot", nil) + break main + + default: + run(runtimeID) + } + } + resultsFile, err := m.outputWriter.SaveMigrationResults(results) if err != nil { return err