Merge pull request #487 from m00g3n/fix-migrator-timeout-issue

fix issue with migrator's timeout

hack/runtime-migrator/cmd/main.go

@@ -5,16 +5,17 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/go-playground/validator/v10"
-	"github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs"
 	"io"
-	v12 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/types"
 	"log/slog"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/go-playground/validator/v10"
+	"github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs"
+	v12 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+
 	"github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	gardener_types "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1"
 	"github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/config"
@@ -28,7 +29,7 @@ import (
 )
 
 const (
-	contextTimeout      = 5 * time.Minute
+	timeoutK8sOperation = 5 * time.Second
 	expirationTime      = 60 * time.Minute
 	runtimeIDAnnotation = "kcp.provisioner.kyma-project.io/runtime-id"
 )
@@ -89,7 +90,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	err = migrator.Do(runtimeIds)
+	err = migrator.Do(context.Background(), runtimeIds)
 	if err != nil {
 		slog.Error(fmt.Sprintf("Failed to migrate runtimes: %v", slog.Any("error", err)))
 		os.Exit(1)

hack/runtime-migrator/cmd/migration.go

@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
+	"log/slog"
+
 	"github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	gardener_types "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1"
 	runtimev1 "github.com/kyma-project/infrastructure-manager/api/v1"
@@ -13,7 +15,6 @@ import (
 	"github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig"
 	"github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"log/slog"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -43,29 +44,26 @@ func NewMigration(migratorConfig config2.Config, converterConfig config.Converte
 	}, nil
 }
 
-func (m Migration) Do(runtimeIDs []string) error {
+func (m Migration) Do(ctx context.Context, runtimeIDs []string) error {
 
-	migratorContext, cancel := context.WithTimeout(context.Background(), contextTimeout)
+	listCtx, cancel := context.WithTimeout(ctx, timeoutK8sOperation)
 	defer cancel()
 
-	shootList, err := m.shootClient.List(migratorContext, v1.ListOptions{})
+	shootList, err := m.shootClient.List(listCtx, v1.ListOptions{})
 	if err != nil {
 		return err
 	}
 
 	results := migration.NewMigratorResults(m.outputWriter.NewResultsDir)
 
 	reportError := func(runtimeID, shootName string, msg string, err error) {
-		var errorMsg string
 
 		if err != nil {
-			errorMsg = fmt.Sprintf("%s: %v", msg, err)
-		} else {
-			errorMsg = fmt.Sprintf(msg)
+			msg = fmt.Sprintf("%s: %v", msg, err)
 		}
 
-		results.ErrorOccurred(runtimeID, shootName, errorMsg)
-		slog.Error(errorMsg, "runtimeID", runtimeID)
+		results.ErrorOccurred(runtimeID, shootName, msg)
+		slog.Error(msg, "runtimeID", runtimeID)
 	}
 
 	reportValidationError := func(runtimeID, shootName string, msg string, err error) {
@@ -84,58 +82,70 @@ func (m Migration) Do(runtimeIDs []string) error {
 		slog.Info(msg, "runtimeID", runtimeID)
 	}
 
-	for _, runtimeID := range runtimeIDs {
+	run := func(runtimeID string) {
 		shoot := findShoot(runtimeID, shootList)
 		if shoot == nil {
 			reportError(runtimeID, "", "Failed to find shoot", nil)
-
-			continue
+			return
 		}
 
-		runtime, err := m.runtimeMigrator.Do(migratorContext, *shoot)
+		migrationCtx, cancel := context.WithTimeout(ctx, timeoutK8sOperation)
+		defer cancel()
+
+		runtime, err := m.runtimeMigrator.Do(migrationCtx, *shoot)
+
 		if err != nil {
 			reportError(runtimeID, shoot.Name, "Failed to migrate runtime", err)
-
-			continue
+			return
 		}
 
 		err = m.outputWriter.SaveRuntimeCR(runtime)
 		if err != nil {
 			reportError(runtimeID, shoot.Name, "Failed to save runtime CR", err)
-
-			continue
+			return
 		}
 
 		shootComparisonResult, err := m.runtimeVerifier.Do(runtime, *shoot)
 		if err != nil {
 			reportValidationError(runtimeID, shoot.Name, "Failed to verify runtime", err)
-
-			continue
+			return
 		}
 
 		if !shootComparisonResult.IsEqual() {
 			err = m.outputWriter.SaveComparisonResult(shootComparisonResult)
 			if err != nil {
 				reportError(runtimeID, shoot.Name, "Failed to save comparison result", err)
-			} else {
-				reportUnwantedUpdateDetected(runtimeID, shoot.Name, "Runtime CR can cause unwanted update in Gardener. Please verify the runtime CR.")
+				return
 			}
 
-			continue
+			reportUnwantedUpdateDetected(runtimeID, shoot.Name, "Runtime CR can cause unwanted update in Gardener. Please verify the runtime CR.")
+			return
 		}
 
 		if !m.isDryRun {
 			err = m.applyRuntimeCR(runtime)
 			if err != nil {
 				reportError(runtimeID, shoot.Name, "Failed to apply Runtime CR", err)
 			}
-
-			continue
+			return
 		}
 
 		reportSuccess(runtimeID, shoot.Name, "Runtime processed successfully")
 	}
 
+main:
+	for _, runtimeID := range runtimeIDs {
+		select {
+		case <-ctx.Done():
+			// application context was canceled
+			reportError(runtimeID, "", "Failed to find shoot", nil)
+			break main
+
+		default:
+			run(runtimeID)
+		}
+	}
+
 	resultsFile, err := m.outputWriter.SaveMigrationResults(results)
 	if err != nil {
 		return err