diff --git a/cmd/main.go b/cmd/main.go index 8d616bee..3806bdac 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -40,7 +40,9 @@ import ( "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" "github.com/kyma-project/infrastructure-manager/pkg/gardener/shoot/extender/auditlogs" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + k8slabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -48,6 +50,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/util/flowcontrol" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -133,6 +136,7 @@ func main() { HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "f1c68560.kyma-project.io", + Cache: restrictWatchedNamespace(), // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily // when the Manager ends. This requires the binary to immediately end when the // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly @@ -334,3 +338,26 @@ func refreshRuntimeMetrics(restConfig *rest.Config, logger logr.Logger, metrics metrics.SetRuntimeStates(rt) } } + +func restrictWatchedNamespace() cache.Options { + return cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Secret{}: { + Label: k8slabels.Everything(), + Namespaces: map[string]cache.Config{ + "kcp-system": {}, + }, + }, + &infrastructuremanagerv1.Runtime{}: { + Namespaces: map[string]cache.Config{ + "kcp-system": {}, + }, + }, + &infrastructuremanagerv1.GardenerCluster{}: { + Namespaces: map[string]cache.Config{ + "kcp-system": {}, + }, + }, + }, + } +} diff --git a/hack/runtime-migrator/README.md b/hack/runtime-migrator/README.md index f0874ba4..be41f54b 100644 --- a/hack/runtime-migrator/README.md +++ b/hack/runtime-migrator/README.md @@ -54,29 +54,29 @@ cat ./runtime-migrator \ ### Output example ``` -2024/11/21 14:53:24 INFO Starting runtime-migrator -2024/11/21 14:53:24 gardener-kubeconfig-path: /Users/myuser/gardener-kubeconfig.yml -2024/11/21 14:53:24 kcp-kubeconfig-path: /Users/myuser/kcp-kubeconfig.yml -2024/11/21 14:53:24 gardener-project-name: kyma-stage -2024/11/21 14:53:24 output-path: /tmp/ -2024/11/21 14:53:24 dry-run: true -2024/11/21 14:53:24 input-type: txt -2024/11/21 14:53:24 input-file-path: /Users/myuser/migrator-input/runtimeIds.txt -2024/11/21 14:53:24 -2024/11/21 14:53:24 INFO Migrating runtimes -2024/11/21 14:53:24 INFO Reading runtimeIds from input file -2024/11/21 14:53:29 INFO Runtime processed successfully runtimeID=1df09b5b-0347-459d-aa0a-715db8fcaad7 -2024/11/21 14:53:32 INFO Runtime processed successfully runtimeID=ea439a5e-aa59-4e3e-8bfb-9bab1b31371e -2024/11/21 14:53:33 INFO Runtime processed successfully runtimeID=d6eeafee-ffd5-4f23-97dc-a1df197b3b30 -2024/11/21 14:53:37 WARN Runtime CR can cause unwanted update in Gardener runtimeID=99a38a99-e8d7-4b98-a6f2-5a54ed389c4d -2024/11/21 14:53:37 ERROR Failed to find shoot: no shoot with given runtimeID found runtimeID=0a61a3c4-0ea8-4e39-860a-7853f0b6d180 -2024/11/21 14:53:40 ERROR Failed to verify runtime runtimeID=6daf5f59-b0ab-44af-bb8e-7735fd609449 -2024/11/21 14:53:40 INFO Migration completed. Successfully migrated runtimes: 3, Failed migrations: 2, Differences detected: 1 -2024/11/21 14:53:40 INFO Migration results saved in: /tmp/migration-2024-11-21T14:53:24+01:00/migration-results.json +2024/11/22 17:05:28 INFO Starting runtime-migrator +2024/11/22 17:05:28 gardener-kubeconfig-path: /Users/i326211/Downloads/kubeconfig-garden-kyma-stage.yaml +2024/11/22 17:05:28 kcp-kubeconfig-path: /Users/i326211/dev/config/sap +2024/11/22 17:05:28 gardener-project-name: kyma-stage +2024/11/22 17:05:28 output-path: /tmp/ +2024/11/22 17:05:28 dry-run: true +2024/11/22 17:05:28 input-type: txt +2024/11/22 17:05:28 input-file-path: /Users/i326211/dev/source/infrastructure-manager/hack/runtime-migrator/input/runtimes-stage-docs.txt +2024/11/22 17:05:28 +2024/11/22 17:05:33 INFO Migrating runtimes +2024/11/22 17:05:33 INFO Reading runtimeIds from input file +2024/11/22 17:05:43 INFO Runtime processed successfully (dry run) runtimeID=1df09b5b-0347-459d-aa0a-715db8fcaad7 +2024/11/22 17:05:45 INFO Runtime processed successfully (dry run) runtimeID=ea439a5e-aa59-4e3e-8bfb-9bab1b31371e +2024/11/22 17:05:49 INFO Runtime processed successfully (dry run) runtimeID=d6eeafee-ffd5-4f23-97dc-a1df197b3b30 +2024/11/22 17:05:52 WARN Runtime CR can cause unwanted update in Gardener runtimeID=99a38a99-e8d7-4b98-a6f2-5a54ed389c4d +2024/11/22 17:05:52 ERROR Failed to fetch shoot: shoot was deleted or the runtime ID is incorrect runtimeID=0a61a3c4-0ea8-4e39-860a-7853f0b6d180 +2024/11/22 17:05:55 ERROR Failed to verify runtime runtimeID=6daf5f59-b0ab-44af-bb8e-7735fd609449 +2024/11/22 17:05:55 INFO Migration completed. Successfully migrated runtimes: 3, Failed migrations: 2, Differences detected: 1 +2024/11/22 17:05:55 INFO Migration results saved in: /tmp/migration-2024-11-22T17:05:33+01:00/migration-results.json ``` -The migration results are saved in the `/tmp/migration-2024-11-21T14:53:24+01:00/migration-results.json` file. -The runtime custom resources are saved in the `/tmp/migration-2024-11-21T14:53:24+01:00/runtimes` directory. +The migration results are saved in the `/tmp/migration-2024-11-22T17:05:33+01:00/migration-results.json` file. +The runtime custom resources are saved in the `/tmp/migration-2024-11-22T17:05:33+01:00/runtimes` directory. The `migration-results.json` file contains the following content: ```json @@ -85,40 +85,40 @@ The `migration-results.json` file contains the following content: "runtimeId": "1df09b5b-0347-459d-aa0a-715db8fcaad7", "shootName": "c-1228ddd", "status": "Success", - "runtimeCRFilePath": "/tmp/migration-2024-11-21T14:53:24+01:00/runtimes/1df09b5b-0347-459d-aa0a-715db8fcaad7.yaml" + "runtimeCRFilePath": "/tmp/migration-2024-11-22T17:05:33+01:00/runtimes/1df09b5b-0347-459d-aa0a-715db8fcaad7.yaml" }, { "runtimeId": "ea439a5e-aa59-4e3e-8bfb-9bab1b31371e", "shootName": "c3a59d5", "status": "Success", - "runtimeCRFilePath": "/tmp/migration-2024-11-21T14:53:24+01:00/runtimes/ea439a5e-aa59-4e3e-8bfb-9bab1b31371e.yaml" + "runtimeCRFilePath": "/tmp/migration-2024-11-22T17:05:33+01:00/runtimes/ea439a5e-aa59-4e3e-8bfb-9bab1b31371e.yaml" }, { "runtimeId": "d6eeafee-ffd5-4f23-97dc-a1df197b3b30", "shootName": "c141da7", "status": "Success", - "runtimeCRFilePath": "/tmp/migration-2024-11-21T14:53:24+01:00/runtimes/d6eeafee-ffd5-4f23-97dc-a1df197b3b30.yaml" + "runtimeCRFilePath": "/tmp/migration-2024-11-22T17:05:33+01:00/runtimes/d6eeafee-ffd5-4f23-97dc-a1df197b3b30.yaml" }, { "runtimeId": "99a38a99-e8d7-4b98-a6f2-5a54ed389c4d", "shootName": "c-71da0f2", "status": "ValidationDetectedUnwantedUpdate", - "errorMessage": "Runtime may cause unwanted update in Gardener", - "runtimeCRFilePath": "/tmp/migration-2024-11-21T14:53:24+01:00/runtimes/99a38a99-e8d7-4b98-a6f2-5a54ed389c4d.yaml", - "comparisonResultDirPath": "/tmp/migration-2024-11-21T14:53:24+01:00/comparison-results/99a38a99-e8d7-4b98-a6f2-5a54ed389c4d" + "errorMessage": "Runtime may cause unwanted update in Gardener. Please verify the runtime CR.", + "runtimeCRFilePath": "/tmp/migration-2024-11-22T17:05:33+01:00/runtimes/99a38a99-e8d7-4b98-a6f2-5a54ed389c4d.yaml", + "comparisonResultDirPath": "/tmp/migration-2024-11-22T17:05:33+01:00/comparison-results/99a38a99-e8d7-4b98-a6f2-5a54ed389c4d" }, { "runtimeId": "0a61a3c4-0ea8-4e39-860a-7853f0b6d180", "shootName": "", "status": "Error", - "errorMessage": "Failed to find shoot: no shoot with given runtimeID found" + "errorMessage": "Failed to fetch shoot: shoot was deleted or the runtime ID is incorrect" }, { "runtimeId": "6daf5f59-b0ab-44af-bb8e-7735fd609449", "shootName": "c-1f810d0", "status": "ValidationError", "errorMessage": "Failed to verify runtime: audit logs configuration not found: missing region: 'australiaeast' for providerType: 'azure'", - "runtimeCRFilePath": "/tmp/migration-2024-11-21T14:53:24+01:00/runtimes/6daf5f59-b0ab-44af-bb8e-7735fd609449.yaml" + "runtimeCRFilePath": "/tmp/migration-2024-11-22T17:05:33+01:00/runtimes/6daf5f59-b0ab-44af-bb8e-7735fd609449.yaml" } ] @@ -126,7 +126,7 @@ The `migration-results.json` file contains the following content: The following problems were detected in the above example: - The runtime with the `0a61a3c4-0ea8-4e39-860a-7853f0b6d180` identifier was not found ; the identifier may be incorrect, or the corresponding shoot was deleted for some reason. - The validation process for the runtime with the `6daf5f59-b0ab-44af-bb8e-7735fd609449` identifier failed. -- The runtime with the `99a38a99-e8d7-4b98-a6f2-5a54ed389c4d` identifier may cause an unwanted update in the Gardener. The comparison results are saved in the `/tmp/migration-2024-11-21T14:53:24+01:00/comparison-results/99a38a99-e8d7-4b98-a6f2-5a54ed389c4d` directory. +- The runtime with the `99a38a99-e8d7-4b98-a6f2-5a54ed389c4d` identifier may cause an unwanted update in the Gardener. The comparison results are saved in the `/tmp/migration-2024-11-22T17:05:33+01:00/comparison-results/99a38a99-e8d7-4b98-a6f2-5a54ed389c4d` directory. The `/tmp/migration-2024-11-21T14:53:24+01:00/comparison-results/99a38a99-e8d7-4b98-a6f2-5a54ed389c4d"` directory contains the following files: diff --git a/hack/runtime-migrator/cmd/main.go b/hack/runtime-migrator/cmd/main.go index 71e5c25b..264a25a9 100644 --- a/hack/runtime-migrator/cmd/main.go +++ b/hack/runtime-migrator/cmd/main.go @@ -28,7 +28,7 @@ import ( ) const ( - timeoutK8sOperation = 10 * time.Second + timeoutK8sOperation = 20 * time.Second expirationTime = 60 * time.Minute runtimeIDAnnotation = "kcp.provisioner.kyma-project.io/runtime-id" ) @@ -53,45 +53,45 @@ func main() { kcpClient, err := config.CreateKcpClient(&cfg) if err != nil { - slog.Error("Failed to create kcp client: %v ", slog.Any("error", err)) + slog.Error("Failed to create kcp client", slog.Any("error", err)) os.Exit(1) } gardenerShootClient, err := setupGardenerShootClient(cfg.GardenerKubeconfigPath, gardenerNamespace) if err != nil { - slog.Error("Failed to setup Gardener shoot client: %v", slog.Any("error", err)) + slog.Error("Failed to setup Gardener shoot client", slog.Any("error", err)) os.Exit(1) } auditLogConfig, err := getAuditLogConfig(kcpClient) if err != nil { - slog.Error("Failed to get audit log config: %v", slog.Any("error", err)) + slog.Error("Failed to get audit log config", slog.Any("error", err)) os.Exit(1) } converterConfig, err := getConverterConfig(kcpClient) if err != nil { - slog.Error("Failed to get converter config: %v", slog.Any("error", err)) + slog.Error("Failed to get converter config", slog.Any("error", err)) os.Exit(1) } slog.Info("Migrating runtimes") migrator, err := NewMigration(cfg, converterConfig, auditLogConfig, kubeconfigProvider, kcpClient, gardenerShootClient) if err != nil { - slog.Error("Failed to create migrator: %v", slog.Any("error", err)) + slog.Error("Failed to create migrator", slog.Any("error", err)) os.Exit(1) } slog.Info("Reading runtimeIds from input file") runtimeIds, err := getRuntimeIDsFromInputFile(cfg) if err != nil { - slog.Error("Failed to read runtime Ids from input: %v", slog.Any("error", err)) + slog.Error("Failed to read runtime Ids from input", slog.Any("error", err)) os.Exit(1) } err = migrator.Do(context.Background(), runtimeIds) if err != nil { - slog.Error(fmt.Sprintf("Failed to migrate runtimes: %v", slog.Any("error", err))) + slog.Error("Failed to migrate runtimes", slog.Any("error", err)) os.Exit(1) } } diff --git a/hack/runtime-migrator/cmd/migration.go b/hack/runtime-migrator/cmd/migration.go index a496fe96..b8266813 100644 --- a/hack/runtime-migrator/cmd/migration.go +++ b/hack/runtime-migrator/cmd/migration.go @@ -3,6 +3,8 @@ package main import ( "context" "fmt" + "github.com/pkg/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "log/slog" "github.com/gardener/gardener/pkg/apis/core/v1beta1" @@ -36,7 +38,7 @@ func NewMigration(migratorConfig config2.Config, converterConfig config.Converte } return Migration{ - runtimeMigrator: runtime.NewMigrator(migratorConfig, kubeconfigProvider), + runtimeMigrator: runtime.NewMigrator(migratorConfig, kubeconfigProvider, kcpClient), runtimeVerifier: runtime.NewVerifier(converterConfig, auditLogConfig), kcpClient: kcpClient, shootClient: shootClient, @@ -84,9 +86,14 @@ func (m Migration) Do(ctx context.Context, runtimeIDs []string) error { } run := func(runtimeID string) { - shoot := findShoot(runtimeID, shootList) - if shoot == nil { - reportError(runtimeID, "", "Failed to find shoot", errors.New("no shoot with given runtimeID found")) + shoot, err := m.fetchShoot(ctx, shootList, m.shootClient, runtimeID) + if err != nil { + reportError(runtimeID, "", "Failed to fetch shoot", err) + return + } + + if shootIsBeingDeleted(shoot) { + reportError(runtimeID, shoot.Name, "Runtime is being deleted", nil) return } @@ -123,15 +130,17 @@ func (m Migration) Do(ctx context.Context, runtimeIDs []string) error { return } - if !m.isDryRun { + if m.isDryRun { + reportSuccess(runtimeID, shoot.Name, "Runtime processed successfully (dry-run)") + } else { err = m.applyRuntimeCR(runtime) if err != nil { reportError(runtimeID, shoot.Name, "Failed to apply Runtime CR", err) + return } - return - } - reportSuccess(runtimeID, shoot.Name, "Runtime processed successfully") + reportSuccess(runtimeID, shoot.Name, "Runtime have been applied") + } } main: @@ -167,6 +176,32 @@ func findShoot(runtimeID string, shootList *v1beta1.ShootList) *v1beta1.Shoot { return nil } +func (m Migration) fetchShoot(ctx context.Context, shootList *v1beta1.ShootList, shootClient gardener_types.ShootInterface, runtimeID string) (*v1beta1.Shoot, error) { + shoot := findShoot(runtimeID, shootList) + if shoot == nil { + return nil, errors.New("shoot was deleted or the runtimeID is incorrect") + } + + getCtx, cancel := context.WithTimeout(ctx, timeoutK8sOperation) + defer cancel() + + // We are fetching the shoot from the gardener to make sure the runtime didn't get deleted during the migration process + refreshedShoot, err := m.shootClient.Get(getCtx, shoot.Name, v1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, errors.New("shoot was deleted") + } + + return nil, err + } + + return refreshedShoot, nil +} + +func shootIsBeingDeleted(shoot *v1beta1.Shoot) bool { + return !shoot.DeletionTimestamp.IsZero() +} + func (m Migration) applyRuntimeCR(runtime runtimev1.Runtime) error { // TODO: This method covers create scenario only, we should implement update as well return m.kcpClient.Create(context.Background(), &runtime) diff --git a/hack/runtime-migrator/internal/config/config.go b/hack/runtime-migrator/internal/config/config.go index 2f1a60ac..b4d26141 100644 --- a/hack/runtime-migrator/internal/config/config.go +++ b/hack/runtime-migrator/internal/config/config.go @@ -70,22 +70,6 @@ func addToScheme(s *runtime.Scheme) error { type GetClient = func() (client.Client, error) -func (cfg *Config) Client() (client.Client, error) { - restCfg, err := clientcmd.BuildConfigFromFlags("", cfg.KcpKubeconfigPath) - if err != nil { - return nil, fmt.Errorf("unable to fetch rest config: %w", err) - } - - scheme := runtime.NewScheme() - if err := addToScheme(scheme); err != nil { - return nil, err - } - - return client.New(restCfg, client.Options{ - Scheme: scheme, - }) -} - func CreateKcpClient(cfg *Config) (client.Client, error) { restCfg, err := clientcmd.BuildConfigFromFlags("", cfg.KcpKubeconfigPath) if err != nil { diff --git a/hack/runtime-migrator/internal/runtime/migrator.go b/hack/runtime-migrator/internal/runtime/migrator.go index 45574acd..d8b6052c 100644 --- a/hack/runtime-migrator/internal/runtime/migrator.go +++ b/hack/runtime-migrator/internal/runtime/migrator.go @@ -29,10 +29,11 @@ type Migrator struct { kcpClient client.Client } -func NewMigrator(cfg migrator.Config, kubeconfigProvider kubeconfig.Provider) Migrator { +func NewMigrator(cfg migrator.Config, kubeconfigProvider kubeconfig.Provider, kcpClient client.Client) Migrator { return Migrator{ cfg: cfg, kubeconfigProvider: kubeconfigProvider, + kcpClient: kcpClient, } } @@ -45,7 +46,7 @@ func (m Migrator) Do(ctx context.Context, shoot v1beta1.Shoot) (v1.Runtime, erro var oidcConfig = getOidcConfig(shoot) var licenceType = shoot.Annotations["kcp.provisioner.kyma-project.io/licence-type"] - labels, err := getAllRuntimeLabels(ctx, shoot, m.cfg.Client) + labels, err := getAllRuntimeLabels(ctx, shoot, m.kcpClient) if err != nil { return v1.Runtime{}, err } @@ -171,16 +172,10 @@ func getOidcConfig(shoot v1beta1.Shoot) v1beta1.OIDCConfig { return oidcConfig } -func getAllRuntimeLabels(ctx context.Context, shoot v1beta1.Shoot, getClient migrator.GetClient) (map[string]string, error) { +func getAllRuntimeLabels(ctx context.Context, shoot v1beta1.Shoot, kcpClient client.Client) (map[string]string, error) { enrichedRuntimeLabels := map[string]string{} var err error - // add agreed labels from the GardenerCluster CR - k8sClient, clientErr := getClient() - - if clientErr != nil { - return map[string]string{}, errors.Wrap(clientErr, fmt.Sprintf("Failed to get GardenerClient for shoot %s - %s\n", shoot.Name, clientErr)) - } gardenerCluster := v1.GardenerCluster{} kymaID, found := shoot.Annotations["kcp.provisioner.kyma-project.io/runtime-id"] @@ -189,7 +184,7 @@ func getAllRuntimeLabels(ctx context.Context, shoot v1beta1.Shoot, getClient mig } gardenerCRKey := types.NamespacedName{Name: kymaID, Namespace: "kcp-system"} - getGardenerCRerr := k8sClient.Get(ctx, gardenerCRKey, &gardenerCluster) + getGardenerCRerr := kcpClient.Get(ctx, gardenerCRKey, &gardenerCluster) if getGardenerCRerr != nil { var errMsg = fmt.Sprintf("Failed to retrieve GardenerCluster CR for shoot %s\n", shoot.Name) return map[string]string{}, errors.Wrap(getGardenerCRerr, errMsg) @@ -203,7 +198,7 @@ func getAllRuntimeLabels(ctx context.Context, shoot v1beta1.Shoot, getClient mig enrichedRuntimeLabels["kyma-project.io/region"] = gardenerCluster.Labels["kyma-project.io/region"] enrichedRuntimeLabels["kyma-project.io/shoot-name"] = gardenerCluster.Labels["kyma-project.io/shoot-name"] enrichedRuntimeLabels["operator.kyma-project.io/kyma-name"] = gardenerCluster.Labels["operator.kyma-project.io/kyma-name"] - // The runtime CR should be controlled by the KIM in dry-run mode + // The runtime CR should be controlled by the Provisioner enrichedRuntimeLabels["kyma-project.io/controlled-by-provisioner"] = "true" // add custom label for the migrator enrichedRuntimeLabels[migratorLabel] = "true" diff --git a/hack/shoot-comparator/pkg/shoot/matcher.go b/hack/shoot-comparator/pkg/shoot/matcher.go index 6bc61c0a..e90105df 100644 --- a/hack/shoot-comparator/pkg/shoot/matcher.go +++ b/hack/shoot-comparator/pkg/shoot/matcher.go @@ -351,6 +351,7 @@ func newKubeAPIServerMatcher(k v1beta1.Kubernetes) types.GomegaMatcher { "DefaultUnreachableTolerationSeconds": gstruct.Ignore(), "EncryptionConfig": gstruct.Ignore(), "StructuredAuthentication": gstruct.Ignore(), + "StructuredAuthorization": gstruct.Ignore(), }, )) } diff --git a/hack/shoot-comparator/scripts/manifests/job.yaml b/hack/shoot-comparator/scripts/manifests/job.yaml index 4e0ba739..8bf969e5 100644 --- a/hack/shoot-comparator/scripts/manifests/job.yaml +++ b/hack/shoot-comparator/scripts/manifests/job.yaml @@ -47,7 +47,7 @@ spec: # You can specify a date when the comparison should start from. The date should be in RFC3339 format. - --fromDate - 2024-09-19T00:00:00Z - image: europe-docker.pkg.dev/kyma-project/prod/shoot-comparator:v20240919-26f83162 + image: europe-docker.pkg.dev/kyma-project/prod/shoot-comparator:v20241119-8ab6d328 name: compare-shoots resources: {} securityContext: