From f750a3611c31fff55669d91fbc0ef82f74e85710 Mon Sep 17 00:00:00 2001 From: Rafal Foks Date: Wed, 4 Sep 2024 10:23:13 +0200 Subject: [PATCH] Implement wait for Gardener shoot be ready after configuring Audit Log --- internal/auditlogging/auditlogging.go | 2 +- .../fsm/runtime_fsm_configure_auditlog.go | 29 +++++-- .../runtime_fsm_configure_auditlog_test.go | 41 +++++++++- ...runtime_fsm_waiting_for_shoot_reconcile.go | 4 +- .../runtime/runtime_controller_test.go | 2 +- internal/controller/runtime/suite_test.go | 82 +++++++++++++++++-- .../runtime/testdata/auditConfig.json | 2 +- 7 files changed, 140 insertions(+), 22 deletions(-) diff --git a/internal/auditlogging/auditlogging.go b/internal/auditlogging/auditlogging.go index 20c2dd28..5aaf150a 100644 --- a/internal/auditlogging/auditlogging.go +++ b/internal/auditlogging/auditlogging.go @@ -125,7 +125,7 @@ func (al *AuditLog) Enable(ctx context.Context, shoot *gardener.Shoot) (bool, er } } - return true, nil + return annotated, nil } func ApplyAuditLogConfig(shoot *gardener.Shoot, auditConfigFromFile map[string]map[string]AuditLogData, providerType string) (bool, error) { diff --git a/internal/controller/runtime/fsm/runtime_fsm_configure_auditlog.go b/internal/controller/runtime/fsm/runtime_fsm_configure_auditlog.go index b1ea7449..8383080f 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_configure_auditlog.go +++ b/internal/controller/runtime/fsm/runtime_fsm_configure_auditlog.go @@ -14,21 +14,32 @@ func sFnConfigureAuditLog(ctx context.Context, m *fsm, s *systemState) (stateFn, if wasAuditLogEnabled { m.log.Info("Audit Log configured for shoot: " + s.shoot.Name) - s.instance.UpdateStateReady( + s.instance.UpdateStatePending( imv1.ConditionTypeAuditLogConfigured, imv1.ConditionReasonAuditLogConfigured, - "Audit Log configured", + "Unknown", + "Waiting for Gardener shoot to be Ready state after configuration of the Audit Logs", ) - return updateStatusAndStop() + return updateStatusAndRequeueAfter(gardenerRequeueDuration) } - m.log.Error(err, "Failed to configure Audit Log") - s.instance.UpdateStatePending( + if err != nil { + m.log.Error(err, "Failed to configure Audit Log") + s.instance.UpdateStatePending( + imv1.ConditionTypeAuditLogConfigured, + imv1.ConditionReasonAuditLogError, + "False", + err.Error(), + ) + return updateStatusAndRequeueAfter(gardenerRequeueDuration) + } + + s.instance.UpdateStateReady( imv1.ConditionTypeAuditLogConfigured, - imv1.ConditionReasonAuditLogError, - "False", - err.Error(), + imv1.ConditionReasonAuditLogConfigured, + "Audit Log configured successfully", ) - return updateStatusAndRequeueAfter(gardenerRequeueDuration) + + return updateStatusAndStop() } diff --git a/internal/controller/runtime/fsm/runtime_fsm_configure_auditlog_test.go b/internal/controller/runtime/fsm/runtime_fsm_configure_auditlog_test.go index bd0dce9f..deb0bb79 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_configure_auditlog_test.go +++ b/internal/controller/runtime/fsm/runtime_fsm_configure_auditlog_test.go @@ -14,7 +14,7 @@ import ( ) func TestAuditLogState(t *testing.T) { - t.Run("Should set status on Runtime CR when Audit Log was successfully configured", func(t *testing.T) { + t.Run("Should set status on Runtime CR when Audit Log configuration was changed and Shoot enters into reconciliation", func(t *testing.T) { // given ctx := context.Background() auditLog := &mocks.AuditLogging{} @@ -27,9 +27,9 @@ func TestAuditLogState(t *testing.T) { expectedRuntimeConditions := []metav1.Condition{ { Type: string(v1.ConditionTypeAuditLogConfigured), - Status: "True", + Status: "Unknown", Reason: string(v1.ConditionReasonAuditLogConfigured), - Message: "Audit Log configured", + Message: "Waiting for Gardener shoot to be Ready state after configuration of the Audit Logs", }, } @@ -42,6 +42,41 @@ func TestAuditLogState(t *testing.T) { // set the time to its zero value for comparison purposes systemState.instance.Status.Conditions[0].LastTransitionTime = metav1.Time{} + // then + auditLog.AssertExpectations(t) + require.Contains(t, stateFn.name(), "sFnUpdateStatus") + assert.Equal(t, v1.RuntimeStatePending, string(systemState.instance.Status.State)) + assert.Equal(t, expectedRuntimeConditions, systemState.instance.Status.Conditions) + }) + + t.Run("Should set status on Runtime CR when Shoot is in Succeeded state after configuring Audit Log", func(t *testing.T) { + // given + ctx := context.Background() + auditLog := &mocks.AuditLogging{} + shoot := shootForTest() + instance := runtimeForTest() + systemState := &systemState{ + instance: instance, + shoot: shoot, + } + expectedRuntimeConditions := []metav1.Condition{ + { + Type: string(v1.ConditionTypeAuditLogConfigured), + Status: "True", + Reason: string(v1.ConditionReasonAuditLogConfigured), + Message: "Audit Log configured successfully", + }, + } + + auditLog.On("Enable", ctx, shoot).Return(false, nil).Once() + + // when + fsm := &fsm{AuditLogging: auditLog} + stateFn, _, _ := sFnConfigureAuditLog(ctx, fsm, systemState) + + // set the time to its zero value for comparison purposes + systemState.instance.Status.Conditions[0].LastTransitionTime = metav1.Time{} + // then auditLog.AssertExpectations(t) require.Contains(t, stateFn.name(), "sFnUpdateStatus") diff --git a/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go b/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go index b70e09a2..ce700502 100644 --- a/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go +++ b/internal/controller/runtime/fsm/runtime_fsm_waiting_for_shoot_reconcile.go @@ -47,9 +47,9 @@ func sFnWaitForShootReconcile(_ context.Context, m *fsm, s *systemState) (stateF return ensureStatusConditionIsSetAndContinue( &s.instance, imv1.ConditionTypeRuntimeProvisioned, - imv1.ConditionReasonConfigurationCompleted, + imv1.ConditionReasonAuditLogConfigured, "Runtime processing completed successfully", - sFnApplyClusterRoleBindings) + sFnConfigureAuditLog) } m.log.Info("Update did not processed, exiting with no retry") diff --git a/internal/controller/runtime/runtime_controller_test.go b/internal/controller/runtime/runtime_controller_test.go index 8811ef74..ed6fe337 100644 --- a/internal/controller/runtime/runtime_controller_test.go +++ b/internal/controller/runtime/runtime_controller_test.go @@ -157,7 +157,7 @@ var _ = Describe("Runtime Controller", func() { return false } - if !runtime.IsConditionSet(imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonConfigurationCompleted) { + if !runtime.IsConditionSet(imv1.ConditionTypeRuntimeProvisioned, imv1.ConditionReasonAuditLogConfigured) { return false } diff --git a/internal/controller/runtime/suite_test.go b/internal/controller/runtime/suite_test.go index b8ada71a..22c538b2 100644 --- a/internal/controller/runtime/suite_test.go +++ b/internal/controller/runtime/suite_test.go @@ -18,6 +18,10 @@ package runtime import ( "context" + "encoding/json" + "github.com/kyma-project/infrastructure-manager/internal/auditlogging" + v1 "k8s.io/api/autoscaling/v1" + v12 "k8s.io/api/core/v1" "path/filepath" "testing" "time" @@ -140,14 +144,14 @@ var _ = AfterSuite(func() { func setupGardenerTestClientForProvisioning() { baseShoot := getBaseShootForTestingSequence() shoots := fixShootsSequenceForProvisioning(&baseShoot) - seeds := fixSeedsSequence() + seeds := fixSeedsSequenceForProvisioning() setupGardenerClientWithSequence(shoots, seeds) } func setupGardenerTestClientForUpdate() { baseShoot := getBaseShootForTestingSequence() shoots := fixShootsSequenceForUpdate(&baseShoot) - seeds := fixSeedsSequence() + seeds := fixSeedsSequenceForUpdate() setupGardenerClientWithSequence(shoots, seeds) } @@ -205,14 +209,24 @@ func fixShootsSequenceForProvisioning(shoot *gardener_api.Shoot) []*gardener_api readyShoot.Status.LastOperation.State = gardener_api.LastOperationStateSucceeded + processingShootAfterAuditLogs := readyShoot.DeepCopy() + addAuditLogConfigToShoot(processingShootAfterAuditLogs) + processingShootAfterAuditLogs.Status.LastOperation.Type = gardener_api.LastOperationTypeReconcile + processingShootAfterAuditLogs.Status.LastOperation.State = gardener_api.LastOperationStateProcessing + + readyShootAfterAuditLogs := processingShootAfterAuditLogs.DeepCopy() + readyShootAfterAuditLogs.Status.LastOperation.State = gardener_api.LastOperationStateSucceeded + // processedShoot := processingShoot.DeepCopy() // will add specific data later - return []*gardener_api.Shoot{missingShoot, missingShoot, missingShoot, initialisedShoot, dnsShoot, pendingShoot, processingShoot, readyShoot, readyShoot, readyShoot, readyShoot, readyShoot} + return []*gardener_api.Shoot{missingShoot, missingShoot, missingShoot, initialisedShoot, dnsShoot, pendingShoot, processingShoot, readyShoot, readyShoot, readyShoot, readyShoot, readyShoot, processingShootAfterAuditLogs, readyShootAfterAuditLogs, readyShootAfterAuditLogs} } func fixShootsSequenceForUpdate(shoot *gardener_api.Shoot) []*gardener_api.Shoot { pendingShoot := shoot.DeepCopy() + addAuditLogConfigToShoot(pendingShoot) + pendingShoot.Spec.DNS = &gardener_api.DNS{ Domain: ptr.To("test.domain"), } @@ -236,7 +250,7 @@ func fixShootsSequenceForUpdate(shoot *gardener_api.Shoot) []*gardener_api.Shoot // processedShoot := processingShoot.DeepCopy() // will add specific data later - return []*gardener_api.Shoot{pendingShoot, processingShoot, readyShoot, readyShoot, readyShoot} + return []*gardener_api.Shoot{pendingShoot, processingShoot, readyShoot, readyShoot} } func fixShootsSequenceForDelete(shoot *gardener_api.Shoot) []*gardener_api.Shoot { @@ -270,7 +284,22 @@ func fixShootsSequenceForDelete(shoot *gardener_api.Shoot) []*gardener_api.Shoot return []*gardener_api.Shoot{currentShoot, currentShoot, currentShoot, currentShoot, pendingDeleteShoot, nil} } -func fixSeedsSequence() []*gardener_api.Seed { +func fixSeedsSequenceForProvisioning() []*gardener_api.Seed { + seed := &gardener_api.Seed{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-seed", + }, + Spec: gardener_api.SeedSpec{ + Provider: gardener_api.SeedProvider{ + Type: "aws", + }, + }, + } + + return []*gardener_api.Seed{seed, seed} +} + +func fixSeedsSequenceForUpdate() []*gardener_api.Seed { seed := &gardener_api.Seed{ ObjectMeta: metav1.ObjectMeta{ Name: "test-seed", @@ -325,3 +354,46 @@ func fixConverterConfigForTests() gardener_shoot.ConverterConfig { }, } } + +func addAuditLogConfigToShoot(shoot *gardener_api.Shoot) { + shoot.Spec.Kubernetes.KubeAPIServer.AuditConfig = &gardener_api.AuditConfig{ + AuditPolicy: &gardener_api.AuditPolicy{ + ConfigMapRef: &v12.ObjectReference{Name: "policy-config-map"}, + }, + } + + shoot.Spec.Resources = append(shoot.Spec.Resources, gardener_api.NamedResourceReference{ + Name: "auditlog-credentials", + ResourceRef: v1.CrossVersionObjectReference{ + Kind: "Secret", + Name: "auditlog-secret", + APIVersion: "v1", + }, + }) + + const ( + extensionKind = "AuditlogConfig" + extensionVersion = "service.auditlog.extensions.gardener.cloud/v1alpha1" + extensionType = "standard" + ) + + shoot.Spec.Extensions = append(shoot.Spec.Extensions, gardener_api.Extension{ + Type: "shoot-auditlog-service", + }) + + ext := &shoot.Spec.Extensions[len(shoot.Spec.Extensions)-1] + + cfg := auditlogging.AuditlogExtensionConfig{ + TypeMeta: metav1.TypeMeta{ + Kind: extensionKind, + APIVersion: extensionVersion, + }, + Type: extensionType, + TenantID: "79c64792-9c1e-4c1b-9941-ef7560dd3eae", + ServiceURL: "https://auditlog.example.com:3001", + SecretReferenceName: "auditlog-credentials", + } + + ext.ProviderConfig = &runtime.RawExtension{} + ext.ProviderConfig.Raw, _ = json.Marshal(cfg) +} diff --git a/internal/controller/runtime/testdata/auditConfig.json b/internal/controller/runtime/testdata/auditConfig.json index a7a163c3..608d06a2 100644 --- a/internal/controller/runtime/testdata/auditConfig.json +++ b/internal/controller/runtime/testdata/auditConfig.json @@ -3,7 +3,7 @@ "eu-central-1": { "tenantID": "79c64792-9c1e-4c1b-9941-ef7560dd3eae", "serviceURL": "https://auditlog.example.com:3001", - "secretName": "auditlog-secret2" + "secretName": "auditlog-secret" } } }