Skip to content

Commit

Permalink
Merge pull request #293 from koala7659/create-gardener-cr
Browse files Browse the repository at this point in the history
feat: Creating Gardener Cluster CR as part of cluster provisioning
  • Loading branch information
kyma-bot authored Jul 19, 2024
2 parents b8f23ec + 3d66037 commit 1358985
Show file tree
Hide file tree
Showing 11 changed files with 383 additions and 79 deletions.
41 changes: 40 additions & 1 deletion api/v1/runtime_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ limitations under the License.
package v1

import (
"fmt"

gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -33,6 +35,21 @@ const (
AnnotationGardenerCloudDelConfirmation = "confirmation.gardener.cloud/deletion"
)

const (
LabelKymaInstanceID = "kyma-project.io/instance-id"
LabelKymaRuntimeID = "kyma-project.io/runtime-id"
LabelKymaShootName = "kyma-project.io/shootName"
LabelKymaRegion = "kyma-project.io/region"
LabelKymaName = "operator.kyma-project.io/kyma-name"
LabelKymaBrokerPlanID = "kyma-project.io/broker-plan-id"
LabelKymaBrokerPlanName = "kyma-project.io/broker-plan-name"
LabelKymaGlobalAccountID = "kyma-project.io/global-account-id"
LabelKymaSubaccountID = "kyma-project.io/subaccount-id"
LabelKymaManagedBy = "operator.kyma-project.io/managed-by"
LabelKymaInternal = "operator.kyma-project.io/internal"
LabelKymaPlatformRegion = "kyma-project.io/platform-region"
)

const (
RuntimeStateReady = "Ready"
RuntimeStateFailed = "Failed"
Expand All @@ -59,7 +76,8 @@ const (
ConditionReasonShootCreationPending = RuntimeConditionReason("Pending")
ConditionReasonShootCreationCompleted = RuntimeConditionReason("ShootCreationCompleted")

ConditionReasonConfigurationStarted = RuntimeConditionReason("ConfigurationStarted")
ConditionReasonGardenerCRCreated = RuntimeConditionReason("GardenerClusterCRCreated")
ConditionReasonGardenerCRReady = RuntimeConditionReason("GardenerClusterCRReady")
ConditionReasonConfigurationCompleted = RuntimeConditionReason("ConfigurationCompleted")
ConditionReasonConfigurationErr = RuntimeConditionReason("ConfigurationError")

Expand All @@ -68,6 +86,7 @@ const (
ConditionReasonConversionError = RuntimeConditionReason("ConversionErr")
ConditionReasonCreationError = RuntimeConditionReason("CreationErr")
ConditionReasonGardenerError = RuntimeConditionReason("GardenerErr")
ConditionReasonKubernetesAPIErr = RuntimeConditionReason("KubernetesErr")
ConditionReasonSerializationError = RuntimeConditionReason("SerializationErr")
ConditionReasonDeleted = RuntimeConditionReason("Deleted")
)
Expand Down Expand Up @@ -255,3 +274,23 @@ func (k *Runtime) IsConditionSetWithStatus(c RuntimeConditionType, r RuntimeCond
}
return false
}

func (k *Runtime) ValidateRequiredLabels() error {
var requiredLabelKeys = []string{
LabelKymaInstanceID,
LabelKymaRuntimeID,
LabelKymaRegion,
LabelKymaName,
LabelKymaBrokerPlanID,
LabelKymaBrokerPlanName,
LabelKymaGlobalAccountID,
LabelKymaSubaccountID,
}

for _, key := range requiredLabelKeys {
if k.Labels[key] == "" {
return fmt.Errorf("missing required label %s", key)
}
}
return nil
}
3 changes: 2 additions & 1 deletion internal/controller/runtime/fsm/runtime_fsm.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ import (
)

const (
gardenerRequeueDuration = 15 * time.Second
gardenerRequeueDuration = 15 * time.Second
controlPlaneRequeueDuration = 10 * time.Second
)

type stateFn func(context.Context, *fsm, *systemState) (stateFn, *ctrl.Result, error)
Expand Down
106 changes: 106 additions & 0 deletions internal/controller/runtime/fsm/runtime_fsm_create_kubeconfig.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
package fsm

import (
"context"
"fmt"

gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1"
imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
)

func sFnCreateKubeconfig(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) {
m.log.Info("Create Gardener Cluster CR state")

runtimeID := s.instance.Labels[imv1.LabelKymaRuntimeID]

var cluster imv1.GardenerCluster
err := m.Get(ctx, types.NamespacedName{
Namespace: s.instance.Namespace,
Name: runtimeID,
}, &cluster)

if err != nil {
if !k8serrors.IsNotFound(err) {
m.log.Error(err, "GardenerCluster CR read error", "name", runtimeID)
s.instance.UpdateStatePending(imv1.ConditionTypeRuntimeKubeconfigReady, imv1.ConditionReasonKubernetesAPIErr, "False", err.Error())
return updateStatusAndStop()
}

m.log.Info("GardenerCluster CR not found, creating a new one", "Name", runtimeID)
err = m.Create(ctx, makeGardenerClusterForRuntime(s.instance, s.shoot))
if err != nil {
m.log.Error(err, "GardenerCluster CR create error", "name", runtimeID)
s.instance.UpdateStatePending(imv1.ConditionTypeRuntimeKubeconfigReady, imv1.ConditionReasonKubernetesAPIErr, "False", err.Error())
return updateStatusAndStop()
}

m.log.Info("Gardener Cluster CR created, waiting for readiness", "Name", runtimeID)
s.instance.UpdateStatePending(imv1.ConditionTypeRuntimeKubeconfigReady, imv1.ConditionReasonGardenerCRCreated, "Unknown", "Gardener Cluster CR created, waiting for readiness")
return updateStatusAndRequeueAfter(controlPlaneRequeueDuration)
}

if cluster.Status.State != imv1.ReadyState {
m.log.Info("GardenerCluster CR is not ready yet, requeue", "Name", runtimeID, "State", cluster.Status.State)
return requeueAfter(controlPlaneRequeueDuration)
}

m.log.Info("GardenerCluster CR is ready", "Name", runtimeID)

return ensureStatusConditionIsSetAndContinue(&s.instance,
imv1.ConditionTypeRuntimeKubeconfigReady,
imv1.ConditionReasonGardenerCRReady,
"Gardener Cluster CR is ready.",
sFnProcessShoot)
}

func makeGardenerClusterForRuntime(runtime imv1.Runtime, shoot *gardener.Shoot) *imv1.GardenerCluster {
gardenCluster := &imv1.GardenerCluster{
TypeMeta: metav1.TypeMeta{
Kind: "GardenerCluster",
APIVersion: imv1.GroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: runtime.Labels[imv1.LabelKymaRuntimeID],
Namespace: runtime.Namespace,
Annotations: map[string]string{
"skr-domain": *shoot.Spec.DNS.Domain,
},
Labels: map[string]string{
imv1.LabelKymaInstanceID: runtime.Labels[imv1.LabelKymaInstanceID],
imv1.LabelKymaRuntimeID: runtime.Labels[imv1.LabelKymaRuntimeID],
imv1.LabelKymaBrokerPlanID: runtime.Labels[imv1.LabelKymaBrokerPlanID],
imv1.LabelKymaBrokerPlanName: runtime.Labels[imv1.LabelKymaBrokerPlanName],
imv1.LabelKymaGlobalAccountID: runtime.Labels[imv1.LabelKymaGlobalAccountID],
imv1.LabelKymaSubaccountID: runtime.Labels[imv1.LabelKymaSubaccountID], // BTW most likely this value will be missing
imv1.LabelKymaName: runtime.Labels[imv1.LabelKymaName],

// values from Runtime CR fields
imv1.LabelKymaPlatformRegion: runtime.Spec.Shoot.PlatformRegion,
imv1.LabelKymaRegion: runtime.Spec.Shoot.Region,
imv1.LabelKymaShootName: shoot.Name,

// hardcoded values
imv1.LabelKymaManagedBy: "infrastructure-manager",
imv1.LabelKymaInternal: "true",
},
},
Spec: imv1.GardenerClusterSpec{
Shoot: imv1.Shoot{
Name: shoot.Name,
},
Kubeconfig: imv1.Kubeconfig{
Secret: imv1.Secret{
Name: fmt.Sprintf("kubeconfig-%s", runtime.Labels[imv1.LabelKymaRuntimeID]),
Namespace: runtime.Namespace,
Key: "config",
},
},
},
}

return gardenCluster
}
166 changes: 166 additions & 0 deletions internal/controller/runtime/fsm/runtime_fsm_create_kubeconfig_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
package fsm

/*
import (
"context"
"time"
gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1"
imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
. "github.com/onsi/ginkgo/v2" //nolint:revive
. "github.com/onsi/gomega" //nolint:revive
"github.com/onsi/gomega/types"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
util "k8s.io/apimachinery/pkg/util/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
var _ = Describe("KIM sFnCreateKubeconfig", func() {
now := metav1.NewTime(time.Now())
testCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
// GIVEN
testScheme := runtime.NewScheme()
util.Must(imv1.AddToScheme(testScheme))
withTestSchemeAndObjects := func(objs ...client.Object) fakeFSMOpt {
return func(fsm *fsm) error {
return withFakedK8sClient(testScheme, objs...)(fsm)
}
}
testRtWithLables := imv1.Runtime{
ObjectMeta: metav1.ObjectMeta{
Name: "test-instance",
Namespace: "default",
Labels: map[string]string{
imv1.LabelKymaRuntimeID: "059dbc39-fd2b-4186-b0e5-8a1bc8ede5b8",
imv1.LabelKymaInstanceID: "test-instance",
imv1.LabelKymaBrokerPlanID: "broker-plan-id",
imv1.LabelKymaGlobalAccountID: "461f6292-8085-41c8-af0c-e185f39b5e18",
imv1.LabelKymaGlobalSubaccountID: "c5ad84ae-3d1b-4592-bee1-f022661f7b30",
imv1.LabelKymaRegion: "region",
imv1.LabelKymaBrokerPlanName: "aws",
imv1.LabelKymaName: "caadafae-1234-1234-1234-123456789abc",
},
},
}
testRtWithFinalizerNoProvisioningCondition := imv1.Runtime{
ObjectMeta: metav1.ObjectMeta{
Name: "test-instance",
Namespace: "default",
Finalizers: []string{"test-me-plz"},
},
}
testRtWithFinalizerAndProvisioningCondition := imv1.Runtime{
ObjectMeta: metav1.ObjectMeta{
Name: "test-instance",
Namespace: "default",
Finalizers: []string{"test-me-plz"},
},
}
provisioningCondition := metav1.Condition{
Type: string(imv1.ConditionTypeRuntimeProvisioned),
Status: metav1.ConditionUnknown,
LastTransitionTime: now,
Reason: "Test reason",
Message: "Test message",
}
meta.SetStatusCondition(&testRtWithFinalizerAndProvisioningCondition.Status.Conditions, provisioningCondition)
testShoot := gardener.Shoot{
ObjectMeta: metav1.ObjectMeta{
Name: "test-instance",
Namespace: "default",
},
}
testFunction := buildTestFunction(sFnCreateKubeconfig)
// WHEN/THAN
DescribeTable(
"transition graph validation",
testFunction,
Entry(
"should return nothing when CR is being deleted without finalizer and shoot is missing",
testCtx,
must(newFakeFSM, withTestFinalizer),
&systemState{instance: testRtWithDeletionTimestamp},
testOpts{
MatchExpectedErr: BeNil(),
MatchNextFnState: BeNil(),
},
),
Entry(
"should return sFnUpdateStatus when CR is being deleted with finalizer and shoot is missing - Remove finalizer",
testCtx,
must(newFakeFSM, withTestFinalizer, withTestSchemeAndObjects(&testRtWithLables)),
&systemState{instance: testRtWithDeletionTimestampAndFinalizer},
testOpts{
MatchExpectedErr: BeNil(),
MatchNextFnState: haveName("sFnProcessShoot"),
},
),
Entry(
"should return sFnDeleteShoot and no error when CR is being deleted with finalizer and shoot exists",
testCtx,
must(newFakeFSM, withTestFinalizer),
&systemState{instance: testRtWithDeletionTimestampAndFinalizer, shoot: &testShoot},
testOpts{
MatchExpectedErr: BeNil(),
MatchNextFnState: haveName("sFnDeleteShoot"),
},
),
Entry(
"should return sFnUpdateStatus and no error when CR has been created without finalizer - Add finalizer",
testCtx,
must(newFakeFSM, withTestFinalizer, withTestSchemeAndObjects(&testRt)),
&systemState{instance: testRt},
testOpts{
MatchExpectedErr: BeNil(),
MatchNextFnState: BeNil(),
StateMatch: []types.GomegaMatcher{haveFinalizer("test-me-plz")},
},
),
Entry(
"should return sFnUpdateStatus and no error when there is no Provisioning Condition - Add condition",
testCtx,
must(newFakeFSM, withTestFinalizer),
&systemState{instance: testRtWithFinalizerNoProvisioningCondition},
testOpts{
MatchExpectedErr: BeNil(),
MatchNextFnState: haveName("sFnUpdateStatus"),
},
),
Entry(
"should return sFnCreateStatus and no error when exists Provisioning Condition and shoot is missing",
testCtx,
must(newFakeFSM, withTestFinalizer),
&systemState{instance: testRtWithFinalizerAndProvisioningCondition},
testOpts{
MatchExpectedErr: BeNil(),
MatchNextFnState: haveName("sFnCreateShoot"),
},
),
Entry(
"should return sFnSelectShootProcessing and no error when exists Provisioning Condition and shoot exists",
testCtx,
must(newFakeFSM, withTestFinalizer),
&systemState{instance: testRtWithFinalizerAndProvisioningCondition, shoot: &testShoot},
testOpts{
MatchExpectedErr: BeNil(),
MatchNextFnState: haveName("sFnSelectShootProcessing"),
},
),
)
})
*/
3 changes: 2 additions & 1 deletion internal/controller/runtime/fsm/runtime_fsm_create_shoot.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
)

func sFnCreateShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) {
m.log.Info("Create shoot")
m.log.Info("Create shoot state")
newShoot, err := convertShoot(&s.instance, m.ConverterConfig)
if err != nil {
m.log.Error(err, "Failed to convert Runtime instance to shoot object")
Expand Down Expand Up @@ -37,6 +37,7 @@ func sFnCreateShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl
"Shoot is pending",
)

// it will be executed only once because created shoot is executed only once
shouldPersistShoot := m.PVCPath != ""
if shouldPersistShoot {
s.shoot = newShoot.DeepCopy()
Expand Down
6 changes: 5 additions & 1 deletion internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,12 @@ func sFnPatchExistingShoot(ctx context.Context, m *fsm, s *systemState) (stateFn
}

func convertShoot(instance *imv1.Runtime, cfg shoot.ConverterConfig) (gardener.Shoot, error) {
if err := instance.ValidateRequiredLabels(); err != nil {
return gardener.Shoot{}, err
}

converter := gardener_shoot.NewConverter(cfg)
shoot, err := converter.ToShoot(*instance) // returned error is always nil BTW
shoot, err := converter.ToShoot(*instance)

if err == nil {
setObjectFields(&shoot)
Expand Down
Loading

0 comments on commit 1358985

Please sign in to comment.