Skip to content

Commit

Permalink
feat: support update label/annotations for underlying pods and improv…
Browse files Browse the repository at this point in the history 
…e restart ops (#8571)

(cherry picked from commit d23e2e8)
  • Loading branch information
@wangyelei
wangyelei committed Dec 5, 2024
1 parent 299a7d9 commit 6d84abc
Show file tree
Hide file tree
Showing 9 changed files with 203 additions and 289 deletions.
134 changes: 133 additions & 1 deletion controllers/apps/cluster_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (

appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1"
dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1"
workloadsv1 "github.com/apecloud/kubeblocks/apis/workloads/v1"
"github.com/apecloud/kubeblocks/pkg/constant"
"github.com/apecloud/kubeblocks/pkg/controller/builder"
"github.com/apecloud/kubeblocks/pkg/generics"
Expand Down Expand Up @@ -1180,7 +1181,7 @@ var _ = Describe("Cluster Controller", func() {

By("update cluster to upgrade component definition")
Expect(testapps.GetAndChangeObj(&testCtx, clusterKey, func(cluster *appsv1.Cluster) {
cluster.Spec.ComponentSpecs[0].ComponentDef = ""
cluster.Spec.ComponentSpecs[0].ComponentDef = newCompDefObj.Name
})()).ShouldNot(HaveOccurred())

By("check cluster and component objects been upgraded")
Expand All @@ -1193,5 +1194,136 @@ var _ = Describe("Cluster Controller", func() {
g.Expect(comp.Spec.ServiceVersion).Should(Equal(defaultServiceVersion))
})).Should(Succeed())
})

Context("cluster component annotations and labels", func() {
BeforeEach(func() {
cleanEnv()
createAllDefinitionObjects()
})

AfterEach(func() {
cleanEnv()
})

addMetaMap := func(metaMap *map[string]string, key string, value string) {
if *metaMap == nil {
*metaMap = make(map[string]string)
}
(*metaMap)[key] = value
}

checkRelatedObject := func(compName string, checkFunc func(g Gomega, obj client.Object)) {
// check related services of the component
defaultSvcName := constant.GenerateComponentServiceName(clusterObj.Name, compName, "")
Eventually(testapps.CheckObj(&testCtx, client.ObjectKey{Name: defaultSvcName,
Namespace: testCtx.DefaultNamespace}, func(g Gomega, svc *corev1.Service) {
checkFunc(g, svc)
})).Should(Succeed())

// check related account secret of the component
rootAccountSecretName := constant.GenerateAccountSecretName(clusterObj.Name, compName, "root")
Eventually(testapps.CheckObj(&testCtx, client.ObjectKey{Name: rootAccountSecretName,
Namespace: testCtx.DefaultNamespace}, func(g Gomega, secret *corev1.Secret) {
checkFunc(g, secret)
})).Should(Succeed())
}

testUpdateAnnoAndLabels := func(compName string,
changeCluster func(cluster *appsv1.Cluster),
checkWorkloadFunc func(g Gomega, labels, annotations map[string]string),
checkRelatedObjFunc func(g Gomega, obj client.Object)) {
Expect(testapps.ChangeObj(&testCtx, clusterObj, func(obj *appsv1.Cluster) {
changeCluster(obj)
})).Should(Succeed())

By("check component has updated")
workloadName := constant.GenerateWorkloadNamePattern(clusterObj.Name, defaultCompName)
Eventually(testapps.CheckObj(&testCtx, client.ObjectKey{Name: workloadName,
Namespace: testCtx.DefaultNamespace}, func(g Gomega, compObj *appsv1.Component) {
checkWorkloadFunc(g, compObj.Spec.Labels, compObj.Spec.Annotations)
})).Should(Succeed())

By("check related objects annotations and labels")
checkRelatedObject(defaultCompName, func(g Gomega, obj client.Object) {
checkRelatedObjFunc(g, obj)
})

By("InstanceSet.spec.template.annotations/labels need to be consistent with component")
// The labels and annotations of the Pod will be kept consistent with those of the InstanceSet
Eventually(testapps.CheckObj(&testCtx, client.ObjectKey{Name: workloadName, Namespace: testCtx.DefaultNamespace},
func(g Gomega, instanceSet *workloadsv1.InstanceSet) {
checkWorkloadFunc(g, instanceSet.Spec.Template.GetLabels(), instanceSet.Spec.Template.GetAnnotations())
})).Should(Succeed())
}

It("test add/override annotations and labels", func() {
By("creating a cluster")
clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, "").
WithRandomName().
AddComponent(defaultCompName, compDefObj.Name).
SetServiceVersion(defaultServiceVersion).
SetReplicas(3).
Create(&testCtx).
GetObject()

By("add annotations and labels")
key1 := "key1"
value1 := "value1"
testUpdateAnnoAndLabels(defaultCompName,
func(cluster *appsv1.Cluster) {
addMetaMap(&cluster.Spec.ComponentSpecs[0].Annotations, key1, value1)
addMetaMap(&cluster.Spec.ComponentSpecs[0].Labels, key1, value1)
},
func(g Gomega, labels, annotations map[string]string) {
g.Expect(labels[key1]).Should(Equal(value1))
g.Expect(annotations[key1]).Should(Equal(value1))
},
func(g Gomega, obj client.Object) {
g.Expect(obj.GetLabels()[key1]).Should(Equal(value1))
g.Expect(obj.GetAnnotations()[key1]).Should(Equal(value1))
})

By("override annotations and labels")
value2 := "value2"
testUpdateAnnoAndLabels(defaultCompName,
func(cluster *appsv1.Cluster) {
addMetaMap(&cluster.Spec.ComponentSpecs[0].Annotations, key1, value2)
addMetaMap(&cluster.Spec.ComponentSpecs[0].Labels, key1, value2)
},
func(g Gomega, labels, annotations map[string]string) {
g.Expect(labels[key1]).Should(Equal(value2))
g.Expect(annotations[key1]).Should(Equal(value2))
},
func(g Gomega, obj client.Object) {
g.Expect(obj.GetLabels()[key1]).Should(Equal(value2))
g.Expect(obj.GetAnnotations()[key1]).Should(Equal(value2))
})

By("delete the annotations and labels, but retain the deleted annotations and labels for related objects")
key2 := "key2"
testUpdateAnnoAndLabels(defaultCompName,
func(cluster *appsv1.Cluster) {
cluster.Spec.ComponentSpecs[0].Annotations = map[string]string{
key2: value2,
}
cluster.Spec.ComponentSpecs[0].Labels = map[string]string{
key2: value2,
}
},
func(g Gomega, labels, annotations map[string]string) {
g.Expect(labels).ShouldNot(HaveKey(key1))
g.Expect(annotations).ShouldNot(HaveKey(key1))
g.Expect(labels[key2]).Should(Equal(value2))
g.Expect(annotations[key2]).Should(Equal(value2))
},
func(g Gomega, obj client.Object) {
g.Expect(obj.GetLabels()[key1]).Should(Equal(value2))
g.Expect(obj.GetAnnotations()[key1]).Should(Equal(value2))
g.Expect(obj.GetLabels()[key2]).Should(Equal(value2))
g.Expect(obj.GetAnnotations()[key2]).Should(Equal(value2))
})

})
})
})
})
7 changes: 4 additions & 3 deletions controllers/apps/transformer_component_account.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,11 +163,12 @@ func (t *componentAccountTransformer) buildAccountSecretWithPassword(ctx *compon
synthesizeComp *component.SynthesizedComponent, account appsv1.SystemAccount, password []byte) (*corev1.Secret, error) {
secretName := constant.GenerateAccountSecretName(synthesizeComp.ClusterName, synthesizeComp.Name, account.Name)
secret := builder.NewSecretBuilder(synthesizeComp.Namespace, secretName).
AddLabelsInMap(constant.GetCompLabels(synthesizeComp.ClusterName, synthesizeComp.Name)).
AddLabelsInMap(synthesizeComp.DynamicLabels).
// Priority: static < dynamic < built-in
AddLabelsInMap(synthesizeComp.StaticLabels).
AddAnnotationsInMap(synthesizeComp.DynamicAnnotations).
AddLabelsInMap(synthesizeComp.DynamicLabels).
AddLabelsInMap(constant.GetCompLabels(synthesizeComp.ClusterName, synthesizeComp.Name)).
AddAnnotationsInMap(synthesizeComp.StaticAnnotations).
AddAnnotationsInMap(synthesizeComp.DynamicAnnotations).
PutData(constant.AccountNameForSecret, []byte(account.Name)).
PutData(constant.AccountPasswdForSecret, password).
SetImmutable(true).
Expand Down
32 changes: 13 additions & 19 deletions controllers/apps/transformer_component_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import (
"github.com/apecloud/kubeblocks/pkg/controller/instanceset"
"github.com/apecloud/kubeblocks/pkg/controller/model"
"github.com/apecloud/kubeblocks/pkg/controller/multicluster"
intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
)

var (
Expand Down Expand Up @@ -201,12 +202,13 @@ func (t *componentServiceTransformer) buildService(comp *appsv1.Component,

serviceFullName := constant.GenerateComponentServiceName(synthesizeComp.ClusterName, synthesizeComp.Name, service.ServiceName)
builder := builder.NewServiceBuilder(namespace, serviceFullName).
AddLabelsInMap(constant.GetCompLabels(clusterName, compName)).
AddLabelsInMap(synthesizeComp.DynamicLabels).
// Priority: static < dynamic < built-in
AddLabelsInMap(synthesizeComp.StaticLabels).
AddAnnotationsInMap(service.Annotations).
AddAnnotationsInMap(synthesizeComp.DynamicAnnotations).
AddLabelsInMap(synthesizeComp.DynamicLabels).
AddLabelsInMap(constant.GetCompLabels(clusterName, compName)).
AddAnnotationsInMap(synthesizeComp.StaticAnnotations).
AddAnnotationsInMap(synthesizeComp.DynamicAnnotations).
AddAnnotationsInMap(service.Annotations).
SetSpec(&service.Spec).
AddSelectorsInMap(t.builtinSelector(comp)).
Optimize4ExternalTraffic()
Expand Down Expand Up @@ -307,33 +309,25 @@ func (t *componentServiceTransformer) createOrUpdateService(ctx graph.TransformC
}

newSvc := originSvc.DeepCopy()
newSvc.Spec = service.Spec
intctrlutil.MergeMetadataMapInplace(service.Labels, &newSvc.Labels)
intctrlutil.MergeMetadataMapInplace(service.Annotations, &newSvc.Annotations)

// if skip immutable check, update the service directly
if skipImmutableCheckForComponentService(originSvc) {
newSvc.Spec = service.Spec
resolveServiceDefaultFields(&originSvc.Spec, &newSvc.Spec)
if !reflect.DeepEqual(originSvc, newSvc) {
graphCli.Update(dag, originSvc, newSvc, inDataContext4G())
}
return nil
}
// otherwise only support to update the override params defined in cluster.spec.componentSpec[].services

overrideMutableParams := func(originSvc, newSvc *corev1.Service) {
newSvc.Spec.Type = originSvc.Spec.Type
newSvc.Name = originSvc.Name
newSvc.Spec.Selector = originSvc.Spec.Selector
newSvc.Annotations = originSvc.Annotations
}

// modify mutable field of newSvc to check if it is overridable
overrideMutableParams(originSvc, newSvc)
if !reflect.DeepEqual(originSvc, newSvc) {
overrideMutableParams := func() {
// other fields are immutable, we can't update the service
return nil
newSvc.Spec.Type = service.Spec.Type
newSvc.Spec.Selector = service.Spec.Selector
}

overrideMutableParams(service, newSvc)
overrideMutableParams()
if !reflect.DeepEqual(originSvc, newSvc) {
graphCli.Update(dag, originSvc, newSvc, inDataContext4G())
}
Expand Down
28 changes: 2 additions & 26 deletions controllers/apps/transformer_component_workload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,11 +151,6 @@ func (t *componentWorkloadTransformer) runningInstanceSetObject(ctx graph.Transf

func (t *componentWorkloadTransformer) reconcileWorkload(ctx context.Context, cli client.Reader,
synthesizedComp *component.SynthesizedComponent, comp *appsv1.Component, runningITS, protoITS *workloads.InstanceSet) error {
if runningITS != nil {
*protoITS.Spec.Selector = *runningITS.Spec.Selector
protoITS.Spec.Template.Labels = intctrlutil.MergeMetadataMaps(runningITS.Spec.Template.Labels, synthesizedComp.DynamicLabels)
}

buildInstanceSetPlacementAnnotation(comp, protoITS)

if err := t.reconcileReplicasStatus(ctx, cli, synthesizedComp, runningITS, protoITS); err != nil {
Expand Down Expand Up @@ -406,21 +401,6 @@ func buildPodSpecVolumeMounts(synthesizeComp *component.SynthesizedComponent) {
// 1. new an object targetObj by copying from oldObj
// 2. merge all fields can be updated from newObj into targetObj
func copyAndMergeITS(oldITS, newITS *workloads.InstanceSet) *workloads.InstanceSet {
// mergeAnnotations keeps the original annotations.
mergeMetadataMap := func(originalMap map[string]string, targetMap *map[string]string) {
if targetMap == nil || originalMap == nil {
return
}
if *targetMap == nil {
*targetMap = map[string]string{}
}
for k, v := range originalMap {
// if the annotation not exist in targetAnnotations, copy it from original.
if _, ok := (*targetMap)[k]; !ok {
(*targetMap)[k] = v
}
}
}

updateUpdateStrategy := func(itsObj, itsProto *workloads.InstanceSet) {
var objMaxUnavailable *intstr.IntOrString
Expand Down Expand Up @@ -450,12 +430,8 @@ func copyAndMergeITS(oldITS, newITS *workloads.InstanceSet) *workloads.InstanceS
return strings.HasPrefix(k, "monitor.kubeblocks.io")
})
}
mergeMetadataMap(itsObjCopy.Annotations, &itsProto.Annotations)
itsObjCopy.Annotations = itsProto.Annotations

// keep the original template annotations.
// if annotations exist and are replaced, the its will be updated.
mergeMetadataMap(itsObjCopy.Spec.Template.Annotations, &itsProto.Spec.Template.Annotations)
intctrlutil.MergeMetadataMapInplace(itsProto.Annotations, &itsObjCopy.Annotations)
intctrlutil.MergeMetadataMapInplace(itsProto.Labels, &itsObjCopy.Labels)
itsObjCopy.Spec.Template = *itsProto.Spec.Template.DeepCopy()
itsObjCopy.Spec.Replicas = itsProto.Spec.Replicas
itsObjCopy.Spec.Roles = itsProto.Spec.Roles
Expand Down
26 changes: 16 additions & 10 deletions pkg/controller/factory/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,35 +51,39 @@ func BuildInstanceSet(synthesizedComp *component.SynthesizedComponent, component
)

podBuilder := builder.NewPodBuilder("", "").
AddLabelsInMap(constant.GetCompLabels(clusterName, compName, synthesizedComp.Labels)).
AddLabelsInMap(synthesizedComp.DynamicLabels).
// Priority: static < dynamic < built-in
AddLabelsInMap(synthesizedComp.StaticLabels).
AddAnnotationsInMap(synthesizedComp.DynamicAnnotations).
AddAnnotationsInMap(synthesizedComp.StaticAnnotations)
AddLabelsInMap(synthesizedComp.DynamicLabels).
AddLabelsInMap(constant.GetCompLabels(clusterName, compName, synthesizedComp.Labels)).
AddAnnotationsInMap(synthesizedComp.StaticAnnotations).
AddAnnotationsInMap(synthesizedComp.DynamicAnnotations)
template := corev1.PodTemplateSpec{
ObjectMeta: podBuilder.GetObject().ObjectMeta,
Spec: *synthesizedComp.PodSpec.DeepCopy(),
}

itsName := constant.GenerateWorkloadNamePattern(clusterName, compName)
itsBuilder := builder.NewInstanceSetBuilder(namespace, itsName).
AddLabelsInMap(constant.GetCompLabels(clusterName, compName)).
AddLabelsInMap(synthesizedComp.StaticLabels).
AddLabelsInMap(constant.GetCompLabels(clusterName, compName)).
AddAnnotations(constant.KubeBlocksGenerationKey, synthesizedComp.Generation).
AddAnnotations(constant.CRDAPIVersionAnnotationKey, workloads.GroupVersion.String()).
AddAnnotationsInMap(map[string]string{
constant.AppComponentLabelKey: compDefName,
constant.KBAppServiceVersionKey: synthesizedComp.ServiceVersion,
}).
AddAnnotationsInMap(getMonitorAnnotations(synthesizedComp, componentDef)).
AddAnnotationsInMap(synthesizedComp.StaticAnnotations).
AddAnnotationsInMap(getMonitorAnnotations(synthesizedComp, componentDef)).
SetTemplate(template).
AddMatchLabelsInMap(constant.GetCompLabels(clusterName, compName)).
SetReplicas(synthesizedComp.Replicas).
SetMinReadySeconds(synthesizedComp.MinReadySeconds)

var vcts []corev1.PersistentVolumeClaim
for _, vct := range synthesizedComp.VolumeClaimTemplates {
// Priority: static < dynamic < built-in
intctrlutil.MergeMetadataMapInplace(synthesizedComp.StaticLabels, &vct.ObjectMeta.Labels)
intctrlutil.MergeMetadataMapInplace(synthesizedComp.StaticAnnotations, &vct.ObjectMeta.Annotations)
intctrlutil.MergeMetadataMapInplace(synthesizedComp.DynamicLabels, &vct.ObjectMeta.Labels)
intctrlutil.MergeMetadataMapInplace(synthesizedComp.DynamicAnnotations, &vct.ObjectMeta.Annotations)
vcts = append(vcts, vctToPVC(vct))
Expand Down Expand Up @@ -229,18 +233,20 @@ func GetRestoreSystemAccountPassword(annotations map[string]string, componentNam
return password
}

// TODO: add dynamicLabels and dynamicAnnotations by @zhangtao

func BuildConfigMapWithTemplate(cluster *appsv1.Cluster,
synthesizedComp *component.SynthesizedComponent,
configs map[string]string,
cmName string,
configTemplateSpec appsv1.ComponentTemplateSpec) *corev1.ConfigMap {
return builder.NewConfigMapBuilder(cluster.Namespace, cmName).
AddLabelsInMap(synthesizedComp.StaticLabels).
AddLabelsInMap(constant.GetCompLabels(cluster.Name, synthesizedComp.Name)).
AddLabels(constant.CMConfigurationTypeLabelKey, constant.ConfigInstanceType).
AddLabels(constant.CMTemplateNameLabelKey, configTemplateSpec.TemplateRef).
AddLabelsInMap(synthesizedComp.StaticLabels).
AddAnnotations(constant.DisableUpgradeInsConfigurationAnnotationKey, strconv.FormatBool(false)).
AddAnnotationsInMap(synthesizedComp.StaticAnnotations).
AddAnnotations(constant.DisableUpgradeInsConfigurationAnnotationKey, strconv.FormatBool(false)).
SetData(configs).
GetObject()
}
Expand Down Expand Up @@ -316,17 +322,17 @@ func setToolsScriptsPath(container *corev1.Container, meta cfgcm.ConfigSpecMeta)

func BuildServiceAccount(synthesizedComp *component.SynthesizedComponent, saName string) *corev1.ServiceAccount {
return builder.NewServiceAccountBuilder(synthesizedComp.Namespace, saName).
AddLabelsInMap(constant.GetCompLabels(synthesizedComp.ClusterName, synthesizedComp.Name)).
AddLabelsInMap(synthesizedComp.StaticLabels).
AddLabelsInMap(constant.GetCompLabels(synthesizedComp.ClusterName, synthesizedComp.Name)).
AddAnnotationsInMap(synthesizedComp.StaticAnnotations).
SetImagePullSecrets(intctrlutil.BuildImagePullSecrets()).
GetObject()
}

func BuildRoleBinding(synthesizedComp *component.SynthesizedComponent, saName string) *rbacv1.RoleBinding {
return builder.NewRoleBindingBuilder(synthesizedComp.Namespace, saName).
AddLabelsInMap(constant.GetCompLabels(synthesizedComp.ClusterName, synthesizedComp.Name)).
AddLabelsInMap(synthesizedComp.StaticLabels).
AddLabelsInMap(constant.GetCompLabels(synthesizedComp.ClusterName, synthesizedComp.Name)).
AddAnnotationsInMap(synthesizedComp.StaticAnnotations).
SetRoleRef(rbacv1.RoleRef{
APIGroup: rbacv1.GroupName,
Expand Down
7 changes: 4 additions & 3 deletions pkg/controller/plan/tls.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,12 @@ func GenerateTLSSecretName(clusterName, componentName string) string {
func BuildTLSSecret(synthesizedComp component.SynthesizedComponent) *v1.Secret {
name := GenerateTLSSecretName(synthesizedComp.ClusterName, synthesizedComp.Name)
return builder.NewSecretBuilder(synthesizedComp.Namespace, name).
AddLabelsInMap(constant.GetCompLabels(synthesizedComp.ClusterName, synthesizedComp.Name)).
AddLabelsInMap(synthesizedComp.DynamicLabels).
// Priority: static < dynamic < built-in
AddLabelsInMap(synthesizedComp.StaticLabels).
AddAnnotationsInMap(synthesizedComp.DynamicAnnotations).
AddLabelsInMap(synthesizedComp.DynamicLabels).
AddLabelsInMap(constant.GetCompLabels(synthesizedComp.ClusterName, synthesizedComp.Name)).
AddAnnotationsInMap(synthesizedComp.StaticAnnotations).
AddAnnotationsInMap(synthesizedComp.DynamicAnnotations).
SetStringData(map[string]string{}).
GetObject()
}
Expand Down
Loading

0 comments on commit 6d84abc

Please sign in to comment.