Skip to content

Commit

Permalink
Add an ability to remove secrets and configmaps when database is gone (…
Browse files Browse the repository at this point in the history 
…#168)
  • Loading branch information
@allanger
allanger authored Jan 12, 2023
1 parent 7250ff5 commit 077cf9d
Show file tree
Hide file tree
Showing 15 changed files with 230 additions and 104 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ TMP_DIR=$$(mktemp -d) ;\
cd $$TMP_DIR ;\
go mod init tmp ;\
echo "Downloading $(2)" ;\
GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
rm -rf $$TMP_DIR ;\
}
endef
1 change: 1 addition & 0 deletions api/v1alpha1/database_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type DatabaseSpec struct {
ConnectionStringTemplate string `json:"connectionStringTemplate,omitempty"`
SecretsTemplates map[string]string `json:"secretsTemplates,omitempty"`
Postgres Postgres `json:"postgres,omitempty"`
Cleanup bool `json:"cleanup,omitempty"`
}

// Postgres struct should be used to provide resource that only applicable to postgres
Expand Down
9 changes: 5 additions & 4 deletions controllers/backup/cronjob.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import (
// GCSBackupCron builds kubernetes cronjob object
// to create database backup regularly with defined schedule from dbcr
// this job will database dump and upload to google bucket storage for backup
func GCSBackupCron(conf *config.Config, dbcr *kciv1alpha1.Database) (*batchv1beta1.CronJob, error) {
func GCSBackupCron(conf *config.Config, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) (*batchv1beta1.CronJob, error) {
cronJobSpec, err := buildCronJobSpec(conf, dbcr)
if err != nil {
return nil, err
Expand All @@ -46,9 +46,10 @@ func GCSBackupCron(conf *config.Config, dbcr *kciv1alpha1.Database) (*batchv1bet
APIVersion: "batch",
},
ObjectMeta: metav1.ObjectMeta{
Name: dbcr.Namespace + "-" + dbcr.Name + "-" + "backup",
Namespace: dbcr.Namespace,
Labels: kci.BaseLabelBuilder(),
Name: dbcr.Namespace + "-" + dbcr.Name + "-" + "backup",
Namespace: dbcr.Namespace,
Labels: kci.BaseLabelBuilder(),
OwnerReferences: ownership,
},
Spec: cronJobSpec,
}, nil
Expand Down
45 changes: 41 additions & 4 deletions controllers/backup/cronjob_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ import (
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestGCSBackupCronGsql(t *testing.T) {
ownership := []metav1.OwnerReference{}
dbcr := &kciv1alpha1.Database{}
dbcr.Namespace = "TestNS"
dbcr.Name = "TestDB"
Expand All @@ -43,15 +45,15 @@ func TestGCSBackupCronGsql(t *testing.T) {
conf := config.LoadConfig()

instance.Spec.Engine = "postgres"
funcCronObject, err := GCSBackupCron(&conf, dbcr)
funcCronObject, err := GCSBackupCron(&conf, dbcr, ownership)
if err != nil {
fmt.Print(err)
}

assert.Equal(t, "postgresbackupimage:latest", funcCronObject.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image)

instance.Spec.Engine = "mysql"
funcCronObject, err = GCSBackupCron(&conf, dbcr)
funcCronObject, err = GCSBackupCron(&conf, dbcr, ownership)
if err != nil {
fmt.Print(err)
}
Expand All @@ -64,6 +66,7 @@ func TestGCSBackupCronGsql(t *testing.T) {
}

func TestGCSBackupCronGeneric(t *testing.T) {
ownership := []metav1.OwnerReference{}
dbcr := &kciv1alpha1.Database{}
dbcr.Namespace = "TestNS"
dbcr.Name = "TestDB"
Expand All @@ -78,15 +81,15 @@ func TestGCSBackupCronGeneric(t *testing.T) {
conf := config.LoadConfig()

instance.Spec.Engine = "postgres"
funcCronObject, err := GCSBackupCron(&conf, dbcr)
funcCronObject, err := GCSBackupCron(&conf, dbcr, ownership)
if err != nil {
fmt.Print(err)
}

assert.Equal(t, "postgresbackupimage:latest", funcCronObject.Spec.JobTemplate.Spec.Template.Spec.Containers[0].Image)

instance.Spec.Engine = "mysql"
funcCronObject, err = GCSBackupCron(&conf, dbcr)
funcCronObject, err = GCSBackupCron(&conf, dbcr, ownership)
if err != nil {
fmt.Print(err)
}
Expand All @@ -96,8 +99,42 @@ func TestGCSBackupCronGeneric(t *testing.T) {
assert.Equal(t, "TestNS", funcCronObject.Namespace)
assert.Equal(t, "TestNS-TestDB-backup", funcCronObject.Name)
assert.Equal(t, "* * * * *", funcCronObject.Spec.Schedule)
assert.Equal(t, len(funcCronObject.OwnerReferences), 0, "Unexpected size of an OwnerReference")
}

func TestGCSBackupCronGenericWithOwnerReference(t *testing.T) {
ownership := []metav1.OwnerReference{}
ownership = append(ownership, metav1.OwnerReference{
APIVersion: "api-version",
Kind: "kind",
Name: "name",
UID: "uid",
})
dbcr := &kciv1alpha1.Database{}
dbcr.Namespace = "TestNS"
dbcr.Name = "TestDB"
instance := &kciv1alpha1.DbInstance{}
instance.Status.Info = map[string]string{"DB_CONN": "TestConnection", "DB_PORT": "1234"}
instance.Spec.Generic = &kciv1alpha1.GenericInstance{BackupHost: "slave.test"}
dbcr.Status.InstanceRef = instance
dbcr.Spec.Instance = "staging"
dbcr.Spec.Backup.Cron = "* * * * *"

os.Setenv("CONFIG_PATH", "./test/backup_config.yaml")
conf := config.LoadConfig()

instance.Spec.Engine = "postgres"
funcCronObject, err := GCSBackupCron(&conf, dbcr, ownership)
if err != nil {
fmt.Print(err)
}

assert.Equal(t, len(funcCronObject.OwnerReferences), 1, "Unexpected size of an OwnerReference")
assert.Equal(t, funcCronObject.OwnerReferences[0].APIVersion, ownership[0].APIVersion, "API Version in the OwnerReference is wrong")
assert.Equal(t, funcCronObject.OwnerReferences[0].Kind, ownership[0].Kind, "Kind in the OwnerReference is wrong")
assert.Equal(t, funcCronObject.OwnerReferences[0].Name, ownership[0].Name, "Name in the OwnerReference is wrong")
assert.Equal(t, funcCronObject.OwnerReferences[0].UID, ownership[0].UID, "UID in the OwnerReference is wrong")
}
func TestGetResourceRequirements(t *testing.T) {
os.Setenv("CONFIG_PATH", "./test/backup_config.yaml")
conf := config.LoadConfig()
Expand Down
88 changes: 45 additions & 43 deletions controllers/database_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,37 +179,47 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c

// database status not true, process phase
if !dbcr.Status.Status {
ownership := []metav1.OwnerReference{}
if dbcr.Spec.Cleanup {
ownership = append(ownership, metav1.OwnerReference{
APIVersion: dbcr.APIVersion,
Kind: dbcr.Kind,
Name: dbcr.Name,
UID: dbcr.GetUID(),
},
)
}

phase := dbcr.Status.Phase
logrus.Infof("DB: namespace=%s, name=%s start %s", dbcr.Namespace, dbcr.Name, phase)

defer promDBsPhaseTime.WithLabelValues(phase).Observe(kci.TimeTrack(time.Now()))
err := r.createDatabase(ctx, dbcr)
err := r.createDatabase(ctx, dbcr, ownership)
if err != nil {
// when database creation failed, don't requeue request. to prevent exceeding api limit (ex: against google api)
return r.manageError(ctx, dbcr, err, false)
}

dbcr.Status.Phase = dbPhaseInstanceAccessSecret
err = r.createInstanceAccessSecret(ctx, dbcr)
if err != nil {

if err = r.createInstanceAccessSecret(ctx, dbcr, ownership); err != nil {
return r.manageError(ctx, dbcr, err, true)
}
dbcr.Status.Phase = dbPhaseProxy
err = r.createProxy(ctx, dbcr)
err = r.createProxy(ctx, dbcr, ownership)
if err != nil {
return r.manageError(ctx, dbcr, err, true)
}
dbcr.Status.Phase = dbPhaseSecretsTemplating
err = r.createTemplatedSecrets(ctx, dbcr)
if err != nil {
if err = r.createTemplatedSecrets(ctx, dbcr, ownership); err != nil {
return r.manageError(ctx, dbcr, err, true)
}
dbcr.Status.Phase = dbPhaseConfigMap
err = r.createInfoConfigMap(ctx, dbcr)
if err != nil {
if err = r.createInfoConfigMap(ctx, dbcr, ownership); err != nil {
return r.manageError(ctx, dbcr, err, true)
}
dbcr.Status.Phase = dbPhaseBackupJob
err = r.createBackupJob(ctx, dbcr)
err = r.createBackupJob(ctx, dbcr, ownership)
if err != nil {
return r.manageError(ctx, dbcr, err, true)
}
Expand Down Expand Up @@ -278,7 +288,7 @@ func (r *DatabaseReconciler) initialize(ctx context.Context, dbcr *kciv1alpha1.D
}

// createDatabase secret, actual database using admin secret
func (r *DatabaseReconciler) createDatabase(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createDatabase(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
databaseSecret, err := r.getDatabaseSecret(ctx, dbcr)
if err != nil {
if k8serrors.IsNotFound(err) {
Expand All @@ -287,7 +297,7 @@ func (r *DatabaseReconciler) createDatabase(ctx context.Context, dbcr *kciv1alph
logrus.Errorf("can not generate credentials for database - %s", err)
return err
}
newDatabaseSecret := kci.SecretBuilder(dbcr.Spec.SecretName, dbcr.Namespace, secretData)
newDatabaseSecret := kci.SecretBuilder(dbcr.Spec.SecretName, dbcr.Namespace, secretData, ownership)
err = r.Create(ctx, newDatabaseSecret)
if err != nil {
// failed to create secret
Expand Down Expand Up @@ -388,7 +398,7 @@ func (r *DatabaseReconciler) deleteDatabase(ctx context.Context, dbcr *kciv1alph
return nil
}

func (r *DatabaseReconciler) createInstanceAccessSecret(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createInstanceAccessSecret(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
if backend, _ := dbcr.GetBackendType(); backend != "google" {
logrus.Debugf("DB: namespace=%s, name=%s %s doesn't need instance access secret skipping...", dbcr.Namespace, dbcr.Name, backend)
return nil
Expand Down Expand Up @@ -422,7 +432,7 @@ func (r *DatabaseReconciler) createInstanceAccessSecret(ctx context.Context, dbc
secretData[credFile] = data

newName := dbcr.InstanceAccessSecretName()
newSecret := kci.SecretBuilder(newName, dbcr.GetNamespace(), secretData)
newSecret := kci.SecretBuilder(newName, dbcr.GetNamespace(), secretData, ownership)

err = r.Create(ctx, newSecret)
if err != nil {
Expand All @@ -442,7 +452,7 @@ func (r *DatabaseReconciler) createInstanceAccessSecret(ctx context.Context, dbc
return nil
}

func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
backend, _ := dbcr.GetBackendType()
if backend == "generic" {
logrus.Infof("DB: namespace=%s, name=%s %s proxy creation is not yet implemented skipping...", dbcr.Namespace, dbcr.Name, backend)
Expand All @@ -455,7 +465,7 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.
}

// create proxy configmap
cm, err := proxy.BuildConfigmap(proxyInterface)
cm, err := proxy.BuildConfigmap(proxyInterface, ownership)
if err != nil {
return err
}
Expand All @@ -478,7 +488,7 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.
}

// create proxy deployment
deploy, err := proxy.BuildDeployment(proxyInterface)
deploy, err := proxy.BuildDeployment(proxyInterface, ownership)
if err != nil {
return err
}
Expand All @@ -499,7 +509,7 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.
}

// create proxy service
svc, err := proxy.BuildService(proxyInterface)
svc, err := proxy.BuildService(proxyInterface, ownership)
if err != nil {
return err
}
Expand Down Expand Up @@ -533,7 +543,7 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.

if isMonitoringEnabled && inCrdList(crdList, "servicemonitors.monitoring.coreos.com") {
// create proxy PromServiceMonitor
promSvcMon, err := proxy.BuildServiceMonitor(proxyInterface)
promSvcMon, err := proxy.BuildServiceMonitor(proxyInterface, ownership)
if err != nil {
return err
}
Expand Down Expand Up @@ -567,15 +577,17 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.
return nil
}

func (r *DatabaseReconciler) createTemplatedSecrets(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createTemplatedSecrets(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
// First of all the password should be taken from secret because it's not stored anywhere else
databaseSecret, err := r.getDatabaseSecret(ctx, dbcr)
if err != nil {
return err
}
// Then parse the secret to get the password
// Connection stirng is deprecated and will be removed soon. So this switch is temporary.
// Connection string is deprecated and will be removed soon. So this switch is temporary.
// Once connection string is removed, the switch and the following if condition are gone
// Connection String doesn't support the cleaning up feature, so the secret with a connection
// string won't be removed after a db resource is removed.
useLegacyConnectionString := false
switch {
case len(dbcr.Spec.ConnectionStringTemplate) > 0 && len(dbcr.Spec.SecretsTemplates) > 0:
Expand Down Expand Up @@ -610,29 +622,31 @@ func (r *DatabaseReconciler) createTemplatedSecrets(ctx context.Context, dbcr *k
}
logrus.Debugf("DB: namespace=%s, name=%s updating credentials secret", dbcr.Namespace, dbcr.Name)
newSecret := addConnectionStringToSecret(dbcr, databaseSecret.Data, dbConnectionString)
return r.Update(ctx, newSecret, &client.UpdateOptions{})
if err = r.Update(ctx, newSecret, &client.UpdateOptions{}); err != nil {
return err
}
return nil
}

dbSecrets, err := generateTemplatedSecrets(dbcr, databaseCred)
if err != nil {
return err
}
// Adding values
newSecret := fillTemplatedSecretData(dbcr, databaseSecret.Data, dbSecrets)
err = r.Update(ctx, newSecret, &client.UpdateOptions{})
if err != nil {
newSecret := fillTemplatedSecretData(dbcr, databaseSecret.Data, dbSecrets, ownership)
if err = r.Update(ctx, newSecret, &client.UpdateOptions{}); err != nil {
return err
}
newSecret = removeObsoleteSecret(dbcr, databaseSecret.Data, dbSecrets)
err = r.Update(ctx, newSecret, &client.UpdateOptions{})
if err != nil {

newSecret = removeObsoleteSecret(dbcr, databaseSecret.Data, dbSecrets, ownership)
if err = r.Update(ctx, newSecret, &client.UpdateOptions{}); err != nil {
return err
}

return nil
}

func (r *DatabaseReconciler) createInfoConfigMap(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createInfoConfigMap(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
instance, err := dbcr.GetInstanceRef()
if err != nil {
return err
Expand All @@ -645,19 +659,7 @@ func (r *DatabaseReconciler) createInfoConfigMap(ctx context.Context, dbcr *kciv
info["DB_HOST"] = proxyStatus.ServiceName
info["DB_PORT"] = strconv.FormatInt(int64(proxyStatus.SQLPort), 10)
}

databaseConfigResource := &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: dbcr.Namespace,
Name: dbcr.Spec.SecretName,
Labels: kci.BaseLabelBuilder(),
},
Data: info,
}
databaseConfigResource := kci.ConfigMapBuilder(dbcr.Spec.SecretName, dbcr.Namespace, info, ownership)

err = r.Create(ctx, databaseConfigResource)
if err != nil {
Expand All @@ -678,13 +680,13 @@ func (r *DatabaseReconciler) createInfoConfigMap(ctx context.Context, dbcr *kciv
return nil
}

func (r *DatabaseReconciler) createBackupJob(ctx context.Context, dbcr *kciv1alpha1.Database) error {
func (r *DatabaseReconciler) createBackupJob(ctx context.Context, dbcr *kciv1alpha1.Database, ownership []metav1.OwnerReference) error {
if !dbcr.Spec.Backup.Enable {
// if not enabled, skip
return nil
}

cronjob, err := backup.GCSBackupCron(r.Conf, dbcr)
cronjob, err := backup.GCSBackupCron(r.Conf, dbcr, ownership)
if err != nil {
return err
}
Expand Down
Loading

0 comments on commit 077cf9d

Please sign in to comment.