diff --git a/apis/vshn/v1/dbaas_vshn_mariadb.go b/apis/vshn/v1/dbaas_vshn_mariadb.go index 66ca5e79fe..a8a94ebd50 100644 --- a/apis/vshn/v1/dbaas_vshn_mariadb.go +++ b/apis/vshn/v1/dbaas_vshn_mariadb.go @@ -2,6 +2,7 @@ package v1 import ( "fmt" + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -78,12 +79,22 @@ type VSHNMariaDBParameters struct { // Security defines the security of a service Security Security `json:"security,omitempty"` + + // +kubebuilder:default=1 + // +kubebuilder:validation:Enum=1;3; + + // Instances configures the number of MariaDB instances for the cluster. + // Each instance contains one MariaDB server. + // These serves will form a Galera cluster. + // An additional ProxySQL statefulset will be deployed to make failovers + // as seamless as possible. + Instances int `json:"instances,omitempty"` } // VSHNMariaDBServiceSpec contains MariaDB DBaaS specific properties type VSHNMariaDBServiceSpec struct { - // +kubebuilder:validation:Enum="10.4";"10.5";"10.6";"10.9";"10.10";"10.11";"11.0";"11.1";"11.2"; - // +kubebuilder:default="11.2" + // +kubebuilder:validation:Enum="10.4";"10.5";"10.6";"10.9";"10.10";"10.11";"11.0";"11.1";"11.2";"11.3";"11.4"; + // +kubebuilder:default="11.4" // Version contains supported version of MariaDB. // Multiple versions are supported. The latest version "11.2" is the default version. @@ -264,7 +275,7 @@ func (v *VSHNMariaDB) GetMonitoring() VSHNMonitoring { } func (v *VSHNMariaDB) GetInstances() int { - return 1 + return v.Spec.Parameters.Instances } func (v *VSHNMariaDB) GetPDBLabels() map[string]string { diff --git a/crds/vshn.appcat.vshn.io_vshnmariadbs.yaml b/crds/vshn.appcat.vshn.io_vshnmariadbs.yaml index 82caf646bc..a9642e5474 100644 --- a/crds/vshn.appcat.vshn.io_vshnmariadbs.yaml +++ b/crds/vshn.appcat.vshn.io_vshnmariadbs.yaml @@ -67,6 +67,18 @@ spec: type: string type: object default: {} + instances: + default: 1 + description: |- + Instances configures the number of MariaDB instances for the cluster. + Each instance contains one MariaDB server. + These serves will form a Galera cluster. + An additional ProxySQL statefulset will be deployed to make failovers + as seamless as possible. + enum: + - 1 + - 3 + type: integer maintenance: description: Maintenance contains settings to control the maintenance of an instance. properties: @@ -4895,7 +4907,7 @@ spec: - guaranteed type: string version: - default: "11.2" + default: "11.4" description: |- Version contains supported version of MariaDB. Multiple versions are supported. The latest version "11.2" is the default version. @@ -4909,6 +4921,8 @@ spec: - "11.0" - "11.1" - "11.2" + - "11.3" + - "11.4" type: string type: object default: {} diff --git a/crds/vshn.appcat.vshn.io_xvshnmariadbs.yaml b/crds/vshn.appcat.vshn.io_xvshnmariadbs.yaml index 6bf2860634..4387fffb74 100644 --- a/crds/vshn.appcat.vshn.io_xvshnmariadbs.yaml +++ b/crds/vshn.appcat.vshn.io_xvshnmariadbs.yaml @@ -111,6 +111,18 @@ spec: (\*|([1-9]|1[0-2])|\*\/([1-9]|1[0-2])) (\*|([0-6])|\*\/([0-6]))$ type: string type: object + instances: + default: 1 + description: |- + Instances configures the number of MariaDB instances for the cluster. + Each instance contains one MariaDB server. + These serves will form a Galera cluster. + An additional ProxySQL statefulset will be deployed to make failovers + as seamless as possible. + enum: + - 1 + - 3 + type: integer maintenance: description: Maintenance contains settings to control the maintenance of an instance. @@ -5623,7 +5635,7 @@ spec: - guaranteed type: string version: - default: "11.2" + default: "11.4" description: |- Version contains supported version of MariaDB. Multiple versions are supported. The latest version "11.2" is the default version. @@ -5637,6 +5649,8 @@ spec: - "11.0" - "11.1" - "11.2" + - "11.3" + - "11.4" type: string type: object size: diff --git a/pkg/comp-functions/functions/common/tls.go b/pkg/comp-functions/functions/common/tls.go index d996d40cd0..19a5a10aa8 100644 --- a/pkg/comp-functions/functions/common/tls.go +++ b/pkg/comp-functions/functions/common/tls.go @@ -13,7 +13,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func CreateTlsCerts(ctx context.Context, ns string, serviceName string, svc *runtime.ServiceRuntime) error { +// CreateTLSCerts creates ssl/tls certificates. Servicename will be concatenated with the given namespace to generate a proper k8s fqdn. +func CreateTLSCerts(ctx context.Context, ns string, serviceName string, svc *runtime.ServiceRuntime, additionalSANs ...string) error { selfSignedIssuer := &cmv1.Issuer{ ObjectMeta: metav1.ObjectMeta{ @@ -132,6 +133,10 @@ func CreateTlsCerts(ctx context.Context, ns string, serviceName string, svc *run }, } + for _, SAN := range additionalSANs { + serverCert.Spec.DNSNames = append(serverCert.Spec.DNSNames, SAN+"."+ns+".svc", SAN+"."+ns+".svc.cluster.local") + } + cd := []xkube.ConnectionDetail{ { ObjectReference: corev1.ObjectReference{ diff --git a/pkg/comp-functions/functions/vshnkeycloak/deploy.go b/pkg/comp-functions/functions/vshnkeycloak/deploy.go index 0afe54ed0b..7e816edfad 100644 --- a/pkg/comp-functions/functions/vshnkeycloak/deploy.go +++ b/pkg/comp-functions/functions/vshnkeycloak/deploy.go @@ -122,7 +122,7 @@ func DeployKeycloak(ctx context.Context, comp *vshnv1.VSHNKeycloak, svc *runtime svc.Log.Info("Creating Keycloak TLS certs") // The helm chart appends `-keycloakx-http` to the http service. - err = common.CreateTlsCerts(ctx, comp.GetInstanceNamespace(), comp.GetName()+"-keycloakx-http", svc) + err = common.CreateTLSCerts(ctx, comp.GetInstanceNamespace(), comp.GetName()+"-keycloakx-http", svc) if err != nil { return runtime.NewWarningResult(fmt.Sprintf("cannot add tls certificate: %s", err)) } diff --git a/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go b/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go index fd200c0c0b..c36d7f4a7a 100644 --- a/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go +++ b/pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go @@ -54,7 +54,13 @@ func DeployMariadb(ctx context.Context, comp *vshnv1.VSHNMariaDB, svc *runtime.S } l.Info("Creating tls certificate for mariadb instance") - err = common.CreateTlsCerts(ctx, comp.GetInstanceNamespace(), comp.GetName(), svc) + err = common.CreateTLSCerts(ctx, comp.GetInstanceNamespace(), comp.GetName(), svc, + "mariadb", + comp.GetName()+"-0."+comp.GetName()+"-headless", + comp.GetName()+"-1."+comp.GetName()+"-headless", + comp.GetName()+"-2."+comp.GetName()+"-headless", + ) + if err != nil { return runtime.NewWarningResult(fmt.Errorf("cannot create tls certificate: %w", err).Error()) } @@ -124,13 +130,13 @@ func getConnectionDetails(comp *vshnv1.VSHNMariaDB, svc *runtime.ServiceRuntime, } mariadbRootPw := secret.Data["mariadb-root-password"] - mariadbHost := comp.GetName() + ".vshn-mariadb-" + comp.GetName() + ".svc.cluster.local" - mariadbURL := fmt.Sprintf("mysql://%s:%s@%s:%s", mariadbUser, mariadbRootPw, mariadbHost, mariadbPort) + // mariadbHost := comp.GetName() + ".vshn-mariadb-" + comp.GetName() + ".svc.cluster.local" + // mariadbURL := fmt.Sprintf("mysql://%s:%s@%s:%s", mariadbUser, mariadbRootPw, mariadbHost, mariadbPort) - svc.SetConnectionDetail("MARIADB_HOST", []byte(mariadbHost)) + // svc.SetConnectionDetail("MARIADB_HOST", []byte(mariadbHost)) svc.SetConnectionDetail("MARIADB_PORT", []byte(mariadbPort)) svc.SetConnectionDetail("MARIADB_USERNAME", []byte(mariadbUser)) - svc.SetConnectionDetail("MARIADB_URL", []byte(mariadbURL)) + // svc.SetConnectionDetail("MARIADB_URL", []byte(mariadbURL)) svc.SetConnectionDetail("MARIADB_PASSWORD", mariadbRootPw) return nil @@ -161,7 +167,7 @@ func newValues(ctx context.Context, svc *runtime.ServiceRuntime, comp *vshnv1.VS values = map[string]interface{}{ "existingSecret": secretName, "fullnameOverride": comp.GetName(), - "replicaCount": 1, + "replicaCount": comp.GetInstances(), "resources": map[string]interface{}{ "requests": map[string]interface{}{ "memory": res.ReqMem.String(), @@ -186,8 +192,11 @@ func newValues(ctx context.Context, svc *runtime.ServiceRuntime, comp *vshnv1.VS "size": res.Disk.String(), "storageClass": comp.Spec.Parameters.StorageClass, }, + // We don't need the startup probe for Galera clusters, as ProxySQL + // will check the state independetly and is usually faster than the probe. + // Also for single instances it unnecessarily slows downt he provisioning. "startupProbe": map[string]interface{}{ - "enabled": true, + "enabled": false, }, "metrics": map[string]interface{}{ "enabled": true, @@ -208,6 +217,9 @@ func newValues(ctx context.Context, svc *runtime.ServiceRuntime, comp *vshnv1.VS "enabled": !svc.GetBoolFromCompositionConfig("isOpenshift"), }, "nodeSelector": nodeSelector, + "podLabels": map[string]string{ + "app": "mariadb", + }, } return values, nil diff --git a/pkg/comp-functions/functions/vshnmariadb/proxysql.go b/pkg/comp-functions/functions/vshnmariadb/proxysql.go new file mode 100644 index 0000000000..eefaebf953 --- /dev/null +++ b/pkg/comp-functions/functions/vshnmariadb/proxysql.go @@ -0,0 +1,354 @@ +package vshnmariadb + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "fmt" + "sort" + "text/template" + + xfnproto "github.com/crossplane/function-sdk-go/proto/v1beta1" + vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1" + "github.com/vshn/appcat/v4/pkg/comp-functions/functions/common" + "github.com/vshn/appcat/v4/pkg/comp-functions/runtime" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" +) + +const ( + configTmpl = `datadir="/var/lib/proxysql" + + admin_variables= + { + admin_credentials="admin:{{ .RootPassword }};radmin:{{ .RootPassword }}" + mysql_ifaces="0.0.0.0:6032" + refresh_interval=2000 + cluster_username="radmin" + cluster_password="{{ .RootPassword }}" + } + + mysql_servers = + ( + { address="{{ .CompName }}-0.{{ .CompName }}-headless.{{ .Namespace }}.svc" , port=3306 , hostgroup=2, weight=120 }, + { address="{{ .CompName }}-1.{{ .CompName }}-headless.{{ .Namespace }}.svc" , port=3306 , hostgroup=2, weight=110 }, + { address="{{ .CompName }}-2.{{ .CompName }}-headless.{{ .Namespace }}.svc" , port=3306 , hostgroup=2, weight=100 } + ) + + mysql_users = + ( + { username = "root", password = "{{ .RootPassword }}", default_hostgroup = 2 }, + {{range $val := .Users}} + { username = "{{ $val.Name }}", password = "{{ $val.Password }}", default_hostgroup = 2 }, + {{end}} + ) + + mysql_galera_hostgroups = + ( + {writer_hostgroup=2,backup_writer_hostgroup=4,reader_hostgroup=3,offline_hostgroup=1,active=1,max_writers=1,writer_is_also_reader=1,max_transactions_behind=100} + ) + + proxysql_servers = + ( + { hostname = "proxysql-0.proxysqlcluster.{{ .Namespace }}.svc", port = 6032, weight = 1 }, + { hostname = "proxysql-1.proxysqlcluster.{{ .Namespace }}.svc", port = 6032, weight = 1 } + )` +) + +type proxySQLConfigParams struct { + CompName string + RootPassword string + Namespace string + Users []proxySQLUsers +} + +type proxySQLUsers struct { + Name string + Password string +} + +// AddProxySQL will add a ProxySQL cluster to the service, if instances > 1. +// This function also creates a main service, which should always be used to connect from the outside. +// This service is necessary to seamlessly scale up and down without changing the IP address. +func AddProxySQL(ctx context.Context, comp *vshnv1.VSHNMariaDB, svc *runtime.ServiceRuntime) *xfnproto.Result { + + err := svc.GetDesiredComposite(comp) + if err != nil { + return runtime.NewFatalResult(fmt.Errorf("cannot get mariadb composite: %s", err)) + } + + svc.Log.Info("Creating main mariadb service") + err = createMainService(comp, svc) + if err != nil { + return runtime.NewWarningResult(fmt.Sprintf("cannot create service: %s", err)) + } + + if comp.GetInstances() == 1 { + // nothing else to do here + return nil + } + + svc.Log.Info("Creating proxySQL TLS certificates") + err = common.CreateTLSCerts(ctx, comp.GetInstanceNamespace(), comp.GetName()+"-proxysql", svc, + "proxysql-0.proxysqlcluster", + "proxysql-1.proxysqlcluster", + "mysql."+comp.GetInstanceNamespace()) + if err != nil { + return runtime.NewWarningResult(fmt.Sprintf("cannot create proxysql certificates: %s", err)) + } + + svc.Log.Info("Creating config for proxySQL") + configHash, err := createProxySQLConfig(comp, svc) + if err != nil { + return runtime.NewWarningResult(fmt.Sprintf("cannot create config: %s", err)) + } + + svc.Log.Info("Creating headless service for proxySQL") + err = createProxySQLHeadlessService(comp, svc) + if err != nil { + return runtime.NewWarningResult(fmt.Sprintf("cannot create headless service: %s", err)) + } + + svc.Log.Info("Creating statefulset for proxySQL") + err = createProxySQLStatefulset(comp, svc, configHash) + if err != nil { + return runtime.NewWarningResult(fmt.Sprintf("cannot create statefulset: %s", err)) + } + + return nil +} + +func createProxySQLConfig(comp *vshnv1.VSHNMariaDB, svc *runtime.ServiceRuntime) (string, error) { + + cd := svc.GetConnectionDetails() + + userList, err := getUserList(comp, svc) + if err != nil { + return "", err + } + + confParams := proxySQLConfigParams{ + CompName: comp.GetName(), + RootPassword: string(cd["MARIADB_PASSWORD"]), + Namespace: comp.GetInstanceNamespace(), + Users: userList, + } + + var buf bytes.Buffer + tmpl, err := template.New("ProxySQLConfig").Parse(configTmpl) + if err != nil { + return "", err + } + + err = tmpl.Execute(&buf, confParams) + if err != nil { + return "", err + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proxysql-config", + Namespace: comp.GetInstanceNamespace(), + }, + Data: map[string][]byte{ + "proxysql.cnf": buf.Bytes(), + }, + } + + hash := md5.Sum(buf.Bytes()) + + return hex.EncodeToString(hash[:]), svc.SetDesiredKubeObject(secret, comp.GetName()+"-proxySQL-config") +} + +func getUserList(comp *vshnv1.VSHNMariaDB, svc *runtime.ServiceRuntime) ([]proxySQLUsers, error) { + users := []proxySQLUsers{} + userMap := map[string]string{} + + for _, access := range comp.Spec.Parameters.Service.Access { + userCD, err := svc.GetObservedComposedResourceConnectionDetails(comp.GetName() + "-userpass-" + *access.User) + if err != nil { + return users, err + } + userMap[*access.User] = string(userCD["userpass"]) + } + + for k, v := range userMap { + users = append(users, proxySQLUsers{ + Name: k, + Password: v, + }) + } + + // Maps are not sorted and return a random order every time + // To avoid restarts on every reconcile due to order changes, we sort the slice + sort.Slice(users, func(i, j int) bool { + return users[i].Name < users[j].Name + }) + + return users, nil +} + +func createProxySQLHeadlessService(comp *vshnv1.VSHNMariaDB, svc *runtime.ServiceRuntime) error { + service := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proxysqlcluster", + Namespace: comp.GetInstanceNamespace(), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "None", + Ports: []corev1.ServicePort{ + { + Port: 6032, + Name: "proxysql-admin", + }, + }, + Selector: map[string]string{ + "app": "proxysql", + }, + }, + } + + return svc.SetDesiredKubeObject(&service, comp.GetName()+"-proxysql-headless-service") +} + +func createProxySQLStatefulset(comp *vshnv1.VSHNMariaDB, svc *runtime.ServiceRuntime, configHash string) error { + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proxysql", + Namespace: comp.GetInstanceNamespace(), + Labels: map[string]string{ + "app": "proxysql", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To[int32](2), + ServiceName: "proxysqlcluster", + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "proxysql", + }, + }, + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "proxysql", + }, + Annotations: map[string]string{ + "configHash": configHash, + }, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + Containers: []corev1.Container{ + { + // TODO: make configurable + Image: "proxysql/proxysql:2.7.1", + Name: "proxysql", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "proxysql-config", + MountPath: "/etc/proxysql.cnf", + SubPath: "proxysql.cnf", + }, + // { + // Name: "proxysql-data", + // MountPath: "/var/lib/proxysql", + // }, + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 6033, + Name: "proxysql-mysql", + }, + { + ContainerPort: 6032, + Name: "proxysql-admin", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "proxysql-config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "proxysql-config", + }, + }, + }, + }, + }, + }, + // If we want dynamic configurations + // VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + // { + // ObjectMeta: metav1.ObjectMeta{ + // Name: "proxysql-data", + // }, + // Spec: corev1.PersistentVolumeClaimSpec{ + // AccessModes: []corev1.PersistentVolumeAccessMode{ + // corev1.ReadWriteOnce, + // }, + // Resources: corev1.VolumeResourceRequirements{ + // Requests: corev1.ResourceList{ + // "storage": resource.MustParse("2Gi"), + // }, + // }, + // }, + // }, + // }, + }, + } + + return svc.SetDesiredKubeObject(sts, comp.GetName()+"-proxysql-sts") +} + +func createMainService(comp *vshnv1.VSHNMariaDB, svc *runtime.ServiceRuntime) error { + target := map[string]string{} + targetPort := 3306 + serviceName := "mariadb" + cd := svc.GetConnectionDetails() + target["app.kubernetes.io/instance"] = comp.GetName() + target["app.kubernetes.io/name"] = "mariadb-galera" + + if comp.GetInstances() == 1 { + target["app"] = "mariadb" + } else { + target["app"] = "proxysql" + targetPort = 6033 + } + + service := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: comp.GetInstanceNamespace(), + }, + Spec: corev1.ServiceSpec{ + Selector: target, + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{ + { + Name: serviceName, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt(targetPort), + Port: 3306, + }, + }, + }, + } + + mariadbHost := serviceName + ".vshn-mariadb-" + comp.GetName() + ".svc.cluster.local" + mariadbURL := fmt.Sprintf("mysql://%s:%s@%s:%s", mariadbUser, cd["MARIADB_PASSWORD"], mariadbHost, mariadbPort) + + svc.SetConnectionDetail("MARIADB_HOST", []byte(mariadbHost)) + svc.SetConnectionDetail("MARIADB_URL", []byte(mariadbURL)) + + return svc.SetDesiredKubeObject(&service, comp.GetName()+"-main-service") +} diff --git a/pkg/comp-functions/functions/vshnmariadb/register.go b/pkg/comp-functions/functions/vshnmariadb/register.go index 4fe1f57eca..d52350c29e 100644 --- a/pkg/comp-functions/functions/vshnmariadb/register.go +++ b/pkg/comp-functions/functions/vshnmariadb/register.go @@ -35,14 +35,18 @@ func init() { Name: "non-sla-prometheus-rules", Execute: nonsla.GenerateNonSLAPromRules[*vshnv1.VSHNMariaDB](nonsla.NewAlertSetBuilder("mariadb", "mariadb").AddAll().GetAlerts()), }, - { - Name: "billing", - Execute: AddServiceBillingLabel, - }, + // { + // Name: "billing", + // Execute: AddServiceBillingLabel, + // }, { Name: "user-management", Execute: UserManagement, }, + { + Name: "proxySQL", + Execute: AddProxySQL, + }, }, }) } diff --git a/pkg/comp-functions/functions/vshnmariadb/user_management.go b/pkg/comp-functions/functions/vshnmariadb/user_management.go index d4dc46ada8..6cb5ede9f4 100644 --- a/pkg/comp-functions/functions/vshnmariadb/user_management.go +++ b/pkg/comp-functions/functions/vshnmariadb/user_management.go @@ -140,10 +140,18 @@ func addProviderConfig(comp *vshnv1.VSHNMariaDB, svc *runtime.ServiceRuntime) { }, } - err := svc.SetDesiredKubeObject(secret, comp.GetName()+"-provider-conf-credentials", - runtime.KubeOptionProtects("namespace-conditions"), - runtime.KubeOptionProtects("cluster"), - runtime.KubeOptionProtects(comp.GetName()+"-netpol")) + opts := []runtime.KubeObjectOption{ + runtime.KubeOptionProtects(comp.GetName() + "-instanceNs"), + runtime.KubeOptionProtects(comp.GetName() + "-release"), + runtime.KubeOptionProtects(comp.GetName() + "-netpol"), + runtime.KubeOptionProtects(comp.GetName() + "-main-service"), + } + + if comp.GetInstances() != 1 { + opts = append(opts, runtime.KubeOptionProtects(comp.GetName()+"-proxysql-sts")) + } + + err := svc.SetDesiredKubeObject(secret, comp.GetName()+"-provider-conf-credentials", opts...) if err != nil { svc.AddResult(runtime.NewWarningResult(fmt.Sprintf("cannot set credential secret for provider-sql: %s", err))) svc.Log.Error(err, "cannot set credential secret for provider-sql")