From 357f7d2e994349cdf142fd0a6cb17d591a60ab0c Mon Sep 17 00:00:00 2001 From: Kirill Garbar Date: Sun, 5 May 2024 21:30:55 +0000 Subject: [PATCH] add auth enabler --- api/v1alpha1/aux_functions.go | 25 ++ api/v1alpha1/etcdcluster_types.go | 12 + api/v1alpha1/etcdcluster_webhook.go | 9 + .../templates/workload/deployment.yml | 6 + .../crd/bases/etcd.aenix.io_etcdclusters.yaml | 10 +- config/manager/manager.yaml | 6 + internal/controller/etcdcluster_controller.go | 227 ++++++++++++++++++ internal/controller/factory/statefulset.go | 16 +- 8 files changed, 305 insertions(+), 6 deletions(-) create mode 100644 api/v1alpha1/aux_functions.go diff --git a/api/v1alpha1/aux_functions.go b/api/v1alpha1/aux_functions.go new file mode 100644 index 00000000..782560b5 --- /dev/null +++ b/api/v1alpha1/aux_functions.go @@ -0,0 +1,25 @@ +package v1alpha1 + +func IsClientSecurityEnabled(c EtcdCluster) bool { + clientSecurityEnabled := false + if c.Spec.Security != nil && c.Spec.Security.TLS.ClientSecret != "" { + clientSecurityEnabled = true + } + return clientSecurityEnabled +} + +func IsServerSecurityEnabled(c EtcdCluster) bool { + serverSecurityEnabled := false + if c.Spec.Security != nil && c.Spec.Security.TLS.ServerSecret != "" { + serverSecurityEnabled = true + } + return serverSecurityEnabled +} + +func IsServerCADefined(c EtcdCluster) bool { + serverCADefined := false + if c.Spec.Security != nil && c.Spec.Security.TLS.ServerTrustedCASecret != "" { + serverCADefined = true + } + return serverCADefined +} diff --git a/api/v1alpha1/etcdcluster_types.go b/api/v1alpha1/etcdcluster_types.go index 205861ff..25976d4c 100644 --- a/api/v1alpha1/etcdcluster_types.go +++ b/api/v1alpha1/etcdcluster_types.go @@ -174,24 +174,36 @@ type SecuritySpec struct { // Section for user-managed tls certificates // +optional TLS TLSSpec `json:"tls,omitempty"` + // Section to enable etcd auth + EnableAuth bool `json:"enableAuth,omitempty"` } // TLSSpec defines user-managed certificates names. type TLSSpec struct { // Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional PeerTrustedCASecret string `json:"peerTrustedCASecret,omitempty"` // Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional PeerSecret string `json:"peerSecret,omitempty"` + // Trusted CA for etcd server certificates for client-server communication. Is necessary to set trust between operator and etcd. + // It is expected to have tls.crt field in the secret. If it is not specified, then insecure communication will be used. + // This secret must be created in the namespace with etcd-operator. + // +optional + ServerTrustedCASecret string `json:"serverTrustedCASecret,omitempty"` // Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). // It is expected to have tls.crt and tls.key fields in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional ServerSecret string `json:"serverSecret,omitempty"` // Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional ClientTrustedCASecret string `json:"clientTrustedCASecret,omitempty"` // Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + // This secret must be created in the namespace with etcd-operator. // +optional ClientSecret string `json:"clientSecret,omitempty"` } diff --git a/api/v1alpha1/etcdcluster_webhook.go b/api/v1alpha1/etcdcluster_webhook.go index 210efb02..18c7971b 100644 --- a/api/v1alpha1/etcdcluster_webhook.go +++ b/api/v1alpha1/etcdcluster_webhook.go @@ -281,6 +281,15 @@ func (r *EtcdCluster) validateSecurity() field.ErrorList { ) } + if security.EnableAuth && (security.TLS.ClientSecret == "" || security.TLS.ServerSecret == "") { + + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "security"), + security.TLS, + "if auth is enabled, client secret and server secret must be provided"), + ) + } + if len(allErrors) > 0 { return allErrors } diff --git a/charts/etcd-operator/templates/workload/deployment.yml b/charts/etcd-operator/templates/workload/deployment.yml index ece5bae8..17848ada 100644 --- a/charts/etcd-operator/templates/workload/deployment.yml +++ b/charts/etcd-operator/templates/workload/deployment.yml @@ -58,6 +58,12 @@ spec: - configMapRef: name: {{ include "etcd-operator.fullname" . }}-env {{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace volumeMounts: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert diff --git a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml index a824d508..43573059 100644 --- a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml +++ b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: etcdclusters.etcd.aenix.io spec: group: etcd.aenix.io @@ -192,6 +192,9 @@ spec: security: description: Security describes security settings of etcd (authentication, certificates, rbac) properties: + enableAuth: + description: Section to enable etcd auth + type: boolean tls: description: Section for user-managed tls certificates properties: @@ -212,6 +215,11 @@ spec: Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). It is expected to have tls.crt and tls.key fields in the secret. type: string + serverTrustedCASecret: + description: |- + Trusted CA for etcd server certificates for client-server communication. Is necessary to set trust between operator and etcd. + It is expected to have tls.crt field in the secret. If it is not specified, then insecure communication will be used. + type: string type: object type: object serviceTemplate: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index b62c4319..d657a5e9 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -99,5 +99,11 @@ spec: requests: cpu: 10m memory: 64Mi + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/internal/controller/etcdcluster_controller.go b/internal/controller/etcdcluster_controller.go index d3834494..1c06058d 100644 --- a/internal/controller/etcdcluster_controller.go +++ b/internal/controller/etcdcluster_controller.go @@ -18,8 +18,14 @@ package controller import ( "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" goerrors "errors" "fmt" + "os" + "slices" + "time" policyv1 "k8s.io/api/policy/v1" "sigs.k8s.io/controller-runtime/pkg/log" @@ -37,6 +43,8 @@ import ( etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" "github.com/aenix-io/etcd-operator/internal/controller/factory" + + clientv3 "go.etcd.io/etcd/client/v3" ) // EtcdClusterReconciler reconciles a EtcdCluster object @@ -83,6 +91,11 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return r.updateStatusOnErr(ctx, instance, fmt.Errorf("cannot create Cluster auxiliary objects: %w", err)) } + reconcileResult, err := r.configureAuth(ctx, instance) + if err != nil { + return reconcileResult, err + } + // set cluster initialization condition factory.SetCondition(instance, factory.NewCondition(etcdaenixiov1alpha1.EtcdConditionInitialized). WithStatus(true). @@ -192,3 +205,217 @@ func (r *EtcdClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&policyv1.PodDisruptionBudget{}). Complete(r) } + +func (r *EtcdClusterReconciler) configureAuth(ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster) (ctrl.Result, error) { + logger := log.FromContext(ctx) + var err error + + cli, reconcileResult, err := r.getEtcdClient(ctx, cluster) + if err != nil { + return reconcileResult, err + } + + reconcileResult, err = testMemberList(ctx, cli) + if err != nil { + return reconcileResult, err + } + + auth := clientv3.NewAuth(cli) + + if cluster.Spec.Security.EnableAuth { + + // Create root role + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _, err = auth.RoleGet(ctx, "root") + cancel() + + if err != nil { + if err.Error() == "etcdserver: role name not found" { + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second) + + _, err = auth.RoleAdd(ctx, "root") + if err != nil { + logger.Error(err, "failed to add role", "role name", "root") + return ctrl.Result{}, err + } else { + logger.Info("role added", "role name", "root") + } + cancel() + } else { + logger.Error(err, "failed to get role", "role name", "root") + return ctrl.Result{}, err + } + } else { + logger.Info("role exists, nothing to do", "role name", "root") + } + + // Create root user + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + rootUserResponse, err := auth.UserGet(ctx, "root") + cancel() + + if err != nil { + if err.Error() == "etcdserver: user name not found" { + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second) + + _, err = auth.UserAddWithOptions(ctx, "root", "", &clientv3.UserAddOptions{ + NoPassword: true, + }) + if err != nil { + logger.Error(err, "failed to add user", "user name", "root") + return ctrl.Result{}, err + } else { + logger.Info("user added", "user name", "root") + } + cancel() + + } else { + logger.Error(err, "failed to get user", "user name", "root") + return ctrl.Result{}, err + } + } else { + logger.Info("user exists, nothing to do", "user name", "root") + } + + // Grant root role to root user + if !slices.Contains(rootUserResponse.Roles, "root") { + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + _, err := auth.UserGrantRole(ctx, "root", "root") + cancel() + if err != nil { + logger.Error(err, "failed to grant user to role", "user:role name", "root:root") + return ctrl.Result{}, err + } + logger.Info("user:role granted", "user:role name", "root:root") + } else { + logger.Info("user:role already granted, nothing to do", "user:role name", "root:root") + } + + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + + // Enable auth + _, err = auth.AuthEnable(ctx) + cancel() + if err != nil { + logger.Error(err, "failed to enable auth") + return ctrl.Result{}, err + } else { + logger.Info("auth enabled") + } + } else { + // Disable auth + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _, err = auth.AuthDisable(ctx) + cancel() + if err != nil { + logger.Error(err, "failed to disable auth") + return ctrl.Result{}, err + } else { + logger.Info("auth disabled") + } + } + + reconcileResult, err = testMemberList(ctx, cli) + if err != nil { + return reconcileResult, err + } + + return ctrl.Result{}, nil +} + +// This is auxiliary self-test function, that shows that connection to etcd cluster works. +// As soon as operator has functionality to operate etcd-cluster, this function can be removed. +func testMemberList(ctx context.Context, cli *clientv3.Client) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + etcdCluster := clientv3.NewCluster(cli) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + memberList, err := etcdCluster.MemberList(ctx) + cancel() + + if err != nil { + logger.Error(err, "failed to get member list", "endpoints", cli.Endpoints()) + return ctrl.Result{}, err + } else { + logger.Info("member list got", "member list", memberList) + } + return ctrl.Result{}, nil +} + +func (r *EtcdClusterReconciler) getEtcdClient(ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster) (*clientv3.Client, ctrl.Result, error) { + logger := log.FromContext(ctx) + var err error + + operatorNamespace := os.Getenv("POD_NAMESPACE") + + rootSecret := &corev1.Secret{} + serverCASecret := &corev1.Secret{} + if err = r.Get(ctx, client.ObjectKey{Namespace: operatorNamespace, Name: cluster.Spec.Security.TLS.ClientSecret}, rootSecret); err != nil { + logger.Error(err, "failed to get root client secret") + return nil, ctrl.Result{}, err + } else { + logger.Info("secret read", "root client secret", rootSecret) + } + + if err = r.Get(ctx, client.ObjectKey{Namespace: operatorNamespace, Name: cluster.Spec.Security.TLS.ServerTrustedCASecret}, serverCASecret); err != nil { + logger.Error(err, "failed to get server trusted CA secret") + return nil, ctrl.Result{}, err + } else { + logger.Info("secret read", "server trusted CA secret", serverCASecret) + } + + endpoints := []string{factory.GetServerProtocol(*cluster) + cluster.Name + ":2379"} + + caCertPool := &x509.CertPool{} + cert := tls.Certificate{} + + if etcdaenixiov1alpha1.IsServerCADefined(*cluster) { + + pemByte := serverCASecret.Data["tls.crt"] + + caCertPool = x509.NewCertPool() + + for { + var block *pem.Block + block, pemByte = pem.Decode(pemByte) + if block == nil { + break + } + caCert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + logger.Error(err, "failed to parse CA certificate") + return nil, ctrl.Result{}, err + } + + caCertPool.AddCert(caCert) + } + } + + cert, err = tls.X509KeyPair(rootSecret.Data["tls.crt"], rootSecret.Data["tls.key"]) + if err != nil { + logger.Error(err, "failed to parse key pair", "cert", cert) + return nil, ctrl.Result{}, err + } + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 5 * time.Second, + TLS: &tls.Config{ + InsecureSkipVerify: !etcdaenixiov1alpha1.IsServerCADefined(*cluster), + RootCAs: caCertPool, + Certificates: []tls.Certificate{ + cert, + }, + }, + }) + if err != nil { + logger.Error(err, "failed to create etcd client", "endpoints", endpoints) + return nil, ctrl.Result{}, err + } else { + logger.Info("etcd client created", "endpoints", endpoints) + } + + return cli, ctrl.Result{}, nil + +} diff --git a/internal/controller/factory/statefulset.go b/internal/controller/factory/statefulset.go index c29bf5c9..c34b1bae 100644 --- a/internal/controller/factory/statefulset.go +++ b/internal/controller/factory/statefulset.go @@ -270,19 +270,17 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { } serverTlsSettings := []string{} - serverProtocol := "http" if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ServerSecret != "" { serverTlsSettings = []string{ "--cert-file=/etc/etcd/pki/server/cert/tls.crt", "--key-file=/etc/etcd/pki/server/cert/tls.key", } - serverProtocol = "https" } clientTlsSettings := []string{} - if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ClientSecret != "" { + if etcdaenixiov1alpha1.IsClientSecurityEnabled(*cluster) { clientTlsSettings = []string{ "--trusted-ca-file=/etc/etcd/pki/client/ca/ca.crt", "--client-cert-auth", @@ -293,10 +291,10 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { "--name=$(POD_NAME)", "--listen-metrics-urls=http://0.0.0.0:2381", "--listen-peer-urls=https://0.0.0.0:2380", - fmt.Sprintf("--listen-client-urls=%s://0.0.0.0:2379", serverProtocol), + fmt.Sprintf("--listen-client-urls=%s0.0.0.0:2379", GetServerProtocol(*cluster)), fmt.Sprintf("--initial-advertise-peer-urls=https://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2380", GetHeadlessServiceName(cluster)), "--data-dir=/var/run/etcd/default.etcd", - fmt.Sprintf("--advertise-client-urls=%s://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2379", serverProtocol, GetHeadlessServiceName(cluster)), + fmt.Sprintf("--advertise-client-urls=%s$(POD_NAME).%s.$(POD_NAMESPACE).svc:2379", GetServerProtocol(*cluster), GetHeadlessServiceName(cluster)), }...) args = append(args, peerTlsSettings...) @@ -389,3 +387,11 @@ func getLivenessProbe() *corev1.Probe { PeriodSeconds: 5, } } + +func GetServerProtocol(c etcdaenixiov1alpha1.EtcdCluster) string { + serverProtocol := "http://" + if etcdaenixiov1alpha1.IsServerSecurityEnabled(c) { + serverProtocol = "https://" + } + return serverProtocol +}