Merge pull request #475 from kube-tarian/agent-ca-cert-issuer

Agent rest server intragated with mTLS certificate and certs vault update

capten/agent/gin-api-server/server.go

@@ -9,11 +9,9 @@ import (
 	"github.com/kube-tarian/kad/capten/agent/internal/config"
 )
 
-var log = logging.NewLogger()
-
-func StartRestServer(rpcapi api.ServerInterface, cfg *config.SericeConfig) {
+func StartRestServer(rpcapi api.ServerInterface, cfg *config.SericeConfig, log logging.Logger) error {
 	r := gin.Default()
 	api.RegisterHandlers(r, rpcapi)
 
-	r.Run(fmt.Sprintf("%s:%d", cfg.Host, cfg.RestPort))
+	return r.RunTLS(fmt.Sprintf("%s:%d", cfg.Host, cfg.RestPort), cfg.CertFileName, cfg.KeyFileName)
 }

capten/agent/internal/app/app.go

@@ -14,6 +14,7 @@ import (
 	"github.com/kube-tarian/kad/capten/agent/internal/job"
 	captenstore "github.com/kube-tarian/kad/capten/common-pkg/capten-store"
 	"github.com/kube-tarian/kad/capten/common-pkg/crossplane"
+	"github.com/kube-tarian/kad/capten/common-pkg/k8s"
 	"github.com/kube-tarian/kad/capten/common-pkg/pb/agentpb"
 	"github.com/kube-tarian/kad/capten/common-pkg/pb/captenpluginspb"
 	"github.com/kube-tarian/kad/capten/common-pkg/pb/clusterpluginspb"
@@ -81,11 +82,17 @@ func Start() {
 		}
 	}()
 
-	err = setupCACertIssuser(cfg.ClusterCAIssuerName)
+	err = k8s.SetupCACertIssuser(cfg.ClusterCAIssuerName, log)
 	if err != nil {
 		log.Fatalf("Failed to setupt CA Cert Issuer in cert-manager %v", err)
 	}
-	go ginapiserver.StartRestServer(rpcapi, cfg)
+
+	go func() {
+		err := ginapiserver.StartRestServer(rpcapi, cfg, log)
+		if err != nil {
+			log.Errorf("Failed to start REST server, %v", err)
+		}
+	}()
 
 	err = registerK8SWatcher(as)
 	if err != nil {

capten/agent/internal/config/config.go

@@ -7,7 +7,7 @@ import (
 type SericeConfig struct {
 	Host                      string `envconfig:"HOST" default:"0.0.0.0"`
 	Port                      int    `envconfig:"PORT" default:"9091"`
-	RestPort                  int    `envconfig:"REST_PORT" default:"9092"`
+	RestPort                  int    `envconfig:"REST_PORT" default:"8443"`
 	Mode                      string `envconfig:"MODE" default:"production"`
 	AuthEnabled               bool   `envconfig:"AUTH_ENABLED" default:"false"`
 	CrossplaneSyncJobEnabled  bool   `envconfig:"CROSSPLANE_SYNC_JOB_ENABLED" default:"true"`
@@ -16,6 +16,8 @@ type SericeConfig struct {
 	TektonSyncJobInterval     string `envconfig:"TEKTON_SYNC_JOB_INTERVAL" default:"@every 1h"`
 	DomainName                string `envconfig:"DOMAIN_NAME" default:"example.com"`
 	ClusterCAIssuerName       string `envconfig:"AGENT_CLUSTER_CA_ISSUER_NAME" default:"agent-ca-issuer"`
+	CertFileName              string `envconfig:"CERT_FILE_NAME" default:"/tmp/certs/tls.crt"`
+	KeyFileName               string `envconfig:"KEY_FILE_NAME" default:"/tmp/certs/tls.key"`
 }
 
 func GetServiceConfig() (*SericeConfig, error) {

capten/common-pkg/cert/generate_certs.go

@@ -51,7 +51,7 @@ func GenerateRootCerts() (*CertificatesData, error) {
 	}, nil
 }
 
-func generateCACert() (*Key, *Cert, error) { //(rootKey *rsa.PrivateKey, rootCertTemplate *x509.Certificate, err error) {
+func generateCACert() (*Key, *Cert, error) {
 	rootKey, err := rsa.GenerateKey(rand.Reader, caBitSize)
 	if err != nil {
 		err = errors.WithMessage(err, "failed to generate RSA key for root certificate")

capten/common-pkg/k8s/cert_issuer.go

@@ -2,9 +2,11 @@ package k8s
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/intelops/go-common/logging"
 	"github.com/kube-tarian/kad/capten/common-pkg/cert"
+	"github.com/kube-tarian/kad/capten/common-pkg/credential"
 	"github.com/pkg/errors"
 
 	certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
@@ -17,15 +19,52 @@ import (
 
 var log = logging.NewLogger()
 
-func CreateOrUpdateClusterIssuer(clusterCAIssuer string) error {
+func SetupCACertIssuser(clusterIssuerName string, log logging.Logger) error {
+	k8sclient, err := NewK8SClient(log)
+	if err != nil {
+		log.Errorf("failed to initalize k8s client, %v", err)
+		return err
+	}
+
+	err = setupCertificateIssuer(k8sclient, clusterIssuerName, log)
+	if err != nil {
+		log.Errorf("Setup Certificates Issuer failed, %v", err)
+		return err
+	}
+	return nil
+}
+
+// Setup agent certificate issuer
+func setupCertificateIssuer(k8sClient *K8SClient, clusterIssuerName string, log logging.Logger) error {
+	// Create Agent Cluster Issuer
+	certsData, err := CreateOrUpdateClusterIssuer(clusterIssuerName, k8sClient, false)
+	if err != nil {
+		return fmt.Errorf("failed to create/update CA Issuer %s in cert-manager: %v", clusterIssuerName, err)
+	}
+
+	// Update Vault
+	err = credential.PutClusterCerts(context.TODO(), "kad-agent", "kad-agent", string(certsData.CaChainCertData), string(certsData.RootKey.KeyData), string(certsData.RootCert.CertData))
+	if err != nil {
+		log.Errorf("Failed to write to vault, %v", err)
+		log.Infof("Continued to start the agent as these certs from vault are not used...")
+	}
+	return nil
+}
+
+func CreateOrUpdateClusterIssuer(clusterCAIssuer string, k8sclient *K8SClient, forceUpdate bool) (*cert.CertificatesData, error) {
 	config, err := rest.InClusterConfig()
 	if err != nil {
-		return errors.WithMessage(err, "error while building kubeconfig")
+		return nil, errors.WithMessage(err, "error while building kubeconfig")
 	}
 
 	cmClient, err := cmclient.NewForConfig(config)
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	certsData, err := cert.GenerateRootCerts()
+	if err != nil {
+		return nil, err
 	}
 
 	issuer := &certmanagerv1.ClusterIssuer{
@@ -42,23 +81,48 @@ func CreateOrUpdateClusterIssuer(clusterCAIssuer string) error {
 	}
 
 	serverIssuer, err := cmClient.CertmanagerV1().ClusterIssuers().Get(context.Background(), issuer.Name, metav1.GetOptions{})
-	if err != nil && k8serrors.IsNotFound(err) {
-		result, err := cmClient.CertmanagerV1().ClusterIssuers().Create(context.Background(), issuer, metav1.CreateOptions{})
-		if err != nil {
-			return errors.WithMessage(err, "error in creating cert issuer")
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			err = CreateOrUpdateClusterCAIssuerSecret(k8sclient, certsData.RootCert.CertData, certsData.RootKey.KeyData, certsData.CaChainCertData)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create/update CA Issuer Secret: %v", err)
+			}
+
+			result, err := cmClient.CertmanagerV1().ClusterIssuers().Create(context.Background(), issuer, metav1.CreateOptions{})
+			if err != nil {
+				return nil, errors.WithMessage(err, "error in creating cert issuer")
+			}
+			log.Debugf("ClusterIssuer %s created successfully", result.Name)
+			return certsData, nil
+		} else if k8serrors.IsAlreadyExists(err) {
+			secret, err := k8sclient.GetSecretObject(cert.CertManagerNamespace, cert.ClusterCACertSecretName)
+			if err != nil {
+				log.Errorf("Failed to read secert %s, %v", cert.ClusterCACertSecretName, err)
+				return nil, err
+			}
+			certsData.CaChainCertData = secret.Data["ca.crt"]
+			certsData.RootCert.CertData = secret.Data[corev1.TLSCertKey]
+			certsData.RootKey.KeyData = secret.Data[corev1.TLSPrivateKeyKey]
+			return certsData, nil
 		}
-		log.Debugf("ClusterIssuer %s created successfully", result.Name)
-		return nil
+		return nil, err
 	}
 
-	serverIssuer.Spec.IssuerConfig.CA.SecretName = cert.ClusterCACertSecretName
-	issuerClient := cmClient.CertmanagerV1().ClusterIssuers()
-	result, err := issuerClient.Update(context.TODO(), serverIssuer, metav1.UpdateOptions{})
-	if err != nil {
-		return errors.WithMessage(err, "error while updating cluster issuer")
+	if forceUpdate {
+		err = CreateOrUpdateClusterCAIssuerSecret(k8sclient, certsData.RootCert.CertData, certsData.RootKey.KeyData, certsData.CaChainCertData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create/update CA Issuer Secret: %v", err)
+		}
+
+		serverIssuer.Spec.IssuerConfig.CA.SecretName = cert.ClusterCACertSecretName
+		issuerClient := cmClient.CertmanagerV1().ClusterIssuers()
+		_, err := issuerClient.Update(context.TODO(), serverIssuer, metav1.UpdateOptions{})
+		if err != nil {
+			return nil, errors.WithMessage(err, "error while updating cluster issuer")
+		}
 	}
-	log.Debugf("ClusterIssuer %s updated successfully", result.Name)
-	return nil
+	log.Debugf("ClusterIssuer %s updated successfully", issuer.Name)
+	return certsData, nil
 }
 
 func CreateOrUpdateClusterCAIssuerSecret(k8sClient *K8SClient, caCertData, caKeyData, caCertChainData []byte) error {

capten/common-pkg/k8s/client.go

@@ -172,6 +172,10 @@ func (k *K8SClient) GetSecretData(namespace, secretName string) (*SecretData, er
 	}, nil
 }
 
+func (k *K8SClient) GetSecretObject(namespace, secretName string) (*v1.Secret, error) {
+	return k.Clientset.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{})
+}
+
 func (k *K8SClient) CreateOrUpdateSecret(
 	ctx context.Context,
 	namespace, secretName string,

charts/kad/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.4
+version: 1.0.5
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "2.0.2"
+appVersion: "2.0.3"

charts/kad/templates/agent-certificate.yaml

@@ -0,0 +1,18 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "kad.fullname" . }}-agent-server-mtls
+spec:
+  commonName: {{ include "kad.fullname" . }}-agent-server-mtls
+  issuerRef:
+    kind: ClusterIssuer
+    name: {{ .Values.agent.clusterIssuerReference }}
+  privateKey:
+    algorithm: RSA
+    encoding: PKCS1
+    size: 2048
+  secretName: {{ include "kad.fullname" . }}-agent-server-mtls
+  usages:
+  - digital signature
+  - key encipherment
+  - server auth