feat: add bootstrap secrets

apis/apps/v2beta1/emqx_types.go

@@ -103,7 +103,19 @@ type BootstrapAPIKey struct {
 	Key string `json:"key"`
 	// +kubebuilder:validation:MinLength:=3
 	// +kubebuilder:validation:MaxLength:=32
-	Secret string `json:"secret"`
+	Secret    string     `json:"secret"`
+	SecretRef *SecretRef `json:"secretRef"`
+}
+
+type SecretRef struct {
+	Key    *KeyRef `json:"key"`
+	Secret *KeyRef `json:"secret"`
+}
+
+type KeyRef struct {
+	SecretName string `json:"secretName"`
+	// +kubebuilder:validation:Pattern:=`^[a-zA-Z\d-_]+$`
+	SecretKey string `json:"secretKey"`
 }
 
 type Config struct {

config/crd/bases/apps.emqx.io_emqxes.yaml

@@ -6492,9 +6492,38 @@ spec:
                       maxLength: 32
                       minLength: 3
                       type: string
+                    secretRef:
+                      properties:
+                        key:
+                          properties:
+                            secretKey:
+                              pattern: ^[a-zA-Z\d-_]+$
+                              type: string
+                            secretName:
+                              type: string
+                          required:
+                          - secretKey
+                          - secretName
+                          type: object
+                        secret:
+                          properties:
+                            secretKey:
+                              pattern: ^[a-zA-Z\d-_]+$
+                              type: string
+                            secretName:
+                              type: string
+                          required:
+                          - secretKey
+                          - secretName
+                          type: object
+                      required:
+                      - key
+                      - secret
+                      type: object
                   required:
                   - key
                   - secret
+                  - secretRef
                   type: object
                 type: array
               clusterDomain:

controllers/apps/v2beta1/add_bootstrap_resource.go

@@ -7,6 +7,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -23,7 +24,7 @@ type addBootstrap struct {
 func (a *addBootstrap) reconcile(ctx context.Context, instance *appsv2beta1.EMQX, _ innerReq.RequesterInterface) subResult {
 	for _, resource := range []client.Object{
 		generateNodeCookieSecret(instance),
-		generateBootstrapAPIKeySecret(instance),
+		generateBootstrapAPIKeySecret(a.Client, ctx, instance),
 	} {
 		if err := ctrl.SetControllerReference(instance, resource, a.Scheme); err != nil {
 			return subResult{err: emperror.Wrap(err, "failed to set controller reference")}
@@ -63,12 +64,46 @@ func generateNodeCookieSecret(instance *appsv2beta1.EMQX) *corev1.Secret {
 	}
 }
 
-func generateBootstrapAPIKeySecret(instance *appsv2beta1.EMQX) *corev1.Secret {
+// ReadSecret reads a secret from the Kubernetes cluster.
+func ReadSecret(k8sClient client.Client, ctx context.Context, namespace string, name string, key string) (string, error) {
+	// Define a new Secret object
+	secret := &corev1.Secret{}
+
+	// Define the Secret Name and Namespace
+	secretName := types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}
+
+	// Use the client to fetch the Secret
+	if err := k8sClient.Get(ctx, secretName, secret); err != nil {
+		return "", err
+	}
+
+	// secret.Data is a map[string][]byte
+	secretValue := string(secret.Data[key])
+
+	return secretValue, nil
+}
+
+func generateBootstrapAPIKeySecret(k8sClient client.Client, ctx context.Context, instance *appsv2beta1.EMQX) *corev1.Secret {
 	bootstrapAPIKeys := ""
+
 	for _, apiKey := range instance.Spec.BootstrapAPIKeys {
 		bootstrapAPIKeys += apiKey.Key + ":" + apiKey.Secret + "\n"
+		if apiKey.SecretRef != nil {
+			// Read key and secret values from the refenced secrets
+			keyValue, err := ReadSecret(k8sClient, ctx, instance.Namespace, apiKey.SecretRef.Key.SecretName, apiKey.SecretRef.Key.SecretKey)
+			if err != nil {
+				continue
+			}
+			secretValue, err := ReadSecret(k8sClient, ctx, instance.Namespace, apiKey.SecretRef.Secret.SecretName, apiKey.SecretRef.Secret.SecretKey)
+			if err != nil {
+				continue
+			}
+			bootstrapAPIKeys += keyValue + ":" + secretValue + "\n"
+		}
 	}
-
 	defPassword, _ := password.Generate(64, 10, 0, true, true)
 	bootstrapAPIKeys += appsv2beta1.DefaultBootstrapAPIKey + ":" + defPassword
 

controllers/apps/v2beta1/add_bootstrap_resource_suite_test.go

@@ -0,0 +1,66 @@
+package v2beta1
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	appsv2beta1 "github.com/emqx/emqx-operator/apis/apps/v2beta1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var _ = Describe("AddBootstrap", Ordered, Label("bootstrap"), func() {
+	var (
+		instance *appsv2beta1.EMQX
+		a        *addBootstrap
+		ns       *corev1.Namespace
+	)
+	instance = new(appsv2beta1.EMQX)
+	ns = &corev1.Namespace{}
+
+	BeforeEach(func() {
+		a = &addBootstrap{emqxReconciler}
+		ns = &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "controller-v2beta1-add-emqx-bootstrap-test",
+				Labels: map[string]string{
+					"test": "e2e",
+				},
+			},
+		}
+		instance = emqx.DeepCopy()
+		instance.Namespace = ns.Name
+	})
+
+	It("create namespace", func() {
+		Expect(k8sClient.Create(context.TODO(), ns)).Should(Succeed())
+	})
+
+	It("should create bootstrap secrets", func() {
+		// Wait until the bootstrap secrets are created
+		// Call the reconciler.
+		result := a.reconcile(ctx, instance, nil)
+
+		// Make sure there were no errors.
+		Expect(result.err).NotTo(HaveOccurred())
+		// Check the created secrets.
+		cookieSecret := &corev1.Secret{}
+		err := k8sClient.Get(context.Background(), client.ObjectKey{
+			Namespace: ns.Name,
+			Name:      instance.NodeCookieNamespacedName().Name,
+		}, cookieSecret)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(cookieSecret.Data["node_cookie"]).ShouldNot(BeEmpty())
+
+		bootstrapSecret := &corev1.Secret{}
+		err = k8sClient.Get(context.Background(), client.ObjectKey{
+			Namespace: ns.Name,
+			Name:      instance.BootstrapAPIKeyNamespacedName().Name,
+		}, bootstrapSecret)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(bootstrapSecret.Data["bootstrap_api_key"]).ShouldNot(BeEmpty())
+	})
+})

controllers/apps/v2beta1/add_bootstrap_resource_test.go

@@ -1,12 +1,16 @@
 package v2beta1
 
 import (
+	"context"
 	"strings"
 	"testing"
 
 	appsv2beta1 "github.com/emqx/emqx-operator/apis/apps/v2beta1"
 	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
 func TestGenerateNodeCookieSecret(t *testing.T) {
@@ -35,6 +39,18 @@ func TestGenerateNodeCookieSecret(t *testing.T) {
 }
 
 func TestGenerateBootstrapAPIKeySecret(t *testing.T) {
+	// Create a fake client
+	scheme := runtime.NewScheme()
+	err := corev1.AddToScheme(scheme)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build()
+
+	// Create a context
+	ctx := context.Background()
+
 	instance := &appsv2beta1.EMQX{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "emqx",
@@ -50,7 +66,7 @@ func TestGenerateBootstrapAPIKeySecret(t *testing.T) {
 		},
 	}
 
-	got := generateBootstrapAPIKeySecret(instance)
+	got := generateBootstrapAPIKeySecret(fakeClient, ctx, instance)
 	assert.Equal(t, "emqx-bootstrap-api-key", got.Name)
 	data, ok := got.StringData["bootstrap_api_key"]
 	assert.True(t, ok)
@@ -62,3 +78,39 @@ func TestGenerateBootstrapAPIKeySecret(t *testing.T) {
 	}
 	assert.ElementsMatch(t, usernames, []string{appsv2beta1.DefaultBootstrapAPIKey, "test_key"})
 }
+
+func TestReadSecret(t *testing.T) {
+	// Create a fake client
+	scheme := runtime.NewScheme()
+	err := corev1.AddToScheme(scheme)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Define the secret data
+	secretData := map[string][]byte{
+		"key": []byte("value"),
+	}
+
+	// Create a secret
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-secret",
+			Namespace: "default",
+		},
+		Data: secretData,
+	}
+
+	fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(secret).Build()
+
+	// Create a context
+	ctx := context.Background()
+
+	val, err := ReadSecret(fakeClient, ctx, "default", "test-secret", "key")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check the secret value
+	assert.Equal(t, "value", val)
+}