Support ReadOnlyRootFilesystem with DSE 6.8 / DSE 6.9 (#702)

* If the ReadOnlyRootFilesystem is selected, always force the usage of k8ssandra-client for config building * Add DSE specific cassandra base config path for readOnly * Add different mounts for HCD & DSE * Add buildx options to docker-build * Add annotation to control if k8ssandra-client is used for config building regardless if supported or not and add a /opt/dse/resources/dse/conf for dse.yaml output * Add collectd directory for readOnly * Add unit tests and change the CHANGELOG * Add smoke_test_read_only_fs for DSE 6.8.50 and 6.9.2 also * Fix smoke_test_read_only_fs
k8ssandra · Sep 17, 2024 · 3dfee48 · 3dfee48
1 parent 7e644ae
commit 3dfee48
Show file tree

Hide file tree

Showing 8 changed files with 392 additions and 22 deletions.
diff --git a/.github/workflows/kindIntegTest.yml b/.github/workflows/kindIntegTest.yml
@@ -155,7 +155,7 @@ jobs:
     strategy:
       matrix:
         version:
-          - "4.1.5"
+          - "4.1.6"
         integration_test:
           # Single worker tests:
           - additional_serviceoptions
@@ -198,7 +198,7 @@ jobs:
           - scale_up
           - scale_up_stop_resume
           - seed_selection
-          - smoke_test_read_only_fs
+          # - smoke_test_read_only_fs
           #- config_fql # OSS only
           - decommission_dc
         # - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA
@@ -233,25 +233,28 @@ jobs:
         version:
           - "3.11.17"
           - "4.0.13"
-          - "4.1.5"
+          - "4.1.6"
           - "6.8.50"
-          - "6.9.0"
+          - "6.9.2"
           - "1.0.0"
         integration_test:
           - test_all_the_things
+          - smoke_test_read_only_fs
         include:
           - version: 6.8.50
             serverImage: datastax/dse-mgmtapi-6_8:6.8.50-ubi8 # DSE 6.8.50
             serverType: dse
-            integration_test: "test_all_the_things"
-          - version: 6.9.0
-            serverImage: datastax/dse-mgmtapi-6_8:6.9.0-ubi8 # DSE 6.9.0
+          - version: 6.9.2
+            serverImage: datastax/dse-mgmtapi-6_8:6.9.2-ubi # DSE 6.9.2
             serverType: dse
-            integration_test: "test_all_the_things"
           - version: 1.0.0
             serverImage: datastax/hcd:1.0.0-ubi # HCD 1.0.0
             serverType: hcd
-            integration_test: "test_all_the_things"
+        exclude:
+          - version: 3.11.17
+            integration_test: "smoke_test_read_only_fs"
+          - version: 4.0.13
+            integration_test: "smoke_test_read_only_fs"
       fail-fast: true
     runs-on: ubuntu-latest
     env:
@@ -282,7 +285,7 @@ jobs:
     strategy:
       matrix:
         version:
-          - "4.1.5"
+          - "4.1.6"
         integration_test:
           - pvc_expansion
       fail-fast: true

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,7 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti
 ## unreleased
 
 * [FEATURE] [#651](https://github.com/k8ssandra/cass-operator/issues/651) Add tsreload task for DSE deployments and ability to check if sync operation is available on the mgmt-api side
+* [FEATURE] [#701](https://github.com/k8ssandra/cass-operator/issues/701) Allow ReadOnlyRootFilesystem for DSE also with extra mounts to provide support for cass-config-builder setups
 
 ## v1.22.2
 

diff --git a/Makefile b/Makefile
@@ -86,6 +86,8 @@ endif
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+BUILDX_OPTIONS ?=
+
 .PHONY: all
 all: build
 
@@ -160,7 +162,7 @@ run: manifests generate fmt vet ## Run a controller from your host.
 
 .PHONY: docker-build
 docker-build: ## Build docker image with the manager.
-	docker buildx build --build-arg VERSION=${VERSION} -t ${IMG} . --load
+	docker buildx build --build-arg VERSION=${VERSION} -t ${IMG} ${BUILDX_OPTIONS} . --load
 
 .PHONY: docker-kind
 docker-kind: docker-build ## Build docker image and load to kind cluster

diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go
@@ -74,6 +74,9 @@ const (
 	// AllowStorageChangesAnnotation indicates the CassandraDatacenter StorageConfig can be modified for existing datacenters
 	AllowStorageChangesAnnotation = "cassandra.datastax.com/allow-storage-changes"
 
+	// UseClientBuilderAnnotation enforces the usage of new config builder from k8ssandra-client for versions that would otherwise use the cass-config-builder
+	UseClientBuilderAnnotation = "cassandra.datastax.com/use-new-config-builder"
+
 	AllowUpdateAlways AllowUpdateType = "always"
 	AllowUpdateOnce   AllowUpdateType = "once"
 
@@ -985,6 +988,10 @@ func (dc *CassandraDatacenter) DatacenterName() string {
 }
 
 func (dc *CassandraDatacenter) UseClientImage() bool {
+	if metav1.HasAnnotation(dc.ObjectMeta, UseClientBuilderAnnotation) && dc.Annotations[UseClientBuilderAnnotation] == "true" {
+		return true
+	}
+
 	if dc.Spec.ServerType == "hcd" {
 		return true
 	}
@@ -998,3 +1005,10 @@ func (dc *CassandraDatacenter) UseClientImage() bool {
 func (dc *CassandraDatacenter) GenerationChanged() bool {
 	return dc.Status.ObservedGeneration < dc.Generation
 }
+
+func (dc *CassandraDatacenter) ReadOnlyFs() bool {
+	if dc.Spec.ReadOnlyRootFilesystem != nil {
+		return *dc.Spec.ReadOnlyRootFilesystem
+	}
+	return false
+}
diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types_test.go b/apis/cassandra/v1beta1/cassandradatacenter_types_test.go
@@ -5,6 +5,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 )
 
 var internodeEnabledAll = `
@@ -125,3 +127,66 @@ func TestUseClientImage(t *testing.T) {
 		}
 	}
 }
+
+func TestUseClientImageEnforce(t *testing.T) {
+	assert := assert.New(t)
+
+	tests := []struct {
+		serverType string
+		version    string
+	}{
+		{
+			serverType: "cassandra",
+			version:    "4.1.0",
+		},
+		{
+			serverType: "cassandra",
+			version:    "4.1.2",
+		},
+		{
+			serverType: "cassandra",
+			version:    "5.0.0",
+		},
+		{
+			serverType: "cassandra",
+			version:    "3.11.17",
+		},
+		{
+			serverType: "cassandra",
+			version:    "4.0.8",
+		},
+		{
+			serverType: "dse",
+			version:    "6.8.39",
+		},
+		{
+			serverType: "dse",
+			version:    "6.9.0",
+		},
+		{
+			serverType: "hcd",
+			version:    "1.0.0",
+		},
+		{
+			serverType: "dse",
+			version:    "4.1.2",
+		},
+	}
+
+	for _, tt := range tests {
+		dc := CassandraDatacenter{
+			ObjectMeta: metav1.ObjectMeta{
+				Annotations: map[string]string{
+					UseClientBuilderAnnotation: "true",
+				},
+			},
+			Spec: CassandraDatacenterSpec{
+				ServerVersion:          tt.version,
+				ServerType:             tt.serverType,
+				ReadOnlyRootFilesystem: ptr.To[bool](true),
+			},
+		}
+
+		assert.True(dc.UseClientImage())
+	}
+}
diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go
@@ -307,7 +307,7 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe
 
 	volumeDefaults := []corev1.Volume{vServerConfig, vServerLogs}
 
-	if readOnlyFs(dc) {
+	if dc.ReadOnlyFs() {
 		tmp := corev1.Volume{
 			Name: "tmp",
 			VolumeSource: corev1.VolumeSource{
@@ -323,6 +323,34 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe
 		}
 
 		volumeDefaults = append(volumeDefaults, tmp, etcCass)
+
+		if dc.Spec.ServerType == "dse" {
+			dseConf := corev1.Volume{
+				Name: "dse-conf",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			}
+			volumeDefaults = append(volumeDefaults, dseConf)
+
+			if !dc.UseClientImage() {
+				sparkConf := corev1.Volume{
+					Name: "spark-conf",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				}
+
+				collectDConf := corev1.Volume{
+					Name: "collectd-conf",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				}
+
+				volumeDefaults = append(volumeDefaults, sparkConf, collectDConf)
+			}
+		}
 	}
 
 	if dc.UseClientImage() {
@@ -476,6 +504,8 @@ func buildInitContainers(dc *api.CassandraDatacenter, rackName string, baseTempl
 			configContainer.Command = []string{"/bin/sh"}
 			if dc.Spec.ServerType == "cassandra" {
 				configContainer.Args = []string{"-c", "cp -rf /etc/cassandra/* /cassandra-base-config/"}
+			} else if dc.Spec.ServerType == "dse" {
+				configContainer.Args = []string{"-c", "cp -rf /opt/dse/resources/cassandra/conf/* /cassandra-base-config/"}
 			} else if dc.Spec.ServerType == "hcd" {
 				configContainer.Args = []string{"-c", "cp -rf /opt/hcd/resources/cassandra/conf/* /cassandra-base-config/"}
 			}
@@ -649,7 +679,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 		}
 	}
 
-	if readOnlyFs(dc) {
+	if dc.ReadOnlyFs() {
 		cassContainer.SecurityContext = &corev1.SecurityContext{
 			ReadOnlyRootFilesystem: ptr.To[bool](true),
 		}
@@ -680,7 +710,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 		envDefaults = append(envDefaults, corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"})
 	}
 
-	if readOnlyFs(dc) {
+	if dc.ReadOnlyFs() {
 		envDefaults = append(envDefaults, corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"})
 	}
 
@@ -737,7 +767,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 		}
 	}
 
-	if readOnlyFs(dc) {
+	if dc.ReadOnlyFs() {
 		cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
 			Name:      "tmp",
 			MountPath: "/tmp",
@@ -748,6 +778,26 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 				Name:      "etc-cassandra",
 				MountPath: "/opt/hcd/resources/cassandra/conf",
 			})
+		} else if dc.Spec.ServerType == "dse" {
+			cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
+				Name:      "etc-cassandra",
+				MountPath: "/opt/dse/resources/cassandra/conf",
+			})
+			cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
+				Name:      "dse-conf",
+				MountPath: "/opt/dse/resources/dse/conf",
+			})
+
+			if !dc.UseClientImage() {
+				cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
+					Name:      "spark-conf",
+					MountPath: "/opt/dse/resources/spark/conf",
+				})
+				cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
+					Name:      "collectd-conf",
+					MountPath: "/opt/dse/resources/dse/collectd/etc/collectd",
+				})
+			}
 		} else {
 			cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
 				Name:      "etc-cassandra",
@@ -813,10 +863,6 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 	return nil
 }
 
-func readOnlyFs(dc *api.CassandraDatacenter) bool {
-	return dc.Spec.ReadOnlyRootFilesystem != nil && *dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage()
-}
-
 func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) {
 
 	baseTemplate := dc.Spec.PodTemplateSpec.DeepCopy()