diff --git a/e2e/Makefile b/e2e/Makefile index 963e926ed..d36089083 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -48,6 +48,7 @@ endif $(KUSTOMIZE) build . | $(KUBECTL) apply -f - $(KUBECTL) -n moco-system wait --for=condition=available --timeout=180s --all deployments $(KUBECTL) apply -f minio.yaml + $(KUBECTL) apply -f minio-tls.yaml $(KUBECTL) apply -f fake-gcs-server.yaml $(KUBECTL) wait --timeout=60s --for=condition=Ready --all pods diff --git a/e2e/backup_test.go b/e2e/backup_test.go index 1dd187db9..e98f38a02 100644 --- a/e2e/backup_test.go +++ b/e2e/backup_test.go @@ -41,12 +41,9 @@ var _ = Context("backup", func() { job := &batchv1.Job{} err = json.Unmarshal(out, job) g.Expect(err).NotTo(HaveOccurred()) - for _, cond := range job.Status.Conditions { - if cond.Type != batchv1.JobComplete { - continue - } - g.Expect(cond.Status).To(Equal(corev1.ConditionTrue)) - } + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete).To(Equal(corev1.ConditionTrue), "make-bucket has not been finished") }).Should(Succeed()) }) diff --git a/e2e/backup_tls_test.go b/e2e/backup_tls_test.go new file mode 100644 index 000000000..138ba29d6 --- /dev/null +++ b/e2e/backup_tls_test.go @@ -0,0 +1,161 @@ +package e2e + +import ( + "bytes" + _ "embed" + "encoding/json" + "strconv" + "strings" + "text/template" + "time" + + mocov1beta2 "github.com/cybozu-go/moco/api/v1beta2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +//go:embed testdata/makebucket_tls.yaml +var makeBucketTLSYAML string + +//go:embed testdata/backup_tls.yaml +var backupTLSYAML string + +//go:embed testdata/restore_tls.yaml +var restoreTLSYAML string + +var _ = Context("backup-tls", func() { + if doUpgrade { + return + } + + var restorePoint time.Time + + It("should create a bucket", func() { + kubectlSafe([]byte(makeBucketTLSYAML), "apply", "-f", "-") + Eventually(func(g Gomega) { + out, err := kubectl(nil, "get", "jobs", "make-bucket-tls", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete).To(Equal(corev1.ConditionTrue), "make-bucket-tls has not been finished") + }).Should(Succeed()) + }) + + It("should construct a source cluster", func() { + kubectlSafe(fillTemplate(backupTLSYAML), "apply", "-f", "-") + secjson := kubectlSafe(nil, "get", "secret", "-o", "json", "minio-cert") + sec := &corev1.Secret{} + json.Unmarshal(secjson, sec) + sec.SetNamespace("backup") + secjson, err := json.Marshal(sec) + Expect(err).NotTo(HaveOccurred()) + kubectlSafe(secjson, "apply", "-f", "-") + Eventually(func(g Gomega) { + cluster, err := getCluster("backup-tls", "source") + g.Expect(err).NotTo(HaveOccurred()) + condHealthy, err := getClusterCondition(cluster, mocov1beta2.ConditionHealthy) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condHealthy.Status).To(Equal(metav1.ConditionTrue)) + }).Should(Succeed()) + + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-e", "CREATE DATABASE test") + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "-e", "CREATE TABLE t (id INT NOT NULL AUTO_INCREMENT, data VARCHAR(32) NOT NULL, PRIMARY KEY (id), KEY key1 (data), KEY key2 (data, id)) ENGINE=InnoDB") + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('aaa')") + }) + + It("should take a full dump", func() { + kubectlSafe(nil, "-n", "backup-tls", "create", "job", "--from=cronjob/moco-backup-source", "backup-tls-1") + Eventually(func(g Gomega) { + out, err := kubectl(nil, "-n", "backup-tls", "get", "jobs", "backup-tls-1", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete).To(Equal(corev1.ConditionTrue), "backup-tls-1 has not been finished") + }).Should(Succeed()) + }) + + It("should take an incremental backup", func() { + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('bbb')") + time.Sleep(1100 * time.Millisecond) + restorePoint = time.Now().UTC() + time.Sleep(1100 * time.Millisecond) + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-admin", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "FLUSH LOCAL BINARY LOGS") + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('ccc')") + time.Sleep(100 * time.Millisecond) + + kubectlSafe(nil, "-n", "backup-tls", "create", "job", "--from=cronjob/moco-backup-source", "backup-tls-2") + Eventually(func(g Gomega) { + out, err := kubectl(nil, "-n", "backup-tls", "get", "jobs", "backup-tls-2", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete).To(Equal(corev1.ConditionTrue), "backup-tls-2 has not been finished") + }).Should(Succeed()) + + cluster, err := getCluster("backup-tls", "source") + Expect(err).NotTo(HaveOccurred()) + Expect(cluster.Status.Backup.BinlogSize).NotTo(Equal(int64(0))) + }) + + It("should destroy the source then restore the backup data", func() { + kubectlSafe(nil, "-n", "backup-tls", "delete", "mysqlclusters", "source") + + tmpl, err := template.New("").Parse(restoreTLSYAML) + Expect(err).NotTo(HaveOccurred()) + buf := new(bytes.Buffer) + err = tmpl.Execute(buf, struct { + MySQLVersion string + RestorePoint string + }{ + mysqlVersion, + restorePoint.Format(time.RFC3339), + }) + Expect(err).NotTo(HaveOccurred()) + + kubectlSafe(buf.Bytes(), "apply", "-f", "-") + Eventually(func(g Gomega) { + cluster, err := getCluster("backup-tls", "target") + g.Expect(err).NotTo(HaveOccurred()) + condHealthy, err := getClusterCondition(cluster, mocov1beta2.ConditionHealthy) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condHealthy.Status).To(Equal(metav1.ConditionTrue), "target is not healthy") + }).Should(Succeed()) + + out := kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "target", "--", + "-N", "-D", "test", "-e", "SELECT COUNT(*) FROM t") + count, err := strconv.Atoi(strings.TrimSpace(string(out))) + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(2)) + }) + + It("should delete clusters", func() { + kubectlSafe(nil, "delete", "-n", "backup-tls", "mysqlclusters", "--all") + + Eventually(func(g Gomega) { + out, err := kubectl(nil, "get", "-n", "backup-tls", "pod", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + pods := &corev1.PodList{} + err = json.Unmarshal(out, pods) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(pods.Items)).To(BeNumerically(">", 0), "wait until all Pods are deleted") + }).Should(Succeed()) + }) +}) diff --git a/e2e/minio-tls.yaml b/e2e/minio-tls.yaml new file mode 100644 index 000000000..770cf9bae --- /dev/null +++ b/e2e/minio-tls.yaml @@ -0,0 +1,70 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + namespace: default + name: default-selfsigned-issuer +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + namespace: default + name: minio-cert +spec: + commonName: minio cert + issuerRef: + kind: Issuer + name: default-selfsigned-issuer + secretName: minio-cert + dnsNames: + - minio-tls.default.svc +--- +apiVersion: v1 +kind: Service +metadata: + namespace: default + name: minio-tls +spec: + ports: + - name: minio + port: 9000 + targetPort: minio + protocol: TCP + selector: + name: minio-tls +--- +apiVersion: v1 +kind: Pod +metadata: + namespace: default + name: minio-tls + labels: + name: minio-tls +spec: + containers: + - name: minio + image: minio/minio + args: + - server + - /data + ports: + - name: minio + containerPort: 9000 + protocol: TCP + volumeMounts: + - name: data + mountPath: /data + - name: secret-volume + mountPath: /root/.minio/certs + volumes: + - name: data + emptyDir: {} + - name: secret-volume + secret: + secretName: minio-cert + items: + - key: tls.crt + path: public.crt + - key: tls.key + path: private.key diff --git a/e2e/minio.yaml b/e2e/minio.yaml index 1b00b0efa..3c7bbb712 100644 --- a/e2e/minio.yaml +++ b/e2e/minio.yaml @@ -1,25 +1,3 @@ -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - namespace: default - name: default-selfsigned-issuer -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - namespace: default - name: minio-cert -spec: - commonName: minio cert - issuerRef: - kind: Issuer - name: default-selfsigned-issuer - secretName: minio-cert - dnsNames: - - minio.default.svc ---- apiVersion: v1 kind: Service metadata: @@ -55,16 +33,6 @@ spec: volumeMounts: - name: data mountPath: /data - - name: secret-volume - mountPath: /root/.minio/certs volumes: - name: data emptyDir: {} - - name: secret-volume - secret: - secretName: minio-cert - items: - - key: tls.crt - path: public.crt - - key: tls.key - path: private.key diff --git a/e2e/testdata/backup.yaml b/e2e/testdata/backup.yaml index 05167ae22..0ddde2c30 100644 --- a/e2e/testdata/backup.yaml +++ b/e2e/testdata/backup.yaml @@ -31,18 +31,10 @@ spec: value: minioadmin - name: AWS_SECRET_ACCESS_KEY value: minioadmin - volumeMounts: - - mountPath: /minio-cert - name: minio-cert - volumes: - - name: minio-cert - secret: - secretName: minio-cert bucketConfig: bucketName: moco - endpointURL: https://minio.default.svc:9000 + endpointURL: http://minio.default.svc:9000 usePathStyle: true - caCert: /minio-cert/ca.crt workVolume: emptyDir: {} --- diff --git a/e2e/testdata/backup_tls.yaml b/e2e/testdata/backup_tls.yaml new file mode 100644 index 000000000..5fe777473 --- /dev/null +++ b/e2e/testdata/backup_tls.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: backup-tls +--- +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: backup-tls + name: mycnf +data: + innodb_log_file_size: "10M" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: backup-tls + name: backup-owner +--- +apiVersion: moco.cybozu.com/v1beta1 +kind: BackupPolicy +metadata: + namespace: backup-tls + name: daily +spec: + schedule: "@daily" + jobConfig: + serviceAccountName: backup-owner + env: + - name: AWS_ACCESS_KEY_ID + value: minioadmin + - name: AWS_SECRET_ACCESS_KEY + value: minioadmin + volumeMounts: + - mountPath: /minio-cert + name: minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert + bucketConfig: + bucketName: moco + endpointURL: https://minio.default.svc:9000 + usePathStyle: true + caCert: /minio-cert/ca.crt + workVolume: + emptyDir: {} +--- +apiVersion: moco.cybozu.com/v1beta2 +kind: MySQLCluster +metadata: + namespace: backup-tls + name: source +spec: + mysqlConfigMapName: mycnf + replicas: 3 + backupPolicyName: daily + podTemplate: + spec: + containers: + - name: mysqld + image: ghcr.io/cybozu-go/moco/mysql:{{ . }} + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/e2e/testdata/makebucket.yaml b/e2e/testdata/makebucket.yaml index 520321d62..065949917 100644 --- a/e2e/testdata/makebucket.yaml +++ b/e2e/testdata/makebucket.yaml @@ -14,20 +14,8 @@ spec: - --host-bucket=minio.default.svc:9000 - --access_key=minioadmin - --secret_key=minioadmin - - --ssl - - --ca-certs=/minio-cert/ca.crt - mb - s3://moco image: moco-backup:dev imagePullPolicy: IfNotPresent name: make-bucket - volumeMounts: - - name: minio-cert - mountPath: /minio-cert - volumes: - - name: minio-cert - secret: - secretName: minio-cert - items: - - key: ca.crt - path: ca.crt diff --git a/e2e/testdata/makebucket_tls.yaml b/e2e/testdata/makebucket_tls.yaml new file mode 100644 index 000000000..ce83d6193 --- /dev/null +++ b/e2e/testdata/makebucket_tls.yaml @@ -0,0 +1,33 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: make-bucket-tls + namespace: default +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - command: + - s3cmd + - --host=minio-tls.default.svc:9000 + - --host-bucket=minio-tls.default.svc:9000 + - --access_key=minioadmin + - --secret_key=minioadmin + - --ssl + - --ca-certs=/minio-cert/ca.crt + - mb + - s3://moco + image: moco-backup:dev + imagePullPolicy: IfNotPresent + name: make-bucket + volumeMounts: + - name: minio-cert + mountPath: /minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert + items: + - key: ca.crt + path: ca.crt diff --git a/e2e/testdata/restore.yaml b/e2e/testdata/restore.yaml index bf290733b..7042a851a 100644 --- a/e2e/testdata/restore.yaml +++ b/e2e/testdata/restore.yaml @@ -17,18 +17,10 @@ spec: value: minioadmin - name: AWS_SECRET_ACCESS_KEY value: minioadmin - volumeMounts: - - mountPath: /minio-cert - name: minio-cert - volumes: - - name: minio-cert - secret: - secretName: minio-cert bucketConfig: bucketName: moco - endpointURL: https://minio.default.svc:9000 + endpointURL: http://minio.default.svc:9000 usePathStyle: true - caCert: /minio-cert/ca.crt workVolume: emptyDir: {} podTemplate: diff --git a/e2e/testdata/restore_tls.yaml b/e2e/testdata/restore_tls.yaml new file mode 100644 index 000000000..9cf9b93a7 --- /dev/null +++ b/e2e/testdata/restore_tls.yaml @@ -0,0 +1,46 @@ +apiVersion: moco.cybozu.com/v1beta2 +kind: MySQLCluster +metadata: + namespace: backup-tls + name: target +spec: + mysqlConfigMapName: mycnf + replicas: 1 + restore: + sourceName: source + sourcenamespace: backup-tls + restorePoint: "{{ .RestorePoint }}" + jobConfig: + serviceAccountName: backup-owner + env: + - name: AWS_ACCESS_KEY_ID + value: minioadmin + - name: AWS_SECRET_ACCESS_KEY + value: minioadmin + volumeMounts: + - mountPath: /minio-cert + name: minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert + bucketConfig: + bucketName: moco + endpointURL: https://minio.default.svc:9000 + usePathStyle: true + caCert: /minio-cert/ca.crt + workVolume: + emptyDir: {} + podTemplate: + spec: + containers: + - name: mysqld + image: ghcr.io/cybozu-go/moco/mysql:{{ .MySQLVersion }} + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi