From c2f38f5f050e5ed44bb3d11adb2153051fa63a57 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Tue, 9 May 2023 17:53:29 +0800 Subject: [PATCH] ci: add test with k3d (#73) - add basic test with k3d - remove embedded MinIO for query - better support for cache - always select node 0 in StatefulSet for query service --- .github/workflows/ci.yml | 100 ++++++++++++++++++ charts/databend-query/Chart.lock | 9 +- charts/databend-query/Chart.yaml | 8 +- .../databend-query/templates/configmap.yaml | 19 ++-- charts/databend-query/templates/pvc.yaml | 16 --- charts/databend-query/templates/service.yaml | 1 + .../databend-query/templates/statefulset.yaml | 27 ++--- charts/databend-query/values.yaml | 48 ++------- tests/minio.yaml | 11 ++ tests/query-with-minio.yaml | 17 +++ 10 files changed, 166 insertions(+), 90 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 charts/databend-query/templates/pvc.yaml create mode 100644 tests/minio.yaml create mode 100644 tests/query-with-minio.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..01adaab --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,100 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + k3d: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install k3d + shell: bash + run: | + curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + + - name: Install BendSQL + shell: bash + run: | + sudo curl -L -o /etc/apt/sources.list.d/datafuselabs.sources https://repo.databend.rs/deb/datafuselabs.sources + sudo apt-get update + sudo apt-get install -y bendsql + + - name: Create k3d cluster + run: | + k3d cluster create databend + kubectl cluster-info + + - uses: azure/setup-helm@v3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Helm Deps + run: | + helm repo add minio https://charts.min.io/ + helm repo add bitnami https://charts.bitnami.com/bitnami + helm plugin install https://github.com/databus23/helm-diff + + - name: Install Databend Meta + shell: bash + working-directory: charts/databend-meta + run: | + helm dependency build + helm lint . + helm diff upgrade --install databend-meta . \ + --namespace databend-meta \ + --set bootstrap=true \ + --set replicaCount=3 \ + --set persistence.size=1Gi + helm upgrade --install databend-meta . \ + --namespace databend-meta --create-namespace \ + --set bootstrap=true \ + --set replicaCount=3 \ + --set persistence.size=1Gi \ + --wait + kubectl get pods -n databend-meta + + - name: Install MinIO + shell: bash + run: | + helm upgrade --install minio minio/minio \ + --namespace minio --create-namespace \ + --values tests/minio.yaml \ + --wait + + - name: Install Databend Query + shell: bash + working-directory: charts/databend-query + run: | + helm dependency build + helm lint . + helm diff upgrade --install cluster1 . \ + --namespace tenant1 \ + --values ../../tests/query-with-minio.yaml + helm upgrade --install cluster1 . \ + --namespace tenant1 --create-namespace \ + --values ../../tests/query-with-minio.yaml \ + --wait + kubectl get pods -n tenant1 + + - name: Checking Cluster Status + shell: bash + working-directory: tests + run: | + kubectl -n tenant1 get svc + addr=$(kubectl -n tenant1 get svc cluster1-databend-query -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo 'select * from system.clusters;' | bendsql -h $addr -u databend -p databend --output table + + - name: Collect Logs + if: always() + run: | + kubectl logs -n databend-meta -l app.kubernetes.io/name=databend-meta + kubectl logs -n tenant1 -l app.kubernetes.io/name=databend-query diff --git a/charts/databend-query/Chart.lock b/charts/databend-query/Chart.lock index d026358..bef1be3 100644 --- a/charts/databend-query/Chart.lock +++ b/charts/databend-query/Chart.lock @@ -1,9 +1,6 @@ dependencies: -- name: minio - repository: https://charts.min.io/ - version: 3.6.3 - name: common repository: https://charts.bitnami.com/bitnami - version: 1.16.0 -digest: sha256:b9e914c823b7721af59eb551e5c862e6c04e25b15a5231dbaed7e3aa8830b05a -generated: "2022-06-14T13:03:34.366256+08:00" + version: 1.17.1 +digest: sha256:91bdebcf473f5da3c018dd74f25fab166d4faaa6be86d492f5caa50fc63f93fb +generated: "2023-05-09T16:49:16.183394+08:00" diff --git a/charts/databend-query/Chart.yaml b/charts/databend-query/Chart.yaml index a0d2628..dcd46cb 100644 --- a/charts/databend-query/Chart.yaml +++ b/charts/databend-query/Chart.yaml @@ -25,19 +25,15 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.6.2 +version: 0.7.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.1.26-nightly" +appVersion: "v1.1.30-nightly" dependencies: -- name: minio - version: 3.6.3 - repository: https://charts.min.io/ - condition: minio.enabled - name: common version: 1.x.x repository: https://charts.bitnami.com/bitnami diff --git a/charts/databend-query/templates/configmap.yaml b/charts/databend-query/templates/configmap.yaml index 43f8d4a..383e224 100644 --- a/charts/databend-query/templates/configmap.yaml +++ b/charts/databend-query/templates/configmap.yaml @@ -44,7 +44,7 @@ data: [log.file] on = {{ .Values.config.log.file.enabled }} level = {{ .Values.config.log.file.level | default "INFO" | quote }} - dir = "/var/log/databend" + dir = {{ .Values.config.log.file.dir | default "/var/log/databend" | quote }} [log.stderr] on = {{ .Values.config.log.stderr.enabled }} level = {{ .Values.config.log.stderr.level | default "WARN" | quote }} @@ -61,12 +61,6 @@ data: {{- if eq $storageType "s3"}} [storage.s3] - {{- if .Values.minio.enabled }} - bucket = {{ .Values.config.storage.s3.bucket | quote }} - endpoint_url = "http://{{ printf "%s-%s.%s" .Release.Name "minio" .Release.Namespace }}:{{ .Values.minio.minioAPIPort }}" - access_key_id = {{ .Values.minio.rootUser | quote }} - secret_access_key = {{ .Values.minio.rootPassword | quote }} - {{- else }} {{- range $field, $value := .Values.config.storage.s3 }} {{- if (kindIs "string" $value) }} {{ $field }} = {{ $value | quote }} @@ -74,7 +68,6 @@ data: {{ $field }} = {{ $value }} {{- end }} {{- end }} - {{- end }} {{- else }} [storage.{{ $storageType }}] {{- range $name, $configs := .Values.config.storage }} @@ -90,10 +83,10 @@ data: {{- end }} {{- end }} + {{- if .Values.cache.enabled }} [cache] - data_cache_storage = {{ .Values.config.cache.dataCacheStorage | default "none" | quote }} - {{- if eq .Values.config.cache.dataCacheStorage "disk" }} + data_cache_storage = "disk" [cache.disk] - path = {{ .Values.config.cache.disk.path | default "./.databend/_cache" | quote }} - max_bytes = {{ .Values.config.cache.disk.maxBytes | default 21474836480 | int64 }} - {{- end }} + path = {{ .Values.cache.path | default "/var/lib/databend/cache" | quote }} + max_bytes = {{ .Values.cache.maxBytes | default 21474836480 | int64 }} + {{- end }} diff --git a/charts/databend-query/templates/pvc.yaml b/charts/databend-query/templates/pvc.yaml deleted file mode 100644 index 224e21b..0000000 --- a/charts/databend-query/templates/pvc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{{- if .Values.persistence.enabled }} -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: {{ include "databend-query.fullname" . }} - labels: - {{- include "databend-query.labels" . | nindent 4 }} -spec: - accessModes: ["ReadWriteMany"] - resources: - requests: - storage: {{ .Values.persistence.size }} - {{- if .Values.persistence.storageClass }} - storageClassName: "{{ .Values.persistence.storageClass }}" - {{- end }} -{{- end }} diff --git a/charts/databend-query/templates/service.yaml b/charts/databend-query/templates/service.yaml index d1c915b..615705f 100644 --- a/charts/databend-query/templates/service.yaml +++ b/charts/databend-query/templates/service.yaml @@ -19,3 +19,4 @@ spec: {{- end }} selector: {{- include "databend-query.selectorLabels" . | nindent 4 }} + statefulset.kubernetes.io/pod-name: {{ include "databend-query.fullname" . }}-0 diff --git a/charts/databend-query/templates/statefulset.yaml b/charts/databend-query/templates/statefulset.yaml index 4a435c0..d455ab2 100644 --- a/charts/databend-query/templates/statefulset.yaml +++ b/charts/databend-query/templates/statefulset.yaml @@ -7,6 +7,7 @@ metadata: spec: serviceName: {{ include "databend-query.fullname" . }} replicas: {{ .Values.replicaCount }} + podManagementPolicy: Parallel selector: matchLabels: {{- include "databend-query.selectorLabels" . | nindent 6 }} @@ -102,9 +103,9 @@ spec: - name: config # Note: subPath volume mount will not receive ConfigMap update. mountPath: /etc/databend-query - {{- if .Values.persistence.enabled }} - - name: data - mountPath: {{ .Values.persistence.mountPath }} + {{- if .Values.cache.enabled }} + - name: cache + mountPath: {{ .Values.cache.path | default "/var/lib/databend/cache" | quote }} {{- end }} {{- if .Values.sidecars }} {{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | nindent 8 }} @@ -113,15 +114,17 @@ spec: - name: config configMap: name: {{ include "databend-query.fullname" .}} - {{- if .Values.persistence.enabled }} - - name: data - persistentVolumeClaim: - {{- if .Values.persistence.existingClaim }} - claimName: {{ .Values.persistence.existingClaim }} - {{- else }} - claimName: {{ include "databend-query.fullname" . }} - {{- end }} - {{- end }} + {{- if .Values.cache.enabled }} + volumeClaimTemplates: + - metadata: + name: cache + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{ .Values.cache.storageClass | quote }} + resources: + requests: + storage: {{ .Values.cache.maxBytes | quote }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/databend-query/values.yaml b/charts/databend-query/values.yaml index f9a347d..b3bd303 100644 --- a/charts/databend-query/values.yaml +++ b/charts/databend-query/values.yaml @@ -26,12 +26,15 @@ serviceAccount: service: type: ClusterIP ports: - mysql: 3307 - http: 8000 - clickhouse: 8124 metric: 7070 admin: 8080 flight: 9090 + + http: 8000 + flight-sql: 8900 + mysql: 3307 + ckhttp: 8124 + # Annotations to add to the service annotations: {} @@ -67,7 +70,7 @@ config: file: enabled: false level: "INFO" - dir: "/data/databend-query/_log" + dir: "/var/log/databend" stderr: enabled: true level: "WARN" @@ -85,10 +88,7 @@ config: # [storage] storage: - type: "fs" - # [storage.fs] - fs: - data_path: "/data/databend-query/_data" + type: "s3" # [storage.s3] s3: @@ -111,37 +111,11 @@ config: access_key_id: "" access_key_secret: "" - #[cache] - cache: - dataCacheStorage: "none" - - # [cache.disk] - # disk: - # path: "./.databend/_cache" - # maxBytes: 21474836480 - -persistence: +cache: enabled: false - existingClaim: "" + path: "/var/lib/databend/cache" + maxBytes: 21474836480 storageClass: "" - size: 10Gi - mountPath: /data/databend-query - -# Configuration for minio subchart -# Not recommended for production -minio: - enabled: false - replicas: 5 - rootUser: rootuser - rootPassword: rootpassword - persistence: - enabled: true - storageClass: "" - size: 10Gi - minioAPIPort: 9000 - resources: - requests: - memory: 500Mi ingress: enabled: false diff --git a/tests/minio.yaml b/tests/minio.yaml new file mode 100644 index 0000000..4215513 --- /dev/null +++ b/tests/minio.yaml @@ -0,0 +1,11 @@ +replicas: 3 +rootUser: miniouser +rootPassword: miniopassword +persistence: + enabled: true + storageClass: "" + size: 2Gi +minioAPIPort: 9000 +resources: + requests: + memory: 100Mi diff --git a/tests/query-with-minio.yaml b/tests/query-with-minio.yaml new file mode 100644 index 0000000..fc10363 --- /dev/null +++ b/tests/query-with-minio.yaml @@ -0,0 +1,17 @@ +replicaCount: 3 +config: + query: + tenantId: tenant1 + clusterId: cluster1 + users: + - name: databend + authType: double_sha1_password + authString: 3081f32caef285c232d066033c89a78d88a6d8a5 # databend + storage: + type: s3 + s3: + endpoint_url: "http://minio.minio.svc:9000" + access_key_id: "miniouser" + secret_access_key: "miniopassword" +service: + type: LoadBalancer