ci: add test with k3d (#73)

- add basic test with k3d
- remove embedded MinIO for query
- better support for cache
- always select node 0 in StatefulSet for query service

.github/workflows/ci.yml

@@ -0,0 +1,100 @@
+name: ci
+
+on:
+ push:
+ branches: [main]
+ pull_request:
+ branches: [main]
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+ cancel-in-progress: true
+
+jobs:
+ k3d:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Install k3d
+ shell: bash
+ run: |
+ curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+
+ - name: Install BendSQL
+ shell: bash
+ run: |
+ sudo curl -L -o /etc/apt/sources.list.d/datafuselabs.sources https://repo.databend.rs/deb/datafuselabs.sources
+ sudo apt-get update
+ sudo apt-get install -y bendsql
+
+ - name: Create k3d cluster
+ run: |
+ k3d cluster create databend
+ kubectl cluster-info
+
+ - uses: azure/setup-helm@v3
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Helm Deps
+ run: |
+ helm repo add minio https://charts.min.io/
+ helm repo add bitnami https://charts.bitnami.com/bitnami
+ helm plugin install https://github.com/databus23/helm-diff
+
+ - name: Install Databend Meta
+ shell: bash
+ working-directory: charts/databend-meta
+ run: |
+ helm dependency build
+ helm lint .
+ helm diff upgrade --install databend-meta . \
+ --namespace databend-meta \
+ --set bootstrap=true \
+ --set replicaCount=3 \
+ --set persistence.size=1Gi
+ helm upgrade --install databend-meta . \
+ --namespace databend-meta --create-namespace \
+ --set bootstrap=true \
+ --set replicaCount=3 \
+ --set persistence.size=1Gi \
+ --wait
+ kubectl get pods -n databend-meta
+
+ - name: Install MinIO
+ shell: bash
+ run: |
+ helm upgrade --install minio minio/minio \
+ --namespace minio --create-namespace \
+ --values tests/minio.yaml \
+ --wait
+
+ - name: Install Databend Query
+ shell: bash
+ working-directory: charts/databend-query
+ run: |
+ helm dependency build
+ helm lint .
+ helm diff upgrade --install cluster1 . \
+ --namespace tenant1 \
+ --values ../../tests/query-with-minio.yaml
+ helm upgrade --install cluster1 . \
+ --namespace tenant1 --create-namespace \
+ --values ../../tests/query-with-minio.yaml \
+ --wait
+ kubectl get pods -n tenant1
+
+ - name: Checking Cluster Status
+ shell: bash
+ working-directory: tests
+ run: |
+ kubectl -n tenant1 get svc
+ addr=$(kubectl -n tenant1 get svc cluster1-databend-query -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+ echo 'select * from system.clusters;' | bendsql -h $addr -u databend -p databend --output table
+
+ - name: Collect Logs
+ if: always()
+ run: |
+ kubectl logs -n databend-meta -l app.kubernetes.io/name=databend-meta
+ kubectl logs -n tenant1 -l app.kubernetes.io/name=databend-query

charts/databend-query/Chart.lock

@@ -1,9 +1,6 @@
 dependencies:
-- name: minio
- repository: https://charts.min.io/
- version: 3.6.3
 - name: common
  repository: https://charts.bitnami.com/bitnami
- version: 1.16.0
-digest: sha256:b9e914c823b7721af59eb551e5c862e6c04e25b15a5231dbaed7e3aa8830b05a
-generated: "2022-06-14T13:03:34.366256+08:00"
+ version: 1.17.1
+digest: sha256:91bdebcf473f5da3c018dd74f25fab166d4faaa6be86d492f5caa50fc63f93fb
+generated: "2023-05-09T16:49:16.183394+08:00"

charts/databend-query/Chart.yaml

@@ -25,19 +25,15 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.6.2
+version: 0.7.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v1.1.26-nightly"
+appVersion: "v1.1.30-nightly"
 
 dependencies:
-- name: minio
- version: 3.6.3
- repository: https://charts.min.io/
- condition: minio.enabled
 - name: common
  version: 1.x.x
  repository: https://charts.bitnami.com/bitnami

charts/databend-query/templates/configmap.yaml

@@ -44,7 +44,7 @@ data:
  [log.file]
  on = {{ .Values.config.log.file.enabled }}
  level = {{ .Values.config.log.file.level | default "INFO" | quote }}
- dir = "/var/log/databend"
+ dir = {{ .Values.config.log.file.dir | default "/var/log/databend" | quote }}
  [log.stderr]
  on = {{ .Values.config.log.stderr.enabled }}
  level = {{ .Values.config.log.stderr.level | default "WARN" | quote }}
@@ -61,20 +61,13 @@ data:
 
  {{- if eq $storageType "s3"}}
  [storage.s3]
- {{- if .Values.minio.enabled }}
- bucket = {{ .Values.config.storage.s3.bucket | quote }}
- endpoint_url = "http://{{ printf "%s-%s.%s" .Release.Name "minio" .Release.Namespace }}:{{ .Values.minio.minioAPIPort }}"
- access_key_id = {{ .Values.minio.rootUser | quote }}
- secret_access_key = {{ .Values.minio.rootPassword | quote }}
- {{- else }}
  {{- range $field, $value := .Values.config.storage.s3 }}
  {{- if (kindIs "string" $value) }}
  {{ $field }} = {{ $value | quote }}
  {{- else }}
  {{ $field }} = {{ $value }}
  {{- end }}
  {{- end }}
- {{- end }}
  {{- else }}
  [storage.{{ $storageType }}]
  {{- range $name, $configs := .Values.config.storage }}
@@ -90,10 +83,10 @@ data:
  {{- end }}
  {{- end }}
 
+ {{- if .Values.cache.enabled }}
  [cache]
- data_cache_storage = {{ .Values.config.cache.dataCacheStorage | default "none" | quote }}
- {{- if eq .Values.config.cache.dataCacheStorage "disk" }}
+ data_cache_storage = "disk"
  [cache.disk]
- path = {{ .Values.config.cache.disk.path | default "./.databend/_cache" | quote }}
- max_bytes = {{ .Values.config.cache.disk.maxBytes | default 21474836480 | int64 }}
-  {{- end }}
+  path = {{ .Values.cache.path | default "/var/lib/databend/cache" | quote }}
+  max_bytes = {{ .Values.cache.maxBytes | default 21474836480 | int64 }}
+ {{- end }}

charts/databend-query/templates/service.yaml

@@ -19,3 +19,4 @@ spec:
  {{- end }}
  selector:
  {{- include "databend-query.selectorLabels" . | nindent 4 }}
+ statefulset.kubernetes.io/pod-name: {{ include "databend-query.fullname" . }}-0

charts/databend-query/templates/statefulset.yaml

@@ -7,6 +7,7 @@ metadata:
 spec:
  serviceName: {{ include "databend-query.fullname" . }}
  replicas: {{ .Values.replicaCount }}
+ podManagementPolicy: Parallel
  selector:
  matchLabels:
  {{- include "databend-query.selectorLabels" . | nindent 6 }}
@@ -102,9 +103,9 @@ spec:
  - name: config
  # Note: subPath volume mount will not receive ConfigMap update.
  mountPath: /etc/databend-query
- {{- if .Values.persistence.enabled }}
- - name: data
- mountPath: {{ .Values.persistence.mountPath }}
+ {{- if .Values.cache.enabled }}
+ - name: cache
+ mountPath: {{ .Values.cache.path | default "/var/lib/databend/cache" | quote }}
  {{- end }}
  {{- if .Values.sidecars }}
  {{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | nindent 8 }}
@@ -113,15 +114,17 @@ spec:
  - name: config
  configMap:
  name: {{ include "databend-query.fullname" .}}
- {{- if .Values.persistence.enabled }}
- - name: data
- persistentVolumeClaim:
- {{- if .Values.persistence.existingClaim }}
- claimName: {{ .Values.persistence.existingClaim }}
- {{- else }}
- claimName: {{ include "databend-query.fullname" . }}
- {{- end }}
- {{- end }}
+ {{- if .Values.cache.enabled }}
+ volumeClaimTemplates:
+ - metadata:
+ name: cache
+ spec:
+ accessModes: [ "ReadWriteOnce" ]
+ storageClassName: {{ .Values.cache.storageClass | quote }}
+ resources:
+ requests:
+ storage: {{ .Values.cache.maxBytes | quote }}
+ {{- end }}
  {{- with .Values.nodeSelector }}
  nodeSelector:
  {{- toYaml . | nindent 8 }}

charts/databend-query/values.yaml

@@ -26,12 +26,15 @@ serviceAccount:
 service:
  type: ClusterIP
  ports:
- mysql: 3307
- http: 8000
- clickhouse: 8124
  metric: 7070
  admin: 8080
  flight: 9090
+
+ http: 8000
+ flight-sql: 8900
+ mysql: 3307
+ ckhttp: 8124
+
  # Annotations to add to the service
  annotations: {}
 
@@ -67,7 +70,7 @@ config:
  file:
  enabled: false
  level: "INFO"
- dir: "/data/databend-query/_log"
+ dir: "/var/log/databend"
  stderr:
  enabled: true
  level: "WARN"
@@ -85,10 +88,7 @@ config:
 
  # [storage]
  storage:
- type: "fs"
- # [storage.fs]
- fs:
- data_path: "/data/databend-query/_data"
+ type: "s3"
 
  # [storage.s3]
  s3:
@@ -111,37 +111,11 @@ config:
  access_key_id: ""
  access_key_secret: ""
 
- #[cache]
- cache:
- dataCacheStorage: "none"
-
- # [cache.disk]
- # disk:
- # path: "./.databend/_cache"
- # maxBytes: 21474836480
-
-persistence:
+cache:
  enabled: false
- existingClaim: ""
+ path: "/var/lib/databend/cache"
+ maxBytes: 21474836480
  storageClass: ""
- size: 10Gi
- mountPath: /data/databend-query
-
-# Configuration for minio subchart
-# Not recommended for production
-minio:
- enabled: false
- replicas: 5
- rootUser: rootuser
- rootPassword: rootpassword
- persistence:
- enabled: true
- storageClass: ""
- size: 10Gi
- minioAPIPort: 9000
- resources:
- requests:
- memory: 500Mi
 
 ingress:
  enabled: false

tests/minio.yaml

@@ -0,0 +1,11 @@
+replicas: 3
+rootUser: miniouser
+rootPassword: miniopassword
+persistence:
+ enabled: true
+ storageClass: ""
+ size: 2Gi
+minioAPIPort: 9000
+resources:
+ requests:
+ memory: 100Mi

tests/query-with-minio.yaml

@@ -0,0 +1,17 @@
+replicaCount: 3
+config:
+ query:
+ tenantId: tenant1
+ clusterId: cluster1
+ users:
+ - name: databend
+ authType: double_sha1_password
+ authString: 3081f32caef285c232d066033c89a78d88a6d8a5 # databend
+ storage:
+ type: s3
+ s3:
+ endpoint_url: "http://minio.minio.svc:9000"
+ access_key_id: "miniouser"
+ secret_access_key: "miniopassword"
+service:
+ type: LoadBalancer