cp-ksql-server chart (#111)

charts/cp-ksql-server/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

charts/cp-ksql-server/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Confluent KSQL Server on Kubernetes
+name: cp-ksql-server
+version: 0.1.0

charts/cp-ksql-server/README.md

@@ -0,0 +1,158 @@
+# KSQL Server Helm Chart
+
+This chart bootstraps a deployment of a Confluent KSQL Server.
+
+This is an example deployment which runs KSQL Server in non-interactive
+mode.
+The included queries file `queries.sql` is a stub provided to illustrate one possible approach to mounting queries in the server container via ConfigMap.
+
+## Prerequisites
+
+* Kubernetes 1.9.2+
+* Helm 2.8.2+
+* A healthy and accessible Kafka Cluster
+
+## Developing Environment
+
+* [Pivotal Container Service (PKS)](https://pivotal.io/platform/pivotal-container-service)
+* [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine/)
+
+## Docker Image Source
+
+* [DockerHub -> ConfluentInc](https://hub.docker.com/u/confluentinc/)
+
+## Installing the Chart
+
+### Install along with cp-helm-charts
+
+```console
+git clone https://github.com/confluentinc/cp-helm-charts.git
+helm install cp-helm-charts
+```
+
+To install with a specific name, you can do:
+
+```console
+helm install --name my-confluent cp-helm-charts
+```
+
+### Install with a existing cp-kafka and cp-schema-registry release
+
+```console
+helm install --set cp-zookeeper.url="unhinged-robin-cp-zookeeper:2181",cp-schema-registry.url="lolling-chinchilla-cp-schema-registry:8081" cp-helm-charts/charts/cp-ksql-server
+```
+
+### Installed Components
+
+You can use `helm status <release name>` to view all of the installed components.
+
+For example:
+
+```console{%raw}
+$ helm status excited-lynx
+STATUS: DEPLOYED
+
+RESOURCES:
+==> v1/Service
+NAME                         TYPE       CLUSTER-IP    EXTERNAL-IP  PORT(S)   AGE
+excited-lynx-cp-ksql-server  ClusterIP  10.31.253.70  <none>       8088/TCP  10s
+
+==> v1beta2/Deployment
+NAME                         DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE
+excited-lynx-cp-ksql-server  1        1        1           0          10s
+
+==> v1/Pod(related)
+NAME                                         READY  STATUS  RESTARTS  AGE
+excited-lynx-cp-ksql-server-d4848ff94-x5fmn  2/2    Running   1         10s
+
+==> v1/ConfigMap
+NAME                                                DATA  AGE
+excited-lynx-cp-ksql-server-jmx-configmap           1     10s
+excited-lynx-cp-ksql-server-ksql-queries-configmap  1     10s
+
+
+NOTES:
+This chart installs Confluent KSQL Server.
+
+https://docs.confluent.io/current/ksql/docs
+```
+
+There are
+1. A [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) `excited-lynx-cp-ksql-server` which contains 1 KSQL Server instance [Pod](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/): `excited-lynx-cp-ksql-server-d4848ff94-x5fmn`.
+1. A [Service](https://kubernetes.io/docs/concepts/services-networking/service/) `excited-lynx-cp-kafka-rest` for clients to connect to REST Proxy.
+1. A [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) which contains configuration for Prometheus JMX Exporter.
+1. A [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) which contains SQL queries for the server to run in non-interactive mode.
+
+## Configuration
+
+You can specify each parameter using the `--set key=value[,key=value]` argument to `helm install`.
+
+Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example,
+
+```console
+helm install --name my-ksql-server -f my-values.yaml ./cp-ksql-server
+```
+
+> **Tip**: A default [values.yaml](values.yaml) is provided
+
+### KSQL Server Deployment
+
+The configuration parameters in this section control the resources requested and utilized by the cp-ksql-server chart.
+
+| Parameter | Description | Default |
+| --------- | ----------- | ------- |
+| `replicaCount` | The number of KSQL Server instances. | `1` |
+
+### Image
+
+| Parameter | Description | Default |
+| --------- | ----------- | ------- |
+| `image` | Docker Image of Confluent KSQL Server. | `confluentinc/cp-ksql-server` |
+| `imageTag` | Docker Image Tag of Confluent KSQL Server. | `5.0.0-beta30` |
+| `imagePullPolicy` | Docker Image Tag of Confluent KSQL Server. | `IfNotPresent` |
+
+### Port
+
+| Parameter | Description | Default |
+| --------- | ----------- | ------- |
+| `servicePort` | The port on which the KSQL Server will be available and serving requests. | `8088` |
+
+### Resources
+
+| Parameter | Description | Default |
+| --------- | ----------- | ------- |
+| `resources.requests.cpu` | The amount of CPU to request. | see [values.yaml](values.yaml) for details |
+| `resources.requests.memory` | The amount of memory to request. | see [values.yaml](values.yaml) for details |
+| `resources.requests.limit` | The upper limit CPU usage for a KSQL Server Pod. | see [values.yaml](values.yaml) for details |
+| `resources.requests.limit` | The upper limit memory usage for a KSQL Server Pod. | see [values.yaml](values.yaml) for details |
+
+### JMX Configuration
+
+| Parameter | Description | Default |
+| --------- | ----------- | ------- |
+| `jmx.port` | The jmx port which JMX style metrics are exposed. | `5555` |
+
+### Prometheus JMX Exporter Configuration
+
+| Parameter | Description | Default |
+| --------- | ----------- | ------- |
+| `prometheus.jmx.enabled` | Whether or not to install Prometheus JMX Exporter as a sidecar container and expose JMX metrics to Prometheus. | `true` |
+| `prometheus.jmx.image` | Docker Image for Prometheus JMX Exporter container. | `solsson/kafka-prometheus-jmx-exporter@sha256` |
+| `prometheus.jmx.imageTag` | Docker Image Tag for Prometheus JMX Exporter container. | `a23062396cd5af1acdf76512632c20ea6be76885dfc20cd9ff40fb23846557e8` |
+| `prometheus.jmx.port` | JMX Exporter Port which exposes metrics in Prometheus format for scraping. | `5556` |
+
+### External Access
+
+| Parameter | Description | Default |
+| --------- | ----------- | ------- |
+| `external.enabled` | whether or not to allow external access to KSQL Server | `false` |
+| `external.type` | `Kubernetes Service Type` to expose KSQL Server to external | `LoadBalancer` |
+
+## Dependencies
+
+### Schema Registry (optional)
+
+| Parameter | Description | Default |
+| --------- | ----------- | ------- |
+| `cp-schema-registry.url` | Service name of Schema Registry (Not needed if this is installed along with cp-kafka chart). | `""` |
+| `cp-schema-registry.port` | Port of Schema Registry Service | `8081` |

charts/cp-ksql-server/queries.sql

@@ -0,0 +1,19 @@
+-- From http://docs.confluent.io/current/ksql/docs/tutorials/basics-docker.html#create-a-stream-and-table
+
+-- Create a stream pageviews_original from the Kafka topic pageviews, specifying the value_format of DELIMITED
+CREATE STREAM pageviews_original (viewtime bigint, userid varchar, pageid varchar) WITH (kafka_topic='pageviews', value_format='DELIMITED');
+
+-- Create a table users_original from the Kafka topic users, specifying the value_format of JSON
+CREATE TABLE users_original (registertime BIGINT, gender VARCHAR, regionid VARCHAR, userid VARCHAR) WITH (kafka_topic='users', value_format='JSON', key = 'userid');
+
+-- Create a persistent query by using the CREATE STREAM keywords to precede the SELECT statement
+CREATE STREAM pageviews_enriched AS SELECT users_original.userid AS userid, pageid, regionid, gender FROM pageviews_original LEFT JOIN users_original ON pageviews_original.userid = users_original.userid;
+
+-- Create a new persistent query where a condition limits the streams content, using WHERE
+CREATE STREAM pageviews_female AS SELECT * FROM pageviews_enriched WHERE gender = 'FEMALE';
+
+-- Create a new persistent query where another condition is met, using LIKE
+CREATE STREAM pageviews_female_like_89 WITH (kafka_topic='pageviews_enriched_r8_r9') AS SELECT * FROM pageviews_female WHERE regionid LIKE '%_8' OR regionid LIKE '%_9';
+
+-- Create a new persistent query that counts the pageviews for each region and gender combination in a tumbling window of 30 seconds when the count is greater than one
+CREATE TABLE pageviews_regions WITH (VALUE_FORMAT='avro') AS SELECT gender, regionid , COUNT(*) AS numusers FROM pageviews_enriched WINDOW TUMBLING (size 30 second) GROUP BY gender, regionid HAVING COUNT(*) > 1;

charts/cp-ksql-server/templates/NOTES.txt

@@ -0,0 +1,3 @@
+This chart installs Confluent KSQL Server.
+
+https://docs.confluent.io/current/ksql/docs

charts/cp-ksql-server/templates/_helpers.tpl

@@ -0,0 +1,64 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "cp-ksql-server.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "cp-ksql-server.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "cp-ksql-server.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified kafka headless name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "cp-ksql-server.cp-kafka-headless.fullname" -}}
+{{- $name := "cp-kafka-headless" -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Form the Kafka URL. If Kafka is installed as part of this chart, use k8s service discovery,
+else use user-provided URL
+*/}}
+{{- define "cp-ksql-server.kafka.bootstrapServers" -}}
+{{- if .Values.kafka.bootstrapServers -}}
+{{- .Values.kafka.bootstrapServers -}}
+{{- else -}}
+{{- printf "PLAINTEXT://%s:9092" (include "cp-ksql-server.cp-kafka-headless.fullname" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Default Server Pool Id to Release Name but allow it to be overridden
+*/}}
+{{- define "cp-ksql-server.serviceId" -}}
+{{- if .Values.overrideServiceId -}}
+{{- .Values.overrideServiceId -}}
+{{- else -}}
+{{- .Release.Name -}}
+{{- end -}}
+{{- end -}}

charts/cp-ksql-server/templates/deployment.yaml

@@ -0,0 +1,93 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: {{ template "cp-ksql-server.fullname" . }}
+  labels:
+    app: {{ template "cp-ksql-server.name" . }}
+    chart: {{ template "cp-ksql-server.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ template "cp-ksql-server.name" . }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "cp-ksql-server.name" . }}
+        release: {{ .Release.Name }}
+      {{- if .Values.prometheus.jmx.enabled }}
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: {{ .Values.prometheus.jmx.port | quote }}
+      {{- end }}
+    spec:
+      containers:
+        {{- if .Values.prometheus.jmx.enabled }}
+        - name: prometheus-jmx-exporter
+          image: "{{ .Values.prometheus.jmx.image }}:{{ .Values.prometheus.jmx.imageTag }}"
+          command:
+          - java
+          - -XX:+UnlockExperimentalVMOptions
+          - -XX:+UseCGroupMemoryLimitForHeap
+          - -XX:MaxRAMFraction=1
+          - -XshowSettings:vm
+          - -jar
+          - jmx_prometheus_httpserver.jar
+          - {{ .Values.prometheus.jmx.port | quote }}
+          - /etc/jmx-ksql-server/jmx-ksql-server-prometheus.yml
+          ports:
+          - containerPort: {{ .Values.prometheus.jmx.port }}
+          resources:
+{{ toYaml .Values.prometheus.jmx.resources | indent 12 }}
+          volumeMounts:
+          - name: jmx-config
+            mountPath: /etc/jmx-ksql-server
+        {{- end }}
+        - name: {{ template "cp-ksql-server.name" . }}
+          image: "{{ .Values.image }}:{{ .Values.imageTag }}"
+          imagePullPolicy: "{{ .Values.imagePullPolicy }}"
+          ports:
+            - name: server
+              containerPort: {{ .Values.servicePort}}
+              protocol: TCP
+            {{- if .Values.prometheus.jmx.enabled }}
+            - containerPort: {{ .Values.jmx.port }}
+              name: jmx
+            {{- end }}
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+          volumeMounts:
+          {{- if .Values.ksql.headless }}
+          - name: ksql-queries
+            mountPath: /etc/ksql/queries
+          {{- end }}
+          env:
+          - name: KSQL_BOOTSTRAP_SERVERS
+            value: {{ template "cp-ksql-server.kafka.bootstrapServers" . }}
+          - name: KSQL_KSQL_SERVICE_ID
+            value: {{ template "cp-ksql-server.serviceId" . }}
+          {{- if .Values.ksql.headless }}
+          - name: KSQL_KSQL_QUERIES_FILE
+            value: /etc/ksql/queries/queries.sql
+          {{- else }}
+          - name: KSQL_LISTENERS
+            value: http://0.0.0.0:8088
+          {{- end }}
+          {{- if .Values.jmx.port }}
+          - name: KSQL_JMX_PORT
+            value: "{{ .Values.jmx.port }}"
+          {{- end }}
+      volumes:
+      {{- if .Values.prometheus.jmx.enabled }}
+      - name: jmx-config
+        configMap:
+          name: {{ template "cp-ksql-server.fullname" . }}-jmx-configmap
+      {{- end }}
+      {{- if .Values.ksql.headless }}
+      - name: ksql-queries
+        configMap:
+          name: {{ template "cp-ksql-server.fullname" . }}-ksql-queries-configmap
+      {{- end }}

charts/cp-ksql-server/templates/external-service.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.external.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "cp-ksql-server.fullname" . }}-external
+  labels:
+    app: {{ template "cp-ksql-server.name" . }}
+    chart: {{ template "cp-ksql-server.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  type: {{ .Values.external.type }}
+  ports:
+    - name: server-proxy
+      port: {{ .Values.servicePort }}
+  selector:
+    app: {{ template "cp-ksql-server.name" . }}
+    release: {{ .Release.Name }}
+{{- end -}}