From db471d0bb827ccfbfabc129f07edcf05dc5ed5d0 Mon Sep 17 00:00:00 2001
From: dmytrozakharenko <amidum@gmail.com>
Date: Tue, 10 Dec 2024 16:17:27 -0500
Subject: [PATCH] - add kafka topics

---
 charts/dps/files/partial-cloudevents.yaml |  2 +-
 charts/dps/files/valid-cloudevents.yaml   |  2 +-
 charts/dps/templates/kafka-topic.yaml     | 19 +++++++++++++
 charts/dps/values-prod.yaml               |  3 +--
 charts/dps/values.yaml                    | 33 ++++++++++++++++++++---
 5 files changed, 51 insertions(+), 8 deletions(-)
 create mode 100644 charts/dps/templates/kafka-topic.yaml

diff --git a/charts/dps/files/partial-cloudevents.yaml b/charts/dps/files/partial-cloudevents.yaml
index 17e3a20..914d797 100644
--- a/charts/dps/files/partial-cloudevents.yaml
+++ b/charts/dps/files/partial-cloudevents.yaml
@@ -4,7 +4,7 @@ input:
     addresses:
       - ${KAFKA_BOOTSTRAP_SERVERS:localhost}:${KAFKA_BOOTSTRAP_PORT:9092}
     topics:
-      - '${DEVICE_SIGNALS_TOPIC:topic.device.partialcloudevents}'
+      - '${DEVICE_PARTIAL_CE_TOPIC:topic.device.partialcloudevents}'
     consumer_group: "zone.dimo.export.partialcloudevents"
     client_id: ${CONTAINER_NAME:localhost}-dps-status
     rack_id: ${NODE_NAME:localhost}
diff --git a/charts/dps/files/valid-cloudevents.yaml b/charts/dps/files/valid-cloudevents.yaml
index 5f16a93..e6a8896 100644
--- a/charts/dps/files/valid-cloudevents.yaml
+++ b/charts/dps/files/valid-cloudevents.yaml
@@ -4,7 +4,7 @@ input:
     addresses:
       - ${KAFKA_BOOTSTRAP_SERVERS:localhost}:${KAFKA_BOOTSTRAP_PORT:9092}
     topics:
-      - '${DEVICE_SIGNALS_TOPIC:topic.device.validcloudevents}'
+      - '${DEVICE_VALID_CE_TOPIC:topic.device.validcloudevents}'
     consumer_group: "zone.dimo.export.validcloudevents"
     client_id: ${CONTAINER_NAME:localhost}-dps-status
     rack_id: ${NODE_NAME:localhost}
diff --git a/charts/dps/templates/kafka-topic.yaml b/charts/dps/templates/kafka-topic.yaml
new file mode 100644
index 0000000..e0a2811
--- /dev/null
+++ b/charts/dps/templates/kafka-topic.yaml
@@ -0,0 +1,19 @@
+{{- if .Values.kafka.topics -}}
+{{- range $item := .Values.kafka.topics }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: {{ $item.name }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    strimzi.io/cluster: {{$.Values.kafka.clusterName}}
+    {{- include "dps.labels" $ | nindent 4 }}
+spec:
+  config:
+    {{- range $key, $val := $item.config }}
+    {{ $key }} : {{ $val }}
+    {{- end}}
+  topicName: {{ $item.name }}
+---
+{{- end}}
+{{- end}}
diff --git a/charts/dps/values-prod.yaml b/charts/dps/values-prod.yaml
index 908622d..dda4af0 100644
--- a/charts/dps/values-prod.yaml
+++ b/charts/dps/values-prod.yaml
@@ -5,9 +5,8 @@ kafka:
 
 env:
   KAFKA_BOOTSTRAP_SERVERS: kafka-prod-dimo-kafka-kafka-bootstrap
-  DEVICE_STATUS_TOPIC: topic.device.status.enriched
   ENVIRONMENT: prod
-  DEVICES_APIGRPC_ADDR: devices-api-prod:8086
+  LOG_LEVEL: INFO
 
 ingress:
   enabled: false
diff --git a/charts/dps/values.yaml b/charts/dps/values.yaml
index fa2c806..b4497fd 100644
--- a/charts/dps/values.yaml
+++ b/charts/dps/values.yaml
@@ -46,14 +46,14 @@ securityContext:
 
 env:
   KAFKA_BOOTSTRAP_SERVERS: kafka-dev-dimo-kafka-kafka-brokers
-  DEVICE_STATUS_TOPIC: topic.device.status.enriched
-  DEVICE_STATUS_V2_TOPIC: topic.device.status.v2
+  DEVICE_SIGNALS_TOPIC: topic.device.signals
+  DEVICE_VALID_CE_TOPIC: topic.device.validcloudevents
+  DEVICE_PARTIAL_CE_TOPIC: topic.device.partialcloudevents
   ELASTIC_MULTIPLE_NODE: false
   ENVIRONMENT: dev
   CLICKHOUSE_PORT: 9440
   CLICKHOUSE_DATABASE: dimo
-  DEVICES_APIGRPC_ADDR: devices-api-dev:8086
-  LOG_LEVEL: INFO
+  LOG_LEVEL: DEBUG
 
 service:
   type: ClusterIP
@@ -128,3 +128,28 @@ serviceMonitor:
   path: "/metrics"
   port: mon-http
   interval: 30s
+
+kafka:
+  clusterName: kafka-dev-dimo-kafka
+  topics:
+    - name: topic.device.signals
+      config:
+        segment.ms: '3600000'
+        compression.type: producer
+        cleanup.policy: compact,delete
+        min.compaction.lag.ms: '3600000'
+        retention.ms: '604800000'
+    - name: topic.device.validcloudevents
+      config:
+        segment.ms: '3600000'
+        compression.type: producer
+        cleanup.policy: compact,delete
+        min.compaction.lag.ms: '3600000'
+        retention.ms: '604800000'
+    - name: topic.device.partialcloudevents
+      config:
+        segment.ms: '3600000'
+        compression.type: producer
+        cleanup.policy: compact,delete
+        min.compaction.lag.ms: '3600000'
+        retention.ms: '604800000'