- add kafka topics

DIMO-Network · Dec 10, 2024 · db471d0 · db471d0
1 parent 53c0152
commit db471d0
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 8 deletions.
diff --git a/charts/dps/files/partial-cloudevents.yaml b/charts/dps/files/partial-cloudevents.yaml
@@ -4,7 +4,7 @@ input:
     addresses:
       - ${KAFKA_BOOTSTRAP_SERVERS:localhost}:${KAFKA_BOOTSTRAP_PORT:9092}
     topics:
-      - '${DEVICE_SIGNALS_TOPIC:topic.device.partialcloudevents}'
+      - '${DEVICE_PARTIAL_CE_TOPIC:topic.device.partialcloudevents}'
     consumer_group: "zone.dimo.export.partialcloudevents"
     client_id: ${CONTAINER_NAME:localhost}-dps-status
     rack_id: ${NODE_NAME:localhost}

diff --git a/charts/dps/files/valid-cloudevents.yaml b/charts/dps/files/valid-cloudevents.yaml
@@ -4,7 +4,7 @@ input:
     addresses:
       - ${KAFKA_BOOTSTRAP_SERVERS:localhost}:${KAFKA_BOOTSTRAP_PORT:9092}
     topics:
-      - '${DEVICE_SIGNALS_TOPIC:topic.device.validcloudevents}'
+      - '${DEVICE_VALID_CE_TOPIC:topic.device.validcloudevents}'
     consumer_group: "zone.dimo.export.validcloudevents"
     client_id: ${CONTAINER_NAME:localhost}-dps-status
     rack_id: ${NODE_NAME:localhost}

diff --git a/charts/dps/templates/kafka-topic.yaml b/charts/dps/templates/kafka-topic.yaml
@@ -0,0 +1,19 @@
+{{- if .Values.kafka.topics -}}
+{{- range $item := .Values.kafka.topics }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: {{ $item.name }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    strimzi.io/cluster: {{$.Values.kafka.clusterName}}
+    {{- include "dps.labels" $ | nindent 4 }}
+spec:
+  config:
+    {{- range $key, $val := $item.config }}
+    {{ $key }} : {{ $val }}
+    {{- end}}
+  topicName: {{ $item.name }}
+---
+{{- end}}
+{{- end}}
diff --git a/charts/dps/values-prod.yaml b/charts/dps/values-prod.yaml
@@ -5,9 +5,8 @@ kafka:
 
 env:
   KAFKA_BOOTSTRAP_SERVERS: kafka-prod-dimo-kafka-kafka-bootstrap
-  DEVICE_STATUS_TOPIC: topic.device.status.enriched
   ENVIRONMENT: prod
-  DEVICES_APIGRPC_ADDR: devices-api-prod:8086
+  LOG_LEVEL: INFO
 
 ingress:
   enabled: false

diff --git a/charts/dps/values.yaml b/charts/dps/values.yaml
@@ -46,14 +46,14 @@ securityContext:
 
 env:
   KAFKA_BOOTSTRAP_SERVERS: kafka-dev-dimo-kafka-kafka-brokers
-  DEVICE_STATUS_TOPIC: topic.device.status.enriched
-  DEVICE_STATUS_V2_TOPIC: topic.device.status.v2
+  DEVICE_SIGNALS_TOPIC: topic.device.signals
+  DEVICE_VALID_CE_TOPIC: topic.device.validcloudevents
+  DEVICE_PARTIAL_CE_TOPIC: topic.device.partialcloudevents
   ELASTIC_MULTIPLE_NODE: false
   ENVIRONMENT: dev
   CLICKHOUSE_PORT: 9440
   CLICKHOUSE_DATABASE: dimo
-  DEVICES_APIGRPC_ADDR: devices-api-dev:8086
-  LOG_LEVEL: INFO
+  LOG_LEVEL: DEBUG
 
 service:
   type: ClusterIP
@@ -128,3 +128,28 @@ serviceMonitor:
   path: "/metrics"
   port: mon-http
   interval: 30s
+
+kafka:
+  clusterName: kafka-dev-dimo-kafka
+  topics:
+    - name: topic.device.signals
+      config:
+        segment.ms: '3600000'
+        compression.type: producer
+        cleanup.policy: compact,delete
+        min.compaction.lag.ms: '3600000'
+        retention.ms: '604800000'
+    - name: topic.device.validcloudevents
+      config:
+        segment.ms: '3600000'
+        compression.type: producer
+        cleanup.policy: compact,delete
+        min.compaction.lag.ms: '3600000'
+        retention.ms: '604800000'
+    - name: topic.device.partialcloudevents
+      config:
+        segment.ms: '3600000'
+        compression.type: producer
+        cleanup.policy: compact,delete
+        min.compaction.lag.ms: '3600000'
+        retention.ms: '604800000'