Merge pull request #1 from DIMO-Network/sm-1339_nes_service

- add dps service
- add kafka topics

.github/workflows/buildpushdev.yml

@@ -0,0 +1,42 @@
+name: buildpushdev
+on:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    name: buildpushdev
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get short SHA
+        id: slug
+        run: echo "::set-output name=sha7::$(echo ${GITHUB_SHA} | cut -c1-7)"
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/dockerfile
+          push: true
+          platforms: linux/amd64
+          tags: dimozone/dps:${{ steps.slug.outputs.sha7 }}, dimozone/dps:latest
+
+      - name: Update Image Version in the worker HelmChart values.yaml
+        uses: fjogeleit/yaml-update-action@master
+        with:
+          valueFile: "charts/dps/values.yaml"
+          propertyPath: "image.tag"
+          value: ${{ steps.slug.outputs.sha7 }}
+          branch: main
+          message: "Update Image Version to ${{ steps.slug.outputs.sha7 }}"

.github/workflows/buildpushtagged.yml

@@ -0,0 +1,47 @@
+name: buildpush
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  build_test:
+    # The type of runner that the job will run on
+    name: buildpush
+    runs-on: ubuntu-latest
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get tag
+        id: tag
+        uses: dawidd6/action-get-tag@v1
+        with:
+          strip_v: true
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/dockerfile
+          push: true
+          platforms: linux/amd64
+          tags: dimozone/dps:${{steps.tag.outputs.tag}}
+
+      - name: Update Image Version in the related HelmChart values.yaml
+        uses: fjogeleit/yaml-update-action@v0.15.0
+        with:
+          valueFile: "charts/dps/values-prod.yaml"
+          propertyPath: "image.tag"
+          value: ${{steps.tag.outputs.tag}}
+          branch: main
+          message: "Update Image Version to ${{steps.tag.outputs.tag}}"

.github/workflows/lint.yml

@@ -0,0 +1,26 @@
+name: golangci-lint
+on:
+  pull_request:
+    branches: ["**"]
+jobs:
+  golangci:
+    runs-on: "ubuntu-24.04"
+
+    name: lint
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+
+      - name: install golangci-lint
+        run: make tools-golangci-lint
+
+      - name: tidy
+        run: go mod tidy
+
+      - name: build
+        run: go build
+
+      - name: Run golangci-lint
+        run: make lint

.github/workflows/tests.yml

@@ -0,0 +1,28 @@
+name: Tests
+on:
+  pull_request:
+    branches: ["**"]
+  push:
+    branches:
+      - main
+
+jobs:
+  rules-test:
+    runs-on: [ self-hosted, linux ]
+
+    steps:
+      # using go to get arch and os
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.22
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # we need to install promtool to run the tests
+      - name: Install promtool
+        run: make tools
+
+      - name: Run tests
+        run: make test

.gitignore

@@ -7,9 +7,9 @@
 
 # binary only
 bin
-dis
-!dis/
-!dis/*
+dps
+!dps/
+!dps/*
 __debug_bin*
 *.DS_Store
 
@@ -31,4 +31,4 @@ go.work
 .idea/
 
 # ignore file used for test.
-alerts-modified.yaml
+alerts-modified.yaml

Makefile

@@ -0,0 +1,90 @@
+.PHONY: test
+
+PATHINSTBIN = $(abspath ./bin)
+export PATH := $(PATHINSTBIN):$(PATH)
+SHELL := env PATH=$(PATH) $(SHELL)
+
+BIN_NAME					?= dps
+DEFAULT_INSTALL_DIR			:= $(go env GOPATH)/bin
+DEFAULT_ARCH				:= $(shell go env GOARCH)
+DEFAULT_GOOS				:= $(shell go env GOOS)
+ARCH						?= $(DEFAULT_ARCH)
+GOOS						?= $(DEFAULT_GOOS)
+.DEFAULT_GOAL 				:= test
+
+# Dependency versions
+PROMETHEUS_VERSION = 2.47.0
+
+GOOS := $(shell go env GOOS)
+GOARCH := $(shell go env GOARCH)
+
+# List of supported GOOS and GOARCH
+GOOS_LIST := linux darwin
+GOARCH_LIST := amd64 arm64
+
+PROMETHEUS_TAR := prometheus-$(PROMETHEUS_VERSION).$(GOOS)-$(GOARCH).tar.gz
+PROMETHEUS_URL := https://github.com/prometheus/prometheus/releases/download/v$(PROMETHEUS_VERSION)/$(PROMETHEUS_TAR)
+
+help: ## show this help
+	@echo "\nSpecify a subcommand:\n"
+	@grep -hE '^[0-9a-zA-Z_-]+:.*?## .*$$' ${MAKEFILE_LIST} | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[0;36m%-20s\033[m %s\n", $$1, $$2}'
+	@echo ""
+
+build:
+	@CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(ARCH) \
+		go build -o bin/$(BIN_NAME) ./
+
+build-all:## Build target for all supported GOOS and GOARCH
+	@for goos in $(GOOS_LIST); do \
+		for goarch in $(GOARCH_LIST); do \
+			echo "Building for $$goos/$$goarch..."; \
+			CGO_ENABLED=0 GOOS=$$goos GOARCH=$$goarch \
+			go build -o bin/$(BIN_NAME)-$$goos-$$goarch ./; \
+		done \
+	done
+
+clean: ## clean up
+	rm -rf ./bin
+
+install: build
+	@install -d $(INSTALL_DIR)
+	@rm -f $(INSTALL_DIR)/benthos
+	@cp bin/* $(INSTALL_DIR)/
+
+dep:
+	@go mod tidy
+
+lint-benthos: build  ## Run Benthos linter
+	@CLICKHOUSE_HOST="" CLICKHOUSE_PORT="" CLICKHOUSE_SIGNAL_DATABASE="" CLICKHOUSE_INDEX_DATABASE=""  CLICKHOUSE_USER="" CLICKHOUSE_PASSWORD="" \
+		S3_AWS_ACCESS_KEY_ID="" S3_AWS_SECRET_ACCESS_KEY="" S3_CLOUDEVENT_BUCKET="" S3_EPHEMERAL_BUCKET="" \
+	dps lint -r ./charts/dps/files/resources.yaml ./charts/dps/files/config.yaml ./charts/dps/files/*
+
+lint: lint-benthos ## Run linter for benthos config and go code
+	golangci-lint version
+	@golangci-lint run --timeout=30m
+
+format:
+	@golangci-lint run --fix
+
+test-go: ## Run Go tests
+	@go test ./...
+
+test: test-go test-prom ## run all tests
+
+test-prom: ## run prometheus tests
+	sed "s/{{ .Release.Namespace }}/dev/g"  ./charts/dps/templates/alerts.yaml |  sed 's/{{.*}}//g' >  tests/prom/alerts-modified.yaml
+	promtool check rules tests/prom/alerts-modified.yaml
+	promtool test rules tests/prom/rules-tests.yaml
+
+tools-promtool: ## install promtools
+	mkdir -p $(PATHINSTBIN)
+	wget $(PROMETHEUS_URL) -O $(PATHINSTBIN)/$(PROMETHEUS_TAR)
+	tar -xvf $(PATHINSTBIN)/$(PROMETHEUS_TAR) -C $(PATHINSTBIN)
+	cp $(PATHINSTBIN)/prometheus-$(PROMETHEUS_VERSION).$(GOOS)-$(GOARCH)/promtool $(PATHINSTBIN)
+	rm -rf $(PATHINSTBIN)/$(PROMETHEUS_TAR) $(PATHINSTBIN)/prometheus-$(PROMETHEUS_VERSION).$(GOOS)-$(GOARCH)
+
+tools-golangci-lint:
+	@mkdir -p $(PATHINSTBIN)
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(PATHINSTBIN) $(GOLANGCI_VERSION)
+
+make tools: tools-promtool ## install all tools

README.md

@@ -0,0 +1,2 @@
+# dps
+Redpanda service for persisting the data from the devices to the Dimo Node.

charts/dps/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/dps/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: dps
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

charts/dps/files/config.yaml

@@ -0,0 +1,14 @@
+http:
+  address: 0.0.0.0:8888
+
+logger:
+  level: ${LOG_LEVEL:INFO}
+  add_timestamp: true
+  format: json
+  static_fields:
+    'appname': zone.dimo.export.es
+
+metrics:
+  prometheus: {}
+
+shutdown_timeout: 20s