feat: templatize migrations

.github/workflows/push.yaml

@@ -48,3 +48,42 @@ jobs:
  run: make build-all
  - name: Build and push docker image
  run: make build-and-push-signoz-collector
+ image-build-and-push-schema-migrator:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v3
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v2
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
+ with:
+ version: latest
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - uses: benjlevesque/short-sha@v2.2
+ id: short-sha
+ - name: Get branch name
+ id: branch-name
+ uses: tj-actions/branch-names@v5.1
+ - name: Set docker tag environment
+ run: |
+ if [ '${{ steps.branch-name.outputs.is_tag }}' == 'true' ]; then
+ tag="${{ steps.branch-name.outputs.tag }}"
+ tag="${tag:1}"
+ echo "DOCKER_TAG=$tag" >> $GITHUB_ENV
+ else
+ echo "DOCKER_TAG=${{ steps.branch-name.outputs.current_branch }}" >> $GITHUB_ENV
+ fi
+ - name: Install cross-compilation tools
+ run: |
+ set -ex
+ sudo apt-get update
+ sudo apt-get install -y gcc-aarch64-linux-gnu musl-tools
+ - name: Build artifacts for arm64/amd64
+ run: make build-all
+ - name: Build and push docker image
+ run: make build-and-push-signoz-schema-migrator

.github/workflows/release.yaml

@@ -25,9 +25,13 @@ jobs:
  cd .build/linux-amd64
  sha256sum signoz-collector > signoz-collector-linux-amd64.sha256
  tar -czvf signoz-collector-linux-amd64.tar.gz signoz-collector
+ sha256sum signoz-schema-migrator > signoz-schema-migrator-linux-amd64.sha256
+ tar -czvf signoz-schema-migrator-linux-amd64.tar.gz signoz-schema-migrator
  cd ../linux-arm64
  sha256sum signoz-collector > signoz-collector-linux-arm64.sha256
  tar -czvf signoz-collector-linux-arm64.tar.gz signoz-collector
+ sha256sum signoz-schema-migrator > signoz-schema-migrator-linux-arm64.sha256
+ tar -czvf signoz-schema-migrator-linux-arm64.tar.gz signoz-schema-migrator
  - name: Create release and upload assets
  uses: softprops/action-gh-release@v1
  with:
@@ -36,4 +40,8 @@ jobs:
  .build/linux-amd64/signoz-collector-linux-amd64.tar.gz
  .build/linux-arm64/signoz-collector-linux-arm64.sha256
  .build/linux-arm64/signoz-collector-linux-arm64.tar.gz
+ .build/linux-amd64/signoz-schema-migrator-linux-amd64.sha256
+ .build/linux-amd64/signoz-schema-migrator-linux-amd64.tar.gz
+ .build/linux-arm64/signoz-schema-migrator-linux-arm64.sha256
+ .build/linux-arm64/signoz-schema-migrator-linux-arm64.tar.gz
  generate_release_notes: true

Makefile

@@ -1,6 +1,7 @@
 COMMIT_SHA ?= $(shell git rev-parse HEAD)
 REPONAME ?= signoz
 IMAGE_NAME ?= signoz-otel-collector
+MIGRATOR_IMAGE_NAME ?= signoz-schema-migrator
 CONFIG_FILE ?= ./config/default-config.yaml
 DOCKER_TAG ?= latest
 
@@ -34,6 +35,7 @@ test:
 .PHONY: build
 build:
  CGO_ENABLED=1 go build -tags timetzdata -o .build/${GOOS}-${GOARCH}/signoz-collector -ldflags "-linkmode external -extldflags '-static' -s -w ${LD_FLAGS}" ./cmd/signozcollector
+ CGO_ENABLED=1 go build -tags timetzdata -o .build/${GOOS}-${GOARCH}/signoz-schema-migrator -ldflags "-linkmode external -extldflags '-static' -s -w ${LD_FLAGS}" ./cmd/signozschemamigrator
 
 .PHONY: amd64
 amd64:
@@ -73,6 +75,24 @@ build-signoz-collector:
  --no-cache -f cmd/signozcollector/Dockerfile --progress plain \
  --tag $(REPONAME)/$(IMAGE_NAME):$(DOCKER_TAG) .
 
+.PHONY: build-signoz-schema-migrator
+build-signoz-schema-migrator:
+ @echo "------------------"
+ @echo "--> Build schema migrator docker image"
+ @echo "------------------"
+ docker build --build-arg TARGETPLATFORM="linux/amd64" \
+ --no-cache -f cmd/signozschemamigrator/Dockerfile --progress plain \
+ --tag $(REPONAME)/$(MIGRATOR_IMAGE_NAME):$(DOCKER_TAG) .
+
+.PHONY: build-and-push-signoz-schema-migrator
+build-and-push-signoz-schema-migrator:
+ @echo "------------------"
+ @echo "--> Build and push schema migrator docker image"
+ @echo "------------------"
+ docker buildx build --platform linux/amd64,linux/arm64 --progress plain \
+ --no-cache --push -f cmd/signozschemamigrator/Dockerfile \
+ --tag $(REPONAME)/$(MIGRATOR_IMAGE_NAME):$(DOCKER_TAG) .
+
 .PHONY: lint
 lint:
  @echo "Running linters..."

cmd/signozcollector/Dockerfile

@@ -14,12 +14,8 @@ USER ${USER_UID}
 # copy the binaries from the multi-stage build
 COPY .build/${TARGETOS}-${TARGETARCH}/signoz-collector /signoz-collector
 
-# copy the config and migration files
+# copy the config file
 COPY config/default-config.yaml /etc/otel/config.yaml
-COPY exporter/clickhousetracesexporter/migrations /migrations
-COPY exporter/clickhouselogsexporter/migrations /logsmigrations
-
-ENV LOG_MIGRATIONS_FOLDER="/logsmigrations"
 
 # expose OTLP ports for the collector
 EXPOSE 4317 4318

cmd/signozschemamigrator/Dockerfile

@@ -0,0 +1,17 @@
+# use a minimal alpine image
+FROM alpine:3.17
+
+# define arguments and default values
+ARG TARGETOS TARGETARCH
+ARG USER_UID=10001
+
+# create a non-root user for running the migrator
+USER ${USER_UID}
+
+# copy the binaries from the multi-stage build
+COPY .build/${TARGETOS}-${TARGETARCH}/signoz-schema-migrator /signoz-schema-migrator
+COPY migrationmanager ./migrationmanager
+
+# run the binary as the entrypoint and pass the default dsn as a flag
+ENTRYPOINT [ "/signoz-schema-migrator" ]
+CMD ["--dsn", "tcp://localhost:9000"]

cmd/signozschemamigrator/migrate.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+
+ "github.com/SigNoz/signoz-otel-collector/migrationmanager"
+ "github.com/spf13/pflag"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+)
+
+func init() {
+ // init zap logger
+ config := zap.NewProductionConfig()
+ config.EncoderConfig.EncodeLevel = zapcore.LowercaseLevelEncoder
+ config.EncoderConfig.TimeKey = "timestamp"
+ config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+ logger, err := config.Build()
+ if err != nil {
+ log.Fatalf("Failed to initialize zap logger %v", err)
+ }
+ // replace global logger
+ // TODO(dhawal1248): move away from global logger
+ zap.ReplaceGlobals(logger)
+}
+
+func main() {
+ logger := zap.L().With(zap.String("component", "migrate cli"))
+ f := pflag.NewFlagSet("Schema Migrator CLI Options", pflag.ExitOnError)
+
+ f.Usage = func() {
+ fmt.Println(f.FlagUsages())
+ os.Exit(1)
+ }
+
+ f.String("dsn", "", "Clickhouse DSN")
+ f.String("cluster-name", "cluster", "Cluster name to use while running migrations")
+ f.Bool("disable-duration-sort-feature", false, "Flag to disable the duration sort feature. Defaults to false.")
+ f.Bool("disable-timestamp-sort-feature", false, "Flag to disable the timestamp sort feature. Defaults to false.")
+
+ err := f.Parse(os.Args[1:])
+ if err != nil {
+ logger.Fatal("Failed to parse args", zap.Error(err))
+ }
+
+ dsn, err := f.GetString("dsn")
+ if err != nil {
+ logger.Fatal("Failed to get dsn from args", zap.Error(err))
+ }
+
+ clusterName, err := f.GetString("cluster-name")
+ if err != nil {
+ logger.Fatal("Failed to get cluster name from args", zap.Error(err))
+ }
+
+ disableDurationSortFeature, err := f.GetBool("disable-duration-sort-feature")
+ if err != nil {
+ logger.Fatal("Failed to get disable duration sort feature flag from args", zap.Error(err))
+ }
+
+ disableTimestampSortFeature, err := f.GetBool("disable-timestamp-sort-feature")
+ if err != nil {
+ logger.Fatal("Failed to get disable timestamp sort feature flag from args", zap.Error(err))
+ }
+
+ if dsn == "" {
+ logger.Fatal("dsn is a required field")
+ }
+
+ // set cluster env so that golang-migrate can use it
+ // the value of this env would replace all occurences of {{.SIGNOZ_CLUSTER}} in the migration files
+ os.Setenv("SIGNOZ_CLUSTER", clusterName)
+
+ manager, err := migrationmanager.New(dsn, clusterName, disableDurationSortFeature, disableTimestampSortFeature)
+ if err != nil {
+ logger.Fatal("Failed to create migration manager", zap.Error(err))
+ }
+ defer manager.Close()
+
+ err = manager.Migrate(context.Background())
+ if err != nil {
+ logger.Fatal("Failed to run migrations", zap.Error(err))
+ }
+}

example/docker-compose.yml

@@ -0,0 +1,33 @@
+version: "3.8"
+
+services:
+ schema-migrator:
+ image: signoz/signoz-schema-migrator:templatize-migrations
+ extra_hosts:
+ - signoz-host:host-gateway
+ command:
+ - "--dsn=tcp://signoz-host:9000"
+
+ otel-collector:
+ image: signoz/signoz-otel-collector:templatize-migrations
+ deploy:
+ replicas: 3
+ user: root # required for reading docker container logs
+ extra_hosts:
+ - signoz-host:host-gateway
+ command:
+ - "--config=/etc/otel-collector-config.yml"
+ # - "--set=service.telemetry.logs.level=DEBUG"
+ volumes:
+ - ./example-config.yaml:/etc/otel-collector-config.yml
+ - /var/lib/docker/containers:/var/lib/docker/containers:ro
+ - /var/log:/var/log:ro
+ # required to read logs from /var/lib/docker/containers/
+ ports:
+ - "1888" # pprof extension
+ - "8888" # Prometheus metrics exposed by the collector
+ - "13133" # health_check extension
+ - "55679" # zpages extension
+ depends_on:
+ schema-migrator:
+ condition: service_completed_successfully