diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 58a826ef..7ecbfc5e 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -48,3 +48,42 @@ jobs: run: make build-all - name: Build and push docker image run: make build-and-push-signoz-collector + image-build-and-push-schema-migrator: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + version: latest + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - uses: benjlevesque/short-sha@v2.2 + id: short-sha + - name: Get branch name + id: branch-name + uses: tj-actions/branch-names@v5.1 + - name: Set docker tag environment + run: | + if [ '${{ steps.branch-name.outputs.is_tag }}' == 'true' ]; then + tag="${{ steps.branch-name.outputs.tag }}" + tag="${tag:1}" + echo "DOCKER_TAG=$tag" >> $GITHUB_ENV + else + echo "DOCKER_TAG=${{ steps.branch-name.outputs.current_branch }}" >> $GITHUB_ENV + fi + - name: Install cross-compilation tools + run: | + set -ex + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu musl-tools + - name: Build artifacts for arm64/amd64 + run: make build-all + - name: Build and push docker image + run: make build-and-push-signoz-schema-migrator diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 84ef5bbd..47af5ade 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -25,9 +25,13 @@ jobs: cd .build/linux-amd64 sha256sum signoz-collector > signoz-collector-linux-amd64.sha256 tar -czvf signoz-collector-linux-amd64.tar.gz signoz-collector + sha256sum signoz-schema-migrator > signoz-schema-migrator-linux-amd64.sha256 + tar -czvf signoz-schema-migrator-linux-amd64.tar.gz signoz-schema-migrator cd ../linux-arm64 sha256sum signoz-collector > signoz-collector-linux-arm64.sha256 tar -czvf signoz-collector-linux-arm64.tar.gz signoz-collector + sha256sum signoz-schema-migrator > signoz-schema-migrator-linux-arm64.sha256 + tar -czvf signoz-schema-migrator-linux-arm64.tar.gz signoz-schema-migrator - name: Create release and upload assets uses: softprops/action-gh-release@v1 with: @@ -36,4 +40,8 @@ jobs: .build/linux-amd64/signoz-collector-linux-amd64.tar.gz .build/linux-arm64/signoz-collector-linux-arm64.sha256 .build/linux-arm64/signoz-collector-linux-arm64.tar.gz + .build/linux-amd64/signoz-schema-migrator-linux-amd64.sha256 + .build/linux-amd64/signoz-schema-migrator-linux-amd64.tar.gz + .build/linux-arm64/signoz-schema-migrator-linux-arm64.sha256 + .build/linux-arm64/signoz-schema-migrator-linux-arm64.tar.gz generate_release_notes: true diff --git a/Makefile b/Makefile index 216f4eb6..4257c598 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ COMMIT_SHA ?= $(shell git rev-parse HEAD) REPONAME ?= signoz IMAGE_NAME ?= signoz-otel-collector +MIGRATOR_IMAGE_NAME ?= signoz-schema-migrator CONFIG_FILE ?= ./config/default-config.yaml DOCKER_TAG ?= latest @@ -34,6 +35,7 @@ test: .PHONY: build build: CGO_ENABLED=1 go build -tags timetzdata -o .build/${GOOS}-${GOARCH}/signoz-collector -ldflags "-linkmode external -extldflags '-static' -s -w ${LD_FLAGS}" ./cmd/signozcollector + CGO_ENABLED=1 go build -tags timetzdata -o .build/${GOOS}-${GOARCH}/signoz-schema-migrator -ldflags "-linkmode external -extldflags '-static' -s -w ${LD_FLAGS}" ./cmd/signozschemamigrator .PHONY: amd64 amd64: @@ -73,6 +75,24 @@ build-signoz-collector: --no-cache -f cmd/signozcollector/Dockerfile --progress plain \ --tag $(REPONAME)/$(IMAGE_NAME):$(DOCKER_TAG) . +.PHONY: build-signoz-schema-migrator +build-signoz-schema-migrator: + @echo "------------------" + @echo "--> Build schema migrator docker image" + @echo "------------------" + docker build --build-arg TARGETPLATFORM="linux/amd64" \ + --no-cache -f cmd/signozschemamigrator/Dockerfile --progress plain \ + --tag $(REPONAME)/$(MIGRATOR_IMAGE_NAME):$(DOCKER_TAG) . + +.PHONY: build-and-push-signoz-schema-migrator +build-and-push-signoz-schema-migrator: + @echo "------------------" + @echo "--> Build and push schema migrator docker image" + @echo "------------------" + docker buildx build --platform linux/amd64,linux/arm64 --progress plain \ + --no-cache --push -f cmd/signozschemamigrator/Dockerfile \ + --tag $(REPONAME)/$(MIGRATOR_IMAGE_NAME):$(DOCKER_TAG) . + .PHONY: lint lint: @echo "Running linters..." diff --git a/cmd/signozcollector/Dockerfile b/cmd/signozcollector/Dockerfile index 0874bb58..834c295b 100644 --- a/cmd/signozcollector/Dockerfile +++ b/cmd/signozcollector/Dockerfile @@ -14,12 +14,8 @@ USER ${USER_UID} # copy the binaries from the multi-stage build COPY .build/${TARGETOS}-${TARGETARCH}/signoz-collector /signoz-collector -# copy the config and migration files +# copy the config file COPY config/default-config.yaml /etc/otel/config.yaml -COPY exporter/clickhousetracesexporter/migrations /migrations -COPY exporter/clickhouselogsexporter/migrations /logsmigrations - -ENV LOG_MIGRATIONS_FOLDER="/logsmigrations" # expose OTLP ports for the collector EXPOSE 4317 4318 diff --git a/cmd/signozschemamigrator/Dockerfile b/cmd/signozschemamigrator/Dockerfile new file mode 100644 index 00000000..d9e69be3 --- /dev/null +++ b/cmd/signozschemamigrator/Dockerfile @@ -0,0 +1,17 @@ +# use a minimal alpine image +FROM alpine:3.17 + +# define arguments and default values +ARG TARGETOS TARGETARCH +ARG USER_UID=10001 + +# create a non-root user for running the migrator +USER ${USER_UID} + +# copy the binaries from the multi-stage build +COPY .build/${TARGETOS}-${TARGETARCH}/signoz-schema-migrator /signoz-schema-migrator +COPY migrationmanager ./migrationmanager + +# run the binary as the entrypoint and pass the default dsn as a flag +ENTRYPOINT [ "/signoz-schema-migrator" ] +CMD ["--dsn", "tcp://localhost:9000"] diff --git a/cmd/signozschemamigrator/migrate.go b/cmd/signozschemamigrator/migrate.go new file mode 100644 index 00000000..91184b28 --- /dev/null +++ b/cmd/signozschemamigrator/migrate.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/SigNoz/signoz-otel-collector/migrationmanager" + "github.com/spf13/pflag" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func init() { + // init zap logger + config := zap.NewProductionConfig() + config.EncoderConfig.EncodeLevel = zapcore.LowercaseLevelEncoder + config.EncoderConfig.TimeKey = "timestamp" + config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + logger, err := config.Build() + if err != nil { + log.Fatalf("Failed to initialize zap logger %v", err) + } + // replace global logger + // TODO(dhawal1248): move away from global logger + zap.ReplaceGlobals(logger) +} + +func main() { + logger := zap.L().With(zap.String("component", "migrate cli")) + f := pflag.NewFlagSet("Schema Migrator CLI Options", pflag.ExitOnError) + + f.Usage = func() { + fmt.Println(f.FlagUsages()) + os.Exit(1) + } + + f.String("dsn", "", "Clickhouse DSN") + f.String("cluster-name", "cluster", "Cluster name to use while running migrations") + f.Bool("disable-duration-sort-feature", false, "Flag to disable the duration sort feature. Defaults to false.") + f.Bool("disable-timestamp-sort-feature", false, "Flag to disable the timestamp sort feature. Defaults to false.") + + err := f.Parse(os.Args[1:]) + if err != nil { + logger.Fatal("Failed to parse args", zap.Error(err)) + } + + dsn, err := f.GetString("dsn") + if err != nil { + logger.Fatal("Failed to get dsn from args", zap.Error(err)) + } + + clusterName, err := f.GetString("cluster-name") + if err != nil { + logger.Fatal("Failed to get cluster name from args", zap.Error(err)) + } + + disableDurationSortFeature, err := f.GetBool("disable-duration-sort-feature") + if err != nil { + logger.Fatal("Failed to get disable duration sort feature flag from args", zap.Error(err)) + } + + disableTimestampSortFeature, err := f.GetBool("disable-timestamp-sort-feature") + if err != nil { + logger.Fatal("Failed to get disable timestamp sort feature flag from args", zap.Error(err)) + } + + if dsn == "" { + logger.Fatal("dsn is a required field") + } + + // set cluster env so that golang-migrate can use it + // the value of this env would replace all occurences of {{.SIGNOZ_CLUSTER}} in the migration files + os.Setenv("SIGNOZ_CLUSTER", clusterName) + + manager, err := migrationmanager.New(dsn, clusterName, disableDurationSortFeature, disableTimestampSortFeature) + if err != nil { + logger.Fatal("Failed to create migration manager", zap.Error(err)) + } + defer manager.Close() + + err = manager.Migrate(context.Background()) + if err != nil { + logger.Fatal("Failed to run migrations", zap.Error(err)) + } +} diff --git a/example/docker-compose.yml b/example/docker-compose.yml new file mode 100644 index 00000000..2a90afbc --- /dev/null +++ b/example/docker-compose.yml @@ -0,0 +1,33 @@ +version: "3.8" + +services: + schema-migrator: + image: signoz/signoz-schema-migrator:templatize-migrations + extra_hosts: + - signoz-host:host-gateway + command: + - "--dsn=tcp://signoz-host:9000" + + otel-collector: + image: signoz/signoz-otel-collector:templatize-migrations + deploy: + replicas: 3 + user: root # required for reading docker container logs + extra_hosts: + - signoz-host:host-gateway + command: + - "--config=/etc/otel-collector-config.yml" + # - "--set=service.telemetry.logs.level=DEBUG" + volumes: + - ./example-config.yaml:/etc/otel-collector-config.yml + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/log:/var/log:ro + # required to read logs from /var/lib/docker/containers/ + ports: + - "1888" # pprof extension + - "8888" # Prometheus metrics exposed by the collector + - "13133" # health_check extension + - "55679" # zpages extension + depends_on: + schema-migrator: + condition: service_completed_successfully diff --git a/example/example-config.yaml b/example/example-config.yaml new file mode 100644 index 00000000..9a93ffda --- /dev/null +++ b/example/example-config.yaml @@ -0,0 +1,141 @@ +receivers: + otlp/spanmetrics: + protocols: + grpc: + endpoint: "localhost:12345" + otlp: + protocols: + grpc: + http: + jaeger: + protocols: + grpc: + thrift_http: + hostmetrics: + collection_interval: 30s + scrapers: + cpu: + load: + memory: + disk: + filesystem: + network: + filelog/containers: + include: [ "/var/lib/docker/containers/*/*.log" ] + start_at: end + include_file_path: true + include_file_name: false + operators: + # Find out which format is used by docker + - type: router + id: get-format + routes: + - output: parser-docker + expr: 'body matches "^\\{"' + # Parse Docker format + - type: json_parser + id: parser-docker + output: extract_metadata_from_filepath + timestamp: + parse_from: attributes.time + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + + # Extract metadata from file path + - type: regex_parser + id: extract_metadata_from_filepath + regex: '^.*containers/(?P[^_]+)/.*log$' + parse_from: attributes["log.file.path"] + output: parse_body + - type: move + id: parse_body + from: attributes.log + to: body + output: add_source + - type: add + id: add_source + field: resource["source"] + value: "docker" + filelog/syslog: + include: [ "/var/log/*log" ] + start_at: end + include_file_path: true + include_file_name: false + operators: + # Extract metadata from file path + - type: regex_parser + regex: '^.*(?P