added datadog tracer (#2190)

* added datadog tracer

* updated dependencies

* updated ci go version

.github/workflows/build-image.yml

@@ -60,7 +60,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.21
+          go-version: 1.22
 
       - name: Get and verify dependencies
         run: go mod tidy && go mod download && go mod verify
@@ -70,9 +70,9 @@ jobs:
 
       - name: Build app to make sure there are zero issues
         run: |
-          export CGO_ENABLED=0 
+          export CGO_ENABLED=0
           export GOOS=linux
-          export GOARCH=${{ matrix.arch }} 
+          export GOARCH=${{ matrix.arch }}
           go build -o convoy ./cmd
 
       - name: Set up QEMU
@@ -119,4 +119,4 @@ jobs:
         run: |
           docker buildx imagetools create -t ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }} \
             ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }}-amd64 \
-            ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }}-arm64
+            ${{ env.IMAGE_NAME }}:${{ env.RELEASE_VERSION }}-arm64

.github/workflows/go.yml

@@ -10,7 +10,7 @@ jobs:
         if: ${{ !(contains(github.head_ref, 'ui/')) || !(contains(github.head_ref, 'cms/')) }}
         strategy:
             matrix:
-                go-version: [1.21.x]
+                go-version: [1.22.x]
                 os: [ubuntu-latest, macos-latest]
                 postgres-version: ["15"]
                 redis-version: ["6.2.6"]

Dockerfile.dev

@@ -6,7 +6,7 @@ RUN git config --global url."https://".insteadOf git://
 RUN npm install
 RUN npm run build
 
-FROM golang:1.21 as build-env
+FROM golang:1.22 as build-env
 WORKDIR /go/src/frain-dev/convoy
 
 COPY ./go.mod /go/src/frain-dev/convoy

cmd/hooks/hooks.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/frain-dev/convoy/internal/pkg/tracer"
 	"io"
 	"os"
 	"time"
@@ -32,7 +33,6 @@ import (
 	"github.com/frain-dev/convoy/datastore"
 	"github.com/frain-dev/convoy/internal/pkg/cli"
 	"github.com/frain-dev/convoy/internal/pkg/rdb"
-	"github.com/frain-dev/convoy/internal/pkg/tracer"
 	"github.com/frain-dev/convoy/internal/telemetry"
 	"github.com/frain-dev/convoy/pkg/log"
 	redisQueue "github.com/frain-dev/convoy/queue/redis"
@@ -82,11 +82,6 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [
 			return err
 		}
 
-		app.TracerShutdown, err = tracer.Init(cfg.Tracer, cmd.Name())
-		if err != nil {
-			return err
-		}
-
 		var ca cache.Cache
 		var q queue.Queuer
 
@@ -226,6 +221,15 @@ func PreRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args [
 			}
 		}
 
+		app.TracerBackend, err = tracer.Init(cfg.Tracer, cmd.Name(), app.Licenser)
+		if err != nil {
+			return err
+		}
+		if cfg.Tracer.Type == config.DatadogTracerProvider && !app.Licenser.DatadogTracing() {
+			lo.Error("your instance does not have access to datadog tracing, upgrade to access this feature")
+			_ = app.TracerBackend.Shutdown(context.Background())
+		}
+
 		return nil
 	}
 }
@@ -263,7 +267,7 @@ func PostRun(app *cli.App, db *postgres.Postgres) func(cmd *cobra.Command, args
 			os.Exit(0)
 		}
 
-		err = app.TracerShutdown(context.Background())
+		err = app.TracerBackend.Shutdown(context.Background())
 		if err == nil {
 			os.Exit(0)
 		}
@@ -590,6 +594,14 @@ func buildCliConfiguration(cmd *cobra.Command) (*config.Configuration, error) {
 				HeaderValue: headerValue,
 			},
 		}
+	case config.DatadogTracerProvider:
+		agentUrl, err := cmd.Flags().GetString("datadog-agent-url")
+		if err != nil {
+			return nil, err
+		}
+		c.Tracer.Datadog = config.DatadogConfiguration{
+			AgentURL: agentUrl,
+		}
 
 	case config.SentryTracerProvider:
 		dsn, err := cmd.Flags().GetString("sentry-dsn")

cmd/main.go

@@ -67,6 +67,7 @@ func main() {
 	var otelCollectorURL string
 	var otelAuthHeaderName string
 	var otelAuthHeaderValue string
+	var dataDogAgentUrl string
 	var metricsBackend string
 	var prometheusMetricsSampleTime uint64
 
@@ -120,6 +121,7 @@ func main() {
 	c.Flags().StringVar(&otelCollectorURL, "otel-collector-url", "", "OTel collector URL")
 	c.Flags().StringVar(&otelAuthHeaderName, "otel-auth-header-name", "", "OTel backend auth header name")
 	c.Flags().StringVar(&otelAuthHeaderValue, "otel-auth-header-value", "", "OTel backend auth header value")
+	c.Flags().StringVar(&dataDogAgentUrl, "datadog-agent-url", "", "Datadog agent URL")
 
 	// metrics
 	c.Flags().StringVar(&metricsBackend, "metrics-backend", "prometheus", "Metrics backend e.g. prometheus. ('prometheus' feature flag required")

cmd/worker/worker.go

@@ -327,6 +327,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte
 		attemptRepo,
 		circuitBreakerManager,
 		featureFlag,
+		a.TracerBackend,
 	), newTelemetry)
 
 	consumer.RegisterHandlers(convoy.CreateEventProcessor, task.ProcessEventCreation(
@@ -350,6 +351,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte
 		attemptRepo,
 		circuitBreakerManager,
 		featureFlag,
+		a.TracerBackend,
 	), newTelemetry)
 
 	consumer.RegisterHandlers(convoy.CreateBroadcastEventProcessor, task.ProcessBroadcastEventCreation(
@@ -405,7 +407,7 @@ func StartWorker(ctx context.Context, a *cli.App, cfg config.Configuration, inte
 	}
 
 	consumer.RegisterHandlers(convoy.NotificationProcessor, task.ProcessNotifications(sc), nil)
-	consumer.RegisterHandlers(convoy.MetaEventProcessor, task.ProcessMetaEvent(projectRepo, metaEventRepo, dispatcher), nil)
+	consumer.RegisterHandlers(convoy.MetaEventProcessor, task.ProcessMetaEvent(projectRepo, metaEventRepo, dispatcher, a.TracerBackend), nil)
 	consumer.RegisterHandlers(convoy.DeleteArchivedTasksProcessor, task.DeleteArchivedTasks(a.Queue, rd), nil)
 
 	metrics.RegisterQueueMetrics(a.Queue, a.DB, circuitBreakerManager)

config/config.go

@@ -250,9 +250,10 @@ type LoggerConfiguration struct {
 }
 
 type TracerConfiguration struct {
-	Type   TracerProvider      `json:"type" envconfig:"CONVOY_TRACER_PROVIDER"`
-	OTel   OTelConfiguration   `json:"otel"`
-	Sentry SentryConfiguration `json:"sentry"`
+	Type    TracerProvider       `json:"type" envconfig:"CONVOY_TRACER_PROVIDER"`
+	OTel    OTelConfiguration    `json:"otel"`
+	Sentry  SentryConfiguration  `json:"sentry"`
+	Datadog DatadogConfiguration `json:"datadog"`
 }
 
 type OTelConfiguration struct {
@@ -271,6 +272,10 @@ type SentryConfiguration struct {
 	DSN string `json:"dsn" envconfig:"CONVOY_SENTRY_DSN"`
 }
 
+type DatadogConfiguration struct {
+	AgentURL string `json:"agent_url" envconfig:"CONVOY_DATADOG_AGENT_URL"`
+}
+
 type RetentionPolicyConfiguration struct {
 	Policy                   string `json:"policy" envconfig:"CONVOY_RETENTION_POLICY"`
 	IsRetentionPolicyEnabled bool   `json:"enabled" envconfig:"CONVOY_RETENTION_POLICY_ENABLED"`
@@ -326,8 +331,9 @@ const (
 )
 
 const (
-	OTelTracerProvider   TracerProvider = "otel"
-	SentryTracerProvider TracerProvider = "sentry"
+	OTelTracerProvider    TracerProvider = "otel"
+	SentryTracerProvider  TracerProvider = "sentry"
+	DatadogTracerProvider TracerProvider = "datadog"
 )
 
 const (

go.mod

@@ -1,11 +1,12 @@
 module github.com/frain-dev/convoy
 
-go 1.21.0
+go 1.22.0
 
-toolchain go1.22.3
+toolchain go1.22.4
 
 require (
 	cloud.google.com/go/pubsub v1.33.0
+	github.com/DataDog/datadog-go/v5 v5.3.0
 	github.com/Subomi/go-authz v0.2.0
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
 	github.com/aws/aws-sdk-go v1.44.327
@@ -38,7 +39,7 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/keygen-sh/keygen-go/v3 v3.2.0
 	github.com/lib/pq v1.10.7
-	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/mattn/go-sqlite3 v1.14.18
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/mixpanel/mixpanel-go v1.2.1
 	github.com/newrelic/go-agent/v3 v3.20.4
@@ -72,8 +73,9 @@ require (
 	go.opentelemetry.io/otel/sdk v1.24.0
 	go.opentelemetry.io/otel/trace v1.24.0
 	go.uber.org/mock v0.4.0
-	golang.org/x/crypto v0.23.0
+	golang.org/x/crypto v0.25.0
 	google.golang.org/api v0.149.0
+	gopkg.in/DataDog/dd-trace-go.v1 v1.69.1
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 	gopkg.in/guregu/null.v4 v4.0.0
 )
@@ -83,6 +85,12 @@ require (
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AlecAivazis/survey/v2 v2.3.7 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/DataDog/appsec-internal-go v1.8.0 // indirect
+	github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 // indirect
+	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.57.0 // indirect
+	github.com/DataDog/go-libddwaf/v3 v3.4.0 // indirect
+	github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect
+	github.com/DataDog/sketches-go v1.4.5 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/Microsoft/hcsshim v0.11.7 // indirect
@@ -126,14 +134,16 @@ require (
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect
+	github.com/ebitengine/purego v0.6.0-alpha.5 // indirect
 	github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/fsnotify/fsevents v0.2.0 // indirect
 	github.com/fvbommel/sortorder v1.0.2 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-redis/redis/v7 v7.4.1 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/go-viper/mapstructure/v2 v2.0.0 // indirect
 	github.com/gofrs/flock v0.12.0 // indirect
@@ -152,9 +162,12 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect
+	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
-	github.com/in-toto/in-toto-golang v0.5.0 // indirect
+	github.com/in-toto/in-toto-golang v0.9.0 // indirect
 	github.com/invopop/yaml v0.3.1 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
@@ -167,7 +180,7 @@ require (
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mattn/go-shellwords v1.0.12 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
@@ -193,14 +206,17 @@ require (
 	github.com/oasisprotocol/curve25519-voi v0.0.0-20211102120939-d5a936accd94 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
+	github.com/outcaste-io/ristretto v0.2.3 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/perimeterx/marshmallow v1.1.5 // indirect
+	github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc // indirect
 	github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 // indirect
-	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
+	github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect
 	github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b // indirect
 	github.com/shibumi/go-pathspec v1.3.0 // indirect
 	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
@@ -210,6 +226,7 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 // indirect
+	github.com/tinylib/msgp v1.2.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c // indirect
@@ -233,8 +250,11 @@ require (
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect
-	golang.org/x/term v0.20.0 // indirect
+	golang.org/x/mod v0.18.0 // indirect
+	golang.org/x/term v0.22.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect
 	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
@@ -286,16 +306,16 @@ require (
 	github.com/sergi/go-diff v1.0.0 // indirect
 	github.com/spf13/cast v1.5.1 // indirect
 	github.com/vmihailenco/go-tinylfu v0.2.2 // indirect
-	github.com/vmihailenco/msgpack/v5 v5.3.5
+	github.com/vmihailenco/msgpack/v5 v5.4.1
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
+	golang.org/x/sys v0.23.0 // indirect
+	golang.org/x/text v0.16.0
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.17.0 // indirect
+	golang.org/x/tools v0.22.0 // indirect
 	google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 // indirect
 	google.golang.org/grpc v1.60.1
 	google.golang.org/protobuf v1.33.0 // indirect