Add otel instrumentation to http/grpc/sql libraries.

Also add cli arg to the clis that use those libraries to enable exporting
metrics and traces. Docs under pkg/metrics. Default to off and also change
prometheus default to off.

Signed-off-by: Jeff Mendoza <jlm@jlm.name>

cmd/guaccollect/cmd/deps_dev.go

@@ -30,6 +30,7 @@ import (
 	"github.com/guacsec/guac/pkg/handler/collector"
 	"github.com/guacsec/guac/pkg/handler/collector/deps_dev"
 	"github.com/guacsec/guac/pkg/logging"
+	"github.com/guacsec/guac/pkg/metrics"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -53,6 +54,8 @@ type depsDevOptions struct {
 	publishToQueue bool
 	// sets artificial latency on the deps.dev collector (default to nil)
 	addedLatency *time.Duration
+	// enable otel
+	enableOtel bool
 }
 
 var depsDevCmd = &cobra.Command{
@@ -90,13 +93,24 @@ you have access to read and write to the respective blob store.`,
 			viper.GetInt("prometheus-port"),
 			viper.GetBool("publish-to-queue"),
 			viper.GetString("deps-dev-latency"),
+			viper.GetBool("enable-otel"),
 			args,
 		)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
 			_ = cmd.Help()
 			os.Exit(1)
 		}
+
+		var shutdown func(context.Context) error = func(context.Context) error { return nil }
+		if opts.enableOtel {
+			var err error
+			shutdown, err = metrics.SetupOTelSDK(ctx)
+			if err != nil {
+				logger.Fatalf("Error setting up Otel: %v", err)
+			}
+		}
+
 		// Register collector
 		depsDevCollector, err := deps_dev.NewDepsCollector(ctx, opts.dataSource, opts.poll, opts.retrieveDependencies, 30*time.Second, opts.addedLatency)
 		if err != nil {
@@ -117,6 +131,7 @@ you have access to read and write to the respective blob store.`,
 		}
 
 		initializeNATsandCollector(ctx, opts.pubsubAddr, opts.blobAddr, opts.publishToQueue)
+		shutdown(ctx)
 	},
 }
 
@@ -133,6 +148,7 @@ func validateDepsDevFlags(
 	prometheusPort int,
 	pubToQueue bool,
 	addedLatencyStr string,
+	enableOtel bool,
 	args []string,
 ) (depsDevOptions, error) {
 	var opts depsDevOptions
@@ -143,6 +159,7 @@ func validateDepsDevFlags(
 	opts.enablePrometheus = enablePrometheus
 	opts.prometheusPort = prometheusPort
 	opts.publishToQueue = pubToQueue
+	opts.enableOtel = enableOtel
 
 	if addedLatencyStr != "" {
 		addedLatency, err := time.ParseDuration(addedLatencyStr)

cmd/guaccollect/cmd/github.go

@@ -25,6 +25,7 @@ import (
 	csubclient "github.com/guacsec/guac/pkg/collectsub/client"
 	"github.com/guacsec/guac/pkg/collectsub/datasource/csubsource"
 	"github.com/guacsec/guac/pkg/collectsub/datasource/inmemsource"
+	"github.com/guacsec/guac/pkg/metrics"
 
 	"github.com/guacsec/guac/pkg/cli"
 
@@ -62,6 +63,8 @@ type githubOptions struct {
 	ownerRepoName string
 	// enable/disable message publish to queue
 	publishToQueue bool
+	// enable otel
+	enableOtel bool
 }
 
 var githubCmd = &cobra.Command{
@@ -102,13 +105,23 @@ you have access to read and write to the respective blob store.`,
 			viper.GetBool("use-csub"),
 			viper.GetBool("service-poll"),
 			viper.GetBool("publish-to-queue"),
+			viper.GetBool("enable-otel"),
 			args)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
 			_ = cmd.Help()
 			os.Exit(1)
 		}
 
+		var shutdown func(context.Context) error = func(context.Context) error { return nil }
+		if opts.enableOtel {
+			var err error
+			shutdown, err = metrics.SetupOTelSDK(ctx)
+			if err != nil {
+				logger.Fatalf("Error setting up Otel: %v", err)
+			}
+		}
+
 		// GITHUB_TOKEN is the default token name
 		ghc, err := githubclient.NewGithubClient(ctx, os.Getenv("GITHUB_TOKEN"))
 		if err != nil {
@@ -154,6 +167,7 @@ you have access to read and write to the respective blob store.`,
 		}
 
 		initializeNATsandCollector(ctx, opts.pubsubAddr, opts.blobAddr, opts.publishToQueue)
+		shutdown(ctx)
 	},
 }
 
@@ -169,6 +183,7 @@ func validateGithubFlags(
 	useCsub,
 	poll bool,
 	pubToQueue bool,
+	enableOtel bool,
 	args []string,
 ) (githubOptions, error) {
 	var opts githubOptions
@@ -179,6 +194,7 @@ func validateGithubFlags(
 	opts.sbomName = sbomName
 	opts.workflowFileName = workflowFileName
 	opts.publishToQueue = pubToQueue
+	opts.enableOtel = enableOtel
 
 	if useCsub {
 		csubOpts, err := csubclient.ValidateCsubClientFlags(csubAddr, csubTls, csubTlsSkipVerify)

cmd/guaccollect/cmd/license.go

@@ -30,6 +30,7 @@ import (
 	"github.com/guacsec/guac/pkg/certifier/components/root_package"
 	"github.com/guacsec/guac/pkg/cli"
 	"github.com/guacsec/guac/pkg/logging"
+	"github.com/guacsec/guac/pkg/metrics"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -58,6 +59,8 @@ type cdOptions struct {
 	// last time the scan was done in hours, if not set it will return
 	// all packages to check
 	lastScan *int
+	// enable otel
+	enableOtel bool
 }
 
 var cdCmd = &cobra.Command{
@@ -90,6 +93,7 @@ you have access to read and write to the respective blob store.`,
 			viper.GetString("certifier-latency"),
 			viper.GetInt("certifier-batch-size"),
 			viper.GetInt("last-scan"),
+			viper.GetBool("enable-otel"),
 		)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
@@ -100,6 +104,15 @@ you have access to read and write to the respective blob store.`,
 		ctx := logging.WithLogger(context.Background())
 		logger := logging.FromContext(ctx)
 
+		var shutdown func(context.Context) error = func(context.Context) error { return nil }
+		if opts.enableOtel {
+			var err error
+			shutdown, err = metrics.SetupOTelSDK(ctx)
+			if err != nil {
+				logger.Fatalf("Error setting up Otel: %v", err)
+			}
+		}
+
 		if err := certify.RegisterCertifier(clearlydefined.NewClearlyDefinedCertifier, certifier.CertifierClearlyDefined); err != nil {
 			logger.Fatalf("unable to register certifier: %v", err)
 		}
@@ -115,6 +128,7 @@ you have access to read and write to the respective blob store.`,
 		}
 
 		initializeNATsandCertifier(ctx, opts.blobAddr, opts.pubsubAddr, opts.poll, opts.publishToQueue, opts.interval, packageQueryFunc())
+		shutdown(ctx)
 	},
 }
 
@@ -134,7 +148,10 @@ func validateCDFlags(
 	poll bool,
 	pubToQueue bool,
 	certifierLatencyStr string,
-	batchSize int, lastScan int) (cdOptions, error) {
+	batchSize int,
+	lastScan int,
+	enableOtel bool,
+) (cdOptions, error) {
 
 	var opts cdOptions
 
@@ -144,6 +161,7 @@ func validateCDFlags(
 	opts.blobAddr = blobAddr
 	opts.poll = poll
 	opts.publishToQueue = pubToQueue
+	opts.enableOtel = enableOtel
 
 	i, err := time.ParseDuration(interval)
 	if err != nil {

cmd/guaccollect/cmd/osv.go

@@ -38,6 +38,7 @@ import (
 	"github.com/guacsec/guac/pkg/handler/collector"
 	"github.com/guacsec/guac/pkg/handler/processor"
 	"github.com/guacsec/guac/pkg/logging"
+	"github.com/guacsec/guac/pkg/metrics"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -68,6 +69,8 @@ type osvOptions struct {
 	lastScan *int
 	// adds metadata to vulnerabities during collection
 	addVulnMetadata bool
+	// enable otel
+	enableOtel bool
 }
 
 var osvCmd = &cobra.Command{
@@ -101,6 +104,7 @@ you have access to read and write to the respective blob store.`,
 			viper.GetInt("certifier-batch-size"),
 			viper.GetInt("last-scan"),
 			viper.GetBool("add-vuln-metadata"),
+			viper.GetBool("enable-otel"),
 		)
 		if err != nil {
 			fmt.Printf("unable to validate flags: %v\n", err)
@@ -111,6 +115,15 @@ you have access to read and write to the respective blob store.`,
 		ctx := logging.WithLogger(context.Background())
 		logger := logging.FromContext(ctx)
 
+		var shutdown func(context.Context) error = func(context.Context) error { return nil }
+		if opts.enableOtel {
+			var err error
+			shutdown, err = metrics.SetupOTelSDK(ctx)
+			if err != nil {
+				logger.Fatalf("Error setting up Otel: %v", err)
+			}
+		}
+
 		if err := certify.RegisterCertifier(func() certifier.Certifier {
 			certifierOpts := []osv.CertifierOpts{}
 			if opts.addVulnMetadata {
@@ -132,6 +145,7 @@ you have access to read and write to the respective blob store.`,
 		}
 
 		initializeNATsandCertifier(ctx, opts.blobAddr, opts.pubsubAddr, opts.poll, opts.publishToQueue, opts.interval, packageQueryFunc())
+		shutdown(ctx)
 	},
 }
 
@@ -146,6 +160,7 @@ func validateOSVFlags(
 	certifierLatencyStr string,
 	batchSize int, lastScan int,
 	addVulnMetadata bool,
+	enableOtel bool,
 ) (osvOptions, error) {
 	var opts osvOptions
 
@@ -156,6 +171,7 @@ func validateOSVFlags(
 	opts.poll = poll
 	opts.publishToQueue = pubToQueue
 	opts.addVulnMetadata = addVulnMetadata
+	opts.enableOtel = enableOtel
 
 	i, err := time.ParseDuration(interval)
 	if err != nil {

cmd/guaccollect/cmd/root.go

@@ -40,6 +40,7 @@ func init() {
 		"enable-prometheus",
 		"publish-to-queue",
 		"gql-addr",
+		"enable-otel",
 	})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)

cmd/guacgql/cmd/root.go

@@ -35,6 +35,7 @@ var flags = struct {
 	tlsKeyFile  string
 	debug       bool
 	tracegql    bool
+	enableOtel  bool
 }{}
 
 var rootCmd = &cobra.Command{
@@ -48,6 +49,7 @@ var rootCmd = &cobra.Command{
 		flags.tlsKeyFile = viper.GetString("gql-tls-key-file")
 		flags.debug = viper.GetBool("gql-debug")
 		flags.tracegql = viper.GetBool("gql-trace")
+		flags.enableOtel = viper.GetBool("enable-otel")
 
 		startServer(cmd)
 	},
@@ -65,6 +67,7 @@ func init() {
 		"gql-backend",
 		"gql-trace",
 		"enable-prometheus",
+		"enable-otel",
 	})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)

cmd/guacgql/cmd/server.go

@@ -26,22 +26,23 @@ import (
 	"syscall"
 	"time"
 
-	// import all known backends
-	_ "github.com/guacsec/guac/pkg/assembler/backends/neo4j"
-	_ "github.com/guacsec/guac/pkg/assembler/backends/neptune"
-	_ "github.com/guacsec/guac/pkg/assembler/backends/ent/backend"
-	_ "github.com/guacsec/guac/pkg/assembler/backends/keyvalue"
-	_ "github.com/guacsec/guac/pkg/assembler/backends/arangodb"
-
 	"github.com/99designs/gqlgen/graphql/handler/debug"
 	"github.com/99designs/gqlgen/graphql/playground"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+
 	"github.com/guacsec/guac/pkg/assembler/backends"
+	// import all known backends
+	_ "github.com/guacsec/guac/pkg/assembler/backends/arangodb"
+	_ "github.com/guacsec/guac/pkg/assembler/backends/ent/backend"
+	_ "github.com/guacsec/guac/pkg/assembler/backends/keyvalue"
+	_ "github.com/guacsec/guac/pkg/assembler/backends/neo4j"
+	_ "github.com/guacsec/guac/pkg/assembler/backends/neptune"
 	"github.com/guacsec/guac/pkg/assembler/server"
 	"github.com/guacsec/guac/pkg/logging"
 	"github.com/guacsec/guac/pkg/metrics"
 	"github.com/guacsec/guac/pkg/version"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 func startServer(cmd *cobra.Command) {
@@ -80,6 +81,16 @@ func startServer(cmd *cobra.Command) {
 		srvHandler = srv
 	}
 
+	var shutdown func(context.Context) error = func(context.Context) error { return nil }
+	if flags.enableOtel {
+		var err error
+		shutdown, err = metrics.SetupOTelSDK(ctx)
+		if err != nil {
+			logger.Fatalf("Error setting up Otel: %v", err)
+		}
+		srvHandler = otelhttp.NewHandler(srvHandler, "/")
+	}
+
 	if flags.tracegql {
 		tracer := &debug.Tracer{}
 		srv.Use(tracer)
@@ -115,6 +126,7 @@ func startServer(cmd *cobra.Command) {
 	ctx, cf := context.WithCancel(ctx)
 	go func() {
 		_ = server.Shutdown(ctx)
+		_ = shutdown(ctx)
 		done <- true
 	}()
 	select {