feat: add default sampling rate to mlobs logger

api/pkg/inference-logger/logger/mlobs_sink.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"golang.org/x/exp/rand"
 
 	upiv1 "github.com/caraml-dev/universal-prediction-interface/gen/go/grpc/caraml/upi/v1"
 	"github.com/confluentinc/confluent-kafka-go/v2/kafka"
@@ -17,6 +18,10 @@ var (
 	ErrMalformedLogEntry = errors.New("malformed log entry")
 )
 
+const (
+	SamplingRate = 0.02
+)
+
 type MLObsSink struct {
 	logger   *zap.SugaredLogger
 	producer KafkaProducer
@@ -160,17 +165,19 @@ func (m *MLObsSink) buildNewKafkaMessage(predictionLog *upiv1.PredictionLog) (*k
 
 func (m *MLObsSink) Sink(rawLogEntries []*LogEntry) error {
 	for _, rawLogEntry := range rawLogEntries {
-		predictionLog, err := m.newPredictionLog(rawLogEntry)
-		if err != nil {
-			m.logger.Errorf("unable to convert log entry: %v", err)
-		}
-		kafkaMessage, err := m.buildNewKafkaMessage(predictionLog)
-		if err != nil {
-			m.logger.Errorf("unable to build kafka message: %v", err)
-		}
-		err = m.producer.Produce(kafkaMessage, m.producer.Events())
-		if err != nil {
-			m.logger.Errorf("unable to produce kafka message: %v", err)
+		if rand.Float64() <= SamplingRate {
+			predictionLog, err := m.newPredictionLog(rawLogEntry)
+			if err != nil {
+				m.logger.Errorf("unable to convert log entry: %v", err)
+			}
+			kafkaMessage, err := m.buildNewKafkaMessage(predictionLog)
+			if err != nil {
+				m.logger.Errorf("unable to build kafka message: %v", err)
+			}
+			err = m.producer.Produce(kafkaMessage, m.producer.Events())
+			if err != nil {
+				m.logger.Errorf("unable to produce kafka message: %v", err)
+			}
 		}
 	}
 	return nil