chore: add error msg in the logs when gw req fails (#5369)

* chore: add error msg in the logs when gw req fails * addressed comments * addressed comments * update go version and schema version * update go version in docker file * downgrade swagger-editor-validate * update action to apidom-validate
rudderlabs · Dec 18, 2024 · a0ef0f1 · a0ef0f1
1 parent ab867a5
commit a0ef0f1
Show file tree

Hide file tree

Showing 6 changed files with 38 additions and 13 deletions.
diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
@@ -51,7 +51,7 @@ jobs:
         if: ${{ failure() }}
         run: echo 'Not formatted files. Ensure you have run `make fmt` and committed the files locally.'
       - name: Validate OpenAPI definition
-        uses: char0n/swagger-editor-validate@v1
+        uses: char0n/apidom-validate@v1
         with:
           definition-file: gateway/openapi.yaml
 

diff --git a/Dockerfile b/Dockerfile
@@ -2,7 +2,7 @@
 # syntax=docker/dockerfile:1
 
 # GO_VERSION is updated automatically to match go.mod, see Makefile
-ARG GO_VERSION=1.23.3
+ARG GO_VERSION=1.23.4
 ARG ALPINE_VERSION=3.20
 FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS builder
 ARG VERSION

diff --git a/gateway/handle.go b/gateway/handle.go
@@ -709,11 +709,11 @@ func (gw *Handle) internalBatchHandlerFunc() http.HandlerFunc {
 		errorMessage = err.Error()
 		status = response.GetErrorStatusCode(errorMessage)
 		responseBody = response.GetStatus(errorMessage)
-		gw.logger.Infon("response",
+		gw.logger.Errorn("response",
 			logger.NewStringField("ip", kithttputil.GetRequestIP(r)),
 			logger.NewStringField("path", r.URL.Path),
 			logger.NewIntField("status", int64(status)),
-			logger.NewStringField("body", responseBody),
+			obskit.Error(err),
 		)
 		gw.logger.Debugn("response",
 			logger.NewStringField("ip", kithttputil.GetRequestIP(r)),
@@ -763,13 +763,16 @@ func (gw *Handle) extractJobsFromInternalBatchPayload(reqType string, body []byt
 	if err != nil {
 		stat.RequestFailed(response.InvalidJSON)
 		stat.Report(gw.stats)
+		gw.logger.Errorn("invalid json in request",
+			obskit.Error(err))
 		return nil, errors.New(response.InvalidJSON)
 	}
 	gw.requestSizeStat.Observe(float64(len(body)))
 
 	if len(messages) == 0 {
 		stat.RequestFailed(response.NotRudderEvent)
 		stat.Report(gw.stats)
+		gw.logger.Errorn("no messages in request")
 		return nil, errors.New(response.NotRudderEvent)
 	}
 
@@ -779,7 +782,10 @@ func (gw *Handle) extractJobsFromInternalBatchPayload(reqType string, body []byt
 		stat := gwstats.SourceStat{ReqType: reqType}
 		err := gw.streamMsgValidator(&msg)
 		if err != nil {
-			gw.logger.Errorn("invalid message in request", logger.NewErrorField(err))
+			loggerFields := msg.Properties.LoggerFields()
+			loggerFields = append(loggerFields, obskit.Error(err))
+			gw.logger.Errorn("invalid message in request",
+				loggerFields...)
 			stat.RequestEventsFailed(1, response.InvalidStreamMessage)
 			stat.Report(gw.stats)
 			return nil, errors.New(response.InvalidStreamMessage)
@@ -793,6 +799,9 @@ func (gw *Handle) extractJobsFromInternalBatchPayload(reqType string, body []byt
 				if err != nil {
 					stat.RequestEventsFailed(1, response.NotRudderEvent)
 					stat.Report(gw.stats)
+					loggerFields := msg.Properties.LoggerFields()
+					loggerFields = append(loggerFields, obskit.Error(err))
+					gw.logger.Errorn("failed to set type in message", loggerFields...)
 					return nil, errors.New(response.NotRudderEvent)
 				}
 			}
@@ -806,19 +815,27 @@ func (gw *Handle) extractJobsFromInternalBatchPayload(reqType string, body []byt
 			if err != nil {
 				stat.RequestFailed(response.NotRudderEvent)
 				stat.Report(gw.stats)
+				gw.logger.Errorn("failed to set messageID in message",
+					obskit.Error(err))
 				return nil, errors.New(response.NotRudderEvent)
 			}
 		}
 		rudderId, err := getRudderId(userIDFromReq, anonIDFromReq)
 		if err != nil {
 			stat.RequestFailed(response.NotRudderEvent)
 			stat.Report(gw.stats)
+			gw.logger.Errorn("failed to get rudderId",
+				obskit.Error(err))
 			return nil, errors.New(response.NotRudderEvent)
 		}
 		msg.Payload, err = sjson.SetBytes(msg.Payload, "rudderId", rudderId.String())
 		if err != nil {
 			stat.RequestFailed(response.NotRudderEvent)
 			stat.Report(gw.stats)
+			loggerFields := msg.Properties.LoggerFields()
+			loggerFields = append(loggerFields, obskit.Error(err))
+			gw.logger.Errorn("failed to set rudderId in message",
+				loggerFields...)
 			return nil, errors.New(response.NotRudderEvent)
 		}
 		writeKey, ok := gw.getWriteKeyFromSourceID(msg.Properties.SourceID)
@@ -877,13 +894,17 @@ func (gw *Handle) extractJobsFromInternalBatchPayload(reqType string, body []byt
 			err = fmt.Errorf("filling receivedAt: %w", err)
 			stat.RequestEventsFailed(1, err.Error())
 			stat.Report(gw.stats)
+			gw.logger.Errorn("failed to fill receivedAt in message",
+				obskit.Error(err))
 			return nil, fmt.Errorf("filling receivedAt: %w", err)
 		}
 		msg.Payload, err = fillRequestIP(msg.Payload, msg.Properties.RequestIP)
 		if err != nil {
 			err = fmt.Errorf("filling request_ip: %w", err)
 			stat.RequestEventsFailed(1, err.Error())
 			stat.Report(gw.stats)
+			gw.logger.Errorn("failed to fill request_ip in message",
+				obskit.Error(err))
 			return nil, fmt.Errorf("filling request_ip: %w", err)
 		}
 
@@ -899,6 +920,10 @@ func (gw *Handle) extractJobsFromInternalBatchPayload(reqType string, body []byt
 			err = fmt.Errorf("marshalling event batch: %w", err)
 			stat.RequestEventsFailed(1, err.Error())
 			stat.Report(gw.stats)
+			loggerFields := msg.Properties.LoggerFields()
+			loggerFields = append(loggerFields, obskit.Error(err))
+			gw.logger.Errorn("failed to marshal event batch",
+				loggerFields...)
 			return nil, fmt.Errorf("marshalling event batch: %w", err)
 		}
 		jobUUID := uuid.New()

diff --git a/go.mod b/go.mod
@@ -1,6 +1,6 @@
 module github.com/rudderlabs/rudder-server
 
-go 1.23.3
+go 1.23.4
 
 // Addressing snyk vulnerabilities in indirect dependencies
 // When upgrading a dependency, please make sure that
@@ -78,7 +78,7 @@ require (
 	github.com/rudderlabs/compose-test v0.1.3
 	github.com/rudderlabs/rudder-go-kit v0.45.0
 	github.com/rudderlabs/rudder-observability-kit v0.0.3
-	github.com/rudderlabs/rudder-schemas v0.5.3
+	github.com/rudderlabs/rudder-schemas v0.5.4
 	github.com/rudderlabs/rudder-transformer/go v0.0.0-20240910055720-f77d2ab4125a
 	github.com/rudderlabs/sql-tunnels v0.1.7
 	github.com/rudderlabs/sqlconnect-go v1.13.0
@@ -214,7 +214,7 @@ require (
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.22.1 // indirect
+	github.com/go-playground/validator/v10 v10.23.0 // indirect
 	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.1.0 // indirect

diff --git a/go.sum b/go.sum
@@ -587,8 +587,8 @@ github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/Nu
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
 github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
-github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA=
-github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o=
+github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
 github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
@@ -1175,8 +1175,8 @@ github.com/rudderlabs/rudder-go-kit v0.45.0 h1:y8ModVsl2rAdqnqv/di82cddkaEBT1qZv
 github.com/rudderlabs/rudder-go-kit v0.45.0/go.mod h1:NrHCi0KSzHSMFXQu0t2kgJcE4ClAKklVXfb2glADvQ4=
 github.com/rudderlabs/rudder-observability-kit v0.0.3 h1:vZtuZRkGX+6rjaeKtxxFE2YYP6QlmAcVcgecTOjvz+Q=
 github.com/rudderlabs/rudder-observability-kit v0.0.3/go.mod h1:6UjAh3H6rkE0fFLh7z8ZGQEQbKtUkRfhWOf/OUhfqW8=
-github.com/rudderlabs/rudder-schemas v0.5.3 h1:IWWjAo2TzsjwHNhS2EAr1+0MjvA8BoTpJvB2o/GFwNU=
-github.com/rudderlabs/rudder-schemas v0.5.3/go.mod h1:iUpjG/Zb+ioZcNLvXNYXSKQ2LpPlsIDBfxfCDH9ue/E=
+github.com/rudderlabs/rudder-schemas v0.5.4 h1:QzI6vIC38W0jGJu6E0vdwh4IAtSKx8ziqff+JL0xmIE=
+github.com/rudderlabs/rudder-schemas v0.5.4/go.mod h1:oypQtew1d2jBGJdFMT4zf1XOvXgGEcNPtRFy6vRDAv8=
 github.com/rudderlabs/rudder-transformer/go v0.0.0-20240910055720-f77d2ab4125a h1:OZcvpApxEYNkB9UNXrKDUBufQ24Lsr2Cs0pw70tzXBw=
 github.com/rudderlabs/rudder-transformer/go v0.0.0-20240910055720-f77d2ab4125a/go.mod h1:3NGitPz4pYRRZ6Xt09S+8hb0tHK/9pZcKJ3OgOTaSmE=
 github.com/rudderlabs/sql-tunnels v0.1.7 h1:wDCRl6zY4M5gfWazf7XkSTGQS3yjBzUiUgEMBIfHNDA=

diff --git a/suppression-backup-service/Dockerfile b/suppression-backup-service/Dockerfile
@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1
 
 # GO_VERSION is updated automatically to match go.mod, see Makefile
-ARG GO_VERSION=1.23.3
+ARG GO_VERSION=1.23.4
 ARG ALPINE_VERSION=3.20
 FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS builder
 RUN mkdir /app