diff --git a/internal/common/grpc/grpc.go b/internal/common/grpc/grpc.go index 8a3b053b14c..ee1de497bb5 100644 --- a/internal/common/grpc/grpc.go +++ b/internal/common/grpc/grpc.go @@ -1,13 +1,18 @@ package grpc import ( + "runtime/debug" "time" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" + grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + log "github.com/sirupsen/logrus" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/status" "github.com/G-Research/armada/internal/armada/authorization" ) @@ -24,6 +29,10 @@ func CreateGrpcServer(authServices []authorization.AuthService) *grpc.Server { unaryInterceptors = append(unaryInterceptors, grpc_prometheus.UnaryServerInterceptor) streamInterceptors = append(streamInterceptors, grpc_prometheus.StreamServerInterceptor) + recovery := grpc_recovery.WithRecoveryHandler(panicRecoveryHandler) + unaryInterceptors = append(unaryInterceptors, grpc_recovery.UnaryServerInterceptor(recovery)) + streamInterceptors = append(streamInterceptors, grpc_recovery.StreamServerInterceptor(recovery)) + return grpc.NewServer( grpc.KeepaliveParams(keepalive.ServerParameters{ MaxConnectionIdle: 5 * time.Minute, @@ -31,3 +40,8 @@ func CreateGrpcServer(authServices []authorization.AuthService) *grpc.Server { grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(streamInterceptors...)), grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptors...))) } + +func panicRecoveryHandler(p interface{}) (err error) { + log.Errorf("Request triggered panic with cause %v \n%s", p, string(debug.Stack())) + return status.Errorf(codes.Internal, "Internal server error caused by %v", p) +}