diff --git a/internal/globalratelimit/runner/runner.go b/internal/globalratelimit/runner/runner.go
index b8615cbeb83..4d4c41b4feb 100644
--- a/internal/globalratelimit/runner/runner.go
+++ b/internal/globalratelimit/runner/runner.go
@@ -81,7 +81,7 @@ func (r *Runner) Start(ctx context.Context) error {
 	discoveryv3.RegisterAggregatedDiscoveryServiceServer(r.grpc, serverv3.NewServer(ctx, r.cache, cb))
 
 	// Start and listen xDS gRPC config Server.
-	go r.serverXdsConfigServer(ctx)
+	go r.serveXdsConfigServer(ctx)
 
 	// Start message Subscription.
 	go r.subscribeAndTranslate(ctx)
@@ -90,20 +90,23 @@ func (r *Runner) Start(ctx context.Context) error {
 	return nil
 }
 
-func (r *Runner) serverXdsConfigServer(ctx context.Context) {
+func (r *Runner) serveXdsConfigServer(ctx context.Context) {
 	addr := net.JoinHostPort(XdsGrpcSotwConfigServerAddress, strconv.Itoa(ratelimit.XdsGrpcSotwConfigServerPort))
 	l, err := net.Listen("tcp", addr)
 	if err != nil {
 		r.Logger.Error(err, "failed to listen on address", "address", addr)
 		return
 	}
+
+	go func() {
+		<-ctx.Done()
+		r.Logger.Info("grpc server shutting down")
+		r.grpc.Stop()
+	}()
+
 	if err = r.grpc.Serve(l); err != nil {
 		r.Logger.Error(err, "failed to start grpc based xds config server")
 	}
-
-	<-ctx.Done()
-	r.Logger.Info("grpc config server shutting down")
-	r.grpc.Stop()
 }
 
 func (r *Runner) subscribeAndTranslate(ctx context.Context) {
diff --git a/internal/xds/server/runner/runner.go b/internal/xds/server/runner/runner.go
index ee285c133f2..af9117c8841 100644
--- a/internal/xds/server/runner/runner.go
+++ b/internal/xds/server/runner/runner.go
@@ -102,18 +102,20 @@ func (r *Runner) serveXdsServer(ctx context.Context) {
 		r.Logger.Error(err, "failed to listen on address", "address", addr)
 		return
 	}
-	err = r.grpc.Serve(l)
-	if err != nil {
+
+	go func() {
+		<-ctx.Done()
+		r.Logger.Info("grpc server shutting down")
+		// We don't use GracefulStop here because envoy
+		// has long-lived hanging xDS requests. There's no
+		// mechanism to make those pending requests fail,
+		// so we forcibly terminate the TCP sessions.
+		r.grpc.Stop()
+	}()
+
+	if err = r.grpc.Serve(l); err != nil {
 		r.Logger.Error(err, "failed to start grpc based xds server")
 	}
-
-	<-ctx.Done()
-	r.Logger.Info("grpc server shutting down")
-	// We don't use GracefulStop here because envoy
-	// has long-lived hanging xDS requests. There's no
-	// mechanism to make those pending requests fail,
-	// so we forcibly terminate the TCP sessions.
-	r.grpc.Stop()
 }
 
 // registerServer registers the given xDS protocol Server with the gRPC