diff --git a/http3/server.go b/http3/server.go
index ac0f71774d8..097a8005dfb 100644
--- a/http3/server.go
+++ b/http3/server.go
@@ -285,7 +285,7 @@ func (s *Server) init() {
 }
 
 func (s *Server) decreaseConnCount() {
-	if s.connCount.Add(-1) == 0 {
+	if s.connCount.Add(-1) == 0 && s.graceCtx.Err() != nil {
 		close(s.connHandlingDone)
 	}
 }
@@ -762,7 +762,7 @@ func (s *Server) Shutdown(ctx context.Context) error {
 	s.mutex.Unlock()
 
 	if s.connCount.Load() == 0 {
-		return nil
+		return s.Close()
 	}
 	select {
 	case <-s.connHandlingDone: // all connections were closed