Close zookeeper topo connection on disconnect (#17136)

Signed-off-by: shanth96 <shanth.sathiyaseelan@shopify.com>

go/vt/topo/zk2topo/zk_conn.go

@@ -277,6 +277,8 @@ func (c *ZkConn) withRetry(ctx context.Context, action func(conn *zk.Conn) error
 			c.conn = nil
 		}
 		c.mu.Unlock()
+		log.Infof("zk conn: got ErrConnectionClosed for addr %v: closing", c.addr)
+		conn.Close()
 	}
 	return
 }
@@ -327,23 +329,17 @@ func (c *ZkConn) maybeAddAuth(ctx context.Context) {
 // clears out the connection record.
 func (c *ZkConn) handleSessionEvents(conn *zk.Conn, session <-chan zk.Event) {
 	for event := range session {
-		closeRequired := false
 
 		switch event.State {
-		case zk.StateExpired, zk.StateConnecting:
-			closeRequired = true
-			fallthrough
-		case zk.StateDisconnected:
+		case zk.StateDisconnected, zk.StateExpired, zk.StateConnecting:
 			c.mu.Lock()
 			if c.conn == conn {
 				// The ZkConn still references this
 				// connection, let's nil it.
 				c.conn = nil
 			}
 			c.mu.Unlock()
-			if closeRequired {
-				conn.Close()
-			}
+			conn.Close()
 			log.Infof("zk conn: session for addr %v ended: %v", c.addr, event)
 			return
 		}

go/vt/topo/zk2topo/zk_conn_test.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package zk2topo
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/z-division/go-zookeeper/zk"
+
+	"vitess.io/vitess/go/testfiles"
+	"vitess.io/vitess/go/vt/zkctl"
+)
+
+func TestZkConnClosedOnDisconnect(t *testing.T) {
+	zkd, serverAddr := zkctl.StartLocalZk(testfiles.GoVtTopoZk2topoZkID, testfiles.GoVtTopoZk2topoPort)
+	defer zkd.Teardown()
+
+	conn := Connect(serverAddr)
+	defer conn.Close()
+
+	_, _, err := conn.Get(context.Background(), "/")
+	require.NoError(t, err, "Get() failed")
+
+	require.True(t, conn.conn.State().IsConnected(), "Connection not connected")
+
+	oldConn := conn.conn
+
+	// force a disconnect
+	zkd.Shutdown()
+	zkd.Start()
+
+	// do another get to trigger a new connection
+	_, _, err = conn.Get(context.Background(), "/")
+	require.NoError(t, err, "Get() failed")
+
+	// Check that old connection is closed
+	_, _, err = oldConn.Get("/")
+	require.ErrorContains(t, err, "zookeeper is closing")
+
+	require.Equal(t, zk.StateDisconnected, oldConn.State(), "Connection is not in disconnected state")
+}