added a connection ok check before returning connection to the pool (#11

)

* added a connection ok check before returning connection to the pool

Testing Done
- reproduced reconnect issue when connection closed by server by killing server process
- verified the fix

resolves #4

* check buffer flush error

* remove vendor

backend/pool.go

@@ -2,6 +2,7 @@ package backend
 
 import (
 	"errors"
+	"io"
 	"net"
 	"time"
 
@@ -87,8 +88,25 @@ func (cp *Pool) release(conn net.Conn) {
 	}
 }
 
+func (cp *Pool) connOk(conn net.Conn) bool {
+	if conn == nil {
+		return false
+	}
+	if err := conn.SetReadDeadline(time.Now().Add(2 * time.Millisecond)); err != nil {
+		return false
+	}
+	// you have to try reading atleast 1 byte to detect closed connection
+	b1 := make([]byte, 1)
+	_, err := conn.Read(b1)
+	if err != nil && err == io.EOF {
+		log.Info("Connection is closed")
+		return false
+	}
+	return true
+}
+
 func (cp *Pool) Return(conn net.Conn, failed bool) {
-	if failed {
+	if failed || !cp.connOk(conn) {
 		cp.release(conn)
 		return
 	}

backend/writer.go

@@ -45,7 +45,11 @@ func (w *MetricWriter) Write(rq prompb.WriteRequest) error {
 	fail := false
 	bw := bufio.NewWriter(out)
 	defer func() {
-		bw.Flush()
+		err := bw.Flush()
+		if err != nil {
+			log.Infof("failed to write to socket due to %v", err)
+			fail = true
+		}
 		w.pool.Return(out, fail)
 	}()
 	for _, ts := range rq.Timeseries {