ff

src/unit/test_networking.c

@@ -1,4 +1,5 @@
 #include <stdatomic.h>
+#include <limits.h>
 
 #include "../networking.c"
 #include "../server.c"

tests/unit/binbin.tcl

@@ -0,0 +1,74 @@
+proc cluster_get_first_node_in_handshake id {
+    set nodes [get_cluster_nodes $id]
+    foreach n $nodes {
+        if {[cluster_has_flag $n handshake]} {
+            return [dict get $n id]
+        }
+    }
+    return {}
+}
+
+start_cluster 2 0 {tags {external:skip cluster} overrides {loglevel debug cluster-node-timeout 4000 cluster-replica-no-failover yes}} {
+    test "Partial node timeout with two shards" {
+        set cluster_port [find_available_port $::baseport $::portcount]
+        start_server [list overrides [list loglevel debug  cluster-enabled yes cluster-node-timeout 4000 cluster-port $cluster_port]] {
+            # In this test we will trigger a handshake timeout on one side of the handshake.
+            # Node 1 and 2 already know each other, then we make node 0 meet node 1:
+            #
+            # Node 0 -- MEET -> Node 1
+            # Node 0 <- PONG -- Node 1
+            # Node 0 <- PING -- Node 1 [Node 0 will mark the handshake as successful in the above PONG]
+            # Node 0 -- PONG -> Node 1 [we drop this message, so node 1 will eventually mark the handshake as timed out]
+
+            # Drop PONG msgs
+            set CLUSTER_PACKET_TYPE_PONG 1
+            R 1 DEBUG DROP-CLUSTER-PACKET-FILTER $CLUSTER_PACKET_TYPE_PONG
+
+            # Node 0 meets node 1
+            R 0 CLUSTER MEET [srv -1 host] [srv -1 port]
+
+            # Wait for node 0 to know about the other nodes in the cluster
+            wait_for_condition 50 100 {
+                [llength [get_cluster_nodes 0]] == 3
+            } else {
+                fail "Node 0 never learned about node 1 and 2"
+            }
+            # At this point, node 0 learned about the other nodes in the cluster from meeting node 1.
+            wait_for_condition 50 100 {
+                [cluster_get_first_node_in_handshake 0] eq {}
+            } else {
+                fail "Node 1 never exited handshake state"
+            }
+            # At this point, from node 0 point of view, the handshake with node 1 succeeded.
+
+            puts "====== cluster nodes"
+            puts [R 0 CLUSTER NODES]
+            puts [R 1 CLUSTER NODES]
+            puts [R 2 CLUSTER NODES]
+            puts "======"
+
+            wait_for_condition 50 100 {
+                [cluster_get_first_node_in_handshake 1] eq {}
+            } else {
+                fail "Node 1 never exited handshake state"
+            }
+            assert {[llength [R 1 CLUSTER NODES]] == 18}
+            # At this point, from node 1 point of view, the handshake with node 0 timed out.
+
+            # Allow all msgs
+            R 1 DEBUG DROP-CLUSTER-PACKET-FILTER -1
+
+            after 1000
+            puts "Nodes after 1 second"
+            puts [R 0 CLUSTER NODES]
+            puts [R 1 CLUSTER NODES]
+            puts [R 2 CLUSTER NODES]
+
+            after 10000
+            puts "Nodes after 10 seconds"
+            puts [R 0 CLUSTER NODES]
+            puts [R 1 CLUSTER NODES]
+            puts [R 2 CLUSTER NODES]
+        }
+    }
+}