Merge pull request #80 from loxilb-io/set-roles

PR: Faster roles selection on loxilb-lb down event
loxilb-io · Dec 4, 2023 · b2c16ce · b2c16ce
2 parents 2937b5d + dce702e
commit b2c16ce
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 17 deletions.
diff --git a/cmd/loxilb-agent/agent.go b/cmd/loxilb-agent/agent.go
@@ -155,10 +155,11 @@ func run(o *Options) error {
 	loxiLBLiveCh := make(chan *api.LoxiClient, 50)
 	loxiLBPurgeCh := make(chan *api.LoxiClient, 5)
 	loxiLBSelMasterEvent := make(chan bool)
+	loxiLBDeadCh := make(chan bool, 64)
 
 	if len(networkConfig.LoxilbURLs) > 0 {
 		for _, lbURL := range networkConfig.LoxilbURLs {
-			loxilbClient, err := api.NewLoxiClient(lbURL, loxiLBLiveCh, false)
+			loxilbClient, err := api.NewLoxiClient(lbURL, loxiLBLiveCh, loxiLBDeadCh, false)
 			if err != nil {
 				return err
 			}
@@ -179,13 +180,21 @@ func run(o *Options) error {
 	)
 
 	go wait.Until(func() {
-		if len(networkConfig.LoxilbURLs) <= 0 {
-			lbManager.DiscoverLoxiLBServices(loxiLBLiveCh, loxiLBPurgeCh)
-		}
-		lbManager.DiscoverLoxiLBPeerServices(loxiLBLiveCh, loxiLBPurgeCh)
+		select {
+		case <-loxiLBDeadCh:
+			if networkConfig.SetRoles != "" {
+				klog.Infof("Running select-roles")
+				lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent)
+			}
+		default:
+			if len(networkConfig.LoxilbURLs) <= 0 {
+				lbManager.DiscoverLoxiLBServices(loxiLBLiveCh, loxiLBDeadCh, loxiLBPurgeCh)
+			}
+			lbManager.DiscoverLoxiLBPeerServices(loxiLBLiveCh, loxiLBDeadCh, loxiLBPurgeCh)
 
-		if networkConfig.SetRoles != "" {
-			lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent)
+			if networkConfig.SetRoles != "" {
+				lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent)
+			}
 		}
 	}, time.Second*20, stopCh)
 

diff --git a/cmd/loxilb-agent/main.go b/cmd/loxilb-agent/main.go
@@ -37,7 +37,7 @@ func Execute() {
 		Long:  "loxilb-k8s",
 	}
 
-	client, err := api.NewLoxiClient("http://127.0.0.1:11111", nil, false)
+	client, err := api.NewLoxiClient("http://127.0.0.1:11111", nil, nil, false)
 	if err != nil {
 		return
 	}

diff --git a/pkg/agent/manager/loadbalancer/loadbalancer.go b/pkg/agent/manager/loadbalancer/loadbalancer.go
@@ -1243,7 +1243,7 @@ func (m *Manager) addIngress(service *corev1.Service, newIP net.IP) {
 		append(service.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{IP: newIP.String()})
 }
 
-func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBPurgeCh chan *api.LoxiClient) {
+func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBDeadCh chan bool, loxiLBPurgeCh chan *api.LoxiClient) {
 	var tmploxilbClients []*api.LoxiClient
 	// DNS lookup (not used now)
 	// ips, err := net.LookupIP("loxilb-lb-service")
@@ -1270,7 +1270,7 @@ func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, lox
 			}
 		}
 		if !found {
-			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, false)
+			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, loxiLBDeadCh, false)
 			if err2 != nil {
 				continue
 			}
@@ -1293,10 +1293,12 @@ func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, lox
 	m.LoxiClients = tmp
 }
 
-func (m *Manager) DiscoverLoxiLBPeerServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBPurgeCh chan *api.LoxiClient) {
+func (m *Manager) DiscoverLoxiLBPeerServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBDeadCh chan bool, loxiLBPurgeCh chan *api.LoxiClient) {
 	var tmploxilbPeerClients []*api.LoxiClient
 	ips, err := k8s.GetServiceEndPoints(m.kubeClient, "loxilb-peer-service", "kube-system")
-	klog.Infof("loxilb-peer-service end-points:  %v", ips)
+	if len(ips) > 0 {
+		klog.Infof("loxilb-peer-service end-points:  %v", ips)
+	}
 	if err != nil {
 		ips = []net.IP{}
 	}
@@ -1318,7 +1320,7 @@ func (m *Manager) DiscoverLoxiLBPeerServices(loxiLBAliveCh chan *api.LoxiClient,
 			}
 		}
 		if !found {
-			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, true)
+			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, loxiLBDeadCh, true)
 			if err2 != nil {
 				continue
 			}
@@ -1365,7 +1367,7 @@ func (m *Manager) SelectLoxiLBRoles(sendSigCh bool, loxiLBSelMasterEvent chan bo
 				if v.IsAlive {
 					v.MasterLB = true
 					selMaster = true
-					klog.Infof("loxilb-peer(%v) set-role master", v.Url)
+					klog.Infof("loxilb-lb(%v) set-role master", v.Url)
 				}
 			}
 			if selMaster {

diff --git a/pkg/api/client.go b/pkg/api/client.go
@@ -20,13 +20,14 @@ type LoxiClient struct {
 	Host        string
 	Port        string
 	IsAlive     bool
+	DeadSigTs   time.Time
 	DoBGPCfg    bool
 	Purge       bool
 	Stop        chan struct{}
 }
 
 // apiServer is string. what format? http://10.0.0.1 or 10.0.0.1
-func NewLoxiClient(apiServer string, aliveCh chan *LoxiClient, peerOnly bool) (*LoxiClient, error) {
+func NewLoxiClient(apiServer string, aliveCh chan *LoxiClient, deadCh chan bool, peerOnly bool) (*LoxiClient, error) {
 
 	client := &http.Client{}
 
@@ -57,23 +58,31 @@ func NewLoxiClient(apiServer string, aliveCh chan *LoxiClient, peerOnly bool) (*
 		Port:        port,
 		Stop:        stop,
 		PeeringOnly: peerOnly,
+		DeadSigTs:   time.Now(),
 	}
 
-	lc.StartLoxiHealthCheckChan(aliveCh)
+	lc.StartLoxiHealthCheckChan(aliveCh, deadCh)
 
 	klog.Infof("NewLoxiClient Created: %s", apiServer)
 
 	return lc, nil
 }
 
-func (l *LoxiClient) StartLoxiHealthCheckChan(aliveCh chan *LoxiClient) {
+func (l *LoxiClient) StartLoxiHealthCheckChan(aliveCh chan *LoxiClient, deadCh chan bool) {
 	l.IsAlive = false
 
 	go wait.Until(func() {
 		if _, err := l.HealthCheck().Get(context.Background(), ""); err != nil {
 			if l.IsAlive {
 				klog.Infof("LoxiHealthCheckChan: loxilb(%s) is down", l.RestClient.baseURL.String())
 				l.IsAlive = false
+				if time.Duration(time.Since(l.DeadSigTs).Seconds()) >= 3 && l.MasterLB {
+					klog.Infof("LoxiHealthCheckChan: master down")
+					l.DeadSigTs = time.Now()
+					deadCh <- true
+				} else {
+					l.DeadSigTs = time.Now()
+				}
 			}
 		} else {
 			if !l.IsAlive {