Merge branch 'main' of github.com:backguynn/test-kube-loxilb into main

README.md

@@ -98,6 +98,8 @@ metadata:
    # Specify number of secondary networks for multi-homing
    # Only valid for SCTP currently
    # loxilb.io/num-secondary-networks: "2
+   # Specify a static externalIP for this service
+   # loxilb.io/staticIP: "123.123.123.2"
 spec:
   loadBalancerClass: loxilb.io/loxilb
   selector:

cmd/loxilb-agent/agent.go

@@ -155,10 +155,11 @@ func run(o *Options) error {
 	loxiLBLiveCh := make(chan *api.LoxiClient, 50)
 	loxiLBPurgeCh := make(chan *api.LoxiClient, 5)
 	loxiLBSelMasterEvent := make(chan bool)
+	loxiLBDeadCh := make(chan bool, 64)
 
 	if len(networkConfig.LoxilbURLs) > 0 {
 		for _, lbURL := range networkConfig.LoxilbURLs {
-			loxilbClient, err := api.NewLoxiClient(lbURL, loxiLBLiveCh, false)
+			loxilbClient, err := api.NewLoxiClient(lbURL, loxiLBLiveCh, loxiLBDeadCh, false)
 			if err != nil {
 				return err
 			}
@@ -179,12 +180,21 @@ func run(o *Options) error {
 	)
 
 	go wait.Until(func() {
-		if len(networkConfig.LoxilbURLs) <= 0 {
-			lbManager.DiscoverLoxiLBServices(loxiLBLiveCh, loxiLBPurgeCh)
-		}
+		select {
+		case <-loxiLBDeadCh:
+			if networkConfig.SetRoles != "" {
+				klog.Infof("Running select-roles")
+				lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent)
+			}
+		default:
+			if len(networkConfig.LoxilbURLs) <= 0 {
+				lbManager.DiscoverLoxiLBServices(loxiLBLiveCh, loxiLBDeadCh, loxiLBPurgeCh)
+			}
+			lbManager.DiscoverLoxiLBPeerServices(loxiLBLiveCh, loxiLBDeadCh, loxiLBPurgeCh)
 
-		if networkConfig.SetRoles != "" {
-			lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent)
+			if networkConfig.SetRoles != "" {
+				lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent)
+			}
 		}
 	}, time.Second*20, stopCh)
 

cmd/loxilb-agent/main.go

@@ -37,7 +37,7 @@ func Execute() {
 		Long:  "loxilb-k8s",
 	}
 
-	client, err := api.NewLoxiClient("http://127.0.0.1:11111", nil, false)
+	client, err := api.NewLoxiClient("http://127.0.0.1:11111", nil, nil, false)
 	if err != nil {
 		return
 	}

pkg/agent/manager/loadbalancer/loadbalancer.go

@@ -56,6 +56,7 @@ const (
 	LoxiMultusServiceAnnotation = "loxilb.io/multus-nets"
 	numSecIPAnnotation          = "loxilb.io/num-secondary-networks"
 	secIPsAnnotation            = "loxilb.io/secondaryIPs"
+	staticIPAnnotation          = "loxilb.io/staticIP"
 	livenessAnnotation          = "loxilb.io/liveness"
 	lbModeAnnotation            = "loxilb.io/lbmode"
 	lbAddressAnnotation         = "loxilb.io/ipam"
@@ -706,7 +707,7 @@ func (m *Manager) addLoadBalancer(svc *corev1.Service) error {
 			return fmt.Errorf("failed to add loxiLB loadBalancer")
 		}
 		m.lbCache[cacheKey].LbModelList = append(m.lbCache[cacheKey].LbModelList, lbModelList...)
-		if ingSvcPair.InRange && !ingSvcPair.StaticIP {
+		if ingSvcPair.InRange || ingSvcPair.StaticIP {
 			retIngress := corev1.LoadBalancerIngress{Hostname: "llb-" + ingSvcPair.IPString}
 			//retIngress.Ports = append(retIngress.Ports, corev1.PortStatus{Port: ingSvcPair.Port, Protocol: corev1.Protocol(strings.ToUpper(ingSvcPair.Protocol))})
 			svc.Status.LoadBalancer.Ingress = append(svc.Status.LoadBalancer.Ingress, retIngress)
@@ -948,6 +949,18 @@ func (m *Manager) getLBIngressSvcPairs(service *corev1.Service) []SvcPair {
 		}
 	}
 
+	// Check for loxilb specific annotations - StaticIP (user specified)
+	if staticIPStr := service.Annotations[staticIPAnnotation]; staticIPStr != "" {
+		if net.ParseIP(staticIPStr) == nil {
+			klog.Errorf("%s annotation has invalid IP (%s)", staticIPAnnotation, staticIPStr)
+		} else {
+			for _, port := range service.Spec.Ports {
+				sp := SvcPair{staticIPStr, port.Port, strings.ToLower(string(port.Protocol)), false, true, "", port}
+				spairs = append(spairs, sp)
+			}
+		}
+	}
+
 	return spairs
 }
 
@@ -1230,7 +1243,7 @@ func (m *Manager) addIngress(service *corev1.Service, newIP net.IP) {
 		append(service.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{IP: newIP.String()})
 }
 
-func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBPurgeCh chan *api.LoxiClient) {
+func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBDeadCh chan bool, loxiLBPurgeCh chan *api.LoxiClient) {
 	var tmploxilbClients []*api.LoxiClient
 	// DNS lookup (not used now)
 	// ips, err := net.LookupIP("loxilb-lb-service")
@@ -1257,7 +1270,7 @@ func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, lox
 			}
 		}
 		if !found {
-			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, false)
+			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, loxiLBDeadCh, false)
 			if err2 != nil {
 				continue
 			}
@@ -1278,10 +1291,14 @@ func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, lox
 		}
 	}
 	m.LoxiClients = tmp
+}
 
+func (m *Manager) DiscoverLoxiLBPeerServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBDeadCh chan bool, loxiLBPurgeCh chan *api.LoxiClient) {
 	var tmploxilbPeerClients []*api.LoxiClient
-	ips, err = k8s.GetServiceEndPoints(m.kubeClient, "loxilb-peer-service", "kube-system")
-	klog.Infof("loxilb-peer-service end-points:  %v", ips)
+	ips, err := k8s.GetServiceEndPoints(m.kubeClient, "loxilb-peer-service", "kube-system")
+	if len(ips) > 0 {
+		klog.Infof("loxilb-peer-service end-points:  %v", ips)
+	}
 	if err != nil {
 		ips = []net.IP{}
 	}
@@ -1303,7 +1320,7 @@ func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, lox
 			}
 		}
 		if !found {
-			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, true)
+			client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, loxiLBDeadCh, true)
 			if err2 != nil {
 				continue
 			}
@@ -1350,7 +1367,7 @@ func (m *Manager) SelectLoxiLBRoles(sendSigCh bool, loxiLBSelMasterEvent chan bo
 				if v.IsAlive {
 					v.MasterLB = true
 					selMaster = true
-					klog.Infof("loxilb-peer(%v) set-role master", v.Url)
+					klog.Infof("loxilb-lb(%v) set-role master", v.Url)
 				}
 			}
 			if selMaster {
@@ -1487,9 +1504,9 @@ loop:
 				}(&bgpGlobalCfg)
 
 				if err == nil {
-					klog.Infof("set-bgp-global success")
+					klog.Infof("loxilb(%s) set-bgp-global success", aliveClient.Host)
 				} else {
-					klog.Infof("set-bgp-global failed(%s)", err)
+					klog.Infof("loxilb(%s) set-bgp-global failed(%s)", aliveClient.Host, err)
 					m.checkHandleBGPCfgErrors(loxiAliveCh, aliveClient, err)
 				}
 

pkg/api/client.go

@@ -20,13 +20,14 @@ type LoxiClient struct {
 	Host        string
 	Port        string
 	IsAlive     bool
+	DeadSigTs   time.Time
 	DoBGPCfg    bool
 	Purge       bool
 	Stop        chan struct{}
 }
 
 // apiServer is string. what format? http://10.0.0.1 or 10.0.0.1
-func NewLoxiClient(apiServer string, aliveCh chan *LoxiClient, peerOnly bool) (*LoxiClient, error) {
+func NewLoxiClient(apiServer string, aliveCh chan *LoxiClient, deadCh chan bool, peerOnly bool) (*LoxiClient, error) {
 
 	client := &http.Client{}
 
@@ -57,23 +58,31 @@ func NewLoxiClient(apiServer string, aliveCh chan *LoxiClient, peerOnly bool) (*
 		Port:        port,
 		Stop:        stop,
 		PeeringOnly: peerOnly,
+		DeadSigTs:   time.Now(),
 	}
 
-	lc.StartLoxiHealthCheckChan(aliveCh)
+	lc.StartLoxiHealthCheckChan(aliveCh, deadCh)
 
 	klog.Infof("NewLoxiClient Created: %s", apiServer)
 
 	return lc, nil
 }
 
-func (l *LoxiClient) StartLoxiHealthCheckChan(aliveCh chan *LoxiClient) {
+func (l *LoxiClient) StartLoxiHealthCheckChan(aliveCh chan *LoxiClient, deadCh chan bool) {
 	l.IsAlive = false
 
 	go wait.Until(func() {
 		if _, err := l.HealthCheck().Get(context.Background(), ""); err != nil {
 			if l.IsAlive {
 				klog.Infof("LoxiHealthCheckChan: loxilb(%s) is down", l.RestClient.baseURL.String())
 				l.IsAlive = false
+				if time.Duration(time.Since(l.DeadSigTs).Seconds()) >= 3 && l.MasterLB {
+					klog.Infof("LoxiHealthCheckChan: master down")
+					l.DeadSigTs = time.Now()
+					deadCh <- true
+				} else {
+					l.DeadSigTs = time.Now()
+				}
 			}
 		} else {
 			if !l.IsAlive {