diff --git a/cmd/loxilb-agent/agent.go b/cmd/loxilb-agent/agent.go index 07e5c95..bca2f1a 100644 --- a/cmd/loxilb-agent/agent.go +++ b/cmd/loxilb-agent/agent.go @@ -155,10 +155,11 @@ func run(o *Options) error { loxiLBLiveCh := make(chan *api.LoxiClient, 50) loxiLBPurgeCh := make(chan *api.LoxiClient, 5) loxiLBSelMasterEvent := make(chan bool) + loxiLBDeadCh := make(chan bool, 64) if len(networkConfig.LoxilbURLs) > 0 { for _, lbURL := range networkConfig.LoxilbURLs { - loxilbClient, err := api.NewLoxiClient(lbURL, loxiLBLiveCh, false) + loxilbClient, err := api.NewLoxiClient(lbURL, loxiLBLiveCh, loxiLBDeadCh, false) if err != nil { return err } @@ -179,13 +180,21 @@ func run(o *Options) error { ) go wait.Until(func() { - if len(networkConfig.LoxilbURLs) <= 0 { - lbManager.DiscoverLoxiLBServices(loxiLBLiveCh, loxiLBPurgeCh) - } - lbManager.DiscoverLoxiLBPeerServices(loxiLBLiveCh, loxiLBPurgeCh) + select { + case <-loxiLBDeadCh: + if networkConfig.SetRoles != "" { + klog.Infof("Running select-roles") + lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent) + } + default: + if len(networkConfig.LoxilbURLs) <= 0 { + lbManager.DiscoverLoxiLBServices(loxiLBLiveCh, loxiLBDeadCh, loxiLBPurgeCh) + } + lbManager.DiscoverLoxiLBPeerServices(loxiLBLiveCh, loxiLBDeadCh, loxiLBPurgeCh) - if networkConfig.SetRoles != "" { - lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent) + if networkConfig.SetRoles != "" { + lbManager.SelectLoxiLBRoles(true, loxiLBSelMasterEvent) + } } }, time.Second*20, stopCh) diff --git a/cmd/loxilb-agent/main.go b/cmd/loxilb-agent/main.go index 7afe9c0..e2f4086 100644 --- a/cmd/loxilb-agent/main.go +++ b/cmd/loxilb-agent/main.go @@ -37,7 +37,7 @@ func Execute() { Long: "loxilb-k8s", } - client, err := api.NewLoxiClient("http://127.0.0.1:11111", nil, false) + client, err := api.NewLoxiClient("http://127.0.0.1:11111", nil, nil, false) if err != nil { return } diff --git a/pkg/agent/manager/loadbalancer/loadbalancer.go b/pkg/agent/manager/loadbalancer/loadbalancer.go index 42897ec..a8b6623 100644 --- a/pkg/agent/manager/loadbalancer/loadbalancer.go +++ b/pkg/agent/manager/loadbalancer/loadbalancer.go @@ -1243,7 +1243,7 @@ func (m *Manager) addIngress(service *corev1.Service, newIP net.IP) { append(service.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{IP: newIP.String()}) } -func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBPurgeCh chan *api.LoxiClient) { +func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBDeadCh chan bool, loxiLBPurgeCh chan *api.LoxiClient) { var tmploxilbClients []*api.LoxiClient // DNS lookup (not used now) // ips, err := net.LookupIP("loxilb-lb-service") @@ -1270,7 +1270,7 @@ func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, lox } } if !found { - client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, false) + client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, loxiLBDeadCh, false) if err2 != nil { continue } @@ -1293,10 +1293,12 @@ func (m *Manager) DiscoverLoxiLBServices(loxiLBAliveCh chan *api.LoxiClient, lox m.LoxiClients = tmp } -func (m *Manager) DiscoverLoxiLBPeerServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBPurgeCh chan *api.LoxiClient) { +func (m *Manager) DiscoverLoxiLBPeerServices(loxiLBAliveCh chan *api.LoxiClient, loxiLBDeadCh chan bool, loxiLBPurgeCh chan *api.LoxiClient) { var tmploxilbPeerClients []*api.LoxiClient ips, err := k8s.GetServiceEndPoints(m.kubeClient, "loxilb-peer-service", "kube-system") - klog.Infof("loxilb-peer-service end-points: %v", ips) + if len(ips) > 0 { + klog.Infof("loxilb-peer-service end-points: %v", ips) + } if err != nil { ips = []net.IP{} } @@ -1318,7 +1320,7 @@ func (m *Manager) DiscoverLoxiLBPeerServices(loxiLBAliveCh chan *api.LoxiClient, } } if !found { - client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, true) + client, err2 := api.NewLoxiClient("http://"+ip.String()+":11111", loxiLBAliveCh, loxiLBDeadCh, true) if err2 != nil { continue } @@ -1365,7 +1367,7 @@ func (m *Manager) SelectLoxiLBRoles(sendSigCh bool, loxiLBSelMasterEvent chan bo if v.IsAlive { v.MasterLB = true selMaster = true - klog.Infof("loxilb-peer(%v) set-role master", v.Url) + klog.Infof("loxilb-lb(%v) set-role master", v.Url) } } if selMaster { diff --git a/pkg/api/client.go b/pkg/api/client.go index 481d01e..9000377 100644 --- a/pkg/api/client.go +++ b/pkg/api/client.go @@ -20,13 +20,14 @@ type LoxiClient struct { Host string Port string IsAlive bool + DeadSigTs time.Time DoBGPCfg bool Purge bool Stop chan struct{} } // apiServer is string. what format? http://10.0.0.1 or 10.0.0.1 -func NewLoxiClient(apiServer string, aliveCh chan *LoxiClient, peerOnly bool) (*LoxiClient, error) { +func NewLoxiClient(apiServer string, aliveCh chan *LoxiClient, deadCh chan bool, peerOnly bool) (*LoxiClient, error) { client := &http.Client{} @@ -57,16 +58,17 @@ func NewLoxiClient(apiServer string, aliveCh chan *LoxiClient, peerOnly bool) (* Port: port, Stop: stop, PeeringOnly: peerOnly, + DeadSigTs: time.Now(), } - lc.StartLoxiHealthCheckChan(aliveCh) + lc.StartLoxiHealthCheckChan(aliveCh, deadCh) klog.Infof("NewLoxiClient Created: %s", apiServer) return lc, nil } -func (l *LoxiClient) StartLoxiHealthCheckChan(aliveCh chan *LoxiClient) { +func (l *LoxiClient) StartLoxiHealthCheckChan(aliveCh chan *LoxiClient, deadCh chan bool) { l.IsAlive = false go wait.Until(func() { @@ -74,6 +76,13 @@ func (l *LoxiClient) StartLoxiHealthCheckChan(aliveCh chan *LoxiClient) { if l.IsAlive { klog.Infof("LoxiHealthCheckChan: loxilb(%s) is down", l.RestClient.baseURL.String()) l.IsAlive = false + if time.Duration(time.Since(l.DeadSigTs).Seconds()) >= 3 && l.MasterLB { + klog.Infof("LoxiHealthCheckChan: master down") + l.DeadSigTs = time.Now() + deadCh <- true + } else { + l.DeadSigTs = time.Now() + } } } else { if !l.IsAlive {