diff --git a/daemon/client/checks/common.go b/daemon/client/checks/common.go index 68abd3b..79b47c9 100644 --- a/daemon/client/checks/common.go +++ b/daemon/client/checks/common.go @@ -34,6 +34,8 @@ func CheckExternalSystem(url string) (error) { } func CheckNtpd() (error) { + log.Println("Checking output of ntpstat") + out, err := exec.Command("bash", "-c", "ntpstat").Output() if err != nil { msg := "Could not check ntpd status: " + err.Error() diff --git a/daemon/client/checks/docker.go b/daemon/client/checks/docker.go index 9a8b9e8..9dbd291 100644 --- a/daemon/client/checks/docker.go +++ b/daemon/client/checks/docker.go @@ -9,6 +9,8 @@ import ( ) func CheckDockerPool(okSize int) (error) { + log.Println("Checking docker pool used size") + out, err := exec.Command("bash", "-c", "lvs -o data_percent,metadata_percent,LV_NAME --noheadings --units G --nosuffix | grep docker-pool").Output() if err != nil { msg := "Could not parse docker pool size: " + err.Error() diff --git a/daemon/client/checks/openshift.go b/daemon/client/checks/openshift.go index 2fd70a5..82739bc 100644 --- a/daemon/client/checks/openshift.go +++ b/daemon/client/checks/openshift.go @@ -11,6 +11,8 @@ import ( ) func CheckMasterApis(urls string) (error) { + log.Println("Checking master apis. At least one has to be up") + urlArr := strings.Split(urls, ",") oneApiOk := false @@ -31,6 +33,8 @@ func CheckMasterApis(urls string) (error) { } func CheckOcGetNodes() (error) { + log.Println("Checking oc get nodes output") + out, err := exec.Command("bash", "-c", "oc get nodes --show-labels | grep -v monitoring=false").Output() if err != nil { msg := "Could not parse oc get nodes output: " + err.Error() @@ -46,6 +50,8 @@ func CheckOcGetNodes() (error) { } func CheckDnsNslookupOnKubernetes() (error) { + log.Println("Checking nslookup to kubernetes ip") + cmd := exec.Command("nslookup", daemonDNSEndpoint, kubernetesIP) var out bytes.Buffer cmd.Stdout = &out @@ -66,6 +72,8 @@ func CheckDnsNslookupOnKubernetes() (error) { } func CheckDnsServiceNode() (error) { + log.Println("Checking dns to a openshift service") + ips := getIpsForName(daemonDNSServiceA) if (ips == nil) { @@ -76,6 +84,8 @@ func CheckDnsServiceNode() (error) { } func CheckDnsInPod() (error) { + log.Println("Checking dns to a openshift service inside a pod") + ips := getIpsForName(daemonDNSPod) if (ips == nil) { @@ -86,6 +96,8 @@ func CheckDnsInPod() (error) { } func CheckPodHttpAtoB() (error) { + log.Println("Checking if http connection does not work if network not joined") + // This should fail as we do not have access to this project if err := checkHttp("http://" + daemonDNSServiceB + ":8090/hello"); err == nil { errors.New("Pod A could access pod b. This should not be allowed!") @@ -95,6 +107,8 @@ func CheckPodHttpAtoB() (error) { } func CheckPodHttpAtoC(slow bool) (error) { + log.Println("Checking if http connection does work with joined network") + if err := checkHttp("http://" + daemonDNSServiceC + ":8090/" + getEndpoint(slow)); err != nil { return errors.New("Pod A could access pod C. This should not work. Route/Router problem?") } @@ -117,6 +131,8 @@ func CheckHttpService(slow bool) (error) { } func CheckHttpHaProxy(publicUrl string, slow bool) (error) { + log.Println("Checking http via HA-Proxy") + if err := checkHttp(publicUrl + ":80/" + getEndpoint(slow)); err != nil { return errors.New("Could not access pods via haproxy. Route/Router problem?") } @@ -125,6 +141,8 @@ func CheckHttpHaProxy(publicUrl string, slow bool) (error) { } func CheckRegistryHealth(ip string) (error) { + log.Println("Checking registry health") + if err := checkHttp("http://" + ip + ":5000/healthz"); err != nil { return fmt.Errorf("Registry health check failed. %v", err.Error()) } @@ -133,6 +151,8 @@ func CheckRegistryHealth(ip string) (error) { } func CheckHawcularHealth(ip string) (error) { + log.Println("Checking metrics health") + if err := checkHttp("https://" + ip + ":443"); err != nil { return errors.New("Hawcular health check failed") } @@ -141,6 +161,8 @@ func CheckHawcularHealth(ip string) (error) { } func CheckRouterHealth(ip string) (error) { + log.Println("Checking router health", ip) + if err := checkHttp("http://" + ip + ":1936/healthz"); err != nil { return fmt.Errorf("Router health check failed for %v, %v", ip, err.Error()) } @@ -149,6 +171,8 @@ func CheckRouterHealth(ip string) (error) { } func CheckLoggingRestartsCount() (error) { + log.Println("Checking log-container restart count") + out, err := exec.Command("bash", "-c", "oc get pods -n logging -o wide | tr -s ' ' | cut -d ' ' -f 4").Output() if err != nil { msg := "Could not parse logging container restart count: " + err.Error() @@ -176,6 +200,8 @@ func CheckLoggingRestartsCount() (error) { } func CheckRouterRestartCount() (error) { + log.Println("Checking router restart count") + out, err := exec.Command("bash", "-c", "oc get po -n default | grep router | grep -v deploy | tr -s ' ' | cut -d ' ' -f 4").Output() if err != nil { msg := "Could not parse router restart count: " + err.Error() @@ -203,6 +229,8 @@ func CheckRouterRestartCount() (error) { } func CheckEtcdHealth(etcdIps string, etcdCertPath string) (error) { + log.Println("Checking etcd health") + var msg string isOk := true @@ -251,6 +279,8 @@ func checkEtcdHealthWithCertPath(msg *string, certPath string, etcdIps string) b } func CheckLimitsAndQuotas(allowedWithout int) (error) { + log.Println("Checking limits & quotas") + // Count projects projectCount, err := exec.Command("bash", "-c", "oc get projects | wc -l").Output() if err != nil { diff --git a/daemon/client/checks/storage.go b/daemon/client/checks/storage.go index d7ac1a3..05022ce 100644 --- a/daemon/client/checks/storage.go +++ b/daemon/client/checks/storage.go @@ -12,6 +12,8 @@ import ( ) func CheckOpenFileCount() (error) { + log.Println("Checking open files") + out, err := exec.Command("bash", "-c", "cat /proc/sys/fs/file-nr | cut -f1").Output() if err != nil { msg := "Could not evaluate open file count: " + err.Error() @@ -33,6 +35,8 @@ func CheckOpenFileCount() (error) { } func CheckGlusterStatus() (error) { + log.Println("Checking gluster status with gstatus") + out, err := exec.Command("bash", "-c", "gstatus -abw -o json").Output() if err != nil { if (strings.Contains(err.Error(), "exit status 16")) { @@ -71,6 +75,8 @@ func CheckGlusterStatus() (error) { } func CheckVGSizes(okSize int) (error) { + log.Println("Checking VG free size") + out, err := exec.Command("bash", "-c", "vgs -o vg_free,vg_size,VG_NAME --noheadings --units G --nosuffix | grep -v crash").Output() if err != nil { msg := "Could not evaluate VG sizes: " + err.Error() @@ -95,6 +101,8 @@ func CheckVGSizes(okSize int) (error) { } func CheckLVPoolSizes(okSize int) (error) { + log.Println("Checking LV pool used size") + out, err := exec.Command("bash", "-c", "lvs -o data_percent,metadata_percent,LV_NAME --noheadings --units G --nosuffix | grep pool").Output() if err != nil { msg := "Could not evaluate LV pool size: " + err.Error()