From 7f76a4baef1f47c72e0e639dc873fc471baf519d Mon Sep 17 00:00:00 2001 From: sgayangi Date: Sun, 3 Mar 2024 09:49:46 +0530 Subject: [PATCH] Address review comments --- adapter/config/default_config.go | 7 +- adapter/go.mod | 8 +- adapter/go.sum | 13 +- adapter/internal/adapter/adapter.go | 2 +- adapter/internal/discovery/xds/server.go | 11 + .../operator/synchronizer/data_store.go | 5 - adapter/pkg/metrics/metrics.go | 256 ++++-------------- .../commoncontroller/common_controller.go | 6 +- common-controller/go.mod | 8 +- common-controller/go.sum | 9 +- .../internal/config/default_config.go | 7 +- common-controller/pkg/metrics/metrics.go | 191 +------------ common-go-libs/go.mod | 15 +- common-go-libs/go.sum | 27 +- common-go-libs/pkg/metrics/metrics.go | 179 ++++++++++++ common-go-libs/pkg/metrics/types.go | 24 ++ .../gateway-components/common-log-conf.yaml | 6 + .../gateway-components/log-conf.yaml | 6 + .../ratelimiter/ratelimiter-deployment.yaml | 2 +- 19 files changed, 364 insertions(+), 418 deletions(-) create mode 100644 common-go-libs/pkg/metrics/metrics.go create mode 100644 common-go-libs/pkg/metrics/types.go diff --git a/adapter/config/default_config.go b/adapter/config/default_config.go index 1fe3847d70..72dd7cd012 100644 --- a/adapter/config/default_config.go +++ b/adapter/config/default_config.go @@ -33,10 +33,9 @@ var defaultConfig = &Config{ }, Environment: "Default", Metrics: metrics{ - Enabled: true, - Type: "prometheus", - Port: 18006, - CollectionInterval: 5, + Enabled: false, + Type: "prometheus", + Port: 18006, }, }, Envoy: envoy{ diff --git a/adapter/go.mod b/adapter/go.mod index 03faae8b72..de51aaa18f 100644 --- a/adapter/go.mod +++ b/adapter/go.mod @@ -10,9 +10,7 @@ require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/pelletier/go-toml v1.9.5 - github.com/prometheus/client_golang v1.18.0 - github.com/prometheus/procfs v0.12.0 - github.com/shirou/gopsutil/v3 v3.24.1 + github.com/prometheus/client_golang v1.17.0 github.com/sirupsen/logrus v1.9.0 github.com/wso2/apk/common-go-libs v0.0.0-20231208100153-24bee7b4bd81 golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb @@ -62,12 +60,14 @@ require ( github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/shirou/gopsutil/v3 v3.24.2 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/vektah/gqlparser v1.3.1 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/net v0.21.0 // indirect diff --git a/adapter/go.sum b/adapter/go.sum index 0af7a43bee..b59fd5b9a6 100644 --- a/adapter/go.sum +++ b/adapter/go.sum @@ -117,8 +117,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= @@ -128,8 +128,8 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= -github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -159,8 +159,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -214,7 +214,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index 83575ee23a..ec8d8eb3e0 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -160,7 +160,7 @@ func Run(conf *config.Config) { // Start the metrics server if conf.Adapter.Metrics.Enabled && strings.EqualFold(conf.Adapter.Metrics.Type, metrics.PrometheusMetricType) { logger.LoggerAPK.Info("Starting Prometheus Metrics Server ....") - go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port, conf.Adapter.Metrics.CollectionInterval) + go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port) } cache := xds.GetXdsCache() diff --git a/adapter/internal/discovery/xds/server.go b/adapter/internal/discovery/xds/server.go index da5dd92950..cd7607a780 100644 --- a/adapter/internal/discovery/xds/server.go +++ b/adapter/internal/discovery/xds/server.go @@ -813,3 +813,14 @@ func GetEnvoyInternalAPIClusters() int { } return totalClusters } + +// GetEnvoyInternalAPICount method gets the number of APIs deployed in Envoy +func GetEnvoyInternalAPICount() int { + totalCount := 0 + for _, apiMap := range orgAPIMap { + for range apiMap { + totalCount++ + } + } + return totalCount +} diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 1ec062f860..80265707da 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -571,8 +571,3 @@ func (ods *OperatorDataStore) DeleteCachedGateway(gatewayName types.NamespacedNa defer ods.mu.Unlock() delete(ods.gatewayStore, gatewayName) } - -// GetAPICount gets API count -func (ods *OperatorDataStore) GetAPICount() int { - return len(ods.apiStore) -} diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index d62e21ef02..d0f7974713 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -22,227 +22,89 @@ package metrics import ( "fmt" "net/http" - "os" - "runtime" "strconv" - "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/prometheus/procfs" - "github.com/shirou/gopsutil/v3/cpu" - "github.com/shirou/gopsutil/v3/host" - "github.com/shirou/gopsutil/v3/load" - "github.com/shirou/gopsutil/v3/mem" xds "github.com/wso2/apk/adapter/internal/discovery/xds" logger "github.com/wso2/apk/adapter/internal/loggers" - synchronizer "github.com/wso2/apk/adapter/internal/operator/synchronizer" "github.com/wso2/apk/adapter/pkg/logging" + commonmetrics "github.com/wso2/apk/common-go-libs/pkg/metrics" ) var ( prometheusMetricRegistry = prometheus.NewRegistry() - - hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "host_info", - Help: "Host Info", - }, []string{"os"}) - - availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_available_cpu_total", - Help: "Number of available CPUs.", - }) - - freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_free_physical_memory_bytes", - Help: "Amount of free physical memory.", - }) - - totalVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_total_virtual_memory_bytes", - Help: "Amount of total virtual memory.", - }) - usedVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_used_virtual_memory_bytes", - Help: "Amount of used virtual memory.", - }) - - systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_system_cpu_load_percentage", - Help: "System-wide CPU usage as a percentage.", - }) - - loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "os_system_load_average", - Help: "Current load of CPU in the host system for the last {x} minutes", - }, []string{"duration"}) - - processStartTime = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "process_start_time_seconds", - Help: "Start time of the process since unix epoch in seconds.", - }) - - processOpenFDs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "process_open_fds", - Help: "Number of open file descriptors.", - }) - - apis = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "api_count", - Help: "Number of APIs created.", - }) - - internalClusterCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "internal_cluster_count", - Help: "Number of internal clusters created.", - }) - - gwClusterCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "gw_cluster_count", - Help: "Number of gw clusters created.", - }) - - internalRouteCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "internal_route_count", - Help: "Number of internal routes created.", - }) ) -func init() { - // Register the Go collector with the registry - goCollector := collectors.NewGoCollector() - prometheusMetricRegistry.MustRegister(goCollector) - - // Register other metrics - prometheusMetricRegistry.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, usedVirtualMemory, totalVirtualMemory, - systemCPULoad, loadAvg, processStartTime, processOpenFDs, apis, internalClusterCount, internalRouteCount, gwClusterCount) +// AdapterCollector contains the descriptions of the custom metrics exposed by the adapter. +// It also uses the metrics defined in common-go-libs +type AdapterCollector struct { + commonmetrics.Collector + apis *prometheus.Desc + internalClusterCount *prometheus.Desc + internalRouteCount *prometheus.Desc } -// recordMetrics record custom golang metrics -var recordMetrics = func(collectionInterval int32) { - for { - envoyInternalAPIClusterCount := xds.GetEnvoyInternalAPIClusters() - internalClusterCount.Set(float64(envoyInternalAPIClusterCount)) - - envoyInternalAPIRouteCount := xds.GetEnvoyInternalAPIRoutes() - internalRouteCount.Set(float64(envoyInternalAPIRouteCount)) - - gwClusterCount.Set(float64(xds.GetEnvoyGatewayConfigClusters())) - - apiCount := synchronizer.GetOperatorDataStore().GetAPICount() - apis.Set(float64(apiCount)) - - host, err := host.Info() - if handleError(err, "Failed to get host info") { - return - } - hostInfo.WithLabelValues(host.OS).Set(1) - availableCPUs.Set(float64(runtime.NumCPU())) - - v, err := mem.VirtualMemory() - if handleError(err, "Failed to read virtual memory metrics") { - return - } - freePhysicalMemory.Set(float64(v.Free)) - usedVirtualMemory.Set(float64(v.Used)) - totalVirtualMemory.Set(float64(v.Total)) +func adapterMetricsCollector() *AdapterCollector { + return &AdapterCollector{ + Collector: *commonmetrics.CustomMetricsCollector(), + apis: prometheus.NewDesc( + "api_count", + "Number of APIs created.", + nil, nil, + ), + internalClusterCount: prometheus.NewDesc( + "internal_cluster_count", + "Number of internal clusters created.", + nil, nil, + ), + internalRouteCount: prometheus.NewDesc( + "internal_route_count", + "Number of internal routes created.", + nil, nil, + ), + } +} - percentages, err := cpu.Percent(0, false) - if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { - return - } - totalPercentage := 0.0 - for _, p := range percentages { - totalPercentage += p - } - averagePercentage := totalPercentage / float64(len(percentages)) - systemCPULoad.Set(averagePercentage) +// Describe sends all the descriptors of the metrics collected by this Collector +// to the provided channel. +func (collector *AdapterCollector) Describe(ch chan<- *prometheus.Desc) { + collector.Collector.Describe(ch) + ch <- collector.apis + ch <- collector.internalClusterCount + ch <- collector.internalRouteCount +} - pid := os.Getpid() - p, err := procfs.NewProc(pid) - if handleError(err, "Failed to get current process") { - return - } - stat, err := p.Stat() - if handleError(err, "Failed to get process stats") { - return - } - t, err := stat.StartTime() - if handleError(err, "Failed to read process start time") { - return - } - processStartTime.Set(t) - fds, err := p.FileDescriptorsLen() - if handleError(err, "Failed to read file descriptor count") { - return - } - processOpenFDs.Set(float64(fds)) +// Collect collects all the relevant Prometheus metrics. +func (collector *AdapterCollector) Collect(ch chan<- prometheus.Metric) { + collector.Collector.Collect(ch) + var apisCount float64 + var internalClusterCount float64 + var internalRouteCount float64 - avg, err := load.Avg() - if handleError(err, "Failed to read cpu load averages") { - return - } - loadAvg.WithLabelValues("1m").Set(avg.Load1) - loadAvg.WithLabelValues("5m").Set(avg.Load5) - loadAvg.WithLabelValues("15m").Set(avg.Load15) + apiCount := xds.GetEnvoyInternalAPICount() + apisCount = float64(apiCount) - // Sleep before the next measurement - time.Sleep(time.Duration(collectionInterval) * time.Second) - } + internalRouteCount = float64(xds.GetEnvoyInternalAPIRoutes()) + internalClusterCount = float64(xds.GetEnvoyInternalAPIClusters()) + ch <- prometheus.MustNewConstMetric(collector.apis, prometheus.GaugeValue, apisCount) + ch <- prometheus.MustNewConstMetric(collector.internalRouteCount, prometheus.GaugeValue, internalRouteCount) + ch <- prometheus.MustNewConstMetric(collector.internalClusterCount, prometheus.GaugeValue, internalClusterCount) } -func handleError(err error, message string) bool { +// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. +func StartPrometheusMetricsServer(port int32) { + + collector := adapterMetricsCollector() + prometheus.MustRegister(collector) + http.Handle("/metrics", promhttp.Handler()) + err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) if err != nil { logger.LoggerAPK.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintf(message, err.Error()), - Severity: logging.MINOR, - ErrorCode: 1109, + Message: fmt.Sprintln("Prometheus metrics server error:", err), + Severity: logging.MAJOR, + ErrorCode: 1110, }) - return true - } - return false -} - -// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. -func StartPrometheusMetricsServer(port int32, collectionInterval int32) { - done := make(chan struct{}) // Channel to indicate recordMetrics routine exit - - // Start the Prometheus metrics server - go func() { - http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) - err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) - if err != nil { - logger.LoggerAPK.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintln("Prometheus metrics server error:", err), - Severity: logging.MAJOR, - ErrorCode: 1110, - }) - } - }() - - for { - // Start the recordMetrics goroutine - go func() { - defer func() { - if r := recover(); r != nil { - logger.LoggerAPK.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintln("recordMetrics goroutine exited with error:", r), - Severity: logging.MAJOR, - ErrorCode: 1111, - }) - } - }() - recordMetrics(collectionInterval) - done <- struct{}{} // Signal that the goroutine has completed - }() - - // Wait for the previous recordMetrics goroutine to complete - <-done - - // Log and restart the goroutine - logger.LoggerAPK.Info("Restarting recordMetrics goroutine...") - time.Sleep(3 * time.Second) } } diff --git a/common-controller/commoncontroller/common_controller.go b/common-controller/commoncontroller/common_controller.go index 676c87bff1..d270085b22 100644 --- a/common-controller/commoncontroller/common_controller.go +++ b/common-controller/commoncontroller/common_controller.go @@ -206,14 +206,14 @@ func InitCommonControllerServer(conf *config.Config) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - loggers.LoggerAPKOperator.Info("Starting common controller ....") - // Start the metrics server if conf.CommonController.Metrics.Enabled && strings.EqualFold(conf.CommonController.Metrics.Type, metrics.PrometheusMetricType) { loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....") - go metrics.StartPrometheusMetricsServer(conf.CommonController.Metrics.Port, conf.CommonController.Metrics.CollectionInterval) + go metrics.StartPrometheusMetricsServer(conf.CommonController.Metrics.Port) } + loggers.LoggerAPKOperator.Info("Starting common controller ....") + rateLimiterCache := xds.GetRateLimiterCache() rlsSrv := xdsv3.NewServer(ctx, rateLimiterCache, &xds.Callbacks{}) diff --git a/common-controller/go.mod b/common-controller/go.mod index 9013a7ac6a..fa21721907 100644 --- a/common-controller/go.mod +++ b/common-controller/go.mod @@ -6,7 +6,7 @@ require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/prometheus/client_golang v1.18.0 - github.com/prometheus/procfs v0.12.0 + github.com/prometheus/procfs v0.12.0 // indirect github.com/sirupsen/logrus v1.9.0 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 @@ -20,9 +20,8 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/pelletier/go-toml v1.9.5 github.com/redis/go-redis/v9 v9.2.1 - github.com/shirou/gopsutil/v3 v3.24.1 github.com/wso2/apk/adapter v0.0.0-20231207051518-6dd728943082 - github.com/wso2/apk/common-go-libs v0.0.0-20231208100153-24bee7b4bd81 + github.com/wso2/apk/common-go-libs v0.0.0-20240304050809-a382bc6b0d82 google.golang.org/grpc v1.62.0 ) @@ -51,13 +50,14 @@ require ( github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shirou/gopsutil/v3 v3.24.2 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/vektah/gqlparser v1.3.1 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.19.0 // indirect golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect diff --git a/common-controller/go.sum b/common-controller/go.sum index b678ced88b..b98da4f274 100644 --- a/common-controller/go.sum +++ b/common-controller/go.sum @@ -186,8 +186,8 @@ github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= -github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -227,8 +227,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -304,7 +304,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/common-controller/internal/config/default_config.go b/common-controller/internal/config/default_config.go index 269b19bd47..396ec2b16a 100644 --- a/common-controller/internal/config/default_config.go +++ b/common-controller/internal/config/default_config.go @@ -44,10 +44,9 @@ var defaultConfig = &Config{ Persistence: persistence{Type: "K8s"}, }, Metrics: metrics{ - Enabled: true, - Type: "prometheus", - Port: 18010, - CollectionInterval: 5, + Enabled: false, + Type: "prometheus", + Port: 18010, }, }, } diff --git a/common-controller/pkg/metrics/metrics.go b/common-controller/pkg/metrics/metrics.go index ef7824ebb1..c2da1fa33f 100644 --- a/common-controller/pkg/metrics/metrics.go +++ b/common-controller/pkg/metrics/metrics.go @@ -16,200 +16,33 @@ * under the License. */ -// Package metrics holds the implementation for exposing common controller metrics to prometheus +// Package metrics holds the implementation for exposing adapter metrics to prometheus package metrics import ( "fmt" "net/http" - "os" - "runtime" "strconv" - "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/prometheus/procfs" - "github.com/shirou/gopsutil/v3/cpu" - "github.com/shirou/gopsutil/v3/host" - "github.com/shirou/gopsutil/v3/load" - "github.com/shirou/gopsutil/v3/mem" "github.com/wso2/apk/adapter/pkg/logging" logger "github.com/wso2/apk/common-controller/internal/loggers" + metrics "github.com/wso2/apk/common-go-libs/pkg/metrics" ) -var ( - prometheusMetricRegistry = prometheus.NewRegistry() - - hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "host_info", - Help: "Host Info", - }, []string{"os"}) - - availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_available_cpu_total", - Help: "Number of available CPUs.", - }) - - freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_free_physical_memory_bytes", - Help: "Amount of free physical memory.", - }) - - totalVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_total_virtual_memory_bytes", - Help: "Amount of total virtual memory.", - }) - usedVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_used_virtual_memory_bytes", - Help: "Amount of used virtual memory.", - }) - - systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_system_cpu_load_percentage", - Help: "System-wide CPU usage as a percentage.", - }) - - loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "os_system_load_average", - Help: "Current load of CPU in the host system for the last {x} minutes", - }, []string{"duration"}) - - processStartTime = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "process_start_time_seconds", - Help: "Start time of the process since unix epoch in seconds.", - }) - - processOpenFDs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "process_open_fds", - Help: "Number of open file descriptors.", - }) -) - -func init() { - // Register the Go collector with the registry - goCollector := collectors.NewGoCollector() - prometheusMetricRegistry.MustRegister(goCollector) - - // Register other metrics - prometheusMetricRegistry.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, usedVirtualMemory, totalVirtualMemory, - systemCPULoad, loadAvg, processStartTime, processOpenFDs) -} - -// recordMetrics record custom golang metrics -var recordMetrics = func(collectionInterval int32) { - for { - host, err := host.Info() - if handleError(err, "Failed to get host info") { - return - } - hostInfo.WithLabelValues(host.OS).Set(1) - availableCPUs.Set(float64(runtime.NumCPU())) - - v, err := mem.VirtualMemory() - if handleError(err, "Failed to read virtual memory metrics") { - return - } - freePhysicalMemory.Set(float64(v.Free)) - usedVirtualMemory.Set(float64(v.Used)) - totalVirtualMemory.Set(float64(v.Total)) - - percentages, err := cpu.Percent(0, false) - if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { - return - } - totalPercentage := 0.0 - for _, p := range percentages { - totalPercentage += p - } - averagePercentage := totalPercentage / float64(len(percentages)) - systemCPULoad.Set(averagePercentage) - - pid := os.Getpid() - p, err := procfs.NewProc(pid) - if handleError(err, "Failed to get current process") { - return - } - stat, err := p.Stat() - if handleError(err, "Failed to get process stats") { - return - } - t, err := stat.StartTime() - if handleError(err, "Failed to read process start time") { - return - } - processStartTime.Set(t) - fds, err := p.FileDescriptorsLen() - if handleError(err, "Failed to read file descriptor count") { - return - } - processOpenFDs.Set(float64(fds)) - - avg, err := load.Avg() - if handleError(err, "Failed to read cpu load averages") { - return - } - loadAvg.WithLabelValues("1m").Set(avg.Load1) - loadAvg.WithLabelValues("5m").Set(avg.Load5) - loadAvg.WithLabelValues("15m").Set(avg.Load15) - - // Sleep before the next measurement - time.Sleep(time.Duration(collectionInterval) * time.Second) - } - -} +// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. +func StartPrometheusMetricsServer(port int32) { -func handleError(err error, message string) bool { + collector := metrics.CustomMetricsCollector() + prometheus.MustRegister(collector) + http.Handle("/metrics", promhttp.Handler()) + err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) if err != nil { - logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintf(message, err.Error()), - Severity: logging.MINOR, - ErrorCode: 1109, + logger.LoggerAPK.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("Prometheus metrics server error:", err), + Severity: logging.MAJOR, + ErrorCode: 1110, }) - return true - } - return false -} - -// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. -func StartPrometheusMetricsServer(port int32, collectionInterval int32) { - done := make(chan struct{}) // Channel to indicate recordMetrics routine exit - - // Start the Prometheus metrics server - go func() { - http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) - err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) - if err != nil { - logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintln("Prometheus metrics server error:", err), - Severity: logging.MAJOR, - ErrorCode: 1110, - }) - } - }() - - for { - // Start the recordMetrics goroutine - go func() { - defer func() { - if r := recover(); r != nil { - logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintln("recordMetrics goroutine exited with error:", r), - Severity: logging.MAJOR, - ErrorCode: 1111, - }) - } - }() - recordMetrics(collectionInterval) - done <- struct{}{} // Signal that the goroutine has completed - }() - - // Wait for the previous recordMetrics goroutine to complete - <-done - - // Log and restart the goroutine - logger.LoggerAPKOperator.Info("Restarting recordMetrics goroutine...") - time.Sleep(3 * time.Second) } } diff --git a/common-go-libs/go.mod b/common-go-libs/go.mod index 37ab6563a4..fcb0a61691 100644 --- a/common-go-libs/go.mod +++ b/common-go-libs/go.mod @@ -21,6 +21,16 @@ require ( sigs.k8s.io/gateway-api v0.7.1 ) +require ( + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect +) + require ( github.com/agnivade/levenshtein v1.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -55,18 +65,19 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect + github.com/prometheus/client_golang v1.17.0 github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/sergi/go-diff v1.3.1 // indirect + github.com/shirou/gopsutil/v3 v3.24.2 github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/common-go-libs/go.sum b/common-go-libs/go.sum index e4cdd32d2a..b7ff16a467 100644 --- a/common-go-libs/go.sum +++ b/common-go-libs/go.sum @@ -43,6 +43,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= @@ -69,6 +71,7 @@ github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvR github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -95,6 +98,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= @@ -117,6 +122,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -131,6 +138,12 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -147,6 +160,10 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/vektah/gqlparser v1.3.1 h1:8b0IcD3qZKWJQHSzynbDlrtP3IxVydZ2DZepCGofqfU= github.com/vektah/gqlparser v1.3.1/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74= github.com/wso2/apk/adapter v0.0.0-20231207051518-6dd728943082 h1:l+OdeDCNWPgie7L1fCjpfH04mAL3rFi4U+/idE8eduA= @@ -155,6 +172,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -206,17 +225,21 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= diff --git a/common-go-libs/pkg/metrics/metrics.go b/common-go-libs/pkg/metrics/metrics.go new file mode 100644 index 0000000000..40abd594ea --- /dev/null +++ b/common-go-libs/pkg/metrics/metrics.go @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Package metrics holds the implementation for exposing adapter metrics to prometheus +package metrics + +import ( + "fmt" + "runtime" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/host" + "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v3/mem" + "github.com/wso2/apk/adapter/pkg/logging" + logger "github.com/wso2/apk/common-go-libs/loggers" +) + +var ( + prometheusMetricRegistry = prometheus.NewRegistry() +) + +// Collector contains the descriptions of the custom metrics exposed +type Collector struct { + internalRouteCount *prometheus.Desc + hostInfo *prometheus.Desc + availableCPUs *prometheus.Desc + freePhysicalMemory *prometheus.Desc + totalVirtualMemory *prometheus.Desc + usedVirtualMemory *prometheus.Desc + systemCPULoad *prometheus.Desc + loadAvg *prometheus.Desc +} + +// CustomMetricsCollector contains the descriptions of the custom metrics exposed +func CustomMetricsCollector() *Collector { + return &Collector{ + internalRouteCount: prometheus.NewDesc( + "internal_route_count", + "Number of internal routes created.", + nil, nil, + ), + hostInfo: prometheus.NewDesc( + "host_info", + "Host Info", + []string{"os"}, nil, + ), + availableCPUs: prometheus.NewDesc( + "os_available_cpu_total", + "Number of available CPUs.", + nil, nil, + ), + freePhysicalMemory: prometheus.NewDesc( + "os_free_physical_memory_bytes", + "Amount of free physical memory.", + nil, nil, + ), + totalVirtualMemory: prometheus.NewDesc( + "os_total_virtual_memory_bytes", + "Amount of total virtual memory.", + nil, nil, + ), + usedVirtualMemory: prometheus.NewDesc( + "os_used_virtual_memory_bytes", + "Amount of used virtual memory.", + nil, nil, + ), + systemCPULoad: prometheus.NewDesc( + "os_system_cpu_load_percentage", + "System-wide CPU usage as a percentage.", + nil, nil, + ), + loadAvg: prometheus.NewDesc( + "os_system_load_average", + "Current load of CPU in the host system for the last {x} minutes", + []string{"duration"}, nil, + ), + } +} + +// Describe sends all the descriptors of the metrics collected by this Collector +// to the provided channel. +func (collector *Collector) Describe(ch chan<- *prometheus.Desc) { + ch <- collector.internalRouteCount + ch <- collector.hostInfo + ch <- collector.availableCPUs + ch <- collector.freePhysicalMemory + ch <- collector.totalVirtualMemory + ch <- collector.usedVirtualMemory + ch <- collector.systemCPULoad + ch <- collector.loadAvg +} + +// Collect collects all the relevant Prometheus metrics. +func (collector *Collector) Collect(ch chan<- prometheus.Metric) { + + var hostInfoValue float64 + var availableCPUs float64 + var freePhysicalMemory float64 + var totalVirtualMemory float64 + var usedVirtualMemory float64 + var systemCPULoad float64 + + host, err := host.Info() + if handleError(err, "Failed to get host info") { + return + } + hostInfoValue = 1 + availableCPUs = float64(runtime.NumCPU()) + + v, err := mem.VirtualMemory() + if handleError(err, "Failed to read virtual memory metrics") { + return + } + freePhysicalMemory = float64(v.Free) + usedVirtualMemory = float64(v.Used) + totalVirtualMemory = float64(v.Total) + + percentages, err := cpu.Percent(0, false) + if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { + return + } + totalPercentage := 0.0 + for _, p := range percentages { + totalPercentage += p + } + averagePercentage := totalPercentage / float64(len(percentages)) + systemCPULoad = averagePercentage + + avg, err := load.Avg() + if handleError(err, "Failed to read cpu load averages") { + return + } + + ch <- prometheus.MustNewConstMetric(collector.hostInfo, prometheus.GaugeValue, hostInfoValue, host.OS) + ch <- prometheus.MustNewConstMetric(collector.availableCPUs, prometheus.GaugeValue, availableCPUs) + ch <- prometheus.MustNewConstMetric(collector.freePhysicalMemory, prometheus.GaugeValue, freePhysicalMemory) + ch <- prometheus.MustNewConstMetric(collector.usedVirtualMemory, prometheus.GaugeValue, usedVirtualMemory) + ch <- prometheus.MustNewConstMetric(collector.totalVirtualMemory, prometheus.GaugeValue, totalVirtualMemory) + ch <- prometheus.MustNewConstMetric(collector.systemCPULoad, prometheus.GaugeValue, systemCPULoad) + ch <- prometheus.MustNewConstMetric(collector.loadAvg, prometheus.GaugeValue, avg.Load1, "1m") + ch <- prometheus.MustNewConstMetric(collector.loadAvg, prometheus.GaugeValue, avg.Load5, "5m") + ch <- prometheus.MustNewConstMetric(collector.loadAvg, prometheus.GaugeValue, avg.Load15, "15m") +} + +func init() { + // Register the Go collector with the registry + goCollector := collectors.NewGoCollector() + prometheusMetricRegistry.MustRegister(goCollector) +} + +func handleError(err error, message string) bool { + if err != nil { + logger.LoggerAPK.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintf(message, err.Error()), + Severity: logging.MINOR, + ErrorCode: 1109, + }) + return true + } + return false +} diff --git a/common-go-libs/pkg/metrics/types.go b/common-go-libs/pkg/metrics/types.go new file mode 100644 index 0000000000..07761ebaf7 --- /dev/null +++ b/common-go-libs/pkg/metrics/types.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package metrics + +const ( + // PrometheusMetricType prometheus metric type + PrometheusMetricType = "prometheus" +) diff --git a/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml b/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml index 0fa0bb3601..768f3fd45c 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml @@ -10,6 +10,12 @@ data: {{- if and .Values.wso2.apk.dp.environment .Values.wso2.apk.dp.environment.name }} environment = "{{ .Values.wso2.apk.dp.environment.name }}" {{- end }} + {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + [commoncontroller.metrics] + enabled = {{.Values.wso2.apk.metrics.enabled}} + type = "{{.Values.wso2.apk.metrics.type| default "prometheus" }}" + port = 18010 + {{ end}} [commoncontroller.server] label = "ratelimiter" {{ if and .Values.wso2.apk.dp.commonController.configs .Values.wso2.apk.dp.commonController.configs.apiNamespaces }} diff --git a/helm-charts/templates/data-plane/gateway-components/log-conf.yaml b/helm-charts/templates/data-plane/gateway-components/log-conf.yaml index 7b775168b9..2fc6b24fd2 100644 --- a/helm-charts/templates/data-plane/gateway-components/log-conf.yaml +++ b/helm-charts/templates/data-plane/gateway-components/log-conf.yaml @@ -14,6 +14,12 @@ data: [adapter.operator] namespaces = [{{ include "commaJoinedQuotedList" .Values.wso2.apk.dp.adapter.configs.apiNamespaces}}] {{ end}} + {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + [adapter.metrics] + enabled = {{.Values.wso2.apk.metrics.enabled}} + type = "{{.Values.wso2.apk.metrics.type| default "prometheus" }}" + port = 18006 + {{ end}} {{ if and .Values.wso2.apk.dp.gatewayRuntime.deployment .Values.wso2.apk.dp.gatewayRuntime.deployment.router .Values.wso2.apk.dp.gatewayRuntime.deployment.router.configs }} [router] diff --git a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml index 5821cb0c13..9d009a609f 100644 --- a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml +++ b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml @@ -35,7 +35,7 @@ spec: automountServiceAccountToken: false serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: - {{if and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd.enabled }} + {{if and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd .Values.wso2.apk.metrics.statsd.enabled }} - name: statsd image: {{ .Values.wso2.apk.metrics.statsd.image.repository }}:{{ .Values.wso2.apk.metrics.statsd.image.tag }} imagePullPolicy: {{ .Values.wso2.apk.metrics.statsd.imagePullPolicy }}