diff --git a/.readme_assets/kubedata.png b/.readme_assets/kubedata.png new file mode 100644 index 00000000..e2600575 Binary files /dev/null and b/.readme_assets/kubedata.png differ diff --git a/.readme_assets/kubviz.jpeg b/.readme_assets/kubviz.jpeg deleted file mode 100644 index f8bea82b..00000000 Binary files a/.readme_assets/kubviz.jpeg and /dev/null differ diff --git a/.readme_assets/kubvizFeatures.png b/.readme_assets/kubvizFeatures.png new file mode 100644 index 00000000..75f056fa Binary files /dev/null and b/.readme_assets/kubvizFeatures.png differ diff --git a/.readme_assets/kubvizFeatures2.png b/.readme_assets/kubvizFeatures2.png new file mode 100644 index 00000000..d90dc60d Binary files /dev/null and b/.readme_assets/kubvizFeatures2.png differ diff --git a/charts/grafana/Chart.yaml b/charts/grafana/Chart.yaml index fb66e4e5..3424108b 100644 --- a/charts/grafana/Chart.yaml +++ b/charts/grafana/Chart.yaml @@ -19,4 +19,4 @@ name: grafana sources: - https://github.com/grafana/grafana type: application -version: 6.26.13 +version: 6.26.14 diff --git a/charts/grafana/dashboards/custom-dashboard.json b/charts/grafana/dashboards/custom-dashboard.json index c8fe291f..33ebd341 100644 --- a/charts/grafana/dashboards/custom-dashboard.json +++ b/charts/grafana/dashboards/custom-dashboard.json @@ -21,12 +21,15 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, - "iteration": 1651809328098, + "id": 15, "links": [], "liveNow": false, "panels": [ { + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" + }, "description": "This stats panel shows the pod creation events.", "fieldConfig": { "defaults": { @@ -35,7 +38,6 @@ "mode": "fixed" }, "mappings": [], - "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -58,44 +60,38 @@ "x": 0, "y": 0 }, - "id": 6, + "id": 4, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "vertical", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, - "text": { - "valueSize": 150 - }, "textMode": "auto" }, - "pluginVersion": "8.4.6", + "pluginVersion": "9.3.2", "targets": [ { - "database": "default", "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, - "dateColDataType": "", - "dateLoading": false, "dateTimeType": "DATETIME", - "datetimeLoading": false, "extrapolate": true, "format": "table", "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, - "query": "SELECT\n count()\nFROM $table\n\nWHERE kind IN ('Pod') AND reason IN ('Created') AND namespace IN ($namespace)\n\n\n\n\n\n", - "rawQuery": "SELECT\n count()\nFROM default.events\n\nWHERE kind IN ('Pod') AND reason IN ('Created') AND namespace IN ('default','kube-system')", + "query": "SELECT\n count()\nFROM default.events\n\nWHERE Kind IN ('Pod') AND Reason IN ('Created') AND Namespace IN ($namespace)\n", + "rawQuery": "SELECT\n count()\nFROM default.events\n\nWHERE Kind IN ('Pod') AND Reason IN ('Created') AND Namespace IN ('argocd','quality','default','sonarqube','observability')", "refId": "A", "round": "0s", - "skip_comments": true, - "table": "events", - "tableLoading": false + "skip_comments": true } ], "title": "number of pod creation events in Namespace: $namespace", @@ -103,15 +99,20 @@ }, { "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, "description": "This table panal shows all the kubernetes datas.\nif its not showing the data check the time range and refresh.it will take some time to load the data. wait for sometime before clicking refresh another time.", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "custom": { "align": "center", "displayMode": "color-text", - "filterable": true + "filterable": true, + "inspect": false }, "mappings": [], "thresholds": { @@ -128,70 +129,50 @@ ] } }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "eventname" - }, - "properties": [ - { - "id": "custom.width", - "value": 489 - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 15, + "h": 16, "w": 24, "x": 0, "y": 7 }, "id": 2, - "interval": "10s", "options": { "footer": { "fields": "", - "reducer": ["sum"], + "reducer": [ + "sum" + ], "show": false }, - "showHeader": true, - "sortBy": [] + "showHeader": true }, - "pluginVersion": "8.4.6", + "pluginVersion": "9.3.2", "targets": [ { - "database": "default", "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, - "dateColDataType": "", - "dateLoading": false, - "dateTimeColDataType": "event_time", "dateTimeType": "DATETIME", - "datetimeLoading": false, "extrapolate": true, "format": "table", "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, - "query": "SELECT *\r\n \r\nFROM $table\r\n\r\nWHERE namespace IN ($namespace) AND reason IN ($reason) AND $timeFilter AND kind IN ($kind)\r\n", - "rawQuery": "SELECT *\r\n \r\nFROM default.events\r\n\r\nWHERE namespace IN ('kube-system','kubviz','default','monitoring','argocd','istio-system','crossplane-system','ingress-system','tekton-pipelines','kyverno') AND reason IN ('ApplyingManifest','ApplyManifestFailed','SuccessfulCreate','FailedScheduling','Starting','InvalidDiskCapacity','NodeHasSufficientMemory','NodeHasNoDiskPressure','NodeHasSufficientPID','NodeAllocatableEnforced','Scheduled','RegisteredNode','Pulling','Pulled','Created','Started','NodeReady','FailedToUpdateEndpoint','NoPods','RemovingNode','LeaderElection','EnsuringLoadBalancer','EnsuredLoadBalancer','Synced','FailedToUpdateEndpointSlices','Updated','Unhealthy','ApplyRoles','TaintManagerEviction','FailedCreate','FailedMount','BackOff','ResourceUpdated','UpdatedLoadBalancer','FailedGetResourceMetric','Killing','FailedKillPod','UnpackPackage') AND event_time >= toDateTime(1650613877) AND event_time <= toDateTime(1650635477) AND kind IN ('Addon','ReplicaSet','Pod','Node','DaemonSet','Endpoints','PodDisruptionBudget','Lease','Service','Secret','ConfigMap','Namespace','StatefulSet','Application','HorizontalPodAutoscaler','Provider')", + "query": "SELECT * FROM default.events\nWHERE Namespace IN ($namespace) AND Reason IN ($reason) AND Kind IN ($kind) AND ClusterName IN ($clusterName)", + "rawQuery": "SELECT * FROM default.events\nWHERE Namespace IN ('argocd','quality','default','sonarqube','observability') AND Reason IN ('OperationStarted','ResourceUpdated','OperationCompleted','FailedMount','ApplyClusterRoles','ApplyRoles','UpdateFailed','Unhealthy','InstallPackageRevision','Valid','Updated','RenderCRD','BackOff','BindClusterRole','SyncPackage') AND Kind IN ('Application','Pod','ProviderRevision','CompositeResourceDefinition','Namespace','ExternalSecret','Provider','SecretStore') AND ClusterName IN ('kubviz')", "refId": "A", "round": "0s", - "skip_comments": true, - "table": "events", - "tableLoading": false + "skip_comments": true } ], - "title": "kubernetes", + "title": "Kubernetes", "transparent": true, "type": "table" } ], - "refresh": "", - "schemaVersion": 35, + "schemaVersion": 37, "style": "dark", "tags": [], "templating": { @@ -203,15 +184,16 @@ "value": "$__all" }, "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, - "definition": "SELECT namespace FROM default.events", + "definition": "SELECT Namespace FROM default.events", "hide": 0, "includeAll": true, "multi": true, "name": "namespace", "options": [], - "query": "SELECT namespace FROM default.events", + "query": "SELECT Namespace FROM default.events", "refresh": 2, "regex": "", "skipUrlSync": false, @@ -225,15 +207,16 @@ "value": "$__all" }, "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, - "definition": "SELECT reason FROM default.events", + "definition": "SELECT Reason FROM default.events", "hide": 0, "includeAll": true, "multi": true, "name": "reason", "options": [], - "query": "SELECT reason FROM default.events", + "query": "SELECT Reason FROM default.events", "refresh": 2, "regex": "", "skipUrlSync": false, @@ -247,15 +230,39 @@ "value": "$__all" }, "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, - "definition": "SELECT kind FROM default.events", + "definition": "SELECT Kind FROM default.events", "hide": 0, "includeAll": true, "multi": true, "name": "kind", "options": [], - "query": "SELECT kind FROM default.events", + "query": "SELECT Kind FROM default.events", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" + }, + "definition": "SELECT ClusterName FROM default.events", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "clusterName", + "options": [], + "query": "SELECT ClusterName FROM default.events", "refresh": 2, "regex": "", "skipUrlSync": false, @@ -265,15 +272,13 @@ ] }, "time": { - "from": "now-12h", + "from": "now-15m", "to": "now" }, - "timepicker": { - "hidden": false - }, + "timepicker": {}, "timezone": "", - "title": "kubedatas", - "uid": "WnmZG5Jnz", + "title": "Kubedata", + "uid": "Qq-FK1rVz", "version": 1, "weekStart": "" } \ No newline at end of file diff --git a/charts/grafana/dashboards/plugins-dashboard.json b/charts/grafana/dashboards/features-dashboard.json similarity index 72% rename from charts/grafana/dashboards/plugins-dashboard.json rename to charts/grafana/dashboards/features-dashboard.json index c96531bf..34db94e8 100644 --- a/charts/grafana/dashboards/plugins-dashboard.json +++ b/charts/grafana/dashboards/features-dashboard.json @@ -21,15 +21,16 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 8, + "id": 10, "links": [], "liveNow": false, "panels": [ { "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, - "description": "This panel displays the access rights for the current user and all server resources.", + "description": "This panel displays all the resources running in the kubernetes cluster.", "fieldConfig": { "defaults": { "color": { @@ -64,78 +65,6 @@ "x": 0, "y": 0 }, - "id": 10, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "datasource": { - "type": "vertamedia-clickhouse-datasource" - }, - "dateTimeType": "DATETIME", - "extrapolate": true, - "format": "table", - "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", - "intervalFactor": 1, - "query": "SELECT * FROM default.rakkess", - "rawQuery": "SELECT * FROM default.rakkess", - "refId": "A", - "round": "0s", - "skip_comments": true - } - ], - "title": "Rakkess Access Matrix", - "transparent": true, - "type": "table" - }, - { - "datasource": { - "type": "vertamedia-clickhouse-datasource" - }, - "description": "This panel displays all the kubernetes resources from the cluster", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "displayMode": "color-text", - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 8 - }, "id": 8, "options": { "footer": { @@ -151,7 +80,8 @@ "targets": [ { "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, "dateTimeType": "DATETIME", "extrapolate": true, @@ -171,9 +101,10 @@ }, { "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, - "description": "This panel displays the out-off-date images from the cluster.", + "description": "This panel displays all the out-of-date images from the cluster.", "fieldConfig": { "defaults": { "color": { @@ -206,7 +137,7 @@ "h": 9, "w": 24, "x": 0, - "y": 16 + "y": 8 }, "id": 6, "options": { @@ -223,15 +154,16 @@ "targets": [ { "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, "dateTimeType": "DATETIME", "extrapolate": true, "format": "table", "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, - "query": "SELECT * FROM default.outdated_images\nWHERE Versions_Behind > 0", - "rawQuery": "SELECT * FROM default.outdated_images\nWHERE Versions_Behind > 0", + "query": "SELECT * FROM default.outdated_images\nWHERE VersionsBehind > 0", + "rawQuery": "SELECT * FROM default.outdated_images\nWHERE VersionsBehind > 0", "refId": "A", "round": "0s", "skip_comments": true @@ -243,9 +175,10 @@ }, { "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, - "description": "This panel displays all the deleted APIs from the cluster", + "description": "This panel displays all the DeletedAPIs from the cluster.", "fieldConfig": { "defaults": { "color": { @@ -278,7 +211,7 @@ "h": 8, "w": 24, "x": 0, - "y": 25 + "y": 17 }, "id": 4, "options": { @@ -295,7 +228,8 @@ "targets": [ { "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, "dateTimeType": "DATETIME", "extrapolate": true, @@ -315,9 +249,10 @@ }, { "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, - "description": "This panel displays all the deprecatedAPIs from the cluster", + "description": "This panel displays all the DeprecatedAPIs from the cluster.", "fieldConfig": { "defaults": { "color": { @@ -347,10 +282,10 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 9, "w": 24, "x": 0, - "y": 33 + "y": 25 }, "id": 2, "options": { @@ -367,7 +302,8 @@ "targets": [ { "datasource": { - "type": "vertamedia-clickhouse-datasource" + "type": "vertamedia-clickhouse-datasource", + "uid": "vertamedia-clickhouse-datasource" }, "dateTimeType": "DATETIME", "extrapolate": true, @@ -398,8 +334,8 @@ }, "timepicker": {}, "timezone": "", - "title": "Kubviz Plugins", - "uid": "wyP_aLl4z", - "version": 2, + "title": "Kubviz Features", + "uid": "o2M7hbrVk", + "version": 1, "weekStart": "" } \ No newline at end of file diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml index 291b36e3..b3be1975 100644 --- a/charts/grafana/values.yaml +++ b/charts/grafana/values.yaml @@ -543,7 +543,7 @@ dashboards: custom-dashboard: file: dashboards/custom-dashboard.json plugins-dashboard: - file: dashboards/plugins-dashboard.json + file: dashboards/features-dashboard.json # prometheus-stats: # gnetId: 2 # revision: 2 diff --git a/clickhouse/clickhouse_client.go b/clickhouse/clickhouse_client.go index c981bdbe..097f6d86 100644 --- a/clickhouse/clickhouse_client.go +++ b/clickhouse/clickhouse_client.go @@ -72,11 +72,11 @@ func CreateKubePugSchema(connect *sql.DB) { _, err := connect.Exec(` CREATE TABLE IF NOT EXISTS DeprecatedAPIs ( ClusterName String, + ObjectName String, Description String, Kind String, Deprecated UInt8, - Scope String, - ObjectName String + Scope String ) engine=File(TabSeparated) `) if err != nil { @@ -86,13 +86,13 @@ func CreateKubePugSchema(connect *sql.DB) { _, err = connect.Exec(` CREATE TABLE IF NOT EXISTS DeletedAPIs ( ClusterName String, + ObjectName String, Group String, Kind String, Version String, Name String, Deleted UInt8, - Scope String, - ObjectName String + Scope String ) engine=File(TabSeparated) `) if err != nil { @@ -103,7 +103,7 @@ func CreateKubePugSchema(connect *sql.DB) { func CreateKetallSchema(connect *sql.DB) { _, err := connect.Exec(` CREATE TABLE IF NOT EXISTS getall_resources ( - Cluster_Name String, + ClusterName String, Namespace String, Kind String, Resource String, @@ -118,13 +118,13 @@ func CreateKetallSchema(connect *sql.DB) { func CreateOutdatedSchema(connect *sql.DB) { _, err := connect.Exec(` CREATE TABLE IF NOT EXISTS outdated_images ( - Cluster_Name String, + ClusterName String, Namespace String, Pod String, - Current_Image String, - Current_Tag String, - Latest_Version String, - Versions_Behind Int64 + CurrentImage String, + CurrentTag String, + LatestVersion String, + VersionsBehind Int64 ) engine=File(TabSeparated) `) if err != nil { @@ -155,7 +155,7 @@ func InsertRakeesMetrics(connect *sql.DB, metrics model.RakeesMetrics) { func InsertKetallEvent(connect *sql.DB, metrics model.Resource) { var ( tx, _ = connect.Begin() - stmt, _ = tx.Prepare("INSERT INTO getall_resources (Cluster_Name, Namespace, Kind, Resource, Age) VALUES (?, ?, ?, ?, ?)") + stmt, _ = tx.Prepare("INSERT INTO getall_resources (ClusterName, Namespace, Kind, Resource, Age) VALUES (?, ?, ?, ?, ?)") ) defer stmt.Close() if _, err := stmt.Exec( @@ -175,7 +175,7 @@ func InsertKetallEvent(connect *sql.DB, metrics model.Resource) { func InsertOutdatedEvent(connect *sql.DB, metrics model.CheckResultfinal) { var ( tx, _ = connect.Begin() - stmt, _ = tx.Prepare("INSERT INTO outdated_images (Cluster_Name, Namespace, Pod, Current_Image, Current_Tag, Latest_Version, Versions_Behind) VALUES (?, ?, ?, ?, ?, ?, ?)") + stmt, _ = tx.Prepare("INSERT INTO outdated_images (ClusterName, Namespace, Pod, CurrentImage, CurrentTag, LatestVersion, VersionsBehind) VALUES (?, ?, ?, ?, ?, ?, ?)") ) defer stmt.Close() if _, err := stmt.Exec( @@ -197,7 +197,7 @@ func InsertOutdatedEvent(connect *sql.DB, metrics model.CheckResultfinal) { func InsertDeprecatedAPI(connect *sql.DB, deprecatedAPI model.DeprecatedAPI) { var ( tx, _ = connect.Begin() - stmt, _ = tx.Prepare("INSERT INTO DeprecatedAPIs (ClusterName, Description, Kind, Deprecated, Scope, ObjectName) VALUES (?, ?, ?, ?, ?, ?)") + stmt, _ = tx.Prepare("INSERT INTO DeprecatedAPIs (ClusterName, ObjectName, Description, Kind, Deprecated, Scope) VALUES (?, ?, ?, ?, ?, ?)") ) defer stmt.Close() @@ -209,11 +209,11 @@ func InsertDeprecatedAPI(connect *sql.DB, deprecatedAPI model.DeprecatedAPI) { for _, item := range deprecatedAPI.Items { if _, err := stmt.Exec( deprecatedAPI.ClusterName, + item.ObjectName, deprecatedAPI.Description, deprecatedAPI.Kind, deprecated, item.Scope, - item.ObjectName, ); err != nil { log.Fatal(err) } @@ -227,7 +227,7 @@ func InsertDeprecatedAPI(connect *sql.DB, deprecatedAPI model.DeprecatedAPI) { func InsertDeletedAPI(connect *sql.DB, deletedAPI model.DeletedAPI) { var ( tx, _ = connect.Begin() - stmt, _ = tx.Prepare("INSERT INTO DeletedAPIs (ClusterName, Group, Kind, Version, Name, Deleted, Scope, ObjectName) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") + stmt, _ = tx.Prepare("INSERT INTO DeletedAPIs (ClusterName, ObjectName, Group, Kind, Version, Name, Deleted, Scope) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") ) defer stmt.Close() @@ -239,13 +239,13 @@ func InsertDeletedAPI(connect *sql.DB, deletedAPI model.DeletedAPI) { for _, item := range deletedAPI.Items { if _, err := stmt.Exec( deletedAPI.ClusterName, + item.ObjectName, deletedAPI.Group, deletedAPI.Kind, deletedAPI.Version, deletedAPI.Name, deleted, item.Scope, - item.ObjectName, ); err != nil { log.Fatal(err) }