From 2eecd7c5c167e06d7864f2b8048160ad0bb7d226 Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Sat, 29 Jul 2023 08:03:21 +0530 Subject: [PATCH 1/3] added shedule --- agent/kubviz/k8smetrics_agent.go | 104 ++++++++++++++++++++-------- agent/kubviz/trivy_image.go | 69 ++++++++++++++++++ client/pkg/clickhouse/db_client.go | 32 ++++++++- client/pkg/clickhouse/statements.go | 17 ++++- client/pkg/clients/kubviz_client.go | 18 ++++- constants/constants.go | 2 + go.mod | 3 +- go.sum | 12 +++- model/trivy_image.go | 9 +++ 9 files changed, 230 insertions(+), 36 deletions(-) create mode 100644 agent/kubviz/trivy_image.go create mode 100644 model/trivy_image.go diff --git a/agent/kubviz/k8smetrics_agent.go b/agent/kubviz/k8smetrics_agent.go index 14239f97..f140874b 100644 --- a/agent/kubviz/k8smetrics_agent.go +++ b/agent/kubviz/k8smetrics_agent.go @@ -2,8 +2,6 @@ package main import ( "encoding/json" - "github.com/intelops/kubviz/constants" - "github.com/nats-io/nats.go" "log" "os" "strconv" @@ -11,7 +9,11 @@ import ( "sync" "time" + "github.com/go-co-op/gocron" + "github.com/nats-io/nats.go" + "context" + "github.com/intelops/kubviz/model" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,6 +34,14 @@ import ( "k8s.io/client-go/tools/clientcmd" ) +// constants for jetstream +const ( + streamName = "METRICS" + streamSubjects = "METRICS.*" + eventSubject = "METRICS.kubvizevent" + allSubject = "METRICS.all" +) + type RuningEnv int const ( @@ -48,7 +58,8 @@ var ( //for local testing provide the location of kubeconfig // inside the civo file paste your kubeconfig // uncomment this line from Dockerfile.Kubviz (COPY --from=builder /workspace/civo /etc/myapp/civo) - cluster_conf_loc string = os.Getenv("CONFIG_LOCATION") + cluster_conf_loc string = os.Getenv("CONFIG_LOCATION") + schedulingIntervalStr string = os.Getenv("SCHEDULING_INTERVAL") ) func main() { @@ -57,17 +68,18 @@ func main() { outdatedErrChan := make(chan error, 1) kubePreUpgradeChan := make(chan error, 1) getAllResourceChan := make(chan error, 1) + trivyK8sMetricsChan := make(chan error, 1) clusterMetricsChan := make(chan error, 1) kubescoreMetricsChan := make(chan error, 1) - trivyK8sMetricsChan := make(chan error, 1) + trivyImagescanChan := make(chan error, 1) RakeesErrChan := make(chan error, 1) var ( wg sync.WaitGroup config *rest.Config clientset *kubernetes.Clientset ) - // waiting for 7 go routines - wg.Add(7) + // waiting for 4 go routines + wg.Add(6) // connecting with nats ... nc, err := nats.Connect(natsurl, nats.Name("K8s Metrics"), nats.Token(token)) checkErr(err) @@ -92,23 +104,35 @@ func main() { clientset = getK8sClient(config) } // starting all the go routines - go outDatedImages(config, js, &wg, outdatedErrChan) - go KubePreUpgradeDetector(config, js, &wg, kubePreUpgradeChan) - go GetAllResources(config, js, &wg, getAllResourceChan) - go RakeesOutput(config, js, &wg, RakeesErrChan) - go getK8sEvents(clientset) - go publishMetrics(clientset, js, &wg, clusterMetricsChan) - go RunKubeScore(clientset, js, &wg, kubescoreMetricsChan) - go RunTrivyK8sClusterScan(&wg, js, trivyK8sMetricsChan) - wg.Wait() - // once the go routines completes we will close the error channels - close(outdatedErrChan) - close(kubePreUpgradeChan) - close(getAllResourceChan) - close(clusterMetricsChan) - close(kubescoreMetricsChan) - close(RakeesErrChan) - close(trivyK8sMetricsChan) + collectAndPublishMetrics := func() { + go outDatedImages(config, js, &wg, outdatedErrChan) + go KubePreUpgradeDetector(config, js, &wg, kubePreUpgradeChan) + go GetAllResources(config, js, &wg, getAllResourceChan) + go RakeesOutput(config, js, &wg, RakeesErrChan) + go getK8sEvents(clientset) + go RunTrivyImageScans(config, js, &wg, trivyImagescanChan) + go publishMetrics(clientset, js, &wg, clusterMetricsChan) + go RunKubeScore(clientset, js, &wg, kubescoreMetricsChan) + go RunTrivyK8sClusterScan(&wg, js, trivyK8sMetricsChan) + wg.Wait() + // once the go routines completes we will close the error channels + close(outdatedErrChan) + close(kubePreUpgradeChan) + close(getAllResourceChan) + close(clusterMetricsChan) + close(kubescoreMetricsChan) + close(trivyImagescanChan) + close(trivyK8sMetricsChan) + close(RakeesErrChan) + } + collectAndPublishMetrics() + schedulingInterval, err := time.ParseDuration(schedulingIntervalStr) + if err != nil { + log.Fatalf("Failed to parse SCHEDULING_INTERVAL: %v", err) + } + s := gocron.NewScheduler(time.UTC) + s.Every(schedulingInterval).Do(collectAndPublishMetrics) // Adjust the interval as needed + s.StartAsync() // for loop will wait for the error channels // logs if any error occurs for { @@ -133,7 +157,7 @@ func main() { if err != nil { log.Println(err) } - case err := <-RakeesErrChan: + case err := <-trivyImagescanChan: if err != nil { log.Println(err) } @@ -141,11 +165,33 @@ func main() { if err != nil { log.Println(err) } + case err := <-RakeesErrChan: + if err != nil { + log.Println(err) + } } } } +//func setupAgent() { +// configurations, err := config.GetAgentConfigurations() +// if err != nil { +// log.Printf("Failed to get agent config: %v\n", err) +// panic(err) +// } +// //k8s := &K8sData{ +// // Namespace: configurations.SANamespace, +// // ServiceAccountName: configurations.SAName, +// // KubeconfigFileName: constants.KUBECONFIG, +// //} +// //_, err = k8s.GenerateKubeConfiguration() +// if err != nil { +// log.Printf("Failed to generate kubeconfig: %v\n", err) +// panic(err) +// } +//} + // publishMetrics publishes stream of events // with subject "METRICS.created" func publishMetrics(clientset *kubernetes.Clientset, js nats.JetStreamContext, wg *sync.WaitGroup, errCh chan error) { @@ -163,7 +209,7 @@ func publishK8sMetrics(id string, mtype string, mdata *v1.Event, js nats.JetStre ClusterName: ClusterName, } metricsJson, _ := json.Marshal(metrics) - _, err := js.Publish(constants.EventSubject, metricsJson) + _, err := js.Publish(eventSubject, metricsJson) if err != nil { return true, err } @@ -174,16 +220,16 @@ func publishK8sMetrics(id string, mtype string, mdata *v1.Event, js nats.JetStre // createStream creates a stream by using JetStreamContext func createStream(js nats.JetStreamContext) error { // Check if the METRICS stream already exists; if not, create it. - stream, err := js.StreamInfo(constants.StreamName) + stream, err := js.StreamInfo(streamName) log.Printf("Retrieved stream %s", fmt.Sprintf("%v", stream)) if err != nil { log.Printf("Error getting stream %s", err) } if stream == nil { - log.Printf("creating stream %q and subjects %q", constants.StreamName, constants.StreamSubjects) + log.Printf("creating stream %q and subjects %q", streamName, streamSubjects) _, err = js.AddStream(&nats.StreamConfig{ - Name: constants.StreamName, - Subjects: []string{constants.StreamSubjects}, + Name: streamName, + Subjects: []string{streamSubjects}, }) checkErr(err) } diff --git a/agent/kubviz/trivy_image.go b/agent/kubviz/trivy_image.go new file mode 100644 index 00000000..f934bb65 --- /dev/null +++ b/agent/kubviz/trivy_image.go @@ -0,0 +1,69 @@ +package main + +import ( + "encoding/json" + "log" + "strings" + "sync" + + //"github.com/aquasecurity/trivy/pkg/k8s/report" + "github.com/aquasecurity/trivy/pkg/types" + "github.com/google/uuid" + "github.com/intelops/kubviz/constants" + "github.com/intelops/kubviz/model" + "github.com/nats-io/nats.go" + "k8s.io/client-go/rest" +) + +func RunTrivyImageScans(config *rest.Config, js nats.JetStreamContext, wg *sync.WaitGroup, errCh chan error) { + defer wg.Done() + //kubeconfigPath := "/home/nithu/JAD/config" // Set the correct kubeconfig file path or pass nil + images, err := ListImages(config) + if err != nil { + log.Fatal(err) + } + + for _, image := range images { + var report types.Report + out, err := executeCommand("trivy image " + image.PullableImage + " --timeout 60m -f json -q --cache-dir /tmp/.cache") + if err != nil { + log.Printf("Error scanning image %s: %v", image.PullableImage, err) + continue // Move on to the next image in case of an error + } + + parts := strings.SplitN(out, "{", 2) + if len(parts) <= 1 { + log.Println("No output from command", err) + continue // Move on to the next image if there's no output + } + + log.Println("Command logs", parts[0]) + jsonPart := "{" + parts[1] + log.Println("First 200 lines output", jsonPart[:200]) + log.Println("Last 200 lines output", jsonPart[len(jsonPart)-200:]) + + err = json.Unmarshal([]byte(jsonPart), &report) + if err != nil { + log.Printf("Error occurred while Unmarshalling json: %v", err) + continue // Move on to the next image in case of an error + } + publishImageScanReports(report, js, errCh) + // If you want to publish the report or perform any other action with it, you can do it here + + } +} + +func publishImageScanReports(report types.Report, js nats.JetStreamContext, errCh chan error) { + metrics := model.TrivyImage{ + ID: uuid.New().String(), + ClusterName: ClusterName, + Report: report, + } + metricsJson, _ := json.Marshal(metrics) + _, err := js.Publish(constants.TRIVY_IMAGE_SUBJECT, metricsJson) + if err != nil { + errCh <- err + } + log.Printf("Trivy report with ID:%s has been published\n", metrics.ID) + errCh <- nil +} diff --git a/client/pkg/clickhouse/db_client.go b/client/pkg/clickhouse/db_client.go index bfe430f0..ca167578 100644 --- a/client/pkg/clickhouse/db_client.go +++ b/client/pkg/clickhouse/db_client.go @@ -32,6 +32,7 @@ type DBInterface interface { InsertKubvizEvent(model.Metrics) InsertGitEvent(string) InsertKubeScoreMetrics(model.KubeScoreRecommendations) + InsertTrivyImageMetrics(metrics model.TrivyImage) InsertTrivyMetrics(metrics model.Trivy) RetriveKetallEvent() ([]model.Resource, error) RetriveOutdatedEvent() ([]model.CheckResultfinal, error) @@ -69,7 +70,7 @@ func NewDBClient(conf *config.Config) (DBInterface, error) { return nil, err } - tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)} + tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)} for _, table := range tables { if err = splconn.Exec(context.Background(), string(table)); err != nil { return nil, err @@ -352,7 +353,36 @@ func (c *DBClient) InsertTrivyMetrics(metrics model.Trivy) { } } +func (c *DBClient) InsertTrivyImageMetrics(metrics model.TrivyImage) { + for _, result := range metrics.Report.Results { + for _, vulnerability := range result.Vulnerabilities { + var ( + tx, _ = c.conn.Begin() + stmt, _ = tx.Prepare(InsertTrivyImage) + ) + if _, err := stmt.Exec( + metrics.ID, + metrics.ClusterName, + vulnerability.VulnerabilityID, + vulnerability.PkgID, + vulnerability.PkgName, + vulnerability.InstalledVersion, + vulnerability.FixedVersion, + vulnerability.Title, + vulnerability.Severity, + vulnerability.PublishedDate, + vulnerability.LastModifiedDate, + ); err != nil { + log.Fatal(err) + } + if err := tx.Commit(); err != nil { + log.Fatal(err) + } + stmt.Close() + } + } +} func (c *DBClient) Close() { _ = c.conn.Close() } diff --git a/client/pkg/clickhouse/statements.go b/client/pkg/clickhouse/statements.go index 91d623b8..dfd9d673 100644 --- a/client/pkg/clickhouse/statements.go +++ b/client/pkg/clickhouse/statements.go @@ -119,7 +119,21 @@ const trivyTableMisconfig DBStatement = ` misconfig_status String ) engine=File(TabSeparated) ` - +const trivyTableImage DBStatement = ` + CREATE TABLE IF NOT EXISTS trivyimage ( + id UUID, + cluster_name String, + vul_id String, + vul_pkg_id String, + vul_pkg_name String, + vul_installed_version String, + vul_fixed_version String, + vul_title String, + vul_severity String, + vul_published_date DateTime('UTC'), + vul_last_modified_date DateTime('UTC') + ) engine=File(TabSeparated) + ` const dockerHubBuildTable DBStatement = ` CREATE TABLE IF NOT EXISTS dockerhubbuild ( PushedBy String, @@ -143,4 +157,5 @@ const containerDockerhubTable DBStatement = `CREATE table IF NOT EXISTS containe const containerGithubTable DBStatement = `CREATE table IF NOT EXISTS container_github(event JSON) ENGINE = MergeTree ORDER BY tuple();` const InsertKubeScore string = "INSERT INTO kubescore (id, namespace, cluster_name, recommendations) VALUES (?, ?, ?, ?)" const InsertTrivyVul string = "INSERT INTO trivy_vul (id, cluster_name, namespace, kind, name, vul_id, vul_vendor_ids, vul_pkg_id, vul_pkg_name, vul_pkg_path, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?)" +const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, vul_id, vul_pkg_id, vul_pkg_name, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?, ?)" diff --git a/client/pkg/clients/kubviz_client.go b/client/pkg/clients/kubviz_client.go index 0cd2e809..530864de 100644 --- a/client/pkg/clients/kubviz_client.go +++ b/client/pkg/clients/kubviz_client.go @@ -2,9 +2,10 @@ package clients import ( "encoding/json" + "log" + "github.com/intelops/kubviz/constants" "github.com/nats-io/nats.go" - "log" "github.com/intelops/kubviz/client/pkg/clickhouse" "github.com/intelops/kubviz/model" @@ -97,6 +98,21 @@ func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface) { log.Println() }, }, + { + Subject: constants.TRIVY_IMAGE_SUBJECT, + Consumer: constants.Trivy_Image_Consumer, + Handler: func(msg *nats.Msg) { + msg.Ack() + var metrics model.TrivyImage + err := json.Unmarshal(msg.Data, &metrics) + if err != nil { + log.Fatal(err) + } + log.Printf("Trivy Metrics Received: %#v,", metrics) + conn.InsertTrivyImageMetrics(metrics) + log.Println() + }, + }, { Subject: constants.KubvizSubject, Consumer: constants.KubvizConsumer, diff --git a/constants/constants.go b/constants/constants.go index f91041f0..9d76bed9 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -25,4 +25,6 @@ const ( KubvizConsumer = "KUBVIZ_EVENTS_CONSUMER" KubscoreConsumer = "KUBSCORE_CONSUMER" TrivyConsumer = "TRIVY_CONSUMER" + TRIVY_IMAGE_SUBJECT = "METRICS.trivyimage" + Trivy_Image_Consumer = "TRIVY_IMAGE_CONSUMER" ) diff --git a/go.mod b/go.mod index 1ebe913a..e66c9e8c 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/ghodss/yaml v1.0.0 github.com/gin-gonic/gin v1.9.1 github.com/go-chi/chi/v5 v5.0.8 - github.com/go-playground/webhooks/v6 v6.2.0 + github.com/go-co-op/gocron v1.30.1 github.com/google/uuid v1.3.0 github.com/hashicorp/go-version v1.6.0 github.com/kelseyhightower/envconfig v1.4.0 @@ -105,6 +105,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/samber/lo v1.38.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect diff --git a/go.sum b/go.sum index e350058f..2c732848 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,8 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-co-op/gocron v1.30.1 h1:tjWUvJl5KrcwpkEkSXFSQFr4F9h5SfV/m4+RX0cV2fs= +github.com/go-co-op/gocron v1.30.1/go.mod h1:39f6KNSGVOU1LO/ZOoZfcSxwlsJDQOKSu8erN0SH48Y= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= @@ -165,8 +167,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-playground/webhooks/v6 v6.2.0 h1:SV/Euz3xoTc7LQanUtXaYhVQU0rw4DaxNhNKOBZ90JI= -github.com/go-playground/webhooks/v6 v6.2.0/go.mod h1:GCocmfMtpJdkEOM1uG9p2nXzg1kY5X/LtvQgtPHUaaA= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= @@ -174,7 +174,6 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-yaml v1.8.1 h1:JuZRFlqLM5cWF6A+waL8AKVuCcqvKOuhJtUQI+L3ez0= -github.com/gogits/go-gogs-client v0.0.0-20200905025246-8bb8a50cb355/go.mod h1:cY2AIrMgHm6oOHmR7jY+9TtjzSjQ3iG7tURJG3Y6XH0= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -270,6 +269,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -379,7 +379,11 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rubenv/sql-migrate v1.3.1 h1:Vx+n4Du8X8VTYuXbhNxdEUoh6wiJERA0GlWocR5FrbA= @@ -460,6 +464,7 @@ go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyK go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -590,6 +595,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= diff --git a/model/trivy_image.go b/model/trivy_image.go new file mode 100644 index 00000000..2a47e426 --- /dev/null +++ b/model/trivy_image.go @@ -0,0 +1,9 @@ +package model + +import "github.com/aquasecurity/trivy/pkg/types" + +type TrivyImage struct { + ID string + ClusterName string + Report types.Report +} From b58f5b8e942a41cb90ad8bfc4a58be63c0a802db Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Sat, 29 Jul 2023 13:01:37 +0530 Subject: [PATCH 2/3] fix --- agent/kubviz/k8smetrics_agent.go | 37 ++++++-------------------------- agent/kubviz/trivy_image.go | 2 -- 2 files changed, 7 insertions(+), 32 deletions(-) diff --git a/agent/kubviz/k8smetrics_agent.go b/agent/kubviz/k8smetrics_agent.go index f140874b..86119001 100644 --- a/agent/kubviz/k8smetrics_agent.go +++ b/agent/kubviz/k8smetrics_agent.go @@ -14,6 +14,7 @@ import ( "context" + "github.com/intelops/kubviz/constants" "github.com/intelops/kubviz/model" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,12 +36,6 @@ import ( ) // constants for jetstream -const ( - streamName = "METRICS" - streamSubjects = "METRICS.*" - eventSubject = "METRICS.kubvizevent" - allSubject = "METRICS.all" -) type RuningEnv int @@ -79,7 +74,7 @@ func main() { clientset *kubernetes.Clientset ) // waiting for 4 go routines - wg.Add(6) + wg.Add(8) // connecting with nats ... nc, err := nats.Connect(natsurl, nats.Name("K8s Metrics"), nats.Token(token)) checkErr(err) @@ -174,24 +169,6 @@ func main() { } -//func setupAgent() { -// configurations, err := config.GetAgentConfigurations() -// if err != nil { -// log.Printf("Failed to get agent config: %v\n", err) -// panic(err) -// } -// //k8s := &K8sData{ -// // Namespace: configurations.SANamespace, -// // ServiceAccountName: configurations.SAName, -// // KubeconfigFileName: constants.KUBECONFIG, -// //} -// //_, err = k8s.GenerateKubeConfiguration() -// if err != nil { -// log.Printf("Failed to generate kubeconfig: %v\n", err) -// panic(err) -// } -//} - // publishMetrics publishes stream of events // with subject "METRICS.created" func publishMetrics(clientset *kubernetes.Clientset, js nats.JetStreamContext, wg *sync.WaitGroup, errCh chan error) { @@ -209,7 +186,7 @@ func publishK8sMetrics(id string, mtype string, mdata *v1.Event, js nats.JetStre ClusterName: ClusterName, } metricsJson, _ := json.Marshal(metrics) - _, err := js.Publish(eventSubject, metricsJson) + _, err := js.Publish(constants.EventSubject, metricsJson) if err != nil { return true, err } @@ -220,16 +197,16 @@ func publishK8sMetrics(id string, mtype string, mdata *v1.Event, js nats.JetStre // createStream creates a stream by using JetStreamContext func createStream(js nats.JetStreamContext) error { // Check if the METRICS stream already exists; if not, create it. - stream, err := js.StreamInfo(streamName) + stream, err := js.StreamInfo(constants.StreamName) log.Printf("Retrieved stream %s", fmt.Sprintf("%v", stream)) if err != nil { log.Printf("Error getting stream %s", err) } if stream == nil { - log.Printf("creating stream %q and subjects %q", streamName, streamSubjects) + log.Printf("creating stream %q and subjects %q", constants.StreamName, constants.StreamSubjects) _, err = js.AddStream(&nats.StreamConfig{ - Name: streamName, - Subjects: []string{streamSubjects}, + Name: constants.StreamName, + Subjects: []string{constants.StreamSubjects}, }) checkErr(err) } diff --git a/agent/kubviz/trivy_image.go b/agent/kubviz/trivy_image.go index f934bb65..f6d2c132 100644 --- a/agent/kubviz/trivy_image.go +++ b/agent/kubviz/trivy_image.go @@ -6,7 +6,6 @@ import ( "strings" "sync" - //"github.com/aquasecurity/trivy/pkg/k8s/report" "github.com/aquasecurity/trivy/pkg/types" "github.com/google/uuid" "github.com/intelops/kubviz/constants" @@ -17,7 +16,6 @@ import ( func RunTrivyImageScans(config *rest.Config, js nats.JetStreamContext, wg *sync.WaitGroup, errCh chan error) { defer wg.Done() - //kubeconfigPath := "/home/nithu/JAD/config" // Set the correct kubeconfig file path or pass nil images, err := ListImages(config) if err != nil { log.Fatal(err) From e9afd7c1d57c2f075fa4cbec50d06cd5eeccb0a8 Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Sat, 29 Jul 2023 16:50:12 +0530 Subject: [PATCH 3/3] Fix --- client/pkg/clickhouse/db_client.go | 1 + client/pkg/clickhouse/statements.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/client/pkg/clickhouse/db_client.go b/client/pkg/clickhouse/db_client.go index ca167578..0e54a560 100644 --- a/client/pkg/clickhouse/db_client.go +++ b/client/pkg/clickhouse/db_client.go @@ -363,6 +363,7 @@ func (c *DBClient) InsertTrivyImageMetrics(metrics model.TrivyImage) { if _, err := stmt.Exec( metrics.ID, metrics.ClusterName, + metrics.Report.ArtifactName, vulnerability.VulnerabilityID, vulnerability.PkgID, vulnerability.PkgName, diff --git a/client/pkg/clickhouse/statements.go b/client/pkg/clickhouse/statements.go index dfd9d673..12ee7754 100644 --- a/client/pkg/clickhouse/statements.go +++ b/client/pkg/clickhouse/statements.go @@ -123,6 +123,7 @@ const trivyTableImage DBStatement = ` CREATE TABLE IF NOT EXISTS trivyimage ( id UUID, cluster_name String, + artifact_name String, vul_id String, vul_pkg_id String, vul_pkg_name String, @@ -157,5 +158,5 @@ const containerDockerhubTable DBStatement = `CREATE table IF NOT EXISTS containe const containerGithubTable DBStatement = `CREATE table IF NOT EXISTS container_github(event JSON) ENGINE = MergeTree ORDER BY tuple();` const InsertKubeScore string = "INSERT INTO kubescore (id, namespace, cluster_name, recommendations) VALUES (?, ?, ?, ?)" const InsertTrivyVul string = "INSERT INTO trivy_vul (id, cluster_name, namespace, kind, name, vul_id, vul_vendor_ids, vul_pkg_id, vul_pkg_name, vul_pkg_path, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?)" -const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, vul_id, vul_pkg_id, vul_pkg_name, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" +const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artifact_name, vul_id, vul_pkg_id, vul_pkg_name, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES ( ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?, ?)"