issuse #94 obj to table

agent/container/cfg.yaml

@@ -3,4 +3,4 @@ generate:
   chi-server: true
   models: true
   embedded-spec: true
-output: /Users/vijeshdeepan/Desktop/kubviz/agent/container/api/agent.gen.go
+output: agent/container/api/agent.gen.go

agent/container/pkg/handler/docker_event_dockerhub.go

@@ -1,20 +1,32 @@
 package handler
 
 import (
+	"errors"
 	"io"
 	"log"
 	"net/http"
 )
 
+// parse errors
+var (
+	ErrReadingBody   = errors.New("error reading the request body")
+	ErrPublishToNats = errors.New("error while publishing to nats")
+)
+
 func (ah *APIHandler) PostEventDockerHub(w http.ResponseWriter, r *http.Request) {
-	event, err := io.ReadAll(r.Body)
-	if err != nil {
-		log.Printf("Event body read failed: %v", err)
+	defer func() {
+		_, _ = io.Copy(io.Discard, r.Body)
+		_ = r.Body.Close()
+	}()
+	payload, err := io.ReadAll(r.Body)
+	if err != nil || len(payload) == 0 {
+		log.Printf("%v: %v", ErrReadingBody, err)
+		return
 	}
-
-	log.Printf("Received event from docker artifactory: %v", string(event))
-	err = ah.conn.Publish(event, "docker registry")
+	log.Printf("Received event from docker artifactory: %v", string(payload))
+	err = ah.conn.Publish(payload, "Dockerhub_Registry")
 	if err != nil {
-		log.Printf("Publish failed for event: %v, reason: %v", string(event), err)
+		log.Printf("%v: %v", ErrPublishToNats, err)
+		return
 	}
 }

agent/container/pkg/handler/docker_event_github.go

@@ -1,20 +1,37 @@
 package handler
 
 import (
+	"errors"
 	"io"
 	"log"
 	"net/http"
 )
 
+var (
+	ErrMissingGithubEventHeader = errors.New("missing X-GitHub-Event Header")
+)
+
 func (ah *APIHandler) PostEventDockerGithub(w http.ResponseWriter, r *http.Request) {
-	event, err := io.ReadAll(r.Body)
-	if err != nil {
-		log.Printf("Event body read failed: %v", err)
+	defer func() {
+		_, _ = io.Copy(io.Discard, r.Body)
+		_ = r.Body.Close()
+	}()
+	event := r.Header.Get("X-GitHub-Event")
+	if event == "" {
+		log.Printf("%v", ErrMissingGithubEventHeader)
+		return
+	}
+
+	payload, err := io.ReadAll(r.Body)
+	if err != nil || len(payload) == 0 {
+		log.Printf("%v: %v", ErrReadingBody, err)
+		return
 	}
 
-	log.Printf("Received docker event from github artifactory: %v", string(event))
-	err = ah.conn.Publish(event, "Github_Registory")
+	log.Printf("Received docker event from github artifactory: %v", string(payload))
+	err = ah.conn.Publish(payload, "Github_Registry")
 	if err != nil {
-		log.Printf("Publish failed for event: %v, reason: %v", string(event), err)
+		log.Printf("%v: %v", ErrPublishToNats, err)
+		return
 	}
 }

client/pkg/clickhouse/db_client.go

@@ -33,7 +33,7 @@ type DBInterface interface {
 	RetriveOutdatedEvent() ([]model.CheckResultfinal, error)
 	RetriveKubepugEvent() ([]model.Result, error)
 	RetrieveKubvizEvent() ([]model.DbEvent, error)
-	InsertContainerEventDockerHub(string)
+	InsertContainerEventDockerHub(model.DockerHubBuild)
 	InsertContainerEventGithub(string)
 	Close()
 }
@@ -63,7 +63,7 @@ func NewDBClient(conf *config.Config) (DBInterface, error) {
 		}
 		return nil, err
 	}
-	tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, gitTable, kubescoreTable}
+	tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, gitTable, kubescoreTable, dockerHubBuildTable}
 	for _, table := range tables {
 		if err = splconn.Exec(context.Background(), string(table)); err != nil {
 			return nil, err
@@ -379,21 +379,27 @@ func (c *DBClient) RetrieveKubvizEvent() ([]model.DbEvent, error) {
 	return events, nil
 }
 
-func (c *DBClient) InsertContainerEventDockerHub(event string) {
-	ctx := context.Background()
-	batch, err := c.splconn.PrepareBatch(ctx, "INSERT INTO container_dockerhub")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if err = batch.Append(event); err != nil {
+func (c *DBClient) InsertContainerEventDockerHub(build model.DockerHubBuild) {
+	var (
+		tx, _   = c.conn.Begin()
+		stmt, _ = tx.Prepare(string(InsertDockerHubBuild))
+	)
+	defer stmt.Close()
+	if _, err := stmt.Exec(
+		build.PushedBy,
+		build.ImageTag,
+		build.RepositoryName,
+		build.DateCreated,
+		build.Owner,
+		build.Event,
+	); err != nil {
 		log.Fatal(err)
 	}
-
-	if err = batch.Send(); err != nil {
+	if err := tx.Commit(); err != nil {
 		log.Fatal(err)
 	}
 }
+
 func (c *DBClient) InsertContainerEventGithub(event string) {
 	ctx := context.Background()
 	batch, err := c.splconn.PrepareBatch(ctx, "INSERT INTO container_github")

client/pkg/clickhouse/statements.go

@@ -79,6 +79,18 @@ const kubescoreTable DBStatement = `
 			recommendations String
 	    ) engine=File(TabSeparated)
 	`
+
+const dockerHubBuildTable DBStatement = `
+	CREATE TABLE IF NOT EXISTS dockerhubbuild (
+		PushedBy String,
+		ImageTag String,
+		RepositoryName String,
+		DateCreated String,
+		Owner String,
+		Event String
+	) engine=File(TabSeparated)
+	`
+const InsertDockerHubBuild DBStatement = "INSERT INTO dockerhubbuild (PushedBy, ImageTag, RepositoryName, DateCreated, Owner, Event) VALUES (?, ?, ?, ?, ?, ?)"
 const InsertRakees DBStatement = "INSERT INTO rakkess (ClusterName, Name, Create, Delete, List, Update) VALUES (?, ?, ?, ?, ?, ?)"
 const InsertKetall DBStatement = "INSERT INTO getall_resources (ClusterName, Namespace, Kind, Resource, Age) VALUES (?, ?, ?, ?, ?)"
 const InsertOutdated DBStatement = "INSERT INTO outdated_images (ClusterName, Namespace, Pod, CurrentImage, CurrentTag, LatestVersion, VersionsBehind) VALUES (?, ?, ?, ?, ?, ?, ?)"

client/pkg/clients/container_client.go

@@ -2,12 +2,19 @@ package clients
 
 import (
 	"encoding/json"
+	"errors"
 	"log"
+	"time"
 
 	"github.com/intelops/kubviz/client/pkg/clickhouse"
+	"github.com/intelops/kubviz/model"
 	"github.com/nats-io/nats.go"
 )
 
+var (
+	ErrUnmarshalBuildPayload = errors.New("error while unmarshal the dockerhub build payload")
+)
+
 type Container string
 
 // constant variables to use with nats stream and
@@ -18,66 +25,30 @@ const (
 	containerConsumer Container = "container-event-consumer"
 )
 
-// func (n *NATSContext) SubscribeContainerNats(conn clickhouse.DBInterface) {
-// 	n.stream.Subscribe(string(containerSubject), func(msg *nats.Msg) {
-// 		type events struct {
-// 			Events []json.RawMessage `json:"events"`
-// 		}
-
-// 		eventDocker := &events{}
-// 		err := json.Unmarshal(msg.Data, &eventDocker)
-// 		if err == nil {
-// 			log.Println(eventDocker)
-// 			msg.Ack()
-// 			repoName := msg.Header.Get("REPO_NAME")
-// 			type newEvent struct {
-// 				RepoName string          `json:"repoName"`
-// 				Event    json.RawMessage `json:"event"`
-// 			}
-
-// 			for _, event := range eventDocker.Events {
-// 				event := &newEvent{
-// 					RepoName: repoName,
-// 					Event:    event,
-// 				}
-
-// 				eventsJSON, err := json.Marshal(event)
-// 				if err != nil {
-// 					log.Printf("Failed to marshall with repo name going ahead with only event, %v", err)
-// 					eventsJSON = msg.Data
-// 				}
-// 				conn.InsertContainerEvent(string(eventsJSON))
-// 			}
-// 		} else {
-// 			log.Printf("Failed to unmarshal event, %v", err)
-// 			conn.InsertContainerEvent(string(msg.Data))
-// 		}
-
-//			log.Println("Inserted metrics:", string(msg.Data))
-//		}, nats.Durable(string(containerConsumer)), nats.ManualAck())
-//	}
 func (n *NATSContext) SubscribeContainerNats(conn clickhouse.DBInterface) {
 	n.stream.Subscribe(string(containerSubject), func(msg *nats.Msg) {
-		type pubData struct {
-			Metrics json.RawMessage `json:"event"`
-			Repo    string          `json:"repoName"`
-		}
 		msg.Ack()
 		repoName := msg.Header.Get("REPO_NAME")
-		metrics := &pubData{
-			Metrics: json.RawMessage(msg.Data),
-			Repo:    repoName,
-		}
-		data, err := json.Marshal(metrics)
-		if err != nil {
-			log.Fatal(err)
-		}
-		if repoName == "Github_Registory" {
-			conn.InsertContainerEventGithub(string(data))
-			log.Println("Inserted Github Container metrics:", string(msg.Data))
-		} else if repoName == "Dockerhub_Registry" {
-			conn.InsertContainerEventDockerHub(string(data))
+		if repoName == "Dockerhub_Registry" {
+			var pl model.BuildPayload
+			err := json.Unmarshal(msg.Data, &pl)
+			if err != nil {
+				log.Printf("%v", ErrUnmarshalBuildPayload)
+				return
+			}
+			var hub model.DockerHubBuild
+			t := time.Unix(int64(pl.Repository.DateCreated), 0)
+			hub.DateCreated = t.Format("2006-01-02 15:04:05")
+			hub.PushedBy = pl.PushData.Pusher
+			hub.ImageTag = pl.PushData.Tag
+			hub.RepositoryName = pl.Repository.Name
+			hub.Owner = pl.Repository.Owner
+			hub.Event = string(msg.Data)
+			conn.InsertContainerEventDockerHub(hub)
 			log.Println("Inserted DockerHub Container metrics:", string(msg.Data))
+		} else if repoName == "Github_Registry" {
+			conn.InsertContainerEventGithub(string(msg.Data))
+			log.Println("Inserted Github Container metrics:", string(msg.Data))
 		}
 	}, nats.Durable(string(containerConsumer)), nats.ManualAck())
 }

model/dockerhub.go

@@ -0,0 +1,37 @@
+package model
+
+type BuildPayload struct {
+	CallbackURL string `json:"callback_url"`
+	PushData    struct {
+		Images   []string `json:"images"`
+		PushedAt float32  `json:"pushed_at"`
+		Pusher   string   `json:"pusher"`
+		Tag      string   `json:"tag"`
+	} `json:"push_data"`
+	Repository struct {
+		CommentCount    int     `json:"comment_count"`
+		DateCreated     float32 `json:"date_created"`
+		Description     string  `json:"description"`
+		Dockerfile      string  `json:"dockerfile"`
+		FullDescription string  `json:"full_description"`
+		IsOfficial      bool    `json:"is_official"`
+		IsPrivate       bool    `json:"is_private"`
+		IsTrusted       bool    `json:"is_trusted"`
+		Name            string  `json:"name"`
+		Namespace       string  `json:"namespace"`
+		Owner           string  `json:"owner"`
+		RepoName        string  `json:"repo_name"`
+		RepoURL         string  `json:"repo_url"`
+		StarCount       int     `json:"star_count"`
+		Status          string  `json:"status"`
+	} `json:"repository"`
+}
+
+type DockerHubBuild struct {
+	PushedBy       string
+	ImageTag       string
+	RepositoryName string
+	DateCreated    string
+	Owner          string
+	Event          string
+}