Merge pull request #625 from vitessio/store-profile-info-in-db

Attach profile information to an execution

go/exec/exec.go

@@ -279,16 +279,7 @@ func (e *Exec) Prepare() error {
 	}
 
 	// insert new exec in SQL
-	if _, err = e.clientDB.Write(
-		"INSERT INTO execution(uuid, status, source, git_ref, workload, pull_nb, go_version) VALUES(?, ?, ?, ?, ?, ?, ?)",
-		e.UUID.String(),
-		StatusCreated,
-		e.Source,
-		e.GitRef,
-		e.Workload,
-		e.PullNB,
-		e.GolangVersion,
-	); err != nil {
+	if err = e.insert(); err != nil {
 		return err
 	}
 	e.createdInDB = true
@@ -435,17 +426,35 @@ func (e *Exec) handleStepEnd(err error) {
 	}
 }
 
+func (e *Exec) insert() error {
+	_, err := e.clientDB.Write("INSERT INTO execution(uuid, status, source, git_ref, workload, pull_nb, go_version) VALUES(?, ?, ?, ?, ?, ?, ?)",
+		e.UUID.String(),
+		StatusCreated,
+		e.Source,
+		e.GitRef,
+		e.Workload,
+		e.PullNB,
+		e.GolangVersion,
+	)
+	if e.ProfileInformation != nil {
+		_, err = e.clientDB.Write("UPDATE execution SET profile_binary = ?, profile_mode = ? WHERE uuid = ?", e.ProfileInformation.Binary, e.ProfileInformation.Mode, e.UUID.String())
+	}
+	return err
+}
+
 func GetRecentExecutions(client storage.SQLClient) ([]*Exec, error) {
 	var res []*Exec
-	query := "SELECT uuid, status, git_ref, started_at, finished_at, source, workload, pull_nb, go_version FROM execution ORDER BY started_at DESC LIMIT 1000"
+	query := "SELECT uuid, status, git_ref, started_at, finished_at, source, workload, pull_nb, go_version, IFNULL(profile_binary, ''), IFNULL(profile_mode, '') FROM execution ORDER BY started_at DESC LIMIT 1000"
 	result, err := client.Read(query)
 	if err != nil {
 		return nil, err
 	}
 	defer result.Close()
 	for result.Next() {
-		exec := &Exec{}
-		err = result.Scan(&exec.RawUUID, &exec.Status, &exec.GitRef, &exec.StartedAt, &exec.FinishedAt, &exec.Source, &exec.Workload, &exec.PullNB, &exec.GolangVersion)
+		exec := &Exec{
+			ProfileInformation: &ProfileInformation{},
+		}
+		err = result.Scan(&exec.RawUUID, &exec.Status, &exec.GitRef, &exec.StartedAt, &exec.FinishedAt, &exec.Source, &exec.Workload, &exec.PullNB, &exec.GolangVersion, &exec.ProfileInformation.Binary, &exec.ProfileInformation.Mode)
 		if err != nil {
 			return nil, err
 		}
@@ -534,23 +543,6 @@ func GetPreviousFromSourceMacrobenchmark(client storage.SQLClient, source, workl
 	return
 }
 
-// GetLatestDailyJobForMicrobenchmarks will fetch and return the commit sha for which
-// the last daily job for microbenchmarks was run
-func GetLatestDailyJobForMicrobenchmarks(client storage.SQLClient) (gitSha string, err error) {
-	query := "select git_ref from execution where source = \"cron\" and status = \"finished\" and workload = \"micro\" order by started_at desc limit 1"
-	rows, err := client.Read(query)
-	if err != nil {
-		return "", err
-	}
-
-	defer rows.Close()
-	for rows.Next() {
-		err = rows.Scan(&gitSha)
-		return gitSha, err
-	}
-	return "", nil
-}
-
 // GetLatestDailyJobForMacrobenchmarks will fetch and return the commit sha for which
 // the last daily job for macrobenchmarks was run
 func GetLatestDailyJobForMacrobenchmarks(client storage.SQLClient) (gitSha string, err error) {
@@ -579,7 +571,7 @@ func Exists(client storage.SQLClient, gitRef, source, workload, status string) (
 }
 
 func CountMacroBenchmark(client storage.SQLClient, gitRef, source, workload, status, planner string) (int, error) {
-	query := "SELECT count(uuid) FROM execution e, macrobenchmark m WHERE e.status = ? AND e.git_ref = ? AND e.workload = ? AND e.source = ? AND m.vtgate_planner_version = ? AND e.uuid = m.exec_uuid"
+	query := "SELECT count(uuid) FROM execution e, macrobenchmark m WHERE e.profile_binary IS NULL AND e.status = ? AND e.git_ref = ? AND e.workload = ? AND e.source = ? AND m.vtgate_planner_version = ? AND e.uuid = m.exec_uuid"
 	result, err := client.Read(query, status, gitRef, workload, source, planner)
 	if err != nil {
 		return 0, err
@@ -633,7 +625,7 @@ func GetHistory(client storage.SQLClient) ([]*History, error) {
 					source,
 					workload
 				HAVING
-					COUNT(*) >= 10
+					COUNT(*) >= 1
 			) AS subquery
 			GROUP BY
 				git_ref,

go/exec/status.go

@@ -42,7 +42,7 @@ func GetBenchmarkStats(client storage.SQLClient) (BenchmarkStats, error) {
 			(SELECT COUNT(uuid) FROM execution) AS count_status,
 			(SELECT COUNT(DISTINCT git_ref) FROM execution) AS count_commits,
 			(SELECT COUNT(*) FROM execution WHERE started_at >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)) AS count_all,
-			(SELECT AVG(TIMESTAMPDIFF(MINUTE, started_at, finished_at)) AS avg_duration_minutes FROM execution WHERE started_at IS NOT NULL AND finished_at IS NOT NULL AND status NOT IN ('failed', 'started') ORDER BY avg_duration_minutes ASC) AS avg_duration_minutes
+			(SELECT IFNULL(AVG(TIMESTAMPDIFF(MINUTE, started_at, finished_at)), 0) AS avg_duration_minutes FROM execution WHERE profile_binary IS NULL AND started_at IS NOT NULL AND finished_at IS NOT NULL AND status NOT IN ('failed', 'started') ORDER BY avg_duration_minutes ASC) AS avg_duration_minutes
 		FROM 
 			execution
 		LIMIT 1;`)

go/server/api.go

@@ -43,10 +43,12 @@ type ErrorAPI struct {
 }
 
 type ExecutionQueue struct {
-	Source   string `json:"source"`
-	GitRef   string `json:"git_ref"`
-	Workload string `json:"workload"`
-	PullNb   int    `json:"pull_nb"`
+	Source        string `json:"source"`
+	GitRef        string `json:"git_ref"`
+	Workload      string `json:"workload"`
+	PullNb        int    `json:"pull_nb"`
+	ProfileBinary string `json:"profile_binary"`
+	ProfileMode   string `json:"profile_mode"`
 }
 
 type RecentExecutions struct {
@@ -59,6 +61,8 @@ type RecentExecutions struct {
 	GolangVersion string     `json:"golang_version"`
 	StartedAt     *time.Time `json:"started_at"`
 	FinishedAt    *time.Time `json:"finished_at"`
+	ProfileBinary string     `json:"profile_binary"`
+	ProfileMode   string     `json:"profile_mode"`
 }
 
 type ExecutionMetadatas struct {
@@ -102,6 +106,8 @@ func (s *Server) getRecentExecutions(c *gin.Context) {
 			GolangVersion: e.GolangVersion,
 			StartedAt:     e.StartedAt,
 			FinishedAt:    e.FinishedAt,
+			ProfileBinary: e.ProfileInformation.Binary,
+			ProfileMode:   e.ProfileInformation.Mode,
 		})
 		if !slices.Contains(response.Workloads, e.Workload) {
 			response.Workloads = append(response.Workloads, e.Workload)
@@ -124,11 +130,18 @@ func (s *Server) getExecutionsQueue(c *gin.Context) {
 		if e.Executing {
 			continue
 		}
+		var profileBinary, profileMode string
+		if e.identifier.Profile != nil {
+			profileBinary = e.identifier.Profile.Binary
+			profileMode = e.identifier.Profile.Mode
+		}
 		response.Executions = append(response.Executions, ExecutionQueue{
-			Source:   e.identifier.Source,
-			GitRef:   e.identifier.GitRef,
-			Workload: e.identifier.Workload,
-			PullNb:   e.identifier.PullNb,
+			Source:        e.identifier.Source,
+			GitRef:        e.identifier.GitRef,
+			Workload:      e.identifier.Workload,
+			PullNb:        e.identifier.PullNb,
+			ProfileBinary: profileBinary,
+			ProfileMode:   profileMode,
 		})
 		if !slices.Contains(response.Workloads, e.identifier.Workload) {
 			response.Workloads = append(response.Workloads, e.identifier.Workload)

go/tools/macrobench/sql.go

@@ -53,6 +53,7 @@ func getExecutionGroupResults(workload string, ref string, planner PlannerVersio
             metrics AS m ON e.uuid = m.exec_uuid
         WHERE 
             e.status = 'finished'
+            AND e.profile_binary IS NULL
             AND e.git_ref = ? 
             AND info.vtgate_planner_version = ? 
             AND info.workload = ?
@@ -156,6 +157,7 @@ func getExecutionGroupResultsFromLast30Days(workload string, planner PlannerVers
             metrics AS m ON e.uuid = m.exec_uuid
         WHERE 
             e.finished_at BETWEEN DATE(NOW()) - INTERVAL 30 DAY AND DATE(NOW() + INTERVAL 1 DAY)
+            AND e.profile_binary IS NULL
             AND e.source = 'cron'
             AND e.status = 'finished'
             AND info.vtgate_planner_version = ? 
@@ -262,7 +264,8 @@ func getSummaryLast30Days(workload string, planner PlannerVersion, client storag
         JOIN 
             macrobenchmark_results AS results ON info.macrobenchmark_id = results.macrobenchmark_id
         WHERE 
-            e.finished_at BETWEEN DATE(NOW()) - INTERVAL 30 DAY AND DATE(NOW() + INTERVAL 1 DAY) 
+            e.finished_at BETWEEN DATE(NOW()) - INTERVAL 30 DAY AND DATE(NOW() + INTERVAL 1 DAY)
+            AND e.profile_binary IS NULL
             AND e.status = "finished" 
             AND e.source = "cron" 
             AND info.vtgate_planner_version = ? 

website/src/pages/StatusPage/components/ExecutionQueue/Columns.tsx

@@ -29,6 +29,8 @@ export type ExecutionQueueExecution = {
   git_ref: string;
   workload: string;
   pull_nb: number;
+  profile_binary: string;
+  profile_mode: string;
 };
 
 export const columns: ColumnDef<ExecutionQueueExecution>[] = [
@@ -73,4 +75,17 @@ export const columns: ColumnDef<ExecutionQueueExecution>[] = [
       );
     },
   },
+  {
+    header: "Profile",
+    accessorKey: "profile_binary",
+    cell: ({row}) => {
+      if (row.original.profile_mode !== "" && row.original.profile_binary !== "") {
+        return (
+            <div>
+              <Badge variant={"progress"}>{row.original.profile_binary}|{row.original.profile_mode}</Badge>
+            </div>
+        )
+      }
+    }
+  },
 ];

website/src/pages/StatusPage/components/ExecutionQueue/Execution.tsx

@@ -63,6 +63,8 @@ export default function ExecutionQueue() {
         git_ref: value.git_ref,
         workload: value.workload,
         pull_nb: value.pull_nb,
+        profile_binary: value.profile_binary,
+        profile_mode: value.profile_mode,
       };
     });
 

website/src/pages/StatusPage/components/PreviousExecutions/Columns.tsx

@@ -35,6 +35,8 @@ export type PreviousExecutionExecution = {
   golang_version: string;
   started_at: string;
   finished_at: string;
+  profile_binary: string;
+  profile_mode: string;
 };
 
 export type PreviousExecution = {
@@ -170,6 +172,19 @@ export const columns: ColumnDef<PreviousExecutionExecution>[] = [
       return value.includes(row.getValue(id));
     },
   },
+  {
+    header: "Profile",
+    accessorKey: "profile_binary",
+    cell: ({row}) => {
+      if (row.original.profile_mode !== "" && row.original.profile_binary !== "") {
+        return (
+            <div>
+              <Badge variant={"progress"}>{row.original.profile_binary}|{row.original.profile_mode}</Badge>
+            </div>
+        )
+      }
+    }
+  },
   {
     id: "actions",
     cell: ({ row }) => {