benchmark: add support for memory profiling

.gitignore

@@ -37,3 +37,6 @@ charts/gateway-addons-helm/charts/
 
 # VIM
 .*.swp
+
+# Benchmark test profiles
+test/benchmark/profiles/

test/benchmark/benchmark_test.go

@@ -46,6 +46,7 @@ func TestBenchmark(t *testing.T) {
 		"config/httproute.yaml",
 		"config/nighthawk-client.yaml",
 		*suite.ReportSavePath,
+		*suite.ProfilesSaveDir,
 	)
 	if err != nil {
 		t.Fatalf("Failed to create BenchmarkTestSuite: %v", err)

test/benchmark/suite/flags.go

@@ -11,9 +11,10 @@ package suite
 import "flag"
 
 var (
-	RPS            = flag.String("rps", "1000", "The target requests-per-second rate.")
-	Connections    = flag.String("connections", "10", "The maximum allowed number of concurrent connections per event loop. HTTP/1 only.")
-	Duration       = flag.String("duration", "60", "The number of seconds that the test should run.")
-	Concurrency    = flag.String("concurrency", "auto", "The number of concurrent event loops that should be used.")
-	ReportSavePath = flag.String("report-save-path", "", "The path where to save the benchmark test report.")
+	RPS             = flag.String("rps", "1000", "The target requests-per-second rate.")
+	Connections     = flag.String("connections", "10", "The maximum allowed number of concurrent connections per event loop. HTTP/1 only.")
+	Duration        = flag.String("duration", "60", "The number of seconds that the test should run.")
+	Concurrency     = flag.String("concurrency", "auto", "The number of concurrent event loops that should be used.")
+	ReportSavePath  = flag.String("report-save-path", "", "The path where to save the benchmark test report.")
+	ProfilesSaveDir = flag.String("profiles-save-dir", "./profiles", "The dir where to save the benchmark test profiles.")
 )

test/benchmark/suite/render.go

@@ -92,7 +92,7 @@ func RenderReport(writer io.Writer, name, description string, reports []*Benchma
 
 	writeSection(writer, name, titleLevel, description)
 
-	writeSection(writer, "Results", titleLevel+1, "Click to see the full results.")
+	writeSection(writer, "Results", titleLevel+1, "Expand to see the full results.")
 	renderResultsTable(writer, reports)
 
 	writeSection(writer, "Metrics", titleLevel+1, "")
@@ -101,6 +101,9 @@ func RenderReport(writer io.Writer, name, description string, reports []*Benchma
 		return err
 	}
 
+	writeSection(writer, "Profiles", titleLevel+1, "")
+	renderProfilesTable(writer, "Memory", "heap", titleLevel+2, reports)
+
 	return nil
 }
 
@@ -141,7 +144,7 @@ func renderEnvSettingsTable(writer io.Writer) {
 		},
 	}
 
-	renderMetricsTableHeader(table, headers)
+	renderTableHeader(table, headers)
 
 	writeTableRow(table, headers, func(_ int, h ReportTableHeader) string {
 		env := strings.Replace(strings.ToUpper(h.Name), " ", "_", -1)
@@ -190,7 +193,7 @@ func renderMetricsTable(writer io.Writer, headerSettings []ReportTableHeader, re
 		}
 	}
 
-	renderMetricsTableHeader(table, headers)
+	renderTableHeader(table, headers)
 
 	for _, report := range reports {
 		mfCP, err := parseMetrics(report.RawCPMetrics)
@@ -234,7 +237,18 @@ func renderMetricsTable(writer io.Writer, headerSettings []ReportTableHeader, re
 	return nil
 }
 
-func renderMetricsTableHeader(table *tabwriter.Writer, headers []ReportTableHeader) {
+func renderProfilesTable(writer io.Writer, target, key string, titleLevel int, reports []*BenchmarkReport) {
+	writeSection(writer, target, titleLevel, "")
+
+	for _, report := range reports {
+		// The image is not be rendered yet, so it is a placeholder for path.
+		// The image will be rendered after the test has finished.
+		writeSection(writer, report.Name, titleLevel+1,
+			fmt.Sprintf("![%s-%s](%s.png)", key, report.Name, report.ProfilesPath[key]))
+	}
+}
+
+func renderTableHeader(table *tabwriter.Writer, headers []ReportTableHeader) {
 	writeTableRow(table, headers, func(_ int, h ReportTableHeader) string {
 		if h.Metric != nil && len(h.Metric.DisplayUnit) > 0 {
 			return fmt.Sprintf("%s (%s)", h.Name, h.Metric.DisplayUnit)

test/benchmark/suite/report.go

@@ -14,6 +14,8 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"path"
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
@@ -25,8 +27,10 @@ import (
 )
 
 const (
-	localMetricsPort        = 0
-	controlPlaneMetricsPort = 19001
+	localMetricsPort  = 0
+	localProfilesPort = 0
+	podMetricsPort    = 19001
+	podProfilesPort   = 19000
 )
 
 type BenchmarkReport struct {
@@ -35,19 +39,24 @@ type BenchmarkReport struct {
 	RawCPMetrics []byte
 	RawDPMetrics map[string][]byte
 
+	ProfilesOutputDir string
+	ProfilesPath      map[string]string
+
 	kubeClient kube.CLIClient
 }
 
-func NewBenchmarkReport(name string) (*BenchmarkReport, error) {
+func NewBenchmarkReport(name, profilesOutputDir string) (*BenchmarkReport, error) {
 	kubeClient, err := kube.NewCLIClient(options.DefaultConfigFlags.ToRawKubeConfigLoader())
 	if err != nil {
 		return nil, err
 	}
 
 	return &BenchmarkReport{
-		Name:         name,
-		RawDPMetrics: make(map[string][]byte),
-		kubeClient:   kubeClient,
+		Name:              name,
+		RawDPMetrics:      make(map[string][]byte),
+		ProfilesPath:      make(map[string]string),
+		ProfilesOutputDir: profilesOutputDir,
+		kubeClient:        kubeClient,
 	}, nil
 }
 
@@ -64,7 +73,7 @@ func (r *BenchmarkReport) Print(t *testing.T, name string) {
 }
 
 func (r *BenchmarkReport) Collect(t *testing.T, ctx context.Context, job *types.NamespacedName) error {
-	if err := r.GetBenchmarkResult(t, ctx, job); err != nil {
+	if err := r.GetResults(t, ctx, job); err != nil {
 		return err
 	}
 
@@ -76,10 +85,14 @@ func (r *BenchmarkReport) Collect(t *testing.T, ctx context.Context, job *types.
 		return err
 	}
 
+	if err := r.GetProfiles(t, ctx); err != nil {
+		return err
+	}
+
 	return nil
 }
 
-func (r *BenchmarkReport) GetBenchmarkResult(t *testing.T, ctx context.Context, job *types.NamespacedName) error {
+func (r *BenchmarkReport) GetResults(t *testing.T, ctx context.Context, job *types.NamespacedName) error {
 	pods, err := r.kubeClient.Kube().CoreV1().Pods(job.Namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + job.Name})
 
 	if len(pods.Items) < 1 {
@@ -105,23 +118,13 @@ func (r *BenchmarkReport) GetBenchmarkResult(t *testing.T, ctx context.Context,
 }
 
 func (r *BenchmarkReport) GetControlPlaneMetrics(t *testing.T, ctx context.Context) error {
-	egPods, err := r.kubeClient.Kube().CoreV1().Pods("envoy-gateway-system").
-		List(ctx, metav1.ListOptions{LabelSelector: "control-plane=envoy-gateway"})
+	egPod, err := r.fetchEnvoyGatewayPod(t, ctx)
 	if err != nil {
 		return err
 	}
 
-	if len(egPods.Items) < 1 {
-		return fmt.Errorf("failed to get any pods for envoy-gateway")
-	}
-
-	if len(egPods.Items) > 1 {
-		t.Logf("Got %d pod(s), using the first one as default envoy-gateway pod", len(egPods.Items))
-	}
-
-	egPod := &egPods.Items[0]
-	metrics, err := r.getMetricsFromPortForwarder(
-		t, &types.NamespacedName{Name: egPod.Name, Namespace: egPod.Namespace}, "/metrics",
+	metrics, err := r.getResponseFromPortForwarder(
+		t, &types.NamespacedName{Name: egPod.Name, Namespace: egPod.Namespace}, localMetricsPort, podMetricsPort, "/metrics",
 	)
 	if err != nil {
 		return err
@@ -133,21 +136,14 @@ func (r *BenchmarkReport) GetControlPlaneMetrics(t *testing.T, ctx context.Conte
 }
 
 func (r *BenchmarkReport) GetDataPlaneMetrics(t *testing.T, ctx context.Context) error {
-	epPods, err := r.kubeClient.Kube().CoreV1().Pods("envoy-gateway-system").
-		List(ctx, metav1.ListOptions{LabelSelector: "gateway.envoyproxy.io/owning-gateway-namespace=benchmark-test,gateway.envoyproxy.io/owning-gateway-name=benchmark"})
+	epPods, err := r.fetchEnvoyProxyPodList(t, ctx)
 	if err != nil {
 		return err
 	}
 
-	if len(epPods.Items) < 1 {
-		return fmt.Errorf("failed to get any pods for envoy-proxies")
-	}
-
-	t.Logf("Got %d pod(s) from data-plane", len(epPods.Items))
-
 	for _, epPod := range epPods.Items {
 		podNN := &types.NamespacedName{Name: epPod.Name, Namespace: epPod.Namespace}
-		metrics, err := r.getMetricsFromPortForwarder(t, podNN, "/stats/prometheus")
+		metrics, err := r.getResponseFromPortForwarder(t, podNN, localMetricsPort, podMetricsPort, "/stats/prometheus")
 		if err != nil {
 			return err
 		}
@@ -158,6 +154,29 @@ func (r *BenchmarkReport) GetDataPlaneMetrics(t *testing.T, ctx context.Context)
 	return nil
 }
 
+func (r *BenchmarkReport) GetProfiles(t *testing.T, ctx context.Context) error {
+	egPod, err := r.fetchEnvoyGatewayPod(t, ctx)
+	if err != nil {
+		return err
+	}
+
+	// Memory heap profiles.
+	heapProf, err := r.getResponseFromPortForwarder(
+		t, &types.NamespacedName{Name: egPod.Name, Namespace: egPod.Namespace}, localProfilesPort, podProfilesPort, "/debug/pprof/heap",
+	)
+	if err != nil {
+		return err
+	}
+
+	heapProfPath := path.Join(r.ProfilesOutputDir, fmt.Sprintf("heap.%s.pprof", r.Name))
+	if err = os.WriteFile(heapProfPath, heapProf, 0644); err != nil {
+		return fmt.Errorf("failed to write profiles %s: %v", heapProfPath, err)
+	}
+	r.ProfilesPath["heap"] = heapProfPath
+
+	return nil
+}
+
 // getLogsFromPod scrapes the logs directly from the pod (default container).
 func (r *BenchmarkReport) getLogsFromPod(ctx context.Context, pod *types.NamespacedName) ([]byte, error) {
 	podLogOpts := corev1.PodLogOptions{}
@@ -179,9 +198,9 @@ func (r *BenchmarkReport) getLogsFromPod(ctx context.Context, pod *types.Namespa
 	return buf.Bytes(), nil
 }
 
-// getMetricsFromPortForwarder retrieves metrics from pod by request url, like `/metrics`.
-func (r *BenchmarkReport) getMetricsFromPortForwarder(t *testing.T, pod *types.NamespacedName, url string) ([]byte, error) {
-	fw, err := kube.NewLocalPortForwarder(r.kubeClient, *pod, localMetricsPort, controlPlaneMetricsPort)
+// getResponseFromPortForwarder gets response by sending endpoint request to pod port-forwarder.
+func (r *BenchmarkReport) getResponseFromPortForwarder(t *testing.T, pod *types.NamespacedName, localPort, podPort int, endpoint string) ([]byte, error) {
+	fw, err := kube.NewLocalPortForwarder(r.kubeClient, *pod, localPort, podPort)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build port forwarder for pod %s: %v", pod.String(), err)
 	}
@@ -193,27 +212,63 @@ func (r *BenchmarkReport) getMetricsFromPortForwarder(t *testing.T, pod *types.N
 	}
 
 	var out []byte
-	// Retrieving metrics from Pod.
+	// Retrieving response by requesting Pod with url endpoint.
 	go func() {
 		defer fw.Stop()
 
-		url := fmt.Sprintf("http://%s%s", fw.Address(), url)
+		url := fmt.Sprintf("http://%s%s", fw.Address(), endpoint)
 		resp, err := http.Get(url)
 		if err != nil {
 			t.Errorf("failed to request %s: %v", url, err)
 			return
 		}
 
-		metrics, err := io.ReadAll(resp.Body)
+		body, err := io.ReadAll(resp.Body)
 		if err != nil {
-			t.Errorf("failed to read metrics: %v", err)
+			t.Errorf("failed to read response: %v", err)
 			return
 		}
 
-		out = metrics
+		out = body
 	}()
 
 	fw.WaitForStop()
 
 	return out, nil
 }
+
+func (r *BenchmarkReport) fetchEnvoyGatewayPod(t *testing.T, ctx context.Context) (*corev1.Pod, error) {
+	egPods, err := r.kubeClient.Kube().CoreV1().
+		Pods("envoy-gateway-system").
+		List(ctx, metav1.ListOptions{LabelSelector: "control-plane=envoy-gateway"})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(egPods.Items) < 1 {
+		return nil, fmt.Errorf("failed to get any pods for envoy-gateway")
+	}
+
+	if len(egPods.Items) > 1 {
+		t.Logf("Got %d pod(s), using the first one as default envoy-gateway pod", len(egPods.Items))
+	}
+
+	return &egPods.Items[0], nil
+}
+
+func (r *BenchmarkReport) fetchEnvoyProxyPodList(t *testing.T, ctx context.Context) (*corev1.PodList, error) {
+	epPods, err := r.kubeClient.Kube().CoreV1().
+		Pods("envoy-gateway-system").
+		List(ctx, metav1.ListOptions{LabelSelector: "gateway.envoyproxy.io/owning-gateway-namespace=benchmark-test,gateway.envoyproxy.io/owning-gateway-name=benchmark"})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(epPods.Items) < 1 {
+		return nil, fmt.Errorf("failed to get any pods for envoy-proxies")
+	}
+
+	t.Logf("Got %d pod(s) from data-plane", len(epPods.Items))
+
+	return epPods, nil
+}

test/benchmark/suite/suite.go

@@ -34,11 +34,12 @@ const (
 )
 
 type BenchmarkTestSuite struct {
-	Client         client.Client
-	TimeoutConfig  config.TimeoutConfig
-	ControllerName string
-	Options        BenchmarkOptions
-	ReportSavePath string
+	Client          client.Client
+	TimeoutConfig   config.TimeoutConfig
+	ControllerName  string
+	Options         BenchmarkOptions
+	ReportSavePath  string
+	ProfilesSaveDir string
 
 	// Resources template for supported benchmark targets.
 	GatewayTemplate    *gwapiv1.Gateway
@@ -50,7 +51,7 @@ type BenchmarkTestSuite struct {
 }
 
 func NewBenchmarkTestSuite(client client.Client, options BenchmarkOptions,
-	gatewayManifest, httpRouteManifest, benchmarkClientManifest, reportPath string) (*BenchmarkTestSuite, error) {
+	gatewayManifest, httpRouteManifest, benchmarkClientManifest, reportPath, profilesDir string) (*BenchmarkTestSuite, error) {
 	var (
 		gateway         = new(gwapiv1.Gateway)
 		httproute       = new(gwapiv1.HTTPRoute)
@@ -91,12 +92,24 @@ func NewBenchmarkTestSuite(client client.Client, options BenchmarkOptions,
 	container := &benchmarkClient.Spec.Template.Spec.Containers[0]
 	container.Args = append(container.Args, staticArgs...)
 
+	// Ensure the profile output directory.
+	if _, err = os.Stat(profilesDir); err != nil {
+		if os.IsNotExist(err) {
+			if err = os.MkdirAll(profilesDir, os.ModePerm); err != nil {
+				return nil, fmt.Errorf("failed to create dir %s: %v", profilesDir, err)
+			}
+		} else {
+			return nil, fmt.Errorf("failed to get stat of dir %s: %v", profilesDir, err)
+		}
+	}
+
 	return &BenchmarkTestSuite{
 		Client:             client,
 		Options:            options,
 		TimeoutConfig:      timeoutConfig,
 		ControllerName:     DefaultControllerName,
 		ReportSavePath:     reportPath,
+		ProfilesSaveDir:    profilesDir,
 		GatewayTemplate:    gateway,
 		HTTPRouteTemplate:  httproute,
 		BenchmarkClientJob: benchmarkClient,
@@ -190,7 +203,7 @@ func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name,
 
 	t.Logf("Running benchmark test: %s successfully", name)
 
-	report, err := NewBenchmarkReport(name)
+	report, err := NewBenchmarkReport(name, b.ProfilesSaveDir)
 	if err != nil {
 		return nil, err
 	}