Push metrics from testsuite

cmd/testsuite/cmd/test.go

@@ -31,6 +31,8 @@ func testCmd(app *testsuite.App) *cobra.Command {
  cmd.Flags().String("tests", "", "Test file pattern, e.g., './testcases/*.yaml'.")
  cmd.Flags().String("junit", "", "Write a JUnit test report to this path.")
  cmd.Flags().String("benchmark", "", "Write a benchmark test report to this path.")
+ cmd.Flags().String("prometheusPushgatewayUrl", "", "Push metrics to Prometheus pushgateway at this url.")
+ cmd.Flags().String("prometheusPushgatewayJobName", "armada-testsuite", "Metrics are annotated with with job=prometheusPushGatewayJobName.")
  return cmd
 }
 
@@ -64,6 +66,18 @@ func testCmdRunE(app *testsuite.App) func(cmd *cobra.Command, args []string) err
  return errors.New("benchmark report not currently supported")
  }
 
+ prometheusPushgatewayUrl, err := cmd.Flags().GetString("prometheusPushgatewayUrl")
+ if err != nil {
+ return errors.WithStack(err)
+ }
+ app.Params.PrometheusPushGatewayUrl = prometheusPushgatewayUrl
+
+ prometheusPushgatewayJobName, err := cmd.Flags().GetString("prometheusPushgatewayJobName")
+ if err != nil {
+ return errors.WithStack(err)
+ }
+ app.Params.PrometheusPushGatewayJobName = prometheusPushgatewayJobName
+
  // Create a context that is cancelled on SIGINT/SIGTERM.
  // Ensures test jobs are cancelled on ctrl-C.
  ctx, cancel := context.WithCancel(context.Background())

internal/testsuite/app.go

@@ -17,6 +17,8 @@ import (
  "github.com/hashicorp/go-multierror"
  "github.com/mattn/go-zglob"
  "github.com/pkg/errors"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/push"
  "github.com/renstrom/shortuuid"
  "golang.org/x/sync/errgroup"
  apimachineryYaml "k8s.io/apimachinery/pkg/util/yaml"
@@ -29,6 +31,8 @@ import (
  "github.com/armadaproject/armada/pkg/client"
 )
 
+const metricsPrefix = "armada_testsuite_"
+
 type App struct {
  // Parameters passed to the CLI by the user.
  Params *Params
@@ -45,7 +49,13 @@ type App struct {
 // and that they can be provided either dynamically on a command line, or
 // statically in a config file that's reused between command runs.
 type Params struct {
+ // Armada connection details.
  ApiConnectionDetails *client.ApiConnectionDetails
+ // If non-empty, push metrics containing test results to a Prometheus push gateway with this url.
+ PrometheusPushGatewayUrl string
+ // Exported metrics are annotated with job=PrometheusPushGatewayJobName.
+ // Must be non-empty.
+ PrometheusPushGatewayJobName string
 }
 
 // New instantiates an App with default parameters, including standard output
@@ -107,7 +117,7 @@ func TestSpecFromFilePath(filePath string) (*api.TestSpec, error) {
  return nil, err
  }
 
- // Randomise jobSetName for each test to ensure we're only getting events for this run.
+ // Randomise job set for each test to ensure we're only getting events for this run.
  fileName := filepath.Base(filePath)
  fileName = strings.TrimSuffix(fileName, filepath.Ext(fileName))
  testSpec.JobSetId = fileName + "-" + shortuuid.New()
@@ -126,13 +136,119 @@ type TestSuiteReport struct {
  TestCaseReports []*TestCaseReport
 }
 
+func (tsr *TestSuiteReport) Describe(c chan<- *prometheus.Desc) {
+ for _, tcr := range tsr.TestCaseReports {
+ tcr.Describe(c)
+ }
+}
+
+func (tsr *TestSuiteReport) Collect(c chan<- prometheus.Metric) {
+ for _, tcr := range tsr.TestCaseReports {
+ tcr.Collect(c)
+ }
+}
+
 type TestCaseReport struct {
  Out *bytes.Buffer
  Start time.Time
  Finish time.Time
  FailureReason string
  BenchmarkReport *eventbenchmark.TestCaseBenchmarkReport
  TestSpec *api.TestSpec
+
+ // Prometheus metric descriptions.
+ // Test start time in seconds since the epoch.
+ startTimePrometheusDesc *prometheus.Desc
+ // Test finish time in seconds since the epoch.
+ finishTimePrometheusDesc *prometheus.Desc
+ // Outputs 1 on test timeout.
+ testTimeoutPrometheusDesc *prometheus.Desc
+ // Outputs 1 on test failure, not including timeouts.
+ testFailurePrometheusDesc *prometheus.Desc
+}
+
+func NewTestCaseReport(testSpec *api.TestSpec) *TestCaseReport {
+ rv := &TestCaseReport{
+ Start: time.Now(),
+ TestSpec: testSpec,
+ }
+ rv.initialiseMetrics()
+ return rv
+}
+
+func (r *TestCaseReport) initialiseMetrics() {
+ r.startTimePrometheusDesc = prometheus.NewDesc(
+ metricsPrefix+"test_start_time",
+ "The time at which a test started.",
+ []string{"testcase"},
+ nil,
+ )
+ r.finishTimePrometheusDesc = prometheus.NewDesc(
+ metricsPrefix+"test_finish_time",
+ "The time at which a test finished.",
+ []string{"testcase"},
+ nil,
+ )
+ r.testTimeoutPrometheusDesc = prometheus.NewDesc(
+ metricsPrefix+"test_timeout",
+ "Outputs 1 on test timeout and 0 otherwise.",
+ []string{"testcase"},
+ nil,
+ )
+ r.testFailurePrometheusDesc = prometheus.NewDesc(
+ metricsPrefix+"test_failure",
+ "Outputs 1 on test failure, not including timeout, and 0 otherwise.",
+ []string{"testcase"},
+ nil,
+ )
+}
+
+func (r *TestCaseReport) Describe(c chan<- *prometheus.Desc) {
+ c <- r.startTimePrometheusDesc
+ c <- r.finishTimePrometheusDesc
+ c <- r.testTimeoutPrometheusDesc
+ c <- r.testFailurePrometheusDesc
+}
+
+func (r *TestCaseReport) Collect(c chan<- prometheus.Metric) {
+ c <- prometheus.MustNewConstMetric(
+ r.startTimePrometheusDesc,
+ prometheus.CounterValue,
+ float64(r.Start.Unix()),
+ r.TestSpec.Name,
+ )
+ c <- prometheus.MustNewConstMetric(
+ r.finishTimePrometheusDesc,
+ prometheus.CounterValue,
+ float64(r.Finish.Unix()),
+ r.TestSpec.Name,
+ )
+
+ // Test failures always contain either "unexpected event for job" or "error asserting failure reason".
+ // TODO(albin): Improve this.
+ testFailure := 0.0
+ if strings.Contains(r.FailureReason, "unexpected event for job") || strings.Contains(r.FailureReason, "error asserting failure reason") {
+ testFailure = 1.0
+ }
+ c <- prometheus.MustNewConstMetric(
+ r.testFailurePrometheusDesc,
+ prometheus.GaugeValue,
+ testFailure,
+ r.TestSpec.Name,
+ )
+
+ // We assume that any other failures are due to timeout.
+ // TODO(albin): Improve this.
+ testTimeout := 0.0
+ if r.FailureReason != "" && testFailure == 0 {
+ testTimeout = 1.0
+ }
+ c <- prometheus.MustNewConstMetric(
+ r.testTimeoutPrometheusDesc,
+ prometheus.GaugeValue,
+ testTimeout,
+ r.TestSpec.Name,
+ )
 }
 
 func (report *TestSuiteReport) NumSuccesses() int {
@@ -203,9 +319,26 @@ func (a *App) RunTests(ctx context.Context, testSpecs []*api.TestSpec) (*TestSui
  return nil, err
  }
 
+ // Optionally push metrics.
+ if a.Params.PrometheusPushGatewayUrl != "" {
+ if err := pushTestSuiteReportMetrics(rv, a.Params.PrometheusPushGatewayUrl, a.Params.PrometheusPushGatewayJobName); err != nil {
+ return nil, err
+ }
+ }
  return rv, nil
 }
 
+func pushTestSuiteReportMetrics(tsr *TestSuiteReport, url, job string) error {
+ pusher := push.New(url, job)
+ pusher.Collector(tsr)
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ if err := pusher.PushContext(ctx); err != nil {
+ return errors.WithStack(err)
+ }
+ return nil
+}
+
 // UnmarshalTestCase unmarshalls bytes into a TestSpec.
 func UnmarshalTestCase(yamlBytes []byte, testSpec *api.TestSpec) error {
  var result *multierror.Error

internal/testsuite/testrunner.go

@@ -49,11 +49,8 @@ func (report *TestCaseReport) JunitTestCase() junit.Testcase {
 }
 
 func (srv *TestRunner) Run(ctx context.Context) (err error) {
- report := &TestCaseReport{
- Out: &bytes.Buffer{},
- Start: time.Now(),
- TestSpec: srv.testSpec,
- }
+ report := NewTestCaseReport(srv.testSpec)
+ report.Out = &bytes.Buffer{}
  out := io.MultiWriter(srv.Out, report.Out)
 
  fmt.Fprintf(out, "test case started %s\n", srv.testSpec.ShortString())