Add run_command benchcomp visualization

This allows users to write their own custom visualization script to run
after running the benchmarks. Prior to this commit, visualizations had
to be checked into the Kani repository.

When `run_command` is specified as a visualization, benchcomp runs the
specified command and passes the result of the run as a JSON file on
stdin. The command can then process the result however it likes.

This resolves #2518.

docs/src/SUMMARY.md

@@ -35,7 +35,6 @@
  - [Performance comparisons](./performance-comparisons.md)
  - [`benchcomp` command line](./benchcomp-cli.md)
  - [`benchcomp` configuration file](./benchcomp-conf.md)
- - [Custom visualizations](./benchcomp-viz.md)
  - [Custom parsers](./benchcomp-parse.md)
 
 - [Limitations](./limitations.md)

tools/benchcomp/benchcomp/visualizers/__init__.py

@@ -3,6 +3,8 @@
 
 
 import dataclasses
+import json
+import subprocess
 import textwrap
 
 import jinja2
@@ -12,8 +14,42 @@
 import benchcomp.visualizers.utils as viz_utils
 
 
-# TODO The doc comment should appear in the help output, which should list all
-# available checks.
+
+@dataclasses.dataclass
+class run_command:
+ """Run an executable command, passing the result as a JSON file on stdin.
+
+ This allows you to write your own visualization, which reads a result file
+ on stdin and does something with it, e.g. writing out a graph or other
+ output file.
+
+ Sample configuration:
+
+ ```
+ visualize:
+ - type: run_command
+ command: ./my_visualization.py
+ ```
+ """
+
+ command: str
+
+
+ def __call__(self, results):
+ results = json.dumps(results, indent=2)
+ try:
+ proc = subprocess.Popen(
+ self.command, shell=True, text=True, stdin=subprocess.PIPE)
+ _, _ = proc.communicate(input=results)
+ except subprocess.CalledProcessError as exc:
+ logging.warning(
+ "visualization command '%s' exited with code %d",
+ self.command, exc.returncode)
+ except (OSError, subprocess.SubprocessError) as exe:
+ logging.error(
+ "visualization command '%s' failed: %s", self.command, str(exe))
+
+
 
 @dataclasses.dataclass
 class error_on_regression:

tools/benchcomp/test/test_regression.py

@@ -10,6 +10,7 @@
 import tempfile
 import textwrap
 import unittest
+import uuid
 
 import yaml
 
@@ -737,3 +738,105 @@ def test_command_parser(self):
 
  for item in ["benchmarks", "metrics"]:
  self.assertIn(item, result)
+
+
+ def test_run_command_visualization(self):
+ """Ensure that the run_command visualization can execute a command"""
+
+ with tempfile.TemporaryDirectory() as tmp:
+ out_file = pathlib.Path(tmp) / str(uuid.uuid4())
+ run_bc = Benchcomp({
+ "variants": {
+ "v1": {
+ "config": {
+ "command_line": "true",
+ "directory": tmp,
+ }
+ },
+ "v2": {
+ "config": {
+ "command_line": "true",
+ "directory": tmp,
+ }
+ }
+ },
+ "run": {
+ "suites": {
+ "suite_1": {
+ "parser": {
+ "command": """
+ echo '{
+ "benchmarks": {},
+ "metrics": {}
+ }'
+ """
+ },
+ "variants": ["v2", "v1"]
+ }
+ }
+ },
+ "visualize": [{
+ "type": "run_command",
+ "command": f"cat - > {out_file}"
+ }],
+ })
+ run_bc()
+ self.assertEqual(
+ run_bc.proc.returncode, 0, msg=run_bc.stderr)
+
+ with open(out_file) as handle:
+ result = yaml.safe_load(handle)
+
+ for item in ["benchmarks", "metrics"]:
+ self.assertIn(item, result)
+
+
+ def test_run_failing_command_visualization(self):
+ """Ensure that benchcomp terminates normally even when run_command visualization doesn't"""
+
+ with tempfile.TemporaryDirectory() as tmp:
+ out_file = pathlib.Path(tmp) / str(uuid.uuid4())
+ run_bc = Benchcomp({
+ "variants": {
+ "v1": {
+ "config": {
+ "command_line": "true",
+ "directory": tmp,
+ }
+ },
+ "v2": {
+ "config": {
+ "command_line": "true",
+ "directory": tmp,
+ }
+ }
+ },
+ "run": {
+ "suites": {
+ "suite_1": {
+ "parser": {
+ "command": """
+ echo '{
+ "benchmarks": {},
+ "metrics": {}
+ }'
+ """
+ },
+ "variants": ["v2", "v1"]
+ }
+ }
+ },
+ "visualize": [{
+ "type": "run_command",
+ "command": f"cat - > {out_file}; false"
+ }],
+ })
+ run_bc()
+ self.assertEqual(
+ run_bc.proc.returncode, 0, msg=run_bc.stderr)
+
+ with open(out_file) as handle:
+ result = yaml.safe_load(handle)
+
+ for item in ["benchmarks", "metrics"]:
+ self.assertIn(item, result)