From b9792ec81a6a92fc1177212621eee9a8367909b4 Mon Sep 17 00:00:00 2001 From: Kareem Khazem Date: Tue, 20 Jun 2023 16:51:38 +0100 Subject: [PATCH 1/4] Add run_command benchcomp visualization This allows users to write their own custom visualization script to run after running the benchmarks. Prior to this commit, visualizations had to be checked into the Kani repository. When `run_command` is specified as a visualization, benchcomp runs the specified command and passes the result of the run as a JSON file on stdin. The command can then process the result however it likes. This resolves #2518. --- docs/src/SUMMARY.md | 1 - .../benchcomp/visualizers/__init__.py | 40 ++++++- tools/benchcomp/test/test_regression.py | 103 ++++++++++++++++++ 3 files changed, 141 insertions(+), 3 deletions(-) diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 8ff6608445a9..d4609a87c23b 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -35,7 +35,6 @@ - [Performance comparisons](./performance-comparisons.md) - [`benchcomp` command line](./benchcomp-cli.md) - [`benchcomp` configuration file](./benchcomp-conf.md) - - [Custom visualizations](./benchcomp-viz.md) - [Custom parsers](./benchcomp-parse.md) - [Limitations](./limitations.md) diff --git a/tools/benchcomp/benchcomp/visualizers/__init__.py b/tools/benchcomp/benchcomp/visualizers/__init__.py index da9effcba426..df06c791bc4c 100644 --- a/tools/benchcomp/benchcomp/visualizers/__init__.py +++ b/tools/benchcomp/benchcomp/visualizers/__init__.py @@ -3,6 +3,8 @@ import dataclasses +import json +import subprocess import textwrap import jinja2 @@ -12,8 +14,42 @@ import benchcomp.visualizers.utils as viz_utils -# TODO The doc comment should appear in the help output, which should list all -# available checks. + +@dataclasses.dataclass +class run_command: + """Run an executable command, passing the result as a JSON file on stdin. + + This allows you to write your own visualization, which reads a result file + on stdin and does something with it, e.g. writing out a graph or other + output file. + + Sample configuration: + + ``` + visualize: + - type: run_command + command: ./my_visualization.py + ``` + """ + + command: str + + + def __call__(self, results): + results = json.dumps(results, indent=2) + try: + proc = subprocess.Popen( + self.command, shell=True, text=True, stdin=subprocess.PIPE) + _, _ = proc.communicate(input=results) + except subprocess.CalledProcessError as exc: + logging.warning( + "visualization command '%s' exited with code %d", + self.command, exc.returncode) + except (OSError, subprocess.SubprocessError) as exe: + logging.error( + "visualization command '%s' failed: %s", self.command, str(exe)) + + @dataclasses.dataclass class error_on_regression: diff --git a/tools/benchcomp/test/test_regression.py b/tools/benchcomp/test/test_regression.py index a5b02a5b09b1..1ead636ca7fb 100644 --- a/tools/benchcomp/test/test_regression.py +++ b/tools/benchcomp/test/test_regression.py @@ -10,6 +10,7 @@ import tempfile import textwrap import unittest +import uuid import yaml @@ -737,3 +738,105 @@ def test_command_parser(self): for item in ["benchmarks", "metrics"]: self.assertIn(item, result) + + + def test_run_command_visualization(self): + """Ensure that the run_command visualization can execute a command""" + + with tempfile.TemporaryDirectory() as tmp: + out_file = pathlib.Path(tmp) / str(uuid.uuid4()) + run_bc = Benchcomp({ + "variants": { + "v1": { + "config": { + "command_line": "true", + "directory": tmp, + } + }, + "v2": { + "config": { + "command_line": "true", + "directory": tmp, + } + } + }, + "run": { + "suites": { + "suite_1": { + "parser": { + "command": """ + echo '{ + "benchmarks": {}, + "metrics": {} + }' + """ + }, + "variants": ["v2", "v1"] + } + } + }, + "visualize": [{ + "type": "run_command", + "command": f"cat - > {out_file}" + }], + }) + run_bc() + self.assertEqual( + run_bc.proc.returncode, 0, msg=run_bc.stderr) + + with open(out_file) as handle: + result = yaml.safe_load(handle) + + for item in ["benchmarks", "metrics"]: + self.assertIn(item, result) + + + def test_run_failing_command_visualization(self): + """Ensure that benchcomp terminates normally even when run_command visualization doesn't""" + + with tempfile.TemporaryDirectory() as tmp: + out_file = pathlib.Path(tmp) / str(uuid.uuid4()) + run_bc = Benchcomp({ + "variants": { + "v1": { + "config": { + "command_line": "true", + "directory": tmp, + } + }, + "v2": { + "config": { + "command_line": "true", + "directory": tmp, + } + } + }, + "run": { + "suites": { + "suite_1": { + "parser": { + "command": """ + echo '{ + "benchmarks": {}, + "metrics": {} + }' + """ + }, + "variants": ["v2", "v1"] + } + } + }, + "visualize": [{ + "type": "run_command", + "command": f"cat - > {out_file}; false" + }], + }) + run_bc() + self.assertEqual( + run_bc.proc.returncode, 0, msg=run_bc.stderr) + + with open(out_file) as handle: + result = yaml.safe_load(handle) + + for item in ["benchmarks", "metrics"]: + self.assertIn(item, result) From 0b2f370078bcc52468f738a18f3afcc0b11a36f3 Mon Sep 17 00:00:00 2001 From: Kareem Khazem Date: Thu, 22 Jun 2023 12:03:21 +0100 Subject: [PATCH 2/4] Update tools/benchcomp/benchcomp/visualizers/__init__.py Co-authored-by: Qinheping Hu --- tools/benchcomp/benchcomp/visualizers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/benchcomp/benchcomp/visualizers/__init__.py b/tools/benchcomp/benchcomp/visualizers/__init__.py index df06c791bc4c..ae9540695248 100644 --- a/tools/benchcomp/benchcomp/visualizers/__init__.py +++ b/tools/benchcomp/benchcomp/visualizers/__init__.py @@ -17,7 +17,7 @@ @dataclasses.dataclass class run_command: - """Run an executable command, passing the result as a JSON file on stdin. + """Run an executable command, passing the performance metrics as JSON on stdin. This allows you to write your own visualization, which reads a result file on stdin and does something with it, e.g. writing out a graph or other From 8c8fdd6e855bfd73703f8ad9c83e531e0e0ad49d Mon Sep 17 00:00:00 2001 From: Kareem Khazem Date: Fri, 23 Jun 2023 12:44:28 +0100 Subject: [PATCH 3/4] Do not refer to config schema in docs --- docs/src/benchcomp-conf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/benchcomp-conf.md b/docs/src/benchcomp-conf.md index 489ae34a5550..77236d0917bf 100644 --- a/docs/src/benchcomp-conf.md +++ b/docs/src/benchcomp-conf.md @@ -1,7 +1,7 @@ # `benchcomp` configuration file `benchcomp`'s operation is controlled through a YAML file---`benchcomp.yaml` by default or a file passed to the `-c/--config` option. -This page describes the file's schema and lists the different parsers and visualizations that are available. +This page lists the different visualizations that are available. ## Built-in visualizations From 561dff4708e1e86556f179553f17c249f241736c Mon Sep 17 00:00:00 2001 From: Kareem Khazem Date: Fri, 23 Jun 2023 12:53:12 +0100 Subject: [PATCH 4/4] Make benchcomp terminate with 1 if visualization fails --- tools/benchcomp/benchcomp/visualizers/__init__.py | 10 ++++++---- tools/benchcomp/test/test_regression.py | 10 ++-------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/tools/benchcomp/benchcomp/visualizers/__init__.py b/tools/benchcomp/benchcomp/visualizers/__init__.py index ae9540695248..af973379f19d 100644 --- a/tools/benchcomp/benchcomp/visualizers/__init__.py +++ b/tools/benchcomp/benchcomp/visualizers/__init__.py @@ -41,13 +41,15 @@ def __call__(self, results): proc = subprocess.Popen( self.command, shell=True, text=True, stdin=subprocess.PIPE) _, _ = proc.communicate(input=results) - except subprocess.CalledProcessError as exc: - logging.warning( - "visualization command '%s' exited with code %d", - self.command, exc.returncode) except (OSError, subprocess.SubprocessError) as exe: logging.error( "visualization command '%s' failed: %s", self.command, str(exe)) + viz_utils.EXIT_CODE = 1 + if proc.returncode: + logging.error( + "visualization command '%s' exited with code %d", + self.command, proc.returncode) + viz_utils.EXIT_CODE = 1 diff --git a/tools/benchcomp/test/test_regression.py b/tools/benchcomp/test/test_regression.py index 1ead636ca7fb..c5cb61ae8190 100644 --- a/tools/benchcomp/test/test_regression.py +++ b/tools/benchcomp/test/test_regression.py @@ -792,7 +792,7 @@ def test_run_command_visualization(self): def test_run_failing_command_visualization(self): - """Ensure that benchcomp terminates normally even when run_command visualization doesn't""" + """Ensure that benchcomp terminates with a non-zero return code when run_command visualization fails""" with tempfile.TemporaryDirectory() as tmp: out_file = pathlib.Path(tmp) / str(uuid.uuid4()) @@ -832,11 +832,5 @@ def test_run_failing_command_visualization(self): }], }) run_bc() - self.assertEqual( + self.assertNotEqual( run_bc.proc.returncode, 0, msg=run_bc.stderr) - - with open(out_file) as handle: - result = yaml.safe_load(handle) - - for item in ["benchmarks", "metrics"]: - self.assertIn(item, result)