Make benchcomp terminate with 1 if visualization fails

tools/benchcomp/benchcomp/visualizers/__init__.py

@@ -41,13 +41,15 @@ def __call__(self, results):
  proc = subprocess.Popen(
  self.command, shell=True, text=True, stdin=subprocess.PIPE)
  _, _ = proc.communicate(input=results)
- except subprocess.CalledProcessError as exc:
- logging.warning(
- "visualization command '%s' exited with code %d",
- self.command, exc.returncode)
  except (OSError, subprocess.SubprocessError) as exe:
  logging.error(
  "visualization command '%s' failed: %s", self.command, str(exe))
+ viz_utils.EXIT_CODE = 1
+ if proc.returncode:
+ logging.error(
+ "visualization command '%s' exited with code %d",
+ self.command, proc.returncode)
+ viz_utils.EXIT_CODE = 1
 
 
 

tools/benchcomp/test/test_regression.py

@@ -792,7 +792,7 @@ def test_run_command_visualization(self):
 
 
  def test_run_failing_command_visualization(self):
- """Ensure that benchcomp terminates normally even when run_command visualization doesn't"""
+ """Ensure that benchcomp terminates with a non-zero return code when run_command visualization fails"""
 
  with tempfile.TemporaryDirectory() as tmp:
  out_file = pathlib.Path(tmp) / str(uuid.uuid4())
@@ -832,11 +832,5 @@ def test_run_failing_command_visualization(self):
  }],
  })
  run_bc()
- self.assertEqual(
+ self.assertNotEqual(
  run_bc.proc.returncode, 0, msg=run_bc.stderr)
-
- with open(out_file) as handle:
- result = yaml.safe_load(handle)
-
- for item in ["benchmarks", "metrics"]:
- self.assertIn(item, result)