Don't do all metrics

model-checking · Mar 14, 2024 · 5b2ca09 · 5b2ca09
1 parent 5bfe221
commit 5b2ca09
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 3 deletions.
diff --git a/tools/benchcomp/benchcomp/visualizers/__init__.py b/tools/benchcomp/benchcomp/visualizers/__init__.py
@@ -244,7 +244,7 @@ def _get_template():
         return textwrap.dedent("""\
             {% for metric, benchmarks in d["metrics"].items() %}
             ## {{ metric }}
-            {% if scatterplot and d["scaled_variants"][metric]|length == 2 %}
+            {% if scatterplot and metric in d["scaled_metrics"] and d["scaled_variants"][metric]|length == 2 %}
             ```mermaid
             %%{init: { "quadrantChart": { "chartWidth": 400, "chartHeight": 400, "pointRadius": 2, "pointLabelFontSize": 3 }, "themeVariables": { "quadrant1Fill": "#FFFFFF", "quadrant2Fill": "#FFFFFF", "quadrant3Fill": "#FFFFFF", "quadrant4Fill": "#FFFFFF", "quadrant1TextFill": "#FFFFFF", "quadrant2TextFill": "#FFFFFF", "quadrant3TextFill": "#FFFFFF", "quadrant4TextFill": "#FFFFFF", "quadrantInternalBorderStrokeFill": "#FFFFFF" } }%%
             quadrantChart
@@ -282,6 +282,10 @@ def _compute_scaled_metric(data_for_metric, log_scaling):
         max_value = None
         for bench, bench_result in data_for_metric.items():
             for variant, variant_result in bench_result.items():
+                if isinstance(variant_result, (bool, str)):
+                    return None
+                if not isinstance(variant_result, (int, float)):
+                    return None
                 if min_value is None or variant_result < min_value:
                     min_value = variant_result
                 if max_value is None or variant_result > max_value:
@@ -302,12 +306,12 @@ def _compute_scaled_metric(data_for_metric, log_scaling):
                     else:
                         abs_value = variant_result
                     ret[bench][variant] = (abs_value - min_value) / value_range
+        return ret
 
 
     @staticmethod
     def _organize_results_into_metrics(results, log_scaling):
         ret = {metric: {} for metric in results["metrics"]}
-        ret_scaled = {metric: {} for metric in results["metrics"]}
         for bench, bench_result in results["benchmarks"].items():
             for variant, variant_result in bench_result["variants"].items():
                 for metric, value in variant_result["metrics"].items():
@@ -325,8 +329,10 @@ def _organize_results_into_metrics(results, log_scaling):
                             variant: variant_result["metrics"][metric]
                         }
         for metric, bench_result in ret.items():
-            ret_scaled[metric] = dump_markdown_results_table._compute_scaled_metric(
+            scaled = dump_markdown_results_table._compute_scaled_metric(
                     bench_result, log_scaling)
+            if scaled is not None:
+                ret_scaled[metric] = scaled
         return (ret, ret_scaled)
 
 

diff --git a/tools/benchcomp/test/test_regression.py b/tools/benchcomp/test/test_regression.py
@@ -436,6 +436,7 @@ def test_markdown_results_table(self):
                 "visualize": [{
                     "type": "dump_markdown_results_table",
                     "out_file": "-",
+                    "scatterplot": "linear",
                     "extra_columns": {
                         "runtime": [{
                             "column_name": "ratio",