diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fa54eda0..b0c2ca1f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -115,3 +115,12 @@ jobs:
with:
config: .github/memory_statistics_config.json
check_against: docs/doxygen/include/size_table.md
+ proof_ci:
+ runs-on: cbmc_ubuntu-latest_16-core
+ steps:
+ - name: Set up CBMC runner
+ uses: FreeRTOS/CI-CD-Github-Actions/set_up_cbmc_runner@main
+ - name: Run CBMC
+ uses: FreeRTOS/CI-CD-Github-Actions/run_cbmc@main
+ with:
+ proofs_dir: test/cbmc/proofs
diff --git a/test/cbmc/proofs/lib/print_tool_versions.py b/test/cbmc/proofs/lib/print_tool_versions.py
new file mode 100755
index 00000000..bdeb429e
--- /dev/null
+++ b/test/cbmc/proofs/lib/print_tool_versions.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT-0
+
+
+import logging
+import pathlib
+import shutil
+import subprocess
+
+
+_TOOLS = [
+ "cadical",
+ "cbmc",
+ "cbmc-viewer",
+ "cbmc-starter-kit-update",
+ "kissat",
+ "litani",
+]
+
+
+def _format_versions(table):
+ lines = [
+ "
",
+ 'Tool Versions |
',
+ ]
+ for tool, version in table.items():
+ if version:
+ v_str = f'{version}
'
+ else:
+ v_str = 'not found'
+ lines.append(
+ f'{tool}: | '
+ f'{v_str} |
')
+ lines.append("
")
+ return "\n".join(lines)
+
+
+def _get_tool_versions():
+ ret = {}
+ for tool in _TOOLS:
+ err = f"Could not determine version of {tool}: "
+ ret[tool] = None
+ if not shutil.which(tool):
+ logging.error("%s'%s' not found on $PATH", err, tool)
+ continue
+ cmd = [tool, "--version"]
+ proc = subprocess.Popen(cmd, text=True, stdout=subprocess.PIPE)
+ try:
+ out, _ = proc.communicate(timeout=10)
+ except subprocess.TimeoutExpired:
+ logging.error("%s'%s --version' timed out", err, tool)
+ continue
+ if proc.returncode:
+ logging.error(
+ "%s'%s --version' returned %s", err, tool, str(proc.returncode))
+ continue
+ ret[tool] = out.strip()
+ return ret
+
+
+def main():
+ exe_name = pathlib.Path(__file__).name
+ logging.basicConfig(format=f"{exe_name}: %(message)s")
+
+ table = _get_tool_versions()
+ out = _format_versions(table)
+ print(out)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/test/cbmc/proofs/lib/summarize.py b/test/cbmc/proofs/lib/summarize.py
index 154634b4..50dbcc33 100644
--- a/test/cbmc/proofs/lib/summarize.py
+++ b/test/cbmc/proofs/lib/summarize.py
@@ -1,8 +1,28 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: MIT-0
+import argparse
import json
import logging
+import os
+import sys
+
+
+DESCRIPTION = """Print 2 tables in GitHub-flavored Markdown that summarize
+an execution of CBMC proofs."""
+
+
+def get_args():
+ """Parse arguments for summarize script."""
+ parser = argparse.ArgumentParser(description=DESCRIPTION)
+ for arg in [{
+ "flags": ["--run-file"],
+ "help": "path to the Litani run.json file",
+ "required": True,
+ }]:
+ flags = arg.pop("flags")
+ parser.add_argument(*flags, **arg)
+ return parser.parse_args()
def _get_max_length_per_column_list(data):
@@ -56,6 +76,7 @@ def _get_status_and_proof_summaries(run_dict):
run_dict
A dictionary representing a Litani run.
+
Returns
-------
A list of 2 lists.
@@ -70,8 +91,9 @@ def _get_status_and_proof_summaries(run_dict):
count_statuses[status_pretty_name] += 1
except KeyError:
count_statuses[status_pretty_name] = 1
- proof = proof_pipeline["name"]
- proofs.append([proof, status_pretty_name])
+ if proof_pipeline["name"] == "print_tool_versions":
+ continue
+ proofs.append([proof_pipeline["name"], status_pretty_name])
statuses = [["Status", "Count"]]
for status, count in count_statuses.items():
statuses.append([status, str(count)])
@@ -83,10 +105,39 @@ def print_proof_results(out_file):
Print 2 strings that summarize the proof results.
When printing, each string will render as a GitHub flavored Markdown table.
"""
+ output = "## Summary of CBMC proof results\n\n"
+ with open(out_file, encoding='utf-8') as run_json:
+ run_dict = json.load(run_json)
+ status_table, proof_table = _get_status_and_proof_summaries(run_dict)
+ for summary in (status_table, proof_table):
+ output += _get_rendered_table(summary)
+
+ print(output)
+ sys.stdout.flush()
+
+ github_summary_file = os.getenv("GITHUB_STEP_SUMMARY")
+ if github_summary_file:
+ with open(github_summary_file, "a") as handle:
+ print(output, file=handle)
+ handle.flush()
+ else:
+ logging.warning(
+ "$GITHUB_STEP_SUMMARY not set, not writing summary file")
+
+ msg = (
+ "Click the 'Summary' button to view a Markdown table "
+ "summarizing all proof results")
+ if run_dict["status"] != "success":
+ logging.error("Not all proofs passed.")
+ logging.error(msg)
+ sys.exit(1)
+ logging.info(msg)
+
+
+if __name__ == '__main__':
+ args = get_args()
+ logging.basicConfig(format="%(levelname)s: %(message)s")
try:
- with open(out_file, encoding='utf-8') as run_json:
- run_dict = json.load(run_json)
- for summary in _get_status_and_proof_summaries(run_dict):
- print(_get_rendered_table(summary))
+ print_proof_results(args.run_file)
except Exception as ex: # pylint: disable=broad-except
logging.critical("Could not print results. Exception: %s", str(ex))