Render rich output in dashboard, pipeline pages

This feature allows users to render custom HTML data onto the HTML dashboard, allowing Litani jobs to display their results through tables, graphs, and other HTML format. This can be done both for the front page or on the pipeline page. The intention is that individual litani jobs can be used to measure metrics, and then print those metrics out in an easily-viewable format. Jobs that calculate metrics for the entire run can present those metrics as a graph on the front page. Jobs that calculate metrics for a single pipeline (or proof) can display the result on the pipeline page. Users use this feature by adding a tag to a litani job, using the --tags flag. If a job is tagged with front-page-text, Litani will render the job's output onto the front page, in its own section. A tag of literal-stdout will make Litani render the job's output onto the pipeline page, but without any HTML escaping. This feature fixes #81 and fixes #82.
awslabs · Feb 15, 2022 · 4f41b16 · 4f41b16
2 parents 1d545fc + d178b83
commit 4f41b16
Show file tree

Hide file tree

Showing 21 changed files with 570 additions and 14 deletions.
diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,8 @@ test/output
 doc/tmp
 doc/out
 
+examples/rich-output/*.csv
+
 __pycache__/
 
 build.ninja
diff --git a/doc/src/man/litani-add-job.scdoc b/doc/src/man/litani-add-job.scdoc
@@ -125,6 +125,13 @@ configure times.
 	job's list of tags is included in its _run.json_ and is intended to help with
 	analyzing run data.
 
+	If 'literal-stdout' is provided as a tag, then the report will render the
+	job's stdout underneath the boxes that indicate the command and its
+	statistics within the associated pipeline's page. If 'front-page-text' is
+	provided as a tag, then the report will render the job's stdout on the
+	front page of the report. These 2 tags allow users to render custom HTML
+	data, e.g. tables, graphs. Examples can be found at 'examples/rich-output'.
+
 *--pipeline-name* _P_
 	The 'pipeline' that this job is part of. A pipeline is a subgraph of the
 	overall build, representing a single 'end-to-end' set of related jobs. A job's

diff --git a/examples/rich-output/README.md b/examples/rich-output/README.md
@@ -0,0 +1,10 @@
+README
+
+The run-all.py script will run examples that demonstrate how jobs in Litani can
+display rich output either on the dashboard or pipeline pages
+
+To try this out, run
+
+    LITANI=../../litani ./run-all.py
+
+in this directory.
diff --git a/examples/rich-output/assumptions.html b/examples/rich-output/assumptions.html
@@ -0,0 +1,42 @@
+<html>
+    <style>
+        table {
+            font-family: Helvetica, sans-serif;
+            border-collapse: collapse;
+            width: 100%;
+        }
+
+        td, th {
+            border: 1px solid #b0e0e6;
+            text-align: left;
+            padding: 8px;
+        }
+
+        tr:nth-child(odd) {
+            background-color: #b0e0e6;
+        }
+    </style>
+    <body>
+        <div id="assumptions">
+            <table>
+                <tr>
+                    <th>Assumption to check</th>
+                    <th>Difficulty</th>
+                </tr>
+                <tr>
+                    <td>Bound of input size function</td>
+                    <td>Easy</td>
+                </tr>
+                <tr>
+                    <td>Harness</td>
+                    <td>Medium</td>
+                </tr>
+                <tr>
+                    <td>Function specifications</td>
+                    <td>Hard</td>
+                </tr>
+            </table>
+        </div>
+    </body>
+</html>
+
diff --git a/examples/rich-output/file.dat b/examples/rich-output/file.dat
@@ -0,0 +1,7 @@
+pipeline #Assumptions
+"Class\\\_A" 5
+"Class\\\_B" 1
+"Class\\\_C" 2
+"Class\\\_D" 8
+"Class\\\_E" 4
+"Class\\\_F" 0
diff --git a/examples/rich-output/histogram.dat b/examples/rich-output/histogram.dat
@@ -0,0 +1,7 @@
+set terminal svg noenhanced size 565,1096
+set style data histogram
+set style fill solid border
+set style histogram clustered
+set xtics rotate out
+set yrange [ 0 : 10 ]
+plot for [COL=2:2] 'file.dat' using COL:xticlabels(1) title columnheader
diff --git a/examples/rich-output/run-1.sh b/examples/rich-output/run-1.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+LITANI=${LITANI:-litani}
+
+if [ "$1" != "--no-standalone" ]; then
+  "${LITANI}" init --project-name "literal-stdout-examples"
+fi
+
+"${LITANI}" add-job \
+  --pipeline-name "verify assumptions" \
+  --description "list out assumptions" \
+  --command "cat assumptions.html" \
+  --ci-stage test \
+  --phony-outputs assumptions.html \
+  --tags literal-stdout
+
+"${LITANI}" add-job \
+  --pipeline-name "conduct analysis" \
+  --description "generate the histogram" \
+  --command "gnuplot -c histogram.dat" \
+  --ci-stage report \
+  --inputs file.dat \
+  --phony-outputs histogram.svg \
+  --tags literal-stdout
+
+if [ "$1" != "--no-standalone" ]; then
+  "${LITANI}" run-build
+fi
diff --git a/examples/rich-output/run-2.sh b/examples/rich-output/run-2.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+LITANI=${LITANI:-litani}
+
+if [ "$1" != "--no-standalone" ]; then
+  "${LITANI}" init --project-name "front-page-output examples"
+fi
+
+"${LITANI}" add-job \
+    --pipeline-name "fibonacci" \
+    --description "write fibonacci to csv" \
+    --command "python3 scripts/fib.py -o fib.csv" \
+    --ci-stage test \
+    --outputs "fib.csv"
+
+"${LITANI}" add-job \
+    --pipeline-name "fibonacci" \
+    --description "plot fibonacci results" \
+    --command "gnuplot scripts/fib.plt" \
+    --ci-stage report \
+    --inputs "fib.csv" \
+    --tags "front-page-text"
+
+"${LITANI}" add-job \
+    --pipeline-name "fibonacci" \
+    --description "table fibonacci results" \
+    --command "python3 scripts/fib-table.py -i fib.csv" \
+    --ci-stage report \
+    --inputs "fib.csv" \
+    --tags "front-page-text"
+
+if [ "$1" != "--no-standalone" ]; then
+  "${LITANI}" run-build
+fi
diff --git a/examples/rich-output/run-3.sh b/examples/rich-output/run-3.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+LITANI=${LITANI:-litani}
+
+if [ "$1" != "--no-standalone" ]; then
+  "${LITANI}" init \
+    --project-name "front-page-output and literal-stdout examples"
+fi
+
+"${LITANI}" add-job \
+    --pipeline-name "sin" \
+    --description "write sin to csv" \
+    --command "python3 scripts/sin.py -o sin.csv" \
+    --ci-stage test \
+    --outputs "sin.csv"
+
+"${LITANI}" add-job \
+    --pipeline-name "sin" \
+    --description "display sin results" \
+    --command \
+    "gnuplot scripts/sin.plt; python3 scripts/sin-output.py -i sin.csv" \
+    --ci-stage report \
+    --inputs "sin.csv" \
+    --tags front-page-text literal-stdout
+
+if [ "$1" != "--no-standalone" ]; then
+  "${LITANI}" run-build
+fi
diff --git a/examples/rich-output/run-all.py b/examples/rich-output/run-all.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+
+import os
+import pathlib
+import re
+import subprocess
+
+
+litani = os.getenv("LITANI", "litani")
+this_dir = pathlib.Path(__file__).parent
+
+subprocess.run(
+    [litani, "init", "--project-name", "rich-output-examples"], check=True)
+
+pat = re.compile(r"run-\d.sh")
+for fyle in this_dir.iterdir():
+    if pat.match(fyle.name):
+        subprocess.run([fyle, "--no-standalone"], check=True)
+
+subprocess.run([litani, "run-build"], check=True)
diff --git a/examples/rich-output/scripts/fib-table.py b/examples/rich-output/scripts/fib-table.py
@@ -0,0 +1,40 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+import argparse
+import jinja2
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-i", "--input", required=True, help="input csv")
+    args = parser.parse_args()
+
+    data = {}
+    with open(args.input, "r") as handle:
+        for line in handle:
+            values = line.split(',')
+            data[int(values[0].strip())] = int(values[1].strip())
+
+    env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader("./templates"),
+        autoescape=jinja2.select_autoescape(
+            enabled_extensions=('csv'),
+            default_for_string=True))
+    table_template = env.get_template("fib-table.jinja.html")
+    table = table_template.render(data=data)
+    print(table)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/rich-output/scripts/fib.plt b/examples/rich-output/scripts/fib.plt
@@ -0,0 +1,5 @@
+set term svg
+set style fill solid border
+set xlabel "nth fibonacci number"
+set ylabel "value"
+plot "fib.csv" with linespoints
diff --git a/examples/rich-output/scripts/fib.py b/examples/rich-output/scripts/fib.py
@@ -0,0 +1,32 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+import argparse
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-o", "--output", required=True, help="output file")
+    args = parser.parse_args()
+    n = 10
+    fib = [0] * n
+    fib[1] = 1
+    for i in range(2, n):
+        fib[i] = fib[i-1] + fib[i-2]
+    with open(args.output, "w") as handle:
+        for index, value in enumerate(fib, 1):
+            print(f"{index}, {value}", file=handle)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/rich-output/scripts/sin-output.py b/examples/rich-output/scripts/sin-output.py
@@ -0,0 +1,40 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+import argparse
+import jinja2
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-i", "--input", required=True, help="input csv")
+    args = parser.parse_args()
+
+    data = {}
+    with open(args.input, "r") as handle:
+        for line in handle:
+            values = line.split(',')
+            data[float(values[0].strip())] = float(values[1].strip())
+
+    env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader("./templates"),
+        autoescape=jinja2.select_autoescape(
+            enabled_extensions=('csv'),
+            default_for_string=True))
+    table_template = env.get_template("sin-output.jinja.html")
+    output = table_template.render(data=data)
+    print(output)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/rich-output/scripts/sin.plt b/examples/rich-output/scripts/sin.plt
@@ -0,0 +1,5 @@
+set term svg
+set style fill solid border
+set xlabel "x"
+set ylabel "sin(x)"
+plot 0, "sin.csv" smooth csplines