From 5d5ff91aeaf81ae3e193e7615808e712ae675edb Mon Sep 17 00:00:00 2001
From: Oscar Eriksson <oerikss@kth.se>
Date: Sun, 12 Nov 2023 12:19:14 +0100
Subject: [PATCH] Add run_all, timeout, and constant folding to the
 microbenchmarks

---
 test/microbenchmark/run.ml  | 63 +++++++++++++++++++++++--------------
 test/microbenchmark/run_all | 12 +++++++
 2 files changed, 52 insertions(+), 23 deletions(-)
 create mode 100755 test/microbenchmark/run_all

diff --git a/test/microbenchmark/run.ml b/test/microbenchmark/run.ml
index f01fd4d6a..8cdeecb8f 100644
--- a/test/microbenchmark/run.ml
+++ b/test/microbenchmark/run.ml
@@ -2,14 +2,16 @@ open Printf
 
 let menu () =
   printf
-    "Usage: run <benchmark-name-without-postfix-name> <iteration> [excludes]\n\n" ;
+    "Usage: run <benchmark-name-without-postfix-name> <iteration> [excludes] \
+     [timeout]\n\n" ;
   printf "Example: run factorial 1000\n\n" ;
   printf "To exclude certain tests, add excluded numbers. For instance\n" ;
-  printf "command 'run factorial 1000 24' excludes the Miking interpreter\n" ;
-  printf "and OCaml byte code tests (test numbers 2 and 4).\n" ;
+  printf "command 'run factorial 1000 25' excludes the Miking interpreter\n" ;
+  printf "and OCaml byte code tests (test numbers 2 and 5).\n" ;
+  printf "The argument timeout give a timeout in seconds.\n" ;
   exit 1
 
-let measure excludes number str pre_cmd cmd post_cmd =
+let measure excludes number str pre_cmd cmd post_cmd timeout =
   (* printf "\n\npre_cmd: %s\ncmd: %s\npost_cmd: %s\n" pre_cmd cmd post_cmd ; *)
   if String.contains excludes (string_of_int number).[0] then ()
   else (
@@ -17,10 +19,19 @@ let measure excludes number str pre_cmd cmd post_cmd =
     let to_null = " 2>&1 >/dev/null" in
     let _ = Sys.command (pre_cmd ^ to_null) in
     let run () =
+      let command = cmd ^ to_null in
+      let command =
+        match timeout with
+        | Some timeout ->
+            "timeout " ^ string_of_int timeout ^ " " ^ command
+        | None ->
+            command
+      in
       let l1 = Unix.gettimeofday () in
-      let _ = Sys.command (cmd ^ to_null) in
+      let rc = Sys.command command in
       let l2 = Unix.gettimeofday () in
-      printf "%10fs" (l2 -. l1) ;
+      printf "%10f%s" (l2 -. l1)
+        (if rc = 124 && Option.is_some timeout then "s (timeout)" else "s") ;
       flush stdout
     in
     run () ;
@@ -52,28 +63,34 @@ let main =
     if len >= 3 then (Sys.argv.(1), Sys.argv.(2)) else menu ()
   in
   let excludes = if len >= 4 then Sys.argv.(3) else "" in
+  let timeout = if len >= 5 then Some (int_of_string Sys.argv.(4)) else None in
   let name_mc = name ^ ".mc" in
   let _ = Sys.command "rm -rf _build" in
   if Sys.file_exists name_mc then (
-    measure excludes 1 "Boot interpreter:  " ""
+    measure excludes 1 "Boot interpreter:     " ""
       ("boot eval " ^ name_mc ^ " -- " ^ iterations)
-      "" ;
-    measure excludes 2 "Miking interpreter:" ""
+      "" timeout ;
+    measure excludes 2 "Miking interpreter:   " ""
       ("mi eval " ^ name_mc ^ " -- " ^ iterations)
-      "" ;
-    measure excludes 3 "Miking compiler:   " ("mi compile " ^ name_mc)
+      "" timeout ;
+    measure excludes 3 "Miking compiler:      " ("mi compile " ^ name_mc)
       ("./" ^ name ^ " " ^ iterations)
-      ("rm -f " ^ name) ;
-    if Sys.file_exists (name ^ ".ml") then (
-      generate_dune name ;
-      generate_dune_project () ;
-      measure excludes 4 "Ocaml byte code    "
-        ("dune build --root ." ^ " " ^ name ^ ".bc")
-        ("ocamlrun _build/default/" ^ name ^ ".bc" ^ " " ^ iterations)
-        "rm -rf _build" ;
-      measure excludes 5 "OCaml native:      "
-        ("dune build --root ." ^ " " ^ name ^ ".exe")
-        ("./_build/default/" ^ name ^ ".exe" ^ " " ^ iterations)
-        "rm -rf _build && rm dune*" )
+      ("rm -f " ^ name) timeout ;
+    measure excludes 4 "Miking compiler (opt):"
+      ("mi compile --enable-constant-fold " ^ name_mc)
+      ("./" ^ name ^ " " ^ iterations)
+      ("rm -f " ^ name) timeout ;
+    if Sys.file_exists (name ^ ".ml") then
+      ( generate_dune name ;
+        generate_dune_project () ;
+        measure excludes 5 "Ocaml byte code       "
+          ("dune build --root ." ^ " " ^ name ^ ".bc")
+          ("ocamlrun _build/default/" ^ name ^ ".bc" ^ " " ^ iterations)
+          "rm -rf _build" timeout ;
+        measure excludes 6 "OCaml native:         "
+          ("dune build --root ." ^ " " ^ name ^ ".exe")
+          ("./_build/default/" ^ name ^ ".exe" ^ " " ^ iterations)
+          "rm -rf _build && rm dune*" )
+        timeout
     else () )
   else printf "ERROR: Cannot find file %s.mc\n" name
diff --git a/test/microbenchmark/run_all b/test/microbenchmark/run_all
new file mode 100755
index 000000000..4b9b29630
--- /dev/null
+++ b/test/microbenchmark/run_all
@@ -0,0 +1,12 @@
+#!/bin/sh
+for f in *.mc
+do
+    b=$(basename -s .mc "$f")
+    to=20
+    echo "./run $b $@"
+    timeout "$to" ./run "$b" "$@"
+    if [ "$?" -eq 124 ]; then
+        echo "timed out after $to s"
+    fi
+    echo ""
+done