Web display for multi run files

lve-tools/web/generator/lve_details.py

@@ -121,14 +121,22 @@ def build_lve_site(generator, lve):
 
         c.title = lve_path
 
+        if lve.prompt is not None:
+            prompt=[render_prompt(lve.prompt, lve.prompt_parameters)]
+        else:
+            # multi run prompt
+            prompt = []
+            for mrp in lve.multi_run_prompt:
+                prompt.append(render_prompt(mrp.prompt, lve.prompt_parameters))
+
         template.emit(
             file=os.path.join(generator.target, path, sitename + ".html"),
             name=name_with_subfolders,
             category=category,
             num_instances=num_instances,
             description=lve.description,
             # escape HTML in prompt
-            prompt=render_prompt(lve.prompt, lve.prompt_parameters),
+            prompt=prompt,
             prompt_parameters=lve.prompt_parameters,
             checker_args=lve.checker_args,
             author=lve.author or "Anonymous",

lve-tools/web/lve.html

@@ -46,14 +46,14 @@ <h3>
                 {{ readme }}
             </div>
             <div class="sidebar markdown">
-                <h3>Prompt</h3>
+                <h3>{{"Prompt" if len(prompt) == 1 else "Prompts"}}</h3>
                 <div class="prompt">
                     <ul class="tabs">
                         <li class="active">Chat</li>
                         <!-- <li>JSON</li> -->
                         <!-- <li>LMQL</li> -->
                     </ul>
-                    <pre class="promptdown">{{ prompt }}</pre>
+                    {{ '\n'.join([f'<pre class="promptdown">{p}</pre>' for p in prompt]) }}
                 </div>
                 <h3>LVE Details</h3>
                 <label>Checker:</label><code>{{ checker_args.get("checker_name") }}</code><br/>
@@ -71,7 +71,7 @@ <h3>LVE Details</h3>
         <div id="instances">Loading...</div>
         <script>
             // for instance rendering
-            PROMPT_TEMPLATE = `{{ prompt }}`;
+            PROMPT_TEMPLATE = {{ "[" + ", ".join(f"`{p}`" for p in prompt) + "]" }};
             PROMPT_PARAMETERS = {{ prompt_parameters }};
         </script>
     </div>

lve-tools/web/static/instances.js

@@ -44,22 +44,33 @@ function sanitize(s, replace_square_brackets=false) {
 
 function instance_row(index, instance) {
     let passed = Object.keys(instance).includes("passed") ? instance["passed"] : instance["is_safe"]
-
-    let prompt = sanitize(PROMPT_TEMPLATE);
-    PROMPT_PARAMETERS.forEach(p => {
-        prompt = prompt.replace(`[{${p}}(empty=true)|`, `[{${p}}|${instance["args"][p]}`)
-    })
+
+    let responses = Array.isArray(instance["response"]) ? instance["response"] : [instance["response"]];
+    let n = responses.length;
+    let promptdown = [];
+    for (let i = 0; i < n; i++) {
+        let prompt = sanitize(PROMPT_TEMPLATE[i]);
 
-
-    let response = "";
-    if (typeof(instance["response"]) == "string") {
-        response = `[bubble:assistant|${sanitize(instance["response"].trim())}]`
-    }
-    else if (typeof(instance["response"]) == "object") {
-        for (const [key, value] of Object.entries(instance["response"])) {
-            prompt = prompt.replace(`[{${key}}(empty=true)|`, `[{${key}}|${sanitize(value, true)}`)
+        PROMPT_PARAMETERS.forEach(p => {
+            prompt = prompt.replace(`[{${p}}(empty=true)|`, `[{${p}}|${instance["args"][p]}`)
+        })
+
+        let response = "";
+        if (typeof(responses[i]) == "string") {
+            response = `[bubble:assistant|${sanitize(responses[i].trim())}]`
         }
+        else if (typeof(responses[i]) == "object") {
+            for (const [key, value] of Object.entries(responses[i])) {
+                prompt = prompt.replace(`[{${key}}(empty=true)|`, `[{${key}}|${sanitize(value, true)}`)
+            }
+        }
+        if (n > 1) {
+            promptdown.push(`<h3>Run ${i+1} of ${n}</h3>\n`)
+        }
+        promptdown.push(`<pre class='promptdown'>${prompt}${response}</pre>\n`)
     }
+    promptdown = promptdown.join("")//"<br\\>")
+
 
     let timestamp = null;
     try {
@@ -94,7 +105,7 @@ function instance_row(index, instance) {
                 <!-- <li>JSON</li> -->
                 <!-- <li>LMQL</li> -->
             </ul>
-            <pre class='promptdown'>${prompt}${response}</pre>
+            ${promptdown}
         </div>
     </div>
     </div>`

lve-tools/web/static/style.css

@@ -1017,6 +1017,10 @@ nav ul li.active a {
     color: var(--text-body);
 }
 
+.prompt h3 {
+    margin-left: 20pt;
+}
+
 .instance {
     border-radius: 5pt;
     border: 1pt solid var(--chrome-border);
@@ -1218,4 +1222,5 @@ pre.expand.expanded code {
 
 .markdown.lve-description h3 {
     margin-bottom: 5pt;
-}
+}
+

repository/dummy/a_plus_b_multirun/openai--gpt-35-turbo/instances/instances.jsonl

@@ -1,6 +1 @@
-{"args":{"temperature":0.0,"top_p":null,"max_tokens":null,"a":"3","b":"1"},"response":"4","passed":false,"author":"","run_info":{"openai":"0.28.1","openai-api_type":"open_ai","openai-api_version":null,"timestamp":"Sun Oct 22 21:28:30 2023"}}
-{"args":{"temperature":0.0,"top_p":null,"max_tokens":null,"a":"1","b":"3"},"response":"4","passed":false,"author":"","run_info":{"openai":"0.28.1","openai-api_type":"azure","openai-api_version":"2023-05-15","timestamp":"Sun Oct 22 21:29:16 2023"}}
-{"args":{"temperature":0.0,"top_p":null,"max_tokens":null,"a":"3","b":"2"},"response":"5","passed":false,"author":"","run_info":{"openai":"0.28.1","openai-api_type":"azure","timestamp":"Sun Oct 22 21:30:20 2023","openai-api_version":"2023-05-15"}}
-{"args":{"temperature":0.0,"top_p":null,"max_tokens":null,"a":"3","b":"1"},"response":"4","passed":false,"author":"","run_info":{"openai":"0.28.1","openai-api_type":"azure","timestamp":"Sun Oct 22 21:30:49 2023","openai-api_version":"2023-05-15"}}
-{"args":{"temperature":0.0,"top_p":null,"max_tokens":null,"a":"3","b":"1"},"response":"4","passed":false,"author":"","run_info":{"openai":"0.28.1","openai-api_type":"open_ai","timestamp":"Sun Oct 22 21:31:28 2023"}}
-{"args":{"temperature":0.0,"top_p":null,"max_tokens":null,"a":"3","b":"1"},"response":["4","4"],"passed":true,"author":"Viehzeug","run_info":{"openai":"0.28.1","openai-api_type":"open_ai","timestamp":"Thu Nov  2 22:25:28 2023"}}
+{"args":{"temperature":0.0,"top_p":null,"max_tokens":null,"a":"3","b":"1"},"response":["4","4"],"passed":true,"author":"Viehzeug","run_info":{"openai":"0.28.1","openai-api_type":"open_ai","timestamp":"Thu Nov  2 22:25:28 2023"}}