Metric Comparison + API Updates

.gitattributes

@@ -1 +1,2 @@
-*.v linguist-detectable=false
+openlane/smoke_test_design/* linguist-detectable=false
+.github/* linguist-detectable=false

.github/actions/build_nix/action.yml

@@ -35,7 +35,7 @@ runs:
       shell: ${{ inputs.shell }}
       run: |
         echo "#################################################################"
-        nix-shell ${{ inputs.nix_build_args }} --run "pytest --step-rx '.'"
+        nix-shell ${{ inputs.nix_build_args }} --run "pytest --step-rx '.' -n auto"
     - name: Smoke Test
       shell: ${{ inputs.shell }}
       run: |

.github/scripts/compare_main.js

@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+// Copyright 2023 Efabless Corporation
+//
+// Licensed under the Apache License, Version 2.0(the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+//     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+/**
+ * @param {string} verbosity
+ * @param {string} token
+ * @returns {string}
+ */
+function get(verbosity, table_out = null, token = "") {
+    const { spawnSync } = require("child_process");
+
+    let tableOutOpts = table_out ? ["--table-out", table_out] : [];
+
+    let child = spawnSync("python3", ["-m", "openlane.common.metrics", "compare-main", "current", "--table-verbosity", verbosity, "--token", token].concat(tableOutOpts), { encoding: "utf8" });
+
+    let result = "";
+    if (child.status != 0) {
+        throw new Error("Failed to create report: \n\n```\n" + child.stderr + "\n```");
+    } else {
+        result += "Metric comparisons are in beta. Please report bugs under the issues tab.\n---\n";
+        result += "> To create this report yourself, grab the metrics artifact from the CI run, extract them, and invoke `python3 -m openlane.common.metrics compare-main <path to directory>`.\n\n" + child.stdout;
+    }
+
+    return result.trim();
+};
+
+module.exports = get;
+
+if (require.main === module) {
+    console.log(get("ALL", "table.md", process.argv[2]));
+}

.github/scripts/post_metrics.js

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+// Copyright 2023 Efabless Corporation
+//
+// Licensed under the Apache License, Version 2.0(the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+//     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const compareMain = require("./compare_main.js");
+
+/**
+ * 
+ * @param {Octokit} github 
+ * @param {object} context 
+ * @param {string} botUsername 
+ * @param {string} body 
+ */
+async function postOrUpdateComment(github, context, botUsername, body) {
+    const METRIC_REPORT_MARK = "<!-- MARK: METRICS REPORT -->";
+
+    let page = 1;
+    let allComments = [];
+    let comments = [];
+    do {
+        let response = await github.request("GET /repos/{owner}/{repo}/issues/{issue_number}/comments?page={page}&per_page=100", {
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            page,
+        });
+        comments = response.data;
+        allComments = allComments.concat(comments);
+        page += 1;
+    } while (comments.length != 0);
+
+    let found = null;
+    for (let comment of allComments) {
+        if (comment.body.includes(METRIC_REPORT_MARK) && comment.user.login == botUsername) {
+            found = comment;
+            break;
+        }
+    }
+
+    let fn = github.rest.issues.createComment;
+    let request = {
+        issue_number: context.issue.number,
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        body: `${METRIC_REPORT_MARK}\n\n${body}`
+    };
+
+    if (found) {
+        request.comment_id = found.id;
+        fn = github.rest.issues.updateComment;
+    }
+
+    await fn(request);
+}
+
+/**
+ * 
+ * @param {Octokit} github 
+ * @param {object} context 
+ * @param {string} botUsername 
+ * @param {string} botToken 
+ */
+async function main(github, context, botUsername, botToken) {
+    const fs = require("fs");
+
+    let body;
+    try {
+        body = compareMain("ALL", "./tables_all.md", botToken);
+        let tables = fs.readFileSync("./tables_all.md", { encoding: "utf8" });
+
+        let gistResponse = await github.rest.gists.create({
+            public: true,
+            description: `Results for ${context.repo.owner} / ${context.repo.repo}#${context.issue.number} (Run ${context.runId})`,
+            files: {
+                "10-ALL.md": {
+                    content: tables
+                }
+            }
+        });
+
+        body += `\n\nFull tables ► ${gistResponse.data.html_url}\n`;
+    } catch (e) {
+        body = e.message;
+        console.error(e.message)
+    }
+
+    await postOrUpdateComment(github, context, botUsername, body)
+}
+
+module.exports = main;
+
+if (require.main === module) {
+    // Test
+    try {
+        require("@octokit/rest");
+    } catch (error) {
+        console.error("Run 'yarn add @octokit/rest @octokit/plugin-paginate-rest'")
+        process.exit(-1);
+    }
+    const { Octokit } = require("@octokit/rest");
+
+    const context = {
+        repo: {
+            owner: "efabless",
+            repo: "openlane2"
+        },
+        issue: {
+            number: process.argv[3]
+        },
+        runId: "api_test"
+    };
+
+    let octokit = new Octokit({
+        auth: process.argv[2],
+    });
+
+    main(octokit, context, "openlane-bot", process.argv[2]);
+}

.github/test_sets/get_test_matrix.py

@@ -65,7 +65,7 @@ def main(scls, use_json, test_sets):
                 ol_dir, "test", "designs", design_name, config_filename
             )
             run_folder = os.path.join(
-                ol_dir, "test", "designs", design_name, "runs", "CI"
+                ol_dir, "test", "designs", design_name, "runs", f"{pdk}-{scl}"
             )
             designs.append(
                 {

.github/test_sets/test_sets.yml

@@ -2,19 +2,19 @@
   name: fastest_test_set
   designs:
     - inverter
-    - wbqspiflash
-    - APU
-    - s44
-    - zipdiv
-    - blink
-    - name: xtea
-      config_file: config.tcl
-    - usb
-    - usb_cdc_core
-    - name: latch
-      config_file: config_good.json
-    - name: latch
-      config_file: config_bad.json
+    # - wbqspiflash
+    # - APU
+    # - s44
+    # - zipdiv
+    # - blink
+    # - name: xtea
+    #   config_file: config.tcl
+    # - usb
+    # - usb_cdc_core
+    # - name: latch
+    #   config_file: config_good.json
+    # - name: latch
+    #   config_file: config_bad.json
     - spm
     - gcd
     - caravel_upw
@@ -47,16 +47,16 @@
   name: fastest_test_set
   designs:
     - spm
-    - APU
-    - usb
-    - usb_cdc_core
-    - zipdiv
-    - blink
-    - wbqspiflash
-    - name: latch
-      config_file: config_good.json
-    - name: latch
-      config_file: config_bad.json
+    # - APU
+    # - usb
+    # - usb_cdc_core
+    # - zipdiv
+    # - blink
+    # - wbqspiflash
+    # - name: latch
+    #   config_file: config_good.json
+    # - name: latch
+    #   config_file: config_bad.json
 - scl: gf180mcuD/gf180mcu_fd_sc_mcu7t5v0
   name: extended_test_set
   designs: