target/sim: Add benchmark yaml for storing configurations

.gitignore

@@ -14,3 +14,4 @@ modelsim.ini
 transcript
 
 gmon.out
+.ipynb_checkpoints

target/sim/bench.ipynb

@@ -0,0 +1,229 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "ec6a4851-436a-4278-9563-fe60b283a829",
+   "metadata": {},
+   "source": [
+    "# Benchmark"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c018cdf4-4eac-434f-ab48-c86c9ee61541",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!python -m pip install pandas plotly pyyaml"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "263c1118-51c5-4a82-9226-b01df47c40a7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd, numpy as np\n",
+    "import os, glob, datetime, time\n",
+    "import plotly as plotly\n",
+    "import plotly.express as px\n",
+    "import hjson\n",
+    "import yaml"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aff8e0ac-8b91-4ac1-b9f1-5da08ff143e1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Compile hardware for Questa (vsim)\n",
+    "!questa-2022.3 make bin/snitch_cluster.vsim"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16e62def-f903-455e-994a-6661c8b8895d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Compile software\n",
+    "!make DEBUG=ON sw"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ef1ab542-924c-4c02-a507-6044c4b32169",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Post process traces\n",
+    "!make -j traces\n",
+    "!make logs/perf.csv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3caceab4-f9cd-474a-9ce2-8688d05317d8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Read profile data\n",
+    "perf = pd.read_csv('logs/perf.csv', index_col=0)\n",
+    "perf.filter(regex=(\"1_.\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d20fe0e3-3f04-4a05-ab11-3d016dc75e79",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fig = px.scatter(perf, y=['1_total_ipc', '1_fpss_occupancy', '1_fpss_fpu_occupancy', '1_snitch_occupancy'])\n",
+    "fig.update_layout(yaxis_range=[0,1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "71d208c2-0ac9-47c4-ac71-6def321fc682",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def run(cmd, env=None, dryrun=False):\n",
+    "    if dryrun:\n",
+    "        print(cmd)\n",
+    "    else:\n",
+    "        p = subprocess.Popen(cmd, env=env, shell=True)\n",
+    "        retcode = p.wait()\n",
+    "        if retcode != 0:\n",
+    "            sys.exit(retcode)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "28811c2e-63ee-4143-ad82-4d97420b9b68",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!f'make CFG_OVERRIDE={cfg_file} rtl'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4d766185-bd6c-4c21-bbf8-79eded2247f1",
+   "metadata": {},
+   "source": [
+    "# Benchmark Configuration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "96393e95-7390-4157-9314-af5155f46f22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load top-level benchmark config\n",
+    "bench_config_name = \"bench/bench.yaml\"\n",
+    "with open(bench_config_name) as f:\n",
+    "    bench_config = yaml.load(f, Loader=yaml.Loader)\n",
+    "bench_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "826c3a78-a1e4-40d7-ae93-4429cb8a288f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Expand python strings\n",
+    "for app in bench_config['sw']:\n",
+    "    if type(app['sweep']['n']) == str:\n",
+    "        app['sweep']['n'] = eval(app['sweep']['n'])\n",
+    "bench_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f68f9aea-c801-4c1b-a1ab-2c70fc782aa8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# flatten into a table\n",
+    "sw = pd.json_normalize(bench_config['sw'])\n",
+    "for col in sw.columns.tolist():\n",
+    "    if col.startswith('sweep'):\n",
+    "        sw = sw.explode(col)\n",
+    "configs = pd.DataFrame({'hw':bench_config['hw']}).merge(sw, how='cross')\n",
+    "configs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "09539ef4-0b02-4253-a095-26611601820a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"bench/test.yaml\", 'w') as f:\n",
+    "    yaml.dump(bench_config, f)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3213017b-a0ff-4e47-80f4-fa029cda2a4e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "feea31c1-8ef1-414e-8419-9b9239dce30d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "970827bb-8583-4f8a-9202-533eb7f9eaaf",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

target/sim/bench/bench.yaml

@@ -0,0 +1,17 @@
+# All top-level parameters for benchmarking and sweeps
+
+hw:
+    - single-cluster.hjson
+    - full.hjson
+
+sw:
+    - name: axpy
+      config: 'axpy/config.hjson'
+      sweep:
+        n: [24, 48, 100]
+        method: ['baseline', 'optimized']
+    - name: gemm
+      config: 'gemm/config.hjson'
+      sweep:
+        n: 'np.linspace(10, 100, num=4, dtype=int).tolist()'
+        method: ['baseline', 'optimized']