From 01f50683d58a961d517fba40f3ed2e1509f33bbc Mon Sep 17 00:00:00 2001 From: Roger Barton Date: Tue, 24 Oct 2023 13:32:37 +0200 Subject: [PATCH] target/sim: Add benchmark yaml for storing configurations --- .gitignore | 1 + target/sim/bench.ipynb | 229 ++++++++++++++++++++++++++++++++++++ target/sim/bench/bench.yaml | 17 +++ 3 files changed, 247 insertions(+) create mode 100644 target/sim/bench.ipynb create mode 100644 target/sim/bench/bench.yaml diff --git a/.gitignore b/.gitignore index b7f3b4656..9281abdb7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ modelsim.ini transcript gmon.out +.ipynb_checkpoints diff --git a/target/sim/bench.ipynb b/target/sim/bench.ipynb new file mode 100644 index 000000000..831c7ce9d --- /dev/null +++ b/target/sim/bench.ipynb @@ -0,0 +1,229 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ec6a4851-436a-4278-9563-fe60b283a829", + "metadata": {}, + "source": [ + "# Benchmark" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c018cdf4-4eac-434f-ab48-c86c9ee61541", + "metadata": {}, + "outputs": [], + "source": [ + "!python -m pip install pandas plotly pyyaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "263c1118-51c5-4a82-9226-b01df47c40a7", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd, numpy as np\n", + "import os, glob, datetime, time\n", + "import plotly as plotly\n", + "import plotly.express as px\n", + "import hjson\n", + "import yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aff8e0ac-8b91-4ac1-b9f1-5da08ff143e1", + "metadata": {}, + "outputs": [], + "source": [ + "# Compile hardware for Questa (vsim)\n", + "!questa-2022.3 make bin/snitch_cluster.vsim" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16e62def-f903-455e-994a-6661c8b8895d", + "metadata": {}, + "outputs": [], + "source": [ + "# Compile software\n", + "!make DEBUG=ON sw" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ef1ab542-924c-4c02-a507-6044c4b32169", + "metadata": {}, + "outputs": [], + "source": [ + "# Post process traces\n", + "!make -j traces\n", + "!make logs/perf.csv" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3caceab4-f9cd-474a-9ce2-8688d05317d8", + "metadata": {}, + "outputs": [], + "source": [ + "# Read profile data\n", + "perf = pd.read_csv('logs/perf.csv', index_col=0)\n", + "perf.filter(regex=(\"1_.\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d20fe0e3-3f04-4a05-ab11-3d016dc75e79", + "metadata": {}, + "outputs": [], + "source": [ + "fig = px.scatter(perf, y=['1_total_ipc', '1_fpss_occupancy', '1_fpss_fpu_occupancy', '1_snitch_occupancy'])\n", + "fig.update_layout(yaxis_range=[0,1])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71d208c2-0ac9-47c4-ac71-6def321fc682", + "metadata": {}, + "outputs": [], + "source": [ + "def run(cmd, env=None, dryrun=False):\n", + " if dryrun:\n", + " print(cmd)\n", + " else:\n", + " p = subprocess.Popen(cmd, env=env, shell=True)\n", + " retcode = p.wait()\n", + " if retcode != 0:\n", + " sys.exit(retcode)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28811c2e-63ee-4143-ad82-4d97420b9b68", + "metadata": {}, + "outputs": [], + "source": [ + "!f'make CFG_OVERRIDE={cfg_file} rtl'" + ] + }, + { + "cell_type": "markdown", + "id": "4d766185-bd6c-4c21-bbf8-79eded2247f1", + "metadata": {}, + "source": [ + "# Benchmark Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96393e95-7390-4157-9314-af5155f46f22", + "metadata": {}, + "outputs": [], + "source": [ + "# Load top-level benchmark config\n", + "bench_config_name = \"bench/bench.yaml\"\n", + "with open(bench_config_name) as f:\n", + " bench_config = yaml.load(f, Loader=yaml.Loader)\n", + "bench_config" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "826c3a78-a1e4-40d7-ae93-4429cb8a288f", + "metadata": {}, + "outputs": [], + "source": [ + "# Expand python strings\n", + "for app in bench_config['sw']:\n", + " if type(app['sweep']['n']) == str:\n", + " app['sweep']['n'] = eval(app['sweep']['n'])\n", + "bench_config" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f68f9aea-c801-4c1b-a1ab-2c70fc782aa8", + "metadata": {}, + "outputs": [], + "source": [ + "# flatten into a table\n", + "sw = pd.json_normalize(bench_config['sw'])\n", + "for col in sw.columns.tolist():\n", + " if col.startswith('sweep'):\n", + " sw = sw.explode(col)\n", + "configs = pd.DataFrame({'hw':bench_config['hw']}).merge(sw, how='cross')\n", + "configs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "09539ef4-0b02-4253-a095-26611601820a", + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"bench/test.yaml\", 'w') as f:\n", + " yaml.dump(bench_config, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3213017b-a0ff-4e47-80f4-fa029cda2a4e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "feea31c1-8ef1-414e-8419-9b9239dce30d", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "970827bb-8583-4f8a-9202-533eb7f9eaaf", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/target/sim/bench/bench.yaml b/target/sim/bench/bench.yaml new file mode 100644 index 000000000..8f3c7814f --- /dev/null +++ b/target/sim/bench/bench.yaml @@ -0,0 +1,17 @@ +# All top-level parameters for benchmarking and sweeps + +hw: + - single-cluster.hjson + - full.hjson + +sw: + - name: axpy + config: 'axpy/config.hjson' + sweep: + n: [24, 48, 100] + method: ['baseline', 'optimized'] + - name: gemm + config: 'gemm/config.hjson' + sweep: + n: 'np.linspace(10, 100, num=4, dtype=int).tolist()' + method: ['baseline', 'optimized']