misc/Note_calc_evar_score.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example of making EVAR result summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>represention</th>\n",
       "      <th>task</th>\n",
       "      <th>score</th>\n",
       "      <th>run_id</th>\n",
       "      <th>report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MAE</td>\n",
       "      <td>cremad</td>\n",
       "      <td>0.675693</td>\n",
       "      <td>f02a64a1</td>\n",
       "      <td>Linear evaluation: cremad_ar_msm_mae.AR_MSM_MA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MAE</td>\n",
       "      <td>gtzan</td>\n",
       "      <td>0.813793</td>\n",
       "      <td>c18baa69</td>\n",
       "      <td>Linear evaluation: gtzan_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MAE</td>\n",
       "      <td>spcv2</td>\n",
       "      <td>0.923853</td>\n",
       "      <td>04d9d9ad</td>\n",
       "      <td>Linear evaluation: spcv2_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MAE</td>\n",
       "      <td>esc50</td>\n",
       "      <td>0.889000</td>\n",
       "      <td>08be7180</td>\n",
       "      <td>Linear evaluation: esc50_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MAE</td>\n",
       "      <td>us8k</td>\n",
       "      <td>0.840209</td>\n",
       "      <td>53ae2d3f</td>\n",
       "      <td>Linear evaluation: us8k_ar_msm_mae.AR_MSM_MAE_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>MAE</td>\n",
       "      <td>vc1</td>\n",
       "      <td>0.564780</td>\n",
       "      <td>a4284387</td>\n",
       "      <td>Linear evaluation: vc1_ar_msm_mae.AR_MSM_MAE_a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>MAE</td>\n",
       "      <td>voxforge</td>\n",
       "      <td>0.968134</td>\n",
       "      <td>a3a94a79</td>\n",
       "      <td>Linear evaluation: voxforge_ar_msm_mae.AR_MSM_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>MAE</td>\n",
       "      <td>nsynth</td>\n",
       "      <td>0.761475</td>\n",
       "      <td>dba69966</td>\n",
       "      <td>Linear evaluation: nsynth_ar_msm_mae.AR_MSM_MA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>MAE</td>\n",
       "      <td>surge</td>\n",
       "      <td>0.374365</td>\n",
       "      <td>2c086329</td>\n",
       "      <td>Linear evaluation: surge_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>MAE</td>\n",
       "      <td>cremad</td>\n",
       "      <td>0.668601</td>\n",
       "      <td>3ffd61d3</td>\n",
       "      <td>Linear evaluation: cremad_ar_msm_mae.AR_MSM_MA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>MAE</td>\n",
       "      <td>gtzan</td>\n",
       "      <td>0.758621</td>\n",
       "      <td>8c6fe7d3</td>\n",
       "      <td>Linear evaluation: gtzan_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>MAE</td>\n",
       "      <td>spcv2</td>\n",
       "      <td>0.910859</td>\n",
       "      <td>87c0fb58</td>\n",
       "      <td>Linear evaluation: spcv2_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>MAE</td>\n",
       "      <td>esc50</td>\n",
       "      <td>0.871000</td>\n",
       "      <td>0326d39d</td>\n",
       "      <td>Linear evaluation: esc50_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>MAE</td>\n",
       "      <td>us8k</td>\n",
       "      <td>0.822473</td>\n",
       "      <td>8334745f</td>\n",
       "      <td>Linear evaluation: us8k_ar_msm_mae.AR_MSM_MAE_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>MAE</td>\n",
       "      <td>vc1</td>\n",
       "      <td>0.496546</td>\n",
       "      <td>1455ca81</td>\n",
       "      <td>Linear evaluation: vc1_ar_msm_mae.AR_MSM_MAE_1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>MAE</td>\n",
       "      <td>voxforge</td>\n",
       "      <td>0.955978</td>\n",
       "      <td>7f67c7af</td>\n",
       "      <td>Linear evaluation: voxforge_ar_msm_mae.AR_MSM_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>MAE</td>\n",
       "      <td>nsynth</td>\n",
       "      <td>0.765381</td>\n",
       "      <td>7c1c47b0</td>\n",
       "      <td>Linear evaluation: nsynth_ar_msm_mae.AR_MSM_MA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>MAE</td>\n",
       "      <td>surge</td>\n",
       "      <td>0.378000</td>\n",
       "      <td>86efa3bf</td>\n",
       "      <td>Linear evaluation: surge_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>MAE</td>\n",
       "      <td>cremad</td>\n",
       "      <td>0.700838</td>\n",
       "      <td>5907fd7d</td>\n",
       "      <td>Linear evaluation: cremad_ar_msm_mae.AR_MSM_MA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>MAE</td>\n",
       "      <td>gtzan</td>\n",
       "      <td>0.793103</td>\n",
       "      <td>efe8c362</td>\n",
       "      <td>Linear evaluation: gtzan_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>MAE</td>\n",
       "      <td>spcv2</td>\n",
       "      <td>0.930577</td>\n",
       "      <td>2d0c24a6</td>\n",
       "      <td>Linear evaluation: spcv2_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>MAE</td>\n",
       "      <td>esc50</td>\n",
       "      <td>0.891500</td>\n",
       "      <td>b47b89b2</td>\n",
       "      <td>Linear evaluation: esc50_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>MAE</td>\n",
       "      <td>us8k</td>\n",
       "      <td>0.842960</td>\n",
       "      <td>40d94acd</td>\n",
       "      <td>Linear evaluation: us8k_ar_msm_mae.AR_MSM_MAE_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>MAE</td>\n",
       "      <td>vc1</td>\n",
       "      <td>0.560781</td>\n",
       "      <td>0db4d46a</td>\n",
       "      <td>Linear evaluation: vc1_ar_msm_mae.AR_MSM_MAE_0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>MAE</td>\n",
       "      <td>voxforge</td>\n",
       "      <td>0.975266</td>\n",
       "      <td>b7d0ce21</td>\n",
       "      <td>Linear evaluation: voxforge_ar_msm_mae.AR_MSM_...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>MAE</td>\n",
       "      <td>nsynth</td>\n",
       "      <td>0.769043</td>\n",
       "      <td>6d92ba18</td>\n",
       "      <td>Linear evaluation: nsynth_ar_msm_mae.AR_MSM_MA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>MAE</td>\n",
       "      <td>surge</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>84363beb</td>\n",
       "      <td>Linear evaluation: surge_ar_msm_mae.AR_MSM_MAE...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   represention      task     score    run_id  \\\n",
       "0           MAE    cremad  0.675693  f02a64a1   \n",
       "1           MAE     gtzan  0.813793  c18baa69   \n",
       "2           MAE     spcv2  0.923853  04d9d9ad   \n",
       "3           MAE     esc50  0.889000  08be7180   \n",
       "4           MAE      us8k  0.840209  53ae2d3f   \n",
       "5           MAE       vc1  0.564780  a4284387   \n",
       "6           MAE  voxforge  0.968134  a3a94a79   \n",
       "7           MAE    nsynth  0.761475  dba69966   \n",
       "8           MAE     surge  0.374365  2c086329   \n",
       "9           MAE    cremad  0.668601  3ffd61d3   \n",
       "10          MAE     gtzan  0.758621  8c6fe7d3   \n",
       "11          MAE     spcv2  0.910859  87c0fb58   \n",
       "12          MAE     esc50  0.871000  0326d39d   \n",
       "13          MAE      us8k  0.822473  8334745f   \n",
       "14          MAE       vc1  0.496546  1455ca81   \n",
       "15          MAE  voxforge  0.955978  7f67c7af   \n",
       "16          MAE    nsynth  0.765381  7c1c47b0   \n",
       "17          MAE     surge  0.378000  86efa3bf   \n",
       "18          MAE    cremad  0.700838  5907fd7d   \n",
       "19          MAE     gtzan  0.793103  efe8c362   \n",
       "20          MAE     spcv2  0.930577  2d0c24a6   \n",
       "21          MAE     esc50  0.891500  b47b89b2   \n",
       "22          MAE      us8k  0.842960  40d94acd   \n",
       "23          MAE       vc1  0.560781  0db4d46a   \n",
       "24          MAE  voxforge  0.975266  b7d0ce21   \n",
       "25          MAE    nsynth  0.769043  6d92ba18   \n",
       "26          MAE     surge  0.375000  84363beb   \n",
       "\n",
       "                                               report  \n",
       "0   Linear evaluation: cremad_ar_msm_mae.AR_MSM_MA...  \n",
       "1   Linear evaluation: gtzan_ar_msm_mae.AR_MSM_MAE...  \n",
       "2   Linear evaluation: spcv2_ar_msm_mae.AR_MSM_MAE...  \n",
       "3   Linear evaluation: esc50_ar_msm_mae.AR_MSM_MAE...  \n",
       "4   Linear evaluation: us8k_ar_msm_mae.AR_MSM_MAE_...  \n",
       "5   Linear evaluation: vc1_ar_msm_mae.AR_MSM_MAE_a...  \n",
       "6   Linear evaluation: voxforge_ar_msm_mae.AR_MSM_...  \n",
       "7   Linear evaluation: nsynth_ar_msm_mae.AR_MSM_MA...  \n",
       "8   Linear evaluation: surge_ar_msm_mae.AR_MSM_MAE...  \n",
       "9   Linear evaluation: cremad_ar_msm_mae.AR_MSM_MA...  \n",
       "10  Linear evaluation: gtzan_ar_msm_mae.AR_MSM_MAE...  \n",
       "11  Linear evaluation: spcv2_ar_msm_mae.AR_MSM_MAE...  \n",
       "12  Linear evaluation: esc50_ar_msm_mae.AR_MSM_MAE...  \n",
       "13  Linear evaluation: us8k_ar_msm_mae.AR_MSM_MAE_...  \n",
       "14  Linear evaluation: vc1_ar_msm_mae.AR_MSM_MAE_1...  \n",
       "15  Linear evaluation: voxforge_ar_msm_mae.AR_MSM_...  \n",
       "16  Linear evaluation: nsynth_ar_msm_mae.AR_MSM_MA...  \n",
       "17  Linear evaluation: surge_ar_msm_mae.AR_MSM_MAE...  \n",
       "18  Linear evaluation: cremad_ar_msm_mae.AR_MSM_MA...  \n",
       "19  Linear evaluation: gtzan_ar_msm_mae.AR_MSM_MAE...  \n",
       "20  Linear evaluation: spcv2_ar_msm_mae.AR_MSM_MAE...  \n",
       "21  Linear evaluation: esc50_ar_msm_mae.AR_MSM_MAE...  \n",
       "22  Linear evaluation: us8k_ar_msm_mae.AR_MSM_MAE_...  \n",
       "23  Linear evaluation: vc1_ar_msm_mae.AR_MSM_MAE_0...  \n",
       "24  Linear evaluation: voxforge_ar_msm_mae.AR_MSM_...  \n",
       "25  Linear evaluation: nsynth_ar_msm_mae.AR_MSM_MA...  \n",
       "26  Linear evaluation: surge_ar_msm_mae.AR_MSM_MAE...  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "df = pd.read_csv('evar_scores_pretrained.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>task</th>\n",
       "      <th>esc50</th>\n",
       "      <th>us8k</th>\n",
       "      <th>spcv2</th>\n",
       "      <th>vc1</th>\n",
       "      <th>voxforge</th>\n",
       "      <th>cremad</th>\n",
       "      <th>gtzan</th>\n",
       "      <th>nsynth</th>\n",
       "      <th>surge</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>80x208p16x16_0422</th>\n",
       "      <td>0.8710</td>\n",
       "      <td>0.822473</td>\n",
       "      <td>0.910859</td>\n",
       "      <td>0.496546</td>\n",
       "      <td>0.955978</td>\n",
       "      <td>0.668601</td>\n",
       "      <td>0.758621</td>\n",
       "      <td>0.765381</td>\n",
       "      <td>0.378000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80x512p16x16_0425</th>\n",
       "      <td>0.8890</td>\n",
       "      <td>0.840209</td>\n",
       "      <td>0.923853</td>\n",
       "      <td>0.564780</td>\n",
       "      <td>0.968134</td>\n",
       "      <td>0.675693</td>\n",
       "      <td>0.813793</td>\n",
       "      <td>0.761475</td>\n",
       "      <td>0.374365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80x512p16x16_paper</th>\n",
       "      <td>0.8915</td>\n",
       "      <td>0.842960</td>\n",
       "      <td>0.930577</td>\n",
       "      <td>0.560781</td>\n",
       "      <td>0.975266</td>\n",
       "      <td>0.700838</td>\n",
       "      <td>0.793103</td>\n",
       "      <td>0.769043</td>\n",
       "      <td>0.375000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "task                 esc50      us8k     spcv2       vc1  voxforge    cremad  \\\n",
       "weight                                                                         \n",
       "80x208p16x16_0422   0.8710  0.822473  0.910859  0.496546  0.955978  0.668601   \n",
       "80x512p16x16_0425   0.8890  0.840209  0.923853  0.564780  0.968134  0.675693   \n",
       "80x512p16x16_paper  0.8915  0.842960  0.930577  0.560781  0.975266  0.700838   \n",
       "\n",
       "task                   gtzan    nsynth     surge  \n",
       "weight                                            \n",
       "80x208p16x16_0422   0.758621  0.765381  0.378000  \n",
       "80x512p16x16_0425   0.813793  0.761475  0.374365  \n",
       "80x512p16x16_paper  0.793103  0.769043  0.375000  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import io\n",
    "import yaml\n",
    "from pathlib import Path\n",
    "\n",
    "\n",
    "def get_weight(report_str):\n",
    "    try:\n",
    "        report_yaml = report_str.split('\\n')[1]\n",
    "        report = yaml.safe_load(io.StringIO(report_yaml))\n",
    "        weight = Path(report['weight_file']).parent.name\n",
    "    except:\n",
    "        weight = 'unknown'\n",
    "    return weight\n",
    "\n",
    "\n",
    "df['weight'] = df.report.map(get_weight)\n",
    "df = pd.pivot_table(df, index=['weight'], columns=['task'], values=['score'], aggfunc=np.sum)\n",
    "df.columns = df.columns.get_level_values(1)\n",
    "df = df[['esc50', 'us8k', 'spcv2', 'vc1', 'voxforge', 'cremad', 'gtzan', 'nsynth', 'surge']]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| weight             | esc50   | us8k   | spcv2   | vc1   | voxforge   | cremad   | gtzan   | nsynth   | surge   |\n",
      "|:-------------------|:--------|:-------|:--------|:------|:-----------|:---------|:--------|:---------|:--------|\n",
      "| 80x208p16x16_0422  | 87.1%   | 82.2%  | 91.1%   | 49.7% | 95.6%      | 66.9%    | 75.9%   | 76.5%    | 37.8%   |\n",
      "| 80x512p16x16_0425  | 88.9%   | 84.0%  | 92.4%   | 56.5% | 96.8%      | 67.6%    | 81.4%   | 76.1%    | 37.4%   |\n",
      "| 80x512p16x16_paper | 89.1%   | 84.3%  | 93.1%   | 56.1% | 97.5%      | 70.1%    | 79.3%   | 76.9%    | 37.5%   |\n"
     ]
    }
   ],
   "source": [
    "print(df.applymap(lambda x: f'{x*100:.1f}%' if str(x).isnumeric else x).to_markdown())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "c06f8776bb7e34ae9a1ba1d2f1ef614477ad4e75f79cb0748c7774bb4aa9951f"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}