checkpoint/PoseAug_result_summary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nthis file load the event file and analysis the result for the total 16 experiments.\\n'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.python.summary.summary_iterator import summary_iterator\n",
    "import glob\n",
    "import numpy as np\n",
    "\n",
    "'''\n",
    "this file load the event file and analysis the result for the total 16 experiments.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "flag_involve_fake_data_rlt = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "get event list in the checkpoint/poseaug folder\n",
    "'''\n",
    "# get the list of all 16 event file\n",
    "home_dir = './poseaug'\n",
    "path_lst_to_events_file = glob.glob(home_dir+'/*/*/*/*tfevent*')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n"
     ]
    }
   ],
   "source": [
    "print(len(path_lst_to_events_file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# total 16 experiment\n",
    "\n",
    "dataset_namelst = ['detectron_ft_h36m', 'cpn_ft_h36m_dbb', 'hr', 'gt']\n",
    "model_namelst = ['mlp', 'gcn', 'stgcn', 'videopose']\n",
    "\n",
    "# dataset_namelst = ['hr', 'gt']\n",
    "# model_namelst = ['stgcn', 'videopose']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_eventpath_byDM(total_list, d, m):\n",
    "    '''\n",
    "    d: dataset name\n",
    "    m: model name\n",
    "    '''\n",
    "    out = []\n",
    "#     assert len(total_list)==16, 'supposed 16 in total'\n",
    "    for item in total_list:\n",
    "        if '/'+d+'/' in item:\n",
    "            if '/'+m+'/' in item:\n",
    "                out.append(item)\n",
    "    assert len(out) == 1, 'supposed only 1 here'\n",
    "    return out[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_dict = {\n",
    "    's911_p1_fake': 'posenet_H36M_test/p1score_fake',\n",
    "    's911_p1': 'posenet_H36M_test/p1score_real',\n",
    "    'mpi_p1_fake': 'posenet_mpi3d_loader_flip/p1score_fake',\n",
    "    'mpi_p1': 'posenet_mpi3d_loader_flip/p1score_real',\n",
    "}\n",
    "\n",
    "# tag_keylst = ['mpi_p1', 's911_p1']\n",
    "tag_keylst = ['s911_p1','mpi_p1_fake', 'mpi_p1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "extract the value from tag in event file\n",
    "'''\n",
    "            \n",
    "def get_log(event_path, tag):\n",
    "    try:\n",
    "        a = {}\n",
    "        a[tag] = []\n",
    "        a['step']=[]\n",
    "        for e in summary_iterator(event_path):\n",
    "            for v in e.summary.value:\n",
    "                if v.tag == tag:\n",
    "                    a['step'].append(e.step)\n",
    "                    a[tag].append(v.simple_value)\n",
    "                \n",
    "        return a\n",
    "    \n",
    "    # Dirty catch of DataLossError\n",
    "    except:\n",
    "        print('Event file possibly corrupt: {}'.format(event_path))\n",
    "        return None\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "now work on:  detectron_ft_h36m\n",
      "now work on:  detectron_ft_h36m mlp\n",
      "WARNING:tensorflow:From /home/gongkehong/anaconda3/envs/mlenv/lib/python3.6/site-packages/tensorflow_core/python/summary/summary_iterator.py:68: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use eager execution and: \n",
      "`tf.data.TFRecordDataset(path)`\n",
      "now work on:  detectron_ft_h36m gcn\n",
      "now work on:  detectron_ft_h36m stgcn\n",
      "now work on:  detectron_ft_h36m videopose\n",
      "now work on:  cpn_ft_h36m_dbb\n",
      "now work on:  cpn_ft_h36m_dbb mlp\n",
      "now work on:  cpn_ft_h36m_dbb gcn\n",
      "now work on:  cpn_ft_h36m_dbb stgcn\n",
      "now work on:  cpn_ft_h36m_dbb videopose\n",
      "now work on:  hr\n",
      "now work on:  hr mlp\n",
      "now work on:  hr gcn\n",
      "now work on:  hr stgcn\n",
      "now work on:  hr videopose\n",
      "now work on:  gt\n",
      "now work on:  gt mlp\n",
      "now work on:  gt gcn\n",
      "now work on:  gt stgcn\n",
      "now work on:  gt videopose\n"
     ]
    }
   ],
   "source": [
    "# organise into a dict\n",
    "\n",
    "rlt_summary = {}\n",
    "\n",
    "# init\n",
    "for dataset_name in dataset_namelst:\n",
    "    print('now work on: ', dataset_name)\n",
    "    if not dataset_name in rlt_summary:\n",
    "        rlt_summary[dataset_name] = {}\n",
    "    for model_name in model_namelst:\n",
    "        print('now work on: ', dataset_name, model_name)\n",
    "        if not model_name in rlt_summary[dataset_name]:\n",
    "            rlt_summary[dataset_name][model_name] = {}\n",
    "        # extract file list from path\n",
    "        rlt_summary[dataset_name][model_name]['event_path']=get_eventpath_byDM(path_lst_to_events_file\n",
    "                                                                             , dataset_name, model_name)\n",
    "        \n",
    "        for tag_key in tag_keylst:\n",
    "            # get data from tag_key\n",
    "            tmp_tag = tag_dict[tag_key]\n",
    "            tmp_log = get_log(rlt_summary[dataset_name][model_name]['event_path'], tmp_tag)\n",
    "            rlt_summary[dataset_name][model_name][tag_key] = tmp_log[tmp_tag]\n",
    "            if 'step' in rlt_summary[dataset_name][model_name]:\n",
    "                assert rlt_summary[dataset_name][model_name]['step'] == tmp_log['step']\n",
    "            else:\n",
    "                rlt_summary[dataset_name][model_name]['step'] = tmp_log['step']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_min(score_lst):\n",
    "    out={}\n",
    "    idx = np.argmin(score_lst)\n",
    "    score = score_lst[idx]\n",
    "    return {'idx': idx, 'score': round(score,1), 'base':round(score_lst[0], 1), 'improve':-round(score - score_lst[0], 1)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'check the p1 result with/without poseaug'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''check the p1 result with/without poseaug'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "now work on:  detectron_ft_h36m\n",
      "now work on:  cpn_ft_h36m_dbb\n",
      "now work on:  hr\n",
      "now work on:  gt\n"
     ]
    }
   ],
   "source": [
    "# here take every result and get the best one, save in the dict.\n",
    "for dataset_name in dataset_namelst:\n",
    "    print('now work on: ', dataset_name)\n",
    "    for model_name in model_namelst:\n",
    "#         print('now work on: ', dataset_name, model_name)\n",
    "        rlt_summary[dataset_name][model_name]['best_s911_p1'] = get_min(\n",
    "            rlt_summary[dataset_name][model_name]['s911_p1'])\n",
    "        rlt_summary[dataset_name][model_name]['best_mpi_p1'] = get_min(\n",
    "            rlt_summary[dataset_name][model_name]['mpi_p1'])\n",
    "        rlt_summary[dataset_name][model_name]['best_mpi_p1_fake'] = get_min(\n",
    "            rlt_summary[dataset_name][model_name]['mpi_p1_fake'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nnow start use panda to view the result\\n'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "now start use panda to view the result\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "now work on:  detectron_ft_h36m\n",
      "now work on:  cpn_ft_h36m_dbb\n",
      "now work on:  hr\n",
      "now work on:  gt\n",
      "now work on:  detectron_ft_h36m\n",
      "now work on:  cpn_ft_h36m_dbb\n",
      "now work on:  hr\n",
      "now work on:  gt\n",
      "now work on:  detectron_ft_h36m\n",
      "now work on:  cpn_ft_h36m_dbb\n",
      "now work on:  hr\n",
      "now work on:  gt\n"
     ]
    }
   ],
   "source": [
    "# organise into a dict\n",
    "\n",
    "view_dict = {\n",
    "    'best_s911_p1':{},\n",
    "    'best_mpi_p1':{},\n",
    "    'best_mpi_p1_fake':{},\n",
    "}\n",
    "\n",
    "for view_opinion in view_dict:\n",
    "    # init\n",
    "    base_summary = {}\n",
    "    aug_summary = {}\n",
    "    for dataset_name in dataset_namelst:\n",
    "        print('now work on: ', dataset_name)\n",
    "        if not dataset_name in aug_summary:\n",
    "            base_summary[dataset_name] = {}\n",
    "            aug_summary[dataset_name] = {}\n",
    "        for model_name in model_namelst:\n",
    "#             print('now work on: ', dataset_name, model_name)\n",
    "            base_summary[dataset_name][model_name] = rlt_summary[dataset_name][model_name][view_opinion]['base']\n",
    "            aug_summary[dataset_name][model_name] = rlt_summary[dataset_name][model_name][view_opinion]['score']\n",
    "\n",
    "    view_dict[view_opinion] = {\n",
    "        'baseline': base_summary,\n",
    "        'poseaug': aug_summary,\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "if flag_involve_fake_data_rlt:\n",
    "    for dataset_name in dataset_namelst:\n",
    "        for model_name in model_namelst:\n",
    "            view_dict['best_mpi_p1']['poseaug'][dataset_name][model_name] = \\\n",
    "            min(view_dict['best_mpi_p1_fake']['poseaug'][dataset_name][model_name],\\\n",
    "                view_dict['best_mpi_p1']['poseaug'][dataset_name][model_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================================\n",
      "view_opinion:  best_s911_p1\n",
      "------------------------------------------\n",
      "view_key:  baseline\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>detectron_ft_h36m</th>\n",
       "      <th>cpn_ft_h36m_dbb</th>\n",
       "      <th>hr</th>\n",
       "      <th>gt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>60.3</td>\n",
       "      <td>55.1</td>\n",
       "      <td>51.5</td>\n",
       "      <td>41.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gcn</th>\n",
       "      <td>67.3</td>\n",
       "      <td>61.8</td>\n",
       "      <td>56.5</td>\n",
       "      <td>44.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stgcn</th>\n",
       "      <td>62.5</td>\n",
       "      <td>56.4</td>\n",
       "      <td>53.7</td>\n",
       "      <td>41.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>videopose</th>\n",
       "      <td>60.9</td>\n",
       "      <td>55.9</td>\n",
       "      <td>52.2</td>\n",
       "      <td>41.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           detectron_ft_h36m  cpn_ft_h36m_dbb    hr    gt\n",
       "mlp                     60.3             55.1  51.5  41.4\n",
       "gcn                     67.3             61.8  56.5  44.6\n",
       "stgcn                   62.5             56.4  53.7  41.9\n",
       "videopose               60.9             55.9  52.2  41.5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "view_key:  poseaug\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>detectron_ft_h36m</th>\n",
       "      <th>cpn_ft_h36m_dbb</th>\n",
       "      <th>hr</th>\n",
       "      <th>gt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>59.1</td>\n",
       "      <td>53.5</td>\n",
       "      <td>51.3</td>\n",
       "      <td>38.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gcn</th>\n",
       "      <td>65.5</td>\n",
       "      <td>60.5</td>\n",
       "      <td>55.4</td>\n",
       "      <td>41.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stgcn</th>\n",
       "      <td>60.8</td>\n",
       "      <td>54.7</td>\n",
       "      <td>51.5</td>\n",
       "      <td>36.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>videopose</th>\n",
       "      <td>58.5</td>\n",
       "      <td>53.4</td>\n",
       "      <td>50.5</td>\n",
       "      <td>38.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           detectron_ft_h36m  cpn_ft_h36m_dbb    hr    gt\n",
       "mlp                     59.1             53.5  51.3  38.4\n",
       "gcn                     65.5             60.5  55.4  41.0\n",
       "stgcn                   60.8             54.7  51.5  36.4\n",
       "videopose               58.5             53.4  50.5  38.6"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================================\n",
      "view_opinion:  best_mpi_p1\n",
      "------------------------------------------\n",
      "view_key:  baseline\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>detectron_ft_h36m</th>\n",
       "      <th>cpn_ft_h36m_dbb</th>\n",
       "      <th>hr</th>\n",
       "      <th>gt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>92.3</td>\n",
       "      <td>89.2</td>\n",
       "      <td>85.7</td>\n",
       "      <td>83.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gcn</th>\n",
       "      <td>108.7</td>\n",
       "      <td>100.0</td>\n",
       "      <td>96.1</td>\n",
       "      <td>97.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stgcn</th>\n",
       "      <td>95.9</td>\n",
       "      <td>92.4</td>\n",
       "      <td>84.5</td>\n",
       "      <td>87.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>videopose</th>\n",
       "      <td>93.4</td>\n",
       "      <td>91.1</td>\n",
       "      <td>85.9</td>\n",
       "      <td>85.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           detectron_ft_h36m  cpn_ft_h36m_dbb    hr    gt\n",
       "mlp                     92.3             89.2  85.7  83.1\n",
       "gcn                    108.7            100.0  96.1  97.7\n",
       "stgcn                   95.9             92.4  84.5  87.6\n",
       "videopose               93.4             91.1  85.9  85.4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "view_key:  poseaug\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>detectron_ft_h36m</th>\n",
       "      <th>cpn_ft_h36m_dbb</th>\n",
       "      <th>hr</th>\n",
       "      <th>gt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>81.4</td>\n",
       "      <td>80.0</td>\n",
       "      <td>74.8</td>\n",
       "      <td>73.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gcn</th>\n",
       "      <td>91.7</td>\n",
       "      <td>90.1</td>\n",
       "      <td>77.9</td>\n",
       "      <td>80.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stgcn</th>\n",
       "      <td>74.7</td>\n",
       "      <td>75.2</td>\n",
       "      <td>77.9</td>\n",
       "      <td>78.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>videopose</th>\n",
       "      <td>79.9</td>\n",
       "      <td>77.2</td>\n",
       "      <td>75.1</td>\n",
       "      <td>72.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           detectron_ft_h36m  cpn_ft_h36m_dbb    hr    gt\n",
       "mlp                     81.4             80.0  74.8  73.8\n",
       "gcn                     91.7             90.1  77.9  80.0\n",
       "stgcn                   74.7             75.2  77.9  78.0\n",
       "videopose               79.9             77.2  75.1  72.4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "\n",
    "for view_opinion in ['best_s911_p1', 'best_mpi_p1']:\n",
    "    print('==========================================')\n",
    "    print('view_opinion: ', view_opinion)\n",
    "    \n",
    "    for view_key in ['baseline', 'poseaug']:\n",
    "        print('------------------------------------------')\n",
    "        print('view_key: ', view_key)\n",
    "        display(pd.DataFrame.from_dict(view_dict[view_opinion][view_key]))\n",
    "#         print('')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}