Mock_IPL_Auction_Scraping_and_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "from itertools import chain\n",
    "from tqdm import tqdm_notebook as tqdm\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scrape Stats from IPL website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrape_stats_for_year(year):\n",
    "    URL = f\"https://www.iplt20.com/stats/{year}/player-points\"\n",
    "    page = requests.get(URL)\n",
    "    soup = BeautifulSoup(page.content, 'html.parser')\n",
    "    tables = soup.find('table')\n",
    "    trs = tables.find_all('tr')\n",
    "    columns = [th.text.strip() for th in trs[0].find_all('th')]\n",
    "    columns.append(\"team_name\")\n",
    "    columns.append(\"nationality\")\n",
    "    columns.append(\"year\")\n",
    "    data = []\n",
    "    for index, tr in enumerate(trs[1:]):\n",
    "        tds = tr.find_all('td')\n",
    "        stat = [\"\".join(td.text.strip().split()) for td in tds]\n",
    "        class_index = 1 if index > 0 else 2\n",
    "        team_name = tr.get('class')[class_index]\n",
    "        nationality = tr.get('data-nationality')\n",
    "        stat.append(team_name)\n",
    "        stat.append(nationality)\n",
    "        stat.append(year)\n",
    "        data.append(dict(zip(columns, stat)))\n",
    "    return pd.DataFrame(data, columns=columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "d1 = scrape_stats_for_year(2008)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mancunian92/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:2: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1bf63ec9a59f4f83bd0076db07a8f81a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=12.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "allData = []\n",
    "for year in tqdm(range(2008,2020)):\n",
    "    data = scrape_stats_for_year(year)\n",
    "    allData.append(data)\n",
    "stats = pd.concat(allData)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats.Pts = stats.Pts.astype(float)\n",
    "stats.Wkts = stats.Wkts.astype(int)\n",
    "stats.Dots = stats.Dots.astype(int)\n",
    "stats[\"4s\"] = stats[\"4s\"].astype(int)\n",
    "stats[\"6s\"] = stats[\"6s\"].astype(int)\n",
    "stats.Catches = stats.Catches.astype(int)\n",
    "stats.Stumpings = stats.Stumpings.astype(int)\n",
    "stats.Mat = stats.Mat.astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Aggregated Stats over the years for all players"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_stats = stats.groupby(['PLAYER'])['Pts', 'Mat', 'Wkts', 'Dots', '4s', '6s', 'Catches', 'Stumpings'].sum().reset_index(drop=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pts</th>\n",
       "      <th>Wkts</th>\n",
       "      <th>Dots</th>\n",
       "      <th>4s</th>\n",
       "      <th>6s</th>\n",
       "      <th>Catches</th>\n",
       "      <th>Stumpings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>556.000000</td>\n",
       "      <td>556.000000</td>\n",
       "      <td>556.000000</td>\n",
       "      <td>556.000000</td>\n",
       "      <td>556.000000</td>\n",
       "      <td>556.000000</td>\n",
       "      <td>556.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>329.202338</td>\n",
       "      <td>14.154676</td>\n",
       "      <td>112.742806</td>\n",
       "      <td>36.156475</td>\n",
       "      <td>14.584532</td>\n",
       "      <td>9.697842</td>\n",
       "      <td>0.494604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>473.960443</td>\n",
       "      <td>26.616959</td>\n",
       "      <td>211.853988</td>\n",
       "      <td>80.439350</td>\n",
       "      <td>34.320299</td>\n",
       "      <td>15.544950</td>\n",
       "      <td>2.885498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>32.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>120.750000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>21.500000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>438.375000</td>\n",
       "      <td>14.250000</td>\n",
       "      <td>112.500000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>11.250000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2698.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>1249.000000</td>\n",
       "      <td>527.000000</td>\n",
       "      <td>326.000000</td>\n",
       "      <td>101.000000</td>\n",
       "      <td>38.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Pts        Wkts         Dots          4s          6s  \\\n",
       "count   556.000000  556.000000   556.000000  556.000000  556.000000   \n",
       "mean    329.202338   14.154676   112.742806   36.156475   14.584532   \n",
       "std     473.960443   26.616959   211.853988   80.439350   34.320299   \n",
       "min       0.000000    0.000000     0.000000    0.000000    0.000000   \n",
       "25%      32.000000    0.000000     0.000000    0.000000    0.000000   \n",
       "50%     120.750000    3.000000    21.500000    5.000000    1.000000   \n",
       "75%     438.375000   14.250000   112.500000   24.000000   10.000000   \n",
       "max    2698.000000  170.000000  1249.000000  527.000000  326.000000   \n",
       "\n",
       "          Catches   Stumpings  \n",
       "count  556.000000  556.000000  \n",
       "mean     9.697842    0.494604  \n",
       "std     15.544950    2.885498  \n",
       "min      0.000000    0.000000  \n",
       "25%      1.000000    0.000000  \n",
       "50%      4.000000    0.000000  \n",
       "75%     11.250000    0.000000  \n",
       "max    101.000000   38.000000  "
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg_stats.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TOP 10 PLAYERS OVERALL BY POINTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PLAYER</th>\n",
       "      <th>Pts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>ShaneWatson</td>\n",
       "      <td>2698.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>SureshRaina</td>\n",
       "      <td>2532.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>ChrisGayle</td>\n",
       "      <td>2369.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>HarbhajanSingh</td>\n",
       "      <td>2238.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>537</th>\n",
       "      <td>ViratKohli</td>\n",
       "      <td>2119.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>407</th>\n",
       "      <td>RohitSharma</td>\n",
       "      <td>2106.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>DwayneBravo</td>\n",
       "      <td>2038.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>357</th>\n",
       "      <td>PiyushChawla</td>\n",
       "      <td>1932.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>RavindraJadeja</td>\n",
       "      <td>1924.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405</th>\n",
       "      <td>RobinUthappa</td>\n",
       "      <td>1923.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             PLAYER     Pts\n",
       "449     ShaneWatson  2698.0\n",
       "495     SureshRaina  2532.5\n",
       "106      ChrisGayle  2369.0\n",
       "171  HarbhajanSingh  2238.5\n",
       "537      ViratKohli  2119.5\n",
       "407     RohitSharma  2106.5\n",
       "152     DwayneBravo  2038.0\n",
       "357    PiyushChawla  1932.0\n",
       "394  RavindraJadeja  1924.5\n",
       "405    RobinUthappa  1923.5"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.groupby(['PLAYER'])['Pts'].sum().reset_index(drop=False).sort_values('Pts', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Writing to a file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats.to_csv(\"./data/yearly_stats_ipl.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_stats.to_csv(\"./data/sum_agg_stats.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Have all the aggregation stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_agg_stats = stats.groupby(['PLAYER'])['Pts', 'Mat', 'Wkts', 'Dots', '4s', '6s', 'Catches', 'Stumpings'].agg(['sum', 'mean', 'count']).reset_index(drop=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>PLAYER</th>\n",
       "      <th colspan=\"3\" halign=\"left\">Pts</th>\n",
       "      <th colspan=\"3\" halign=\"left\">Mat</th>\n",
       "      <th colspan=\"3\" halign=\"left\">Wkts</th>\n",
       "      <th>...</th>\n",
       "      <th>4s</th>\n",
       "      <th colspan=\"3\" halign=\"left\">6s</th>\n",
       "      <th colspan=\"3\" halign=\"left\">Catches</th>\n",
       "      <th colspan=\"3\" halign=\"left\">Stumpings</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "      <th>...</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABdeVilliers</td>\n",
       "      <td>1712.5</td>\n",
       "      <td>155.681818</td>\n",
       "      <td>11</td>\n",
       "      <td>139</td>\n",
       "      <td>12.636364</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>11</td>\n",
       "      <td>200</td>\n",
       "      <td>18.181818</td>\n",
       "      <td>11</td>\n",
       "      <td>80</td>\n",
       "      <td>7.272727</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AakashChopra</td>\n",
       "      <td>22.5</td>\n",
       "      <td>11.250000</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>3.500000</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AaronFinch</td>\n",
       "      <td>751.0</td>\n",
       "      <td>83.444444</td>\n",
       "      <td>9</td>\n",
       "      <td>75</td>\n",
       "      <td>8.333333</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>9</td>\n",
       "      <td>67</td>\n",
       "      <td>7.444444</td>\n",
       "      <td>9</td>\n",
       "      <td>24</td>\n",
       "      <td>2.666667</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AavishkarSalvi</td>\n",
       "      <td>30.0</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AbdurRazzak</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           PLAYER     Pts                    Mat                  Wkts  \\\n",
       "                      sum        mean count  sum       mean count  sum   \n",
       "0    ABdeVilliers  1712.5  155.681818    11  139  12.636364    11    0   \n",
       "1    AakashChopra    22.5   11.250000     2    7   3.500000     2    0   \n",
       "2      AaronFinch   751.0   83.444444     9   75   8.333333     9    1   \n",
       "3  AavishkarSalvi    30.0   30.000000     1    2   2.000000     1    3   \n",
       "4     AbdurRazzak     5.0    5.000000     1    1   1.000000     1    0   \n",
       "\n",
       "                   ...    4s   6s                  Catches                  \\\n",
       "       mean count  ... count  sum       mean count     sum      mean count   \n",
       "0  0.000000    11  ...    11  200  18.181818    11      80  7.272727    11   \n",
       "1  0.000000     2  ...     2    0   0.000000     2       2  1.000000     2   \n",
       "2  0.111111     9  ...     9   67   7.444444     9      24  2.666667     9   \n",
       "3  3.000000     1  ...     1    0   0.000000     1       1  1.000000     1   \n",
       "4  0.000000     1  ...     1    0   0.000000     1       0  0.000000     1   \n",
       "\n",
       "  Stumpings                  \n",
       "        sum      mean count  \n",
       "0         7  0.636364    11  \n",
       "1         0  0.000000     2  \n",
       "2         0  0.000000     9  \n",
       "3         0  0.000000     1  \n",
       "4         0  0.000000     1  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_agg_stats.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_agg_stats.columns = all_agg_stats.columns.map('_'.join).str.strip('_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_to_select = ['PLAYER', 'Pts_sum', 'Pts_mean', 'Pts_count', 'Mat_sum', 'Mat_mean',\n",
    "       'Wkts_sum', 'Wkts_mean',  'Dots_sum',\n",
    "       'Dots_mean', '4s_sum', '4s_mean', '6s_sum',\n",
    "       '6s_mean',  'Catches_sum', 'Catches_mean',\n",
    "       'Stumpings_sum', 'Stumpings_mean']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_agg_stats = all_agg_stats[cols_to_select]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mancunian92/.local/lib/python3.6/site-packages/pandas/core/frame.py:4238: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  return super().rename(**kwargs)\n"
     ]
    }
   ],
   "source": [
    "all_agg_stats.rename(columns={\"Pts_count\": \"years_played\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_agg_stats.to_csv(\"./data/agg_stats.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Not used anymore as we got official data on it\n",
    "'''\n",
    "\n",
    "def classify_players_on_stats(stats):\n",
    "    '''\n",
    "    A cricket player can be a Batsman, Wicketkeeper, All rounder and a Bowler.\n",
    "    Batsman - He will have more 6's and 4's than everyone\n",
    "    Bowler - He will have more wickets and Dots\n",
    "    All rounder - If a player satisfies both Batsman condition and Bowler condition, then he is an All rounder acc to us\n",
    "    Wicketkeeper - Stumpings can only be got by him\n",
    "    Trying to classify them naively based on stats. \n",
    "    If a player hasn't played much , there wont be any stats. Here, we classify them as unknowns\n",
    "    Possible values -> [BAT, AR, BWL, WK, UNK]\n",
    "    :param stats - The aggregated stats for each player\n",
    "    :returns players - A dictionary of all players and their respective positions.\n",
    "    '''\n",
    "    all_players = stats.PLAYER\n",
    "    players = {}\n",
    "    for player in all_players:\n",
    "        isBatsman = False\n",
    "        isBowler= False\n",
    "        record = stats[stats.PLAYER==player]\n",
    "        if record[\"Stumpings\"].values[0] > 0:\n",
    "            players[player] = \"WK\"\n",
    "            continue\n",
    "        if (record[\"4s\"].values[0] >= np.percentile(stats[\"4s\"], 75)) and (record[\"6s\"].values[0] >= np.percentile(stats[\"6s\"], 75)):\n",
    "            isBatsman = True\n",
    "            players[player] = \"BAT\"\n",
    "        if (record[\"Wkts\"].values[0] >= np.percentile(stats[\"Wkts\"], 75)) and (record[\"Dots\"].values[0] >= np.percentile(stats[\"Dots\"], 50)):\n",
    "            isBowler = True\n",
    "            players[player] = \"BWL\"\n",
    "        if isBatsman and isBowler:\n",
    "            players[player] = \"AR\"\n",
    "        if isBatsman==False and isBowler==False:\n",
    "            players[player] = \"UNK\"\n",
    "    return players"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}