player match log scrapper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import io\n",
    "import os\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "urls = ['https://fbref.com/en/squads/a2d435b3/Leicester-City', 'https://fbref.com/en/squads/47c64c55/Crystal-Palace', 'https://fbref.com/en/squads/822bd0ba/Liverpool', 'https://fbref.com/en/squads/b8fd03ef/Manchester-City', 'https://fbref.com/en/squads/cff3d9bb/Chelsea', 'https://fbref.com/en/squads/19538871/Manchester-United', 'https://fbref.com/en/squads/8cec06e1/Wolverhampton-Wanderers', 'https://fbref.com/en/squads/1df6b87e/Sheffield-United', 'https://fbref.com/en/squads/361ca564/Tottenham-Hotspur', 'https://fbref.com/en/squads/18bb7c10/Arsenal', 'https://fbref.com/en/squads/943e8050/Burnley', 'https://fbref.com/en/squads/d3fd31cc/Everton', 'https://fbref.com/en/squads/b2b47a98/Newcastle-United', 'https://fbref.com/en/squads/33c895d4/Southampton', 'https://fbref.com/en/squads/d07537b9/Brighton--Hove-Albion', 'https://fbref.com/en/squads/7c21e445/West-Ham-United', 'https://fbref.com/en/squads/2abfe087/Watford', 'https://fbref.com/en/squads/4ba7cbea/Bournemouth', 'https://fbref.com/en/squads/8602292d/Aston-Villa', 'https://fbref.com/en/squads/1c781004/Norwich-City']\n",
    "os.mkdir('PlayerData')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for url in urls:\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "    table = soup.find_all('table', id = 'stats_standard_ks_3232')\n",
    "    table = table[0]\n",
    "    teamname = url[37:]\n",
    "    \n",
    "    \n",
    "    for row in table.find_all('tr'):\n",
    "        position = row.find_all(\"td\", {\"data-stat\" : \"position\"})\n",
    "        for pos in position:\n",
    "            if (pos.text == 'FW'):\n",
    "                playername = row.find(\"th\", {\"data-stat\" : \"player\"}).text\n",
    "                url2 = 'https://fbref.com'+row.find_all('a', attrs={'href': re.compile(\"/en/players\")})[1].get('href')\n",
    "                csv_filename = teamname+'-'+playername + '.csv'\n",
    "                txt_filename = playername + '.txt'\n",
    "    \n",
    "                response2 = requests.get(url2)\n",
    "                soup2 = BeautifulSoup(response2.content, 'html.parser')\n",
    "                table2 = soup2.find_all('table', id = 'ks_matchlogs_3232')\n",
    "                table2 = table2[0]\n",
    "            \n",
    "                with io.open (txt_filename, 'w', encoding=\"utf-8\") as r:\n",
    "                    r.write('Day,Round,Venue,Result,Squad,Opponent,Start,Pos,Min,Gls,Ast,PK,PKatt,Sh,SoT,CrdY,CrdR,Touches,Press,Tkl,Int,Blocks,xG,npxG,xA,SCA,GCA,Cmp,Att,Cmp%,PassPrgDist,Carries,CarryPrgDist,Succ,DribAtt,Match Report,')\n",
    "                    r.write('\\n')\n",
    "                    \n",
    "                    for row2 in table2.find_all('tr')[:-1]:\n",
    "                        for cell2 in row2.find_all('td'):\n",
    "                            l = len(row2.find_all('td'))\n",
    "                            if (l != 36):\n",
    "                                continue\n",
    "                            if (cell2.text.find(',') >= 0):\n",
    "                                r.write(cell2.text.replace(',',' '))\n",
    "                            else:\n",
    "                                r.write(cell2.text)\n",
    "                        \n",
    "                            r.write(',')\n",
    "                        r.write('\\n')\n",
    "                    \n",
    "                    \n",
    "                df = pd.read_csv(txt_filename)\n",
    "                df = df.drop(['Match Report','Unnamed: 36'], axis= 1)\n",
    "                df.to_csv('PlayerData/'+csv_filename, index=None)\n",
    "    \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}