Yahoo_Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import json\n",
    "import csv\n",
    "from io import StringIO\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url_esg = 'https://finance.yahoo.com/quote/{}/sustainability?p={}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(r\"C:\\Users\\18143\\OneDrive\\Desktop\\tickers_6123.csv\")   \n",
    "df = pd.DataFrame(data)\n",
    "df=df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ticker = list(df['Ticker'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "stock=Ticker[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "esg_score=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_esg(stock):\n",
    "    response = requests.get(url_esg.format(stock,stock))\n",
    "    soup = BeautifulSoup(response.text,'html.parser')\n",
    "    pattern = re.compile(r'\\s--\\sData\\s--\\s')\n",
    "    try:\n",
    "        script_data = soup.find('script',text=pattern).contents[0]\n",
    "    except:\n",
    "        return None\n",
    "    start = script_data.find(\"context\")-2\n",
    "    json_data = json.loads(script_data[start:-12])\n",
    "    #skipped on displaying keys\n",
    "    #json_data['context'].keys()\n",
    "    #json_data['context']['dispatcher']['stores']['QuoteSummaryStore'].keys()\n",
    "    try:\n",
    "        esgScores = json_data['context']['dispatcher']['stores']['QuoteSummaryStore']['esgScores']\n",
    "    except KeyError:\n",
    "        return None\n",
    "    \n",
    "    stat = ['peerSocialPerformance','socialScore','peerGovernancePerformance','governanceScore','peerEnvironmentPerformance','environmentScore','peerEsgScorePerformance','totalEsg','percentile']\n",
    "    dic = {}\n",
    "\n",
    "    for i in stat:\n",
    "        try:\n",
    "            dic[i]=esgScores[i]\n",
    "        except KeyError:\n",
    "            continue\n",
    "    \n",
    "    score = []\n",
    "    peer = []\n",
    "    for s in dic:\n",
    "        try:\n",
    "            score.append((s,dic[s]['raw']))\n",
    "        except KeyError:\n",
    "            continue\n",
    "    for s in dic:\n",
    "        try: \n",
    "            peer.append((s+'_min',dic[s]['min']))\n",
    "            peer.append((s+'_avg',dic[s]['avg']))\n",
    "            peer.append((s+'_max',dic[s]['max']))\n",
    "        except KeyError:\n",
    "            continue\n",
    "    \n",
    "    score=dict(score)\n",
    "    peer=dict(peer)\n",
    "    \n",
    "    new_dic = {**score,**peer}\n",
    "    new_dic['ticker']=stock\n",
    "    return new_dic\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'socialScore': 7.69,\n",
       " 'governanceScore': 8.94,\n",
       " 'environmentScore': 0.1,\n",
       " 'totalEsg': 16.73,\n",
       " 'percentile': 11.77,\n",
       " 'peerSocialPerformance_min': 2.39,\n",
       " 'peerSocialPerformance_avg': 6.449090909090907,\n",
       " 'peerSocialPerformance_max': 12.2,\n",
       " 'peerGovernancePerformance_min': 3.72,\n",
       " 'peerGovernancePerformance_avg': 7.539090909090912,\n",
       " 'peerGovernancePerformance_max': 14.1,\n",
       " 'peerEnvironmentPerformance_min': 0.04,\n",
       " 'peerEnvironmentPerformance_avg': 3.2439999999999998,\n",
       " 'peerEnvironmentPerformance_max': 10.02,\n",
       " 'peerEsgScorePerformance_min': 7.24,\n",
       " 'peerEsgScorePerformance_avg': 17.05464285714286,\n",
       " 'peerEsgScorePerformance_max': 26.59,\n",
       " 'ticker': 'AAPL'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_esg('AAPL')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in stock:\n",
    "    esg_score.append(get_esg(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#removing None\n",
    "res = [i for i in esg_score if i] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "864"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "na_list = []\n",
    "new_list=[]\n",
    "for i in res:\n",
    "    if len(i) != 18:\n",
    "        na_list.append(i)\n",
    "    else:\n",
    "        new_list.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "527"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "510"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#remove the duplicates\n",
    "updated = [] \n",
    "[updated.append(x) for x in new_list if x not in updated] \n",
    "len(updated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "toCSV = new_list\n",
    "keys = toCSV[0].keys()\n",
    "with open('esg_score_updated.csv', 'w', newline='')  as output_file:\n",
    "    dict_writer = csv.DictWriter(output_file, keys)\n",
    "    dict_writer.writeheader()\n",
    "    dict_writer.writerows(toCSV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "toCSV = res\n",
    "keys = toCSV[0].keys()\n",
    "with open('esg_score_test.csv', 'w', newline='')  as output_file:\n",
    "    dict_writer = csv.DictWriter(output_file, keys)\n",
    "    dict_writer.writeheader()\n",
    "    dict_writer.writerows(toCSV)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}