financial_basics_getdata.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Financial Basics - Get Data\n",
    "\n",
    "This notebook is for the financial basis section.  We will get data that we will use for the rest of the course."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import yfinance as yf\n",
    "import os\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting the Data\n",
    "\n",
    "Packages to consider include yfinance (what we will use here) and pandas-datareader."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# symbols for stocks in the S&P 500\n",
    "symbols = ['MMM','ABT','ABBV','ACN','ATVI','AYI','ADBE','AMD','AAP','AES','AET',\n",
    "    'AMG','AFL','A','APD','AKAM','ALK','ALB','ARE','ALXN','ALGN','ALLE',\n",
    "    'AGN','ADS','LNT','ALL','GOOGL','GOOG','MO','AMZN','AEE','AAL','AEP',\n",
    "    'AXP','AIG','AMT','AWK','AMP','ABC','AME','AMGN','APH','APC','ADI','ANDV',\n",
    "    'ANSS','ANTM','AON','AOS','APA','AIV','AAPL','AMAT','APTV','ADM','ARNC',\n",
    "    'AJG','AIZ','T','ADSK','ADP','AZO','AVB','AVY','BHGE','BLL','BAC','BK',\n",
    "    'BAX','BBT','BDX','BRK.B','BBY','BIIB','BLK','HRB','BA','BWA','BXP','BSX',\n",
    "    'BHF','BMY','AVGO','BF.B','CHRW','CA','COG','CDNS','CPB','COF','CAH','CBOE',\n",
    "    'KMX','CCL','CAT','CBG','CBS','CELG','CNC','CNP','CTL','CERN','CF','SCHW',\n",
    "    'CHTR','CHK','CVX','CMG','CB','CHD','CI','XEC','CINF','CTAS','CSCO','C','CFG',\n",
    "    'CTXS','CLX','CME','CMS','KO','CTSH','CL','CMCSA','CMA','CAG','CXO','COP',\n",
    "    'ED','STZ','COO','GLW','COST','COTY','CCI','CSRA','CSX','CMI','CVS','DHI',\n",
    "    'DHR','DRI','DVA','DE','DAL','XRAY','DVN','DLR','DFS','DISCA','DISCK','DISH',\n",
    "    'DG','DLTR','D','DOV','DWDP','DPS','DTE','DRE','DUK','DXC','ETFC','EMN','ETN',\n",
    "    'EBAY','ECL','EIX','EW','EA','EMR','ETR','EVHC','EOG','EQT','EFX','EQIX','EQR',\n",
    "    'ESS','EL','ES','RE','EXC','EXPE','EXPD','ESRX','EXR','XOM','FFIV','FB','FAST',\n",
    "    'FRT','FDX','FIS','FITB','FE','FISV','FLIR','FLS','FLR','FMC','FL','F','FTV',\n",
    "    'FBHS','BEN','FCX','GPS','GRMN','IT','GD','GE','GGP','GIS','GM','GPC','GILD',\n",
    "    'GPN','GS','GT','GWW','HAL','HBI','HOG','HRS','HIG','HAS','HCA','HCP','HP','HSIC',\n",
    "    'HSY','HES','HPE','HLT','HOLX','HD','HON','HRL','HST','HPQ','HUM','HBAN','HII',\n",
    "    'IDXX','INFO','ITW','ILMN','IR','INTC','ICE','IBM','INCY','IP','IPG','IFF','INTU',\n",
    "    'ISRG','IVZ','IQV','IRM','JEC','JBHT','SJM','JNJ','JCI','JPM','JNPR','KSU','K','KEY',\n",
    "    'KMB','KIM','KMI','KLAC','KSS','KHC','KR','LB','LLL','LH','LRCX','LEG','LEN','LUK',\n",
    "    'LLY','LNC','LKQ','LMT','L','LOW','LYB','MTB','MAC','M','MRO','MPC','MAR','MMC','MLM',\n",
    "    'MAS','MA','MAT','MKC','MCD','MCK','MDT','MRK','MET','MTD','MGM','KORS','MCHP','MU',\n",
    "    'MSFT','MAA','MHK','TAP','MDLZ','MON','MNST','MCO','MS','MOS','MSI','MYL','NDAQ',\n",
    "    'NOV','NAVI','NTAP','NFLX','NWL','NFX','NEM','NWSA','NWS','NEE','NLSN','NKE','NI',\n",
    "    'NBL','JWN','NSC','NTRS','NOC','NCLH','NRG','NUE','NVDA','ORLY','OXY','OMC','OKE',\n",
    "    'ORCL','PCAR','PKG','PH','PDCO','PAYX','PYPL','PNR','PBCT','PEP','PKI','PRGO','PFE',\n",
    "    'PCG','PM','PSX','PNW','PXD','PNC','RL','PPG','PPL','PX','PCLN','PFG','PG','PGR',\n",
    "    'PLD','PRU','PEG','PSA','PHM','PVH','QRVO','PWR','QCOM','DGX','RRC','RJF','RTN','O',\n",
    "    'RHT','REG','REGN','RF','RSG','RMD','RHI','ROK','COL','ROP','ROST','RCL','CRM','SBAC',\n",
    "    'SCG','SLB','SNI','STX','SEE','SRE','SHW','SIG','SPG','SWKS','SLG','SNA','SO','LUV',\n",
    "    'SPGI','SWK','SBUX','STT','SRCL','SYK','STI','SYMC','SYF','SNPS','SYY','TROW','TPR',\n",
    "    'TGT','TEL','FTI','TXN','TXT','TMO','TIF','TWX','TJX','TMK','TSS','TSCO','TDG','TRV',\n",
    "    'TRIP','FOXA','FOX','TSN','UDR','ULTA','USB','UAA','UA','UNP','UAL','UNH','UPS','URI',\n",
    "    'UTX','UHS','UNM','VFC','VLO','VAR','VTR','VRSN','VRSK','VZ','VRTX','VIAB','V','VNO',\n",
    "    'VMC','WMT','WBA','DIS','WM','WAT','WEC','WFC','HCN','WDC','WU','WRK','WY','WHR','WMB',\n",
    "    'WLTW','WYN','WYNN','XEL','XRX','XLNX','XL','XYL','YUM','ZBH','ZION','ZTS']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add symbol for S&P 500 index\n",
    "symbols.append('SPY')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make directory to store data, if doesn't already exist\n",
    "data_path = os.path.join('.','data')\n",
    "if not os.path.isdir(data_path):\n",
    "    os.mkdir(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- AGN: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- APC: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- ARNC: Data doesn't exist for startDate = 1262325600, endDate = 1546236000\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- BHGE: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- BBT: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- BRK.B: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- BF.B: No data found for this date range, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- CBG: No data found for this date range, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- CBS: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- CELG: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- CTL: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- CHK: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- DWDP: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- DPS: No data found for this date range, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- ETFC: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- GGP: No data found for this date range, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- HRS: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- HCP: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- JEC: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- LLL: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- LUK: No data found for this date range, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- KORS: No data found for this date range, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- MYL: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- NBL: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- PX: Data doesn't exist for startDate = 1262325600, endDate = 1546236000\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- PCLN: No data found for this date range, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- RTN: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- RHT: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- STI: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- SYMC: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- TMK: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- TSS: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- FOXA: Data doesn't exist for startDate = 1262325600, endDate = 1546236000\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- FOX: Data doesn't exist for startDate = 1262325600, endDate = 1546236000\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- UTX: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- VIAB: No data found, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- HCN: No data found for this date range, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- WYN: No data found for this date range, symbol may be delisted\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "\n",
      "1 Failed download:\n",
      "- XL: Data doesn't exist for startDate = 1262325600, endDate = 1546236000\n"
     ]
    }
   ],
   "source": [
    "# save each symbol's data down to a separate csv file, if doesn't already exist\n",
    "for symbol in symbols:\n",
    "    filename = os.path.join(data_path, symbol + '.csv')\n",
    "    if not os.path.isfile(filename):\n",
    "        # get a dataframe of this stock's data\n",
    "        stockdata = yf.download(symbol, start='2010-01-01', end='2018-12-31')\n",
    "        # only save down if there are at least 10 rows of data\n",
    "        if stockdata.shape[0] > 10:\n",
    "            stockdata.to_csv(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a dataframe with all of the data, adding a Name column for each symbol\n",
    "all_df = None\n",
    "for symbol in symbols:\n",
    "    filename = os.path.join(data_path, symbol + '.csv')\n",
    "    if os.path.isfile(filename):\n",
    "        df = pd.read_csv(filename)\n",
    "        df['Name'] = symbol\n",
    "        if all_df is None:\n",
    "            all_df = df\n",
    "        else:\n",
    "            all_df = all_df.append(df, ignore_index=True)\n",
    "            \n",
    "all_df.to_csv('./data/sp500full.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'A',\n",
       " 'AAP',\n",
       " 'AAPL',\n",
       " 'ABBV',\n",
       " 'ABC',\n",
       " 'ABT',\n",
       " 'ACN',\n",
       " 'ADBE',\n",
       " 'ADP',\n",
       " 'AES',\n",
       " 'AFL',\n",
       " 'AKAM',\n",
       " 'ALGN',\n",
       " 'ALK',\n",
       " 'ALLE',\n",
       " 'AMD',\n",
       " 'ANSS',\n",
       " 'APTV',\n",
       " 'ATVI',\n",
       " 'AXP',\n",
       " 'BBT',\n",
       " 'BIIB',\n",
       " 'BMY',\n",
       " 'BSX',\n",
       " 'CA',\n",
       " 'CBOE',\n",
       " 'CF',\n",
       " 'CHD',\n",
       " 'CMCSA',\n",
       " 'CPB',\n",
       " 'CSCO',\n",
       " 'DE',\n",
       " 'DHI',\n",
       " 'DIS',\n",
       " 'DISCA',\n",
       " 'DRE',\n",
       " 'DWDP',\n",
       " 'EIX',\n",
       " 'EQIX',\n",
       " 'EQR',\n",
       " 'EVHC',\n",
       " 'FB',\n",
       " 'FFIV',\n",
       " 'FISV',\n",
       " 'FITB',\n",
       " 'FLR',\n",
       " 'GE',\n",
       " 'GGP',\n",
       " 'GILD',\n",
       " 'GOOG',\n",
       " 'GT',\n",
       " 'HAL',\n",
       " 'HCA',\n",
       " 'HES',\n",
       " 'HRS',\n",
       " 'HST',\n",
       " 'IBM',\n",
       " 'IFF',\n",
       " 'IQV',\n",
       " 'JEC',\n",
       " 'KEY',\n",
       " 'KORS',\n",
       " 'KSS',\n",
       " 'LMT',\n",
       " 'LNC',\n",
       " 'LNT',\n",
       " 'MAS',\n",
       " 'MET',\n",
       " 'MMM',\n",
       " 'NDAQ',\n",
       " 'NFX',\n",
       " 'NLSN',\n",
       " 'NTAP',\n",
       " 'NUE',\n",
       " 'NWS',\n",
       " 'NWSA',\n",
       " 'OKE',\n",
       " 'ORLY',\n",
       " 'PEP',\n",
       " 'PFG',\n",
       " 'PPG',\n",
       " 'PRGO',\n",
       " 'PSA',\n",
       " 'QCOM',\n",
       " 'RE',\n",
       " 'RHT',\n",
       " 'RSG',\n",
       " 'SBUX',\n",
       " 'SLB',\n",
       " 'SNPS',\n",
       " 'SPY',\n",
       " 'SWKS',\n",
       " 'SYF',\n",
       " 'SYMC',\n",
       " 'T',\n",
       " 'TAP',\n",
       " 'TJX',\n",
       " 'TRIP',\n",
       " 'TSS',\n",
       " 'TWX',\n",
       " 'TXT',\n",
       " 'UNM',\n",
       " 'UPS',\n",
       " 'USB',\n",
       " 'UTX',\n",
       " 'VIAB',\n",
       " 'VRSK',\n",
       " 'WFC',\n",
       " 'WU',\n",
       " 'XL',\n",
       " 'XLNX',\n",
       " 'XOM',\n",
       " 'XRAY'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a short list of stocks and add a random list of 100 stocks\n",
    "short_symbols = {'MMM', 'ABT', 'ABBV', 'ACN', 'ATVI', 'ADBE', 'AMD', 'AAP', 'AES',\n",
    "  'AFL', 'AKAM', 'IBM', 'GOOG', 'SBUX', 'AAPL', 'SPY'}\n",
    "\n",
    "random_symbols = random.sample(symbols, 100)\n",
    "short_set = short_symbols.union(random_symbols)\n",
    "short_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create dataframe with short list, adding a Name column for each symbol\n",
    "short_df = None\n",
    "for symbol in short_set:\n",
    "    filename = os.path.join(data_path, symbol + '.csv')\n",
    "    if os.path.isfile(filename):\n",
    "        df = pd.read_csv(filename)\n",
    "        df['Name'] = symbol\n",
    "        if short_df is None:\n",
    "            short_df = df\n",
    "        else:\n",
    "            short_df = short_df.append(df, ignore_index=True)\n",
    "            \n",
    "short_df.to_csv('./data/sp500sub.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fineng",
   "language": "python",
   "name": "fineng"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}