NYtimes_API.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pynytimes\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pynytimes import NYTAPI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from my_api_info import get_nytimes_key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "nytapi = NYTAPI(get_nytimes_key(),\n",
    "                parse_dates=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = nytapi.article_search(query=\"stock\",\n",
    "                                results=365,\n",
    "                                dates={'begin':datetime(2023,1,1),\n",
    "                                       'end':datetime(2023,12,30)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "NYdata = pd.DataFrame(results)\n",
    "NYdata['pub_date'] = pd.to_datetime(NYdata['pub_date'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>abstract</th>\n",
       "      <th>web_url</th>\n",
       "      <th>snippet</th>\n",
       "      <th>lead_paragraph</th>\n",
       "      <th>print_section</th>\n",
       "      <th>print_page</th>\n",
       "      <th>source</th>\n",
       "      <th>multimedia</th>\n",
       "      <th>headline</th>\n",
       "      <th>keywords</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>document_type</th>\n",
       "      <th>news_desk</th>\n",
       "      <th>section_name</th>\n",
       "      <th>byline</th>\n",
       "      <th>type_of_material</th>\n",
       "      <th>_id</th>\n",
       "      <th>word_count</th>\n",
       "      <th>uri</th>\n",
       "      <th>subsection_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Investors worried about China’s economy shunne...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/29/business/ho...</td>\n",
       "      <td>Investors worried about China’s economy shunne...</td>\n",
       "      <td>This summer, when Hong Kong’s stock market rou...</td>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Hong Kong Stocks Plunge to Losses fo...</td>\n",
       "      <td>[{'name': 'glocations', 'value': 'Hong Kong', ...</td>\n",
       "      <td>2023-12-29 08:29:18+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Alexandra Stevenson', 'person...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...</td>\n",
       "      <td>779</td>\n",
       "      <td>nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Wall Street’s forecasts mostly missed this yea...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/29/business/de...</td>\n",
       "      <td>Wall Street’s forecasts mostly missed this yea...</td>\n",
       "      <td></td>\n",
       "      <td>B</td>\n",
       "      <td>4</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'What Could Go Right (and Wrong) in t...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'internal-storyl...</td>\n",
       "      <td>2023-12-29 12:39:35+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Bernhard Warner', 'person': [...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/3f960626-f801-5488-a261-24d5863f...</td>\n",
       "      <td>953</td>\n",
       "      <td>nyt://article/3f960626-f801-5488-a261-24d5863f...</td>\n",
       "      <td>DealBook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Inflation, bank failures, strikes, and artific...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/30/business/de...</td>\n",
       "      <td>Inflation, bank failures, strikes, and artific...</td>\n",
       "      <td>It has been a confusing year for the economy a...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': '11 Charts That Explain the Year in B...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'internal-storyl...</td>\n",
       "      <td>2023-12-30 12:51:24+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Christine Zhang', 'person': [...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...</td>\n",
       "      <td>1055</td>\n",
       "      <td>nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...</td>\n",
       "      <td>DealBook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Analysts bullish on 2023 were largely right an...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/26/business/st...</td>\n",
       "      <td>Analysts bullish on 2023 were largely right an...</td>\n",
       "      <td>Twelve months ago, Tom Lee bet that 2023 was g...</td>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Downturn or Not? At Year’s End, Wall...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'Stocks and Bond...</td>\n",
       "      <td>2023-12-26 10:04:45+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Joe Rennison', 'person': [{'f...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...</td>\n",
       "      <td>1100</td>\n",
       "      <td>nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>So-called stock forecasts don’t deserve the na...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/23/business/wa...</td>\n",
       "      <td>So-called stock forecasts don’t deserve the na...</td>\n",
       "      <td>Wall Street strategists are issuing forecasts ...</td>\n",
       "      <td>BU</td>\n",
       "      <td>8</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Wall St. Loves to Guess, but Nobody ...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'Stocks and Bond...</td>\n",
       "      <td>2023-12-23 10:03:30+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>SundayBusiness</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Jeff Sommer', 'person': [{'fi...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...</td>\n",
       "      <td>1108</td>\n",
       "      <td>nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            abstract  \\\n",
       "0  Investors worried about China’s economy shunne...   \n",
       "1  Wall Street’s forecasts mostly missed this yea...   \n",
       "2  Inflation, bank failures, strikes, and artific...   \n",
       "3  Analysts bullish on 2023 were largely right an...   \n",
       "4  So-called stock forecasts don’t deserve the na...   \n",
       "\n",
       "                                             web_url  \\\n",
       "0  https://www.nytimes.com/2023/12/29/business/ho...   \n",
       "1  https://www.nytimes.com/2023/12/29/business/de...   \n",
       "2  https://www.nytimes.com/2023/12/30/business/de...   \n",
       "3  https://www.nytimes.com/2023/12/26/business/st...   \n",
       "4  https://www.nytimes.com/2023/12/23/business/wa...   \n",
       "\n",
       "                                             snippet  \\\n",
       "0  Investors worried about China’s economy shunne...   \n",
       "1  Wall Street’s forecasts mostly missed this yea...   \n",
       "2  Inflation, bank failures, strikes, and artific...   \n",
       "3  Analysts bullish on 2023 were largely right an...   \n",
       "4  So-called stock forecasts don’t deserve the na...   \n",
       "\n",
       "                                      lead_paragraph print_section print_page  \\\n",
       "0  This summer, when Hong Kong’s stock market rou...             B          1   \n",
       "1                                                                B          4   \n",
       "2  It has been a confusing year for the economy a...           NaN        NaN   \n",
       "3  Twelve months ago, Tom Lee bet that 2023 was g...             B          1   \n",
       "4  Wall Street strategists are issuing forecasts ...            BU          8   \n",
       "\n",
       "               source                                         multimedia  \\\n",
       "0  The New York Times  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "1  The New York Times  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2  The New York Times  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "3  The New York Times  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "4  The New York Times  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "\n",
       "                                            headline  \\\n",
       "0  {'main': 'Hong Kong Stocks Plunge to Losses fo...   \n",
       "1  {'main': 'What Could Go Right (and Wrong) in t...   \n",
       "2  {'main': '11 Charts That Explain the Year in B...   \n",
       "3  {'main': 'Downturn or Not? At Year’s End, Wall...   \n",
       "4  {'main': 'Wall St. Loves to Guess, but Nobody ...   \n",
       "\n",
       "                                            keywords  \\\n",
       "0  [{'name': 'glocations', 'value': 'Hong Kong', ...   \n",
       "1  [{'name': 'subject', 'value': 'internal-storyl...   \n",
       "2  [{'name': 'subject', 'value': 'internal-storyl...   \n",
       "3  [{'name': 'subject', 'value': 'Stocks and Bond...   \n",
       "4  [{'name': 'subject', 'value': 'Stocks and Bond...   \n",
       "\n",
       "                   pub_date document_type       news_desk  section_name  \\\n",
       "0 2023-12-29 08:29:18+00:00       article        Business  Business Day   \n",
       "1 2023-12-29 12:39:35+00:00       article        Business  Business Day   \n",
       "2 2023-12-30 12:51:24+00:00       article        Business  Business Day   \n",
       "3 2023-12-26 10:04:45+00:00       article        Business  Business Day   \n",
       "4 2023-12-23 10:03:30+00:00       article  SundayBusiness  Business Day   \n",
       "\n",
       "                                              byline type_of_material  \\\n",
       "0  {'original': 'By Alexandra Stevenson', 'person...             News   \n",
       "1  {'original': 'By Bernhard Warner', 'person': [...             News   \n",
       "2  {'original': 'By Christine Zhang', 'person': [...             News   \n",
       "3  {'original': 'By Joe Rennison', 'person': [{'f...             News   \n",
       "4  {'original': 'By Jeff Sommer', 'person': [{'fi...             News   \n",
       "\n",
       "                                                 _id  word_count  \\\n",
       "0  nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...         779   \n",
       "1  nyt://article/3f960626-f801-5488-a261-24d5863f...         953   \n",
       "2  nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...        1055   \n",
       "3  nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...        1100   \n",
       "4  nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...        1108   \n",
       "\n",
       "                                                 uri subsection_name  \n",
       "0  nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...             NaN  \n",
       "1  nyt://article/3f960626-f801-5488-a261-24d5863f...        DealBook  \n",
       "2  nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...        DealBook  \n",
       "3  nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...             NaN  \n",
       "4  nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...             NaN  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NYdata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp_price=pd.read_csv(\"Download Data - INDEX_US_S&P US_SPX.csv\", parse_dates=['Date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp_price.set_index('Date', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Close</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2023-01-03</th>\n",
       "      <td>3,853.29</td>\n",
       "      <td>3,878.46</td>\n",
       "      <td>3,794.33</td>\n",
       "      <td>3,824.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-01-04</th>\n",
       "      <td>3,840.36</td>\n",
       "      <td>3,873.16</td>\n",
       "      <td>3,815.77</td>\n",
       "      <td>3,852.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-01-05</th>\n",
       "      <td>3,839.74</td>\n",
       "      <td>3,839.74</td>\n",
       "      <td>3,802.42</td>\n",
       "      <td>3,808.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-01-06</th>\n",
       "      <td>3,823.37</td>\n",
       "      <td>3,906.19</td>\n",
       "      <td>3,809.56</td>\n",
       "      <td>3,895.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-01-09</th>\n",
       "      <td>3,910.82</td>\n",
       "      <td>3,950.57</td>\n",
       "      <td>3,890.42</td>\n",
       "      <td>3,892.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-22</th>\n",
       "      <td>4,753.92</td>\n",
       "      <td>4,772.94</td>\n",
       "      <td>4,736.77</td>\n",
       "      <td>4,754.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-26</th>\n",
       "      <td>4,758.86</td>\n",
       "      <td>4,784.72</td>\n",
       "      <td>4,758.45</td>\n",
       "      <td>4,774.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-27</th>\n",
       "      <td>4,773.45</td>\n",
       "      <td>4,785.39</td>\n",
       "      <td>4,768.90</td>\n",
       "      <td>4,781.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-28</th>\n",
       "      <td>4,786.44</td>\n",
       "      <td>4,793.30</td>\n",
       "      <td>4,780.98</td>\n",
       "      <td>4,783.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-29</th>\n",
       "      <td>4,782.88</td>\n",
       "      <td>4,788.43</td>\n",
       "      <td>4,751.99</td>\n",
       "      <td>4,769.83</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                Open      High       Low     Close\n",
       "Date                                              \n",
       "2023-01-03  3,853.29  3,878.46  3,794.33  3,824.14\n",
       "2023-01-04  3,840.36  3,873.16  3,815.77  3,852.97\n",
       "2023-01-05  3,839.74  3,839.74  3,802.42  3,808.10\n",
       "2023-01-06  3,823.37  3,906.19  3,809.56  3,895.08\n",
       "2023-01-09  3,910.82  3,950.57  3,890.42  3,892.09\n",
       "...              ...       ...       ...       ...\n",
       "2023-12-22  4,753.92  4,772.94  4,736.77  4,754.63\n",
       "2023-12-26  4,758.86  4,784.72  4,758.45  4,774.75\n",
       "2023-12-27  4,773.45  4,785.39  4,768.90  4,781.58\n",
       "2023-12-28  4,786.44  4,793.30  4,780.98  4,783.35\n",
       "2023-12-29  4,782.88  4,788.43  4,751.99  4,769.83\n",
       "\n",
       "[250 rows x 4 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sp_price.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "NYdata['Date']=NYdata['pub_date'].dt.date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "NYdata.set_index('Date', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>abstract</th>\n",
       "      <th>web_url</th>\n",
       "      <th>snippet</th>\n",
       "      <th>lead_paragraph</th>\n",
       "      <th>print_section</th>\n",
       "      <th>print_page</th>\n",
       "      <th>source</th>\n",
       "      <th>multimedia</th>\n",
       "      <th>headline</th>\n",
       "      <th>keywords</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>document_type</th>\n",
       "      <th>news_desk</th>\n",
       "      <th>section_name</th>\n",
       "      <th>byline</th>\n",
       "      <th>type_of_material</th>\n",
       "      <th>_id</th>\n",
       "      <th>word_count</th>\n",
       "      <th>uri</th>\n",
       "      <th>subsection_name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2023-12-29</th>\n",
       "      <td>Investors worried about China’s economy shunne...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/29/business/ho...</td>\n",
       "      <td>Investors worried about China’s economy shunne...</td>\n",
       "      <td>This summer, when Hong Kong’s stock market rou...</td>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Hong Kong Stocks Plunge to Losses fo...</td>\n",
       "      <td>[{'name': 'glocations', 'value': 'Hong Kong', ...</td>\n",
       "      <td>2023-12-29 08:29:18+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Alexandra Stevenson', 'person...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...</td>\n",
       "      <td>779</td>\n",
       "      <td>nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-29</th>\n",
       "      <td>Wall Street’s forecasts mostly missed this yea...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/29/business/de...</td>\n",
       "      <td>Wall Street’s forecasts mostly missed this yea...</td>\n",
       "      <td></td>\n",
       "      <td>B</td>\n",
       "      <td>4</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'What Could Go Right (and Wrong) in t...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'internal-storyl...</td>\n",
       "      <td>2023-12-29 12:39:35+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Bernhard Warner', 'person': [...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/3f960626-f801-5488-a261-24d5863f...</td>\n",
       "      <td>953</td>\n",
       "      <td>nyt://article/3f960626-f801-5488-a261-24d5863f...</td>\n",
       "      <td>DealBook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-30</th>\n",
       "      <td>Inflation, bank failures, strikes, and artific...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/30/business/de...</td>\n",
       "      <td>Inflation, bank failures, strikes, and artific...</td>\n",
       "      <td>It has been a confusing year for the economy a...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': '11 Charts That Explain the Year in B...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'internal-storyl...</td>\n",
       "      <td>2023-12-30 12:51:24+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Christine Zhang', 'person': [...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...</td>\n",
       "      <td>1055</td>\n",
       "      <td>nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...</td>\n",
       "      <td>DealBook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-26</th>\n",
       "      <td>Analysts bullish on 2023 were largely right an...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/26/business/st...</td>\n",
       "      <td>Analysts bullish on 2023 were largely right an...</td>\n",
       "      <td>Twelve months ago, Tom Lee bet that 2023 was g...</td>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Downturn or Not? At Year’s End, Wall...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'Stocks and Bond...</td>\n",
       "      <td>2023-12-26 10:04:45+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Joe Rennison', 'person': [{'f...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...</td>\n",
       "      <td>1100</td>\n",
       "      <td>nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-23</th>\n",
       "      <td>So-called stock forecasts don’t deserve the na...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/23/business/wa...</td>\n",
       "      <td>So-called stock forecasts don’t deserve the na...</td>\n",
       "      <td>Wall Street strategists are issuing forecasts ...</td>\n",
       "      <td>BU</td>\n",
       "      <td>8</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Wall St. Loves to Guess, but Nobody ...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'Stocks and Bond...</td>\n",
       "      <td>2023-12-23 10:03:30+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>SundayBusiness</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Jeff Sommer', 'person': [{'fi...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...</td>\n",
       "      <td>1108</td>\n",
       "      <td>nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     abstract  \\\n",
       "Date                                                            \n",
       "2023-12-29  Investors worried about China’s economy shunne...   \n",
       "2023-12-29  Wall Street’s forecasts mostly missed this yea...   \n",
       "2023-12-30  Inflation, bank failures, strikes, and artific...   \n",
       "2023-12-26  Analysts bullish on 2023 were largely right an...   \n",
       "2023-12-23  So-called stock forecasts don’t deserve the na...   \n",
       "\n",
       "                                                      web_url  \\\n",
       "Date                                                            \n",
       "2023-12-29  https://www.nytimes.com/2023/12/29/business/ho...   \n",
       "2023-12-29  https://www.nytimes.com/2023/12/29/business/de...   \n",
       "2023-12-30  https://www.nytimes.com/2023/12/30/business/de...   \n",
       "2023-12-26  https://www.nytimes.com/2023/12/26/business/st...   \n",
       "2023-12-23  https://www.nytimes.com/2023/12/23/business/wa...   \n",
       "\n",
       "                                                      snippet  \\\n",
       "Date                                                            \n",
       "2023-12-29  Investors worried about China’s economy shunne...   \n",
       "2023-12-29  Wall Street’s forecasts mostly missed this yea...   \n",
       "2023-12-30  Inflation, bank failures, strikes, and artific...   \n",
       "2023-12-26  Analysts bullish on 2023 were largely right an...   \n",
       "2023-12-23  So-called stock forecasts don’t deserve the na...   \n",
       "\n",
       "                                               lead_paragraph print_section  \\\n",
       "Date                                                                          \n",
       "2023-12-29  This summer, when Hong Kong’s stock market rou...             B   \n",
       "2023-12-29                                                                B   \n",
       "2023-12-30  It has been a confusing year for the economy a...           NaN   \n",
       "2023-12-26  Twelve months ago, Tom Lee bet that 2023 was g...             B   \n",
       "2023-12-23  Wall Street strategists are issuing forecasts ...            BU   \n",
       "\n",
       "           print_page              source  \\\n",
       "Date                                        \n",
       "2023-12-29          1  The New York Times   \n",
       "2023-12-29          4  The New York Times   \n",
       "2023-12-30        NaN  The New York Times   \n",
       "2023-12-26          1  The New York Times   \n",
       "2023-12-23          8  The New York Times   \n",
       "\n",
       "                                                   multimedia  \\\n",
       "Date                                                            \n",
       "2023-12-29  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-12-29  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-12-30  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-12-26  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-12-23  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "\n",
       "                                                     headline  \\\n",
       "Date                                                            \n",
       "2023-12-29  {'main': 'Hong Kong Stocks Plunge to Losses fo...   \n",
       "2023-12-29  {'main': 'What Could Go Right (and Wrong) in t...   \n",
       "2023-12-30  {'main': '11 Charts That Explain the Year in B...   \n",
       "2023-12-26  {'main': 'Downturn or Not? At Year’s End, Wall...   \n",
       "2023-12-23  {'main': 'Wall St. Loves to Guess, but Nobody ...   \n",
       "\n",
       "                                                     keywords  \\\n",
       "Date                                                            \n",
       "2023-12-29  [{'name': 'glocations', 'value': 'Hong Kong', ...   \n",
       "2023-12-29  [{'name': 'subject', 'value': 'internal-storyl...   \n",
       "2023-12-30  [{'name': 'subject', 'value': 'internal-storyl...   \n",
       "2023-12-26  [{'name': 'subject', 'value': 'Stocks and Bond...   \n",
       "2023-12-23  [{'name': 'subject', 'value': 'Stocks and Bond...   \n",
       "\n",
       "                            pub_date document_type       news_desk  \\\n",
       "Date                                                                 \n",
       "2023-12-29 2023-12-29 08:29:18+00:00       article        Business   \n",
       "2023-12-29 2023-12-29 12:39:35+00:00       article        Business   \n",
       "2023-12-30 2023-12-30 12:51:24+00:00       article        Business   \n",
       "2023-12-26 2023-12-26 10:04:45+00:00       article        Business   \n",
       "2023-12-23 2023-12-23 10:03:30+00:00       article  SundayBusiness   \n",
       "\n",
       "            section_name                                             byline  \\\n",
       "Date                                                                          \n",
       "2023-12-29  Business Day  {'original': 'By Alexandra Stevenson', 'person...   \n",
       "2023-12-29  Business Day  {'original': 'By Bernhard Warner', 'person': [...   \n",
       "2023-12-30  Business Day  {'original': 'By Christine Zhang', 'person': [...   \n",
       "2023-12-26  Business Day  {'original': 'By Joe Rennison', 'person': [{'f...   \n",
       "2023-12-23  Business Day  {'original': 'By Jeff Sommer', 'person': [{'fi...   \n",
       "\n",
       "           type_of_material  \\\n",
       "Date                          \n",
       "2023-12-29             News   \n",
       "2023-12-29             News   \n",
       "2023-12-30             News   \n",
       "2023-12-26             News   \n",
       "2023-12-23             News   \n",
       "\n",
       "                                                          _id  word_count  \\\n",
       "Date                                                                        \n",
       "2023-12-29  nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...         779   \n",
       "2023-12-29  nyt://article/3f960626-f801-5488-a261-24d5863f...         953   \n",
       "2023-12-30  nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...        1055   \n",
       "2023-12-26  nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...        1100   \n",
       "2023-12-23  nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...        1108   \n",
       "\n",
       "                                                          uri subsection_name  \n",
       "Date                                                                           \n",
       "2023-12-29  nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...             NaN  \n",
       "2023-12-29  nyt://article/3f960626-f801-5488-a261-24d5863f...        DealBook  \n",
       "2023-12-30  nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...        DealBook  \n",
       "2023-12-26  nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...             NaN  \n",
       "2023-12-23  nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...             NaN  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NYdata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "NYdata = NYdata[~NYdata.index.duplicated(keep='first')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>abstract</th>\n",
       "      <th>web_url</th>\n",
       "      <th>snippet</th>\n",
       "      <th>lead_paragraph</th>\n",
       "      <th>print_section</th>\n",
       "      <th>print_page</th>\n",
       "      <th>source</th>\n",
       "      <th>multimedia</th>\n",
       "      <th>headline</th>\n",
       "      <th>keywords</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>document_type</th>\n",
       "      <th>news_desk</th>\n",
       "      <th>section_name</th>\n",
       "      <th>byline</th>\n",
       "      <th>type_of_material</th>\n",
       "      <th>_id</th>\n",
       "      <th>word_count</th>\n",
       "      <th>uri</th>\n",
       "      <th>subsection_name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2023-01-02</th>\n",
       "      <td>Today, you will identify the areas of your lif...</td>\n",
       "      <td>https://www.nytimes.com/2023/01/01/well/happin...</td>\n",
       "      <td>Today, you will identify the areas of your lif...</td>\n",
       "      <td>In 1938, researchers at Harvard set out to lea...</td>\n",
       "      <td>D</td>\n",
       "      <td>6</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Day 1: Take Stock of Your Relationsh...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'Content Type: S...</td>\n",
       "      <td>2023-01-02 04:00:06+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Well</td>\n",
       "      <td>Well</td>\n",
       "      <td>{'original': 'By Jancee Dunn', 'person': [{'fi...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/3dfb3527-68cd-5db7-9504-7afc5389...</td>\n",
       "      <td>1172</td>\n",
       "      <td>nyt://article/3dfb3527-68cd-5db7-9504-7afc5389...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-01-04</th>\n",
       "      <td></td>\n",
       "      <td>https://cooking.nytimes.com/recipes/1023769-gr...</td>\n",
       "      <td></td>\n",
       "      <td>Chile sauces at &lt;a href=\"https://www.elcholo.c...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Green Enchilada Sauce', 'kicker': No...</td>\n",
       "      <td>[]</td>\n",
       "      <td>2023-01-04 00:00:00+00:00</td>\n",
       "      <td>recipe</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'original': 'Kevin McKenna, Alexa Weibel', 'p...</td>\n",
       "      <td>Recipe</td>\n",
       "      <td>nyt://recipe/162d357c-e7cc-5f6d-a3eb-1f46ca317fda</td>\n",
       "      <td>0</td>\n",
       "      <td>nyt://recipe/162d357c-e7cc-5f6d-a3eb-1f46ca317fda</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-01-05</th>\n",
       "      <td>The home goods retailer said weak sales and sl...</td>\n",
       "      <td>https://www.nytimes.com/2023/01/05/business/be...</td>\n",
       "      <td>The home goods retailer said weak sales and sl...</td>\n",
       "      <td>Bed Bath &amp; Beyond, the beleaguered home goods ...</td>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Bed Bath &amp; Beyond Warns of Potential...</td>\n",
       "      <td>[{'name': 'organizations', 'value': 'Bed Bath ...</td>\n",
       "      <td>2023-01-05 14:12:37+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Lauren Hirsch and Jordyn Holm...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/86579820-0588-51bc-8125-d4002942...</td>\n",
       "      <td>1226</td>\n",
       "      <td>nyt://article/86579820-0588-51bc-8125-d4002942...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-01-06</th>\n",
       "      <td>The Securities and Exchange Commission ended i...</td>\n",
       "      <td>https://www.nytimes.com/2023/01/06/us/politics...</td>\n",
       "      <td>The Securities and Exchange Commission ended i...</td>\n",
       "      <td>The Securities and Exchange Commission has clo...</td>\n",
       "      <td>A</td>\n",
       "      <td>15</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'S.E.C. Inquiry Into Former Senator’s...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'Insider Trading...</td>\n",
       "      <td>2023-01-06 20:25:30+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Washington</td>\n",
       "      <td>U.S.</td>\n",
       "      <td>{'original': 'By Kate Kelly', 'person': [{'fir...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/482c9261-2dea-5f7b-914f-39bc2ebf...</td>\n",
       "      <td>684</td>\n",
       "      <td>nyt://article/482c9261-2dea-5f7b-914f-39bc2ebf...</td>\n",
       "      <td>Politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-01-08</th>\n",
       "      <td>Sam Bankman-Fried, the disgraced cryptocurrenc...</td>\n",
       "      <td>https://www.nytimes.com/2023/01/08/business/th...</td>\n",
       "      <td>Sam Bankman-Fried, the disgraced cryptocurrenc...</td>\n",
       "      <td>After many consecutive months of jobs reports ...</td>\n",
       "      <td>BU</td>\n",
       "      <td>2</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'The Week in Business: Slowing Job Gr...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'Stocks and Bond...</td>\n",
       "      <td>2023-01-08 12:00:06+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>SundayBusiness</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Marie Solis', 'person': [{'fi...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/bdf2d306-799b-5299-8efb-1289a0ce...</td>\n",
       "      <td>966</td>\n",
       "      <td>nyt://article/bdf2d306-799b-5299-8efb-1289a0ce...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-23</th>\n",
       "      <td>So-called stock forecasts don’t deserve the na...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/23/business/wa...</td>\n",
       "      <td>So-called stock forecasts don’t deserve the na...</td>\n",
       "      <td>Wall Street strategists are issuing forecasts ...</td>\n",
       "      <td>BU</td>\n",
       "      <td>8</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Wall St. Loves to Guess, but Nobody ...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'Stocks and Bond...</td>\n",
       "      <td>2023-12-23 10:03:30+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>SundayBusiness</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Jeff Sommer', 'person': [{'fi...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...</td>\n",
       "      <td>1108</td>\n",
       "      <td>nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-26</th>\n",
       "      <td>Analysts bullish on 2023 were largely right an...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/26/business/st...</td>\n",
       "      <td>Analysts bullish on 2023 were largely right an...</td>\n",
       "      <td>Twelve months ago, Tom Lee bet that 2023 was g...</td>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Downturn or Not? At Year’s End, Wall...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'Stocks and Bond...</td>\n",
       "      <td>2023-12-26 10:04:45+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Joe Rennison', 'person': [{'f...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...</td>\n",
       "      <td>1100</td>\n",
       "      <td>nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-27</th>\n",
       "      <td>After a market rout, gaming companies like Ten...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/27/business/ch...</td>\n",
       "      <td>After a market rout, gaming companies like Ten...</td>\n",
       "      <td>The stock prices of Chinese video game compani...</td>\n",
       "      <td>B</td>\n",
       "      <td>3</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Will China Ease Its New Video Game C...</td>\n",
       "      <td>[{'name': 'glocations', 'value': 'China', 'ran...</td>\n",
       "      <td>2023-12-27 07:56:49+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By John Liu', 'person': [{'first...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/11afe169-2c2e-528b-a728-a56d25a2...</td>\n",
       "      <td>581</td>\n",
       "      <td>nyt://article/11afe169-2c2e-528b-a728-a56d25a2...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-29</th>\n",
       "      <td>Investors worried about China’s economy shunne...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/29/business/ho...</td>\n",
       "      <td>Investors worried about China’s economy shunne...</td>\n",
       "      <td>This summer, when Hong Kong’s stock market rou...</td>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': 'Hong Kong Stocks Plunge to Losses fo...</td>\n",
       "      <td>[{'name': 'glocations', 'value': 'Hong Kong', ...</td>\n",
       "      <td>2023-12-29 08:29:18+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Alexandra Stevenson', 'person...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...</td>\n",
       "      <td>779</td>\n",
       "      <td>nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-12-30</th>\n",
       "      <td>Inflation, bank failures, strikes, and artific...</td>\n",
       "      <td>https://www.nytimes.com/2023/12/30/business/de...</td>\n",
       "      <td>Inflation, bank failures, strikes, and artific...</td>\n",
       "      <td>It has been a confusing year for the economy a...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "      <td>{'main': '11 Charts That Explain the Year in B...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'internal-storyl...</td>\n",
       "      <td>2023-12-30 12:51:24+00:00</td>\n",
       "      <td>article</td>\n",
       "      <td>Business</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>{'original': 'By Christine Zhang', 'person': [...</td>\n",
       "      <td>News</td>\n",
       "      <td>nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...</td>\n",
       "      <td>1055</td>\n",
       "      <td>nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...</td>\n",
       "      <td>DealBook</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>209 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     abstract  \\\n",
       "Date                                                            \n",
       "2023-01-02  Today, you will identify the areas of your lif...   \n",
       "2023-01-04                                                      \n",
       "2023-01-05  The home goods retailer said weak sales and sl...   \n",
       "2023-01-06  The Securities and Exchange Commission ended i...   \n",
       "2023-01-08  Sam Bankman-Fried, the disgraced cryptocurrenc...   \n",
       "...                                                       ...   \n",
       "2023-12-23  So-called stock forecasts don’t deserve the na...   \n",
       "2023-12-26  Analysts bullish on 2023 were largely right an...   \n",
       "2023-12-27  After a market rout, gaming companies like Ten...   \n",
       "2023-12-29  Investors worried about China’s economy shunne...   \n",
       "2023-12-30  Inflation, bank failures, strikes, and artific...   \n",
       "\n",
       "                                                      web_url  \\\n",
       "Date                                                            \n",
       "2023-01-02  https://www.nytimes.com/2023/01/01/well/happin...   \n",
       "2023-01-04  https://cooking.nytimes.com/recipes/1023769-gr...   \n",
       "2023-01-05  https://www.nytimes.com/2023/01/05/business/be...   \n",
       "2023-01-06  https://www.nytimes.com/2023/01/06/us/politics...   \n",
       "2023-01-08  https://www.nytimes.com/2023/01/08/business/th...   \n",
       "...                                                       ...   \n",
       "2023-12-23  https://www.nytimes.com/2023/12/23/business/wa...   \n",
       "2023-12-26  https://www.nytimes.com/2023/12/26/business/st...   \n",
       "2023-12-27  https://www.nytimes.com/2023/12/27/business/ch...   \n",
       "2023-12-29  https://www.nytimes.com/2023/12/29/business/ho...   \n",
       "2023-12-30  https://www.nytimes.com/2023/12/30/business/de...   \n",
       "\n",
       "                                                      snippet  \\\n",
       "Date                                                            \n",
       "2023-01-02  Today, you will identify the areas of your lif...   \n",
       "2023-01-04                                                      \n",
       "2023-01-05  The home goods retailer said weak sales and sl...   \n",
       "2023-01-06  The Securities and Exchange Commission ended i...   \n",
       "2023-01-08  Sam Bankman-Fried, the disgraced cryptocurrenc...   \n",
       "...                                                       ...   \n",
       "2023-12-23  So-called stock forecasts don’t deserve the na...   \n",
       "2023-12-26  Analysts bullish on 2023 were largely right an...   \n",
       "2023-12-27  After a market rout, gaming companies like Ten...   \n",
       "2023-12-29  Investors worried about China’s economy shunne...   \n",
       "2023-12-30  Inflation, bank failures, strikes, and artific...   \n",
       "\n",
       "                                               lead_paragraph print_section  \\\n",
       "Date                                                                          \n",
       "2023-01-02  In 1938, researchers at Harvard set out to lea...             D   \n",
       "2023-01-04  Chile sauces at <a href=\"https://www.elcholo.c...           NaN   \n",
       "2023-01-05  Bed Bath & Beyond, the beleaguered home goods ...             B   \n",
       "2023-01-06  The Securities and Exchange Commission has clo...             A   \n",
       "2023-01-08  After many consecutive months of jobs reports ...            BU   \n",
       "...                                                       ...           ...   \n",
       "2023-12-23  Wall Street strategists are issuing forecasts ...            BU   \n",
       "2023-12-26  Twelve months ago, Tom Lee bet that 2023 was g...             B   \n",
       "2023-12-27  The stock prices of Chinese video game compani...             B   \n",
       "2023-12-29  This summer, when Hong Kong’s stock market rou...             B   \n",
       "2023-12-30  It has been a confusing year for the economy a...           NaN   \n",
       "\n",
       "           print_page              source  \\\n",
       "Date                                        \n",
       "2023-01-02          6  The New York Times   \n",
       "2023-01-04        NaN                 NaN   \n",
       "2023-01-05          1  The New York Times   \n",
       "2023-01-06         15  The New York Times   \n",
       "2023-01-08          2  The New York Times   \n",
       "...               ...                 ...   \n",
       "2023-12-23          8  The New York Times   \n",
       "2023-12-26          1  The New York Times   \n",
       "2023-12-27          3  The New York Times   \n",
       "2023-12-29          1  The New York Times   \n",
       "2023-12-30        NaN  The New York Times   \n",
       "\n",
       "                                                   multimedia  \\\n",
       "Date                                                            \n",
       "2023-01-02  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-01-04  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-01-05  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-01-06  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-01-08  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "...                                                       ...   \n",
       "2023-12-23  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-12-26  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-12-27  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-12-29  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "2023-12-30  [{'rank': 0, 'subtype': 'xlarge', 'caption': N...   \n",
       "\n",
       "                                                     headline  \\\n",
       "Date                                                            \n",
       "2023-01-02  {'main': 'Day 1: Take Stock of Your Relationsh...   \n",
       "2023-01-04  {'main': 'Green Enchilada Sauce', 'kicker': No...   \n",
       "2023-01-05  {'main': 'Bed Bath & Beyond Warns of Potential...   \n",
       "2023-01-06  {'main': 'S.E.C. Inquiry Into Former Senator’s...   \n",
       "2023-01-08  {'main': 'The Week in Business: Slowing Job Gr...   \n",
       "...                                                       ...   \n",
       "2023-12-23  {'main': 'Wall St. Loves to Guess, but Nobody ...   \n",
       "2023-12-26  {'main': 'Downturn or Not? At Year’s End, Wall...   \n",
       "2023-12-27  {'main': 'Will China Ease Its New Video Game C...   \n",
       "2023-12-29  {'main': 'Hong Kong Stocks Plunge to Losses fo...   \n",
       "2023-12-30  {'main': '11 Charts That Explain the Year in B...   \n",
       "\n",
       "                                                     keywords  \\\n",
       "Date                                                            \n",
       "2023-01-02  [{'name': 'subject', 'value': 'Content Type: S...   \n",
       "2023-01-04                                                 []   \n",
       "2023-01-05  [{'name': 'organizations', 'value': 'Bed Bath ...   \n",
       "2023-01-06  [{'name': 'subject', 'value': 'Insider Trading...   \n",
       "2023-01-08  [{'name': 'subject', 'value': 'Stocks and Bond...   \n",
       "...                                                       ...   \n",
       "2023-12-23  [{'name': 'subject', 'value': 'Stocks and Bond...   \n",
       "2023-12-26  [{'name': 'subject', 'value': 'Stocks and Bond...   \n",
       "2023-12-27  [{'name': 'glocations', 'value': 'China', 'ran...   \n",
       "2023-12-29  [{'name': 'glocations', 'value': 'Hong Kong', ...   \n",
       "2023-12-30  [{'name': 'subject', 'value': 'internal-storyl...   \n",
       "\n",
       "                            pub_date document_type       news_desk  \\\n",
       "Date                                                                 \n",
       "2023-01-02 2023-01-02 04:00:06+00:00       article            Well   \n",
       "2023-01-04 2023-01-04 00:00:00+00:00        recipe                   \n",
       "2023-01-05 2023-01-05 14:12:37+00:00       article        Business   \n",
       "2023-01-06 2023-01-06 20:25:30+00:00       article      Washington   \n",
       "2023-01-08 2023-01-08 12:00:06+00:00       article  SundayBusiness   \n",
       "...                              ...           ...             ...   \n",
       "2023-12-23 2023-12-23 10:03:30+00:00       article  SundayBusiness   \n",
       "2023-12-26 2023-12-26 10:04:45+00:00       article        Business   \n",
       "2023-12-27 2023-12-27 07:56:49+00:00       article        Business   \n",
       "2023-12-29 2023-12-29 08:29:18+00:00       article        Business   \n",
       "2023-12-30 2023-12-30 12:51:24+00:00       article        Business   \n",
       "\n",
       "            section_name                                             byline  \\\n",
       "Date                                                                          \n",
       "2023-01-02          Well  {'original': 'By Jancee Dunn', 'person': [{'fi...   \n",
       "2023-01-04           NaN  {'original': 'Kevin McKenna, Alexa Weibel', 'p...   \n",
       "2023-01-05  Business Day  {'original': 'By Lauren Hirsch and Jordyn Holm...   \n",
       "2023-01-06          U.S.  {'original': 'By Kate Kelly', 'person': [{'fir...   \n",
       "2023-01-08  Business Day  {'original': 'By Marie Solis', 'person': [{'fi...   \n",
       "...                  ...                                                ...   \n",
       "2023-12-23  Business Day  {'original': 'By Jeff Sommer', 'person': [{'fi...   \n",
       "2023-12-26  Business Day  {'original': 'By Joe Rennison', 'person': [{'f...   \n",
       "2023-12-27  Business Day  {'original': 'By John Liu', 'person': [{'first...   \n",
       "2023-12-29  Business Day  {'original': 'By Alexandra Stevenson', 'person...   \n",
       "2023-12-30  Business Day  {'original': 'By Christine Zhang', 'person': [...   \n",
       "\n",
       "           type_of_material  \\\n",
       "Date                          \n",
       "2023-01-02             News   \n",
       "2023-01-04           Recipe   \n",
       "2023-01-05             News   \n",
       "2023-01-06             News   \n",
       "2023-01-08             News   \n",
       "...                     ...   \n",
       "2023-12-23             News   \n",
       "2023-12-26             News   \n",
       "2023-12-27             News   \n",
       "2023-12-29             News   \n",
       "2023-12-30             News   \n",
       "\n",
       "                                                          _id  word_count  \\\n",
       "Date                                                                        \n",
       "2023-01-02  nyt://article/3dfb3527-68cd-5db7-9504-7afc5389...        1172   \n",
       "2023-01-04  nyt://recipe/162d357c-e7cc-5f6d-a3eb-1f46ca317fda           0   \n",
       "2023-01-05  nyt://article/86579820-0588-51bc-8125-d4002942...        1226   \n",
       "2023-01-06  nyt://article/482c9261-2dea-5f7b-914f-39bc2ebf...         684   \n",
       "2023-01-08  nyt://article/bdf2d306-799b-5299-8efb-1289a0ce...         966   \n",
       "...                                                       ...         ...   \n",
       "2023-12-23  nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...        1108   \n",
       "2023-12-26  nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...        1100   \n",
       "2023-12-27  nyt://article/11afe169-2c2e-528b-a728-a56d25a2...         581   \n",
       "2023-12-29  nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...         779   \n",
       "2023-12-30  nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...        1055   \n",
       "\n",
       "                                                          uri subsection_name  \n",
       "Date                                                                           \n",
       "2023-01-02  nyt://article/3dfb3527-68cd-5db7-9504-7afc5389...             NaN  \n",
       "2023-01-04  nyt://recipe/162d357c-e7cc-5f6d-a3eb-1f46ca317fda             NaN  \n",
       "2023-01-05  nyt://article/86579820-0588-51bc-8125-d4002942...             NaN  \n",
       "2023-01-06  nyt://article/482c9261-2dea-5f7b-914f-39bc2ebf...        Politics  \n",
       "2023-01-08  nyt://article/bdf2d306-799b-5299-8efb-1289a0ce...             NaN  \n",
       "...                                                       ...             ...  \n",
       "2023-12-23  nyt://article/c842fad6-b1db-5bd7-8142-0c256d17...             NaN  \n",
       "2023-12-26  nyt://article/70d5bf00-69a4-5f68-a5e6-7db124cc...             NaN  \n",
       "2023-12-27  nyt://article/11afe169-2c2e-528b-a728-a56d25a2...             NaN  \n",
       "2023-12-29  nyt://article/cad8aa00-d337-59cd-b827-9cf1f895...             NaN  \n",
       "2023-12-30  nyt://article/2e18c2dc-f929-554d-bbbc-fbce89e5...        DealBook  \n",
       "\n",
       "[209 rows x 20 columns]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NYdata.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "NYdata.to_csv('NYdata.csv', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "Data_combined = pd.merge(NYdata, sp_price, left_index=True, right_index=True, how='outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/g7/0r52jtgn0lx64hw0zmgwgr2h0000gn/T/ipykernel_7914/113215311.py:1: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  Data_combined['abstract'][1]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Data_combined['abstract'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "Data_combined.to_csv('NYdata_SP.csv', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "Data_combined_dict={}\n",
    "Data_combined_dict['train']=Data_combined.loc[Data_combined.index.month<11]\n",
    "Data_combined_dict['validation']=Data_combined.loc[Data_combined.index.month==11]\n",
    "Data_combined_dict['test']=Data_combined.loc[Data_combined.index.month>11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Data_combined_dict['validate'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset, DatasetDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_dict = {key: Dataset.from_pandas(df) for key, df in Data_combined_dict.items()}\n",
    "\n",
    "# Create a DatasetDict object\n",
    "dataset_dict = DatasetDict(datasets_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['abstract', 'web_url', 'snippet', 'lead_paragraph', 'print_section', 'print_page', 'source', 'multimedia', 'headline', 'keywords', 'pub_date', 'document_type', 'news_desk', 'section_name', 'byline', 'type_of_material', '_id', 'word_count', 'uri', 'subsection_name', 'Open', 'High', 'Low', 'Close', 'Date'],\n",
       "        num_rows: 392\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['abstract', 'web_url', 'snippet', 'lead_paragraph', 'print_section', 'print_page', 'source', 'multimedia', 'headline', 'keywords', 'pub_date', 'document_type', 'news_desk', 'section_name', 'byline', 'type_of_material', '_id', 'word_count', 'uri', 'subsection_name', 'Open', 'High', 'Low', 'Close', 'Date'],\n",
       "        num_rows: 25\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['abstract', 'web_url', 'snippet', 'lead_paragraph', 'print_section', 'print_page', 'source', 'multimedia', 'headline', 'keywords', 'pub_date', 'document_type', 'news_desk', 'section_name', 'byline', 'type_of_material', '_id', 'word_count', 'uri', 'subsection_name', 'Open', 'High', 'Low', 'Close', 'Date'],\n",
       "        num_rows: 33\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5bbb6903b5ac4d2c902004052ae62bf5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/392 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb1e72a495ed47adb1f74042d1a113c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/25 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b344ebbf05ca403985517a30c3306f9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/33 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset_dict.save_to_disk('NYdata_SP_datset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-03-08 12:00:20+00:00 2023-03-08\n"
     ]
    }
   ],
   "source": [
    "print(NYdata['pub_date'][361], NYdata['Date'][361])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_from_disk\n",
    "dataset_dict2 = load_from_disk('NYdata_SP_datset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['abstract', 'web_url', 'snippet', 'lead_paragraph', 'print_section', 'print_page', 'source', 'multimedia', 'headline', 'keywords', 'pub_date', 'document_type', 'news_desk', 'section_name', 'byline', 'type_of_material', '_id', 'word_count', 'uri', 'subsection_name', 'Open', 'High', 'Low', 'Close', 'Date'],\n",
       "        num_rows: 392\n",
       "    })\n",
       "    validate: Dataset({\n",
       "        features: ['abstract', 'web_url', 'snippet', 'lead_paragraph', 'print_section', 'print_page', 'source', 'multimedia', 'headline', 'keywords', 'pub_date', 'document_type', 'news_desk', 'section_name', 'byline', 'type_of_material', '_id', 'word_count', 'uri', 'subsection_name', 'Open', 'High', 'Low', 'Close', 'Date'],\n",
       "        num_rows: 25\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['abstract', 'web_url', 'snippet', 'lead_paragraph', 'print_section', 'print_page', 'source', 'multimedia', 'headline', 'keywords', 'pub_date', 'document_type', 'news_desk', 'section_name', 'byline', 'type_of_material', '_id', 'word_count', 'uri', 'subsection_name', 'Open', 'High', 'Low', 'Close', 'Date'],\n",
       "        num_rows: 33\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_dict2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "npss",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}