sentiment_analyzer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RG1_hbnQbPlQ"
      },
      "source": [
        "# Sentiment Analyzer for News Quotes\n",
        "\n",
        "1. Read cleaned data into a pandas dataframe\n",
        "2. Pass quotes from each article into sentiment analyzer\n",
        "3. Save output into new columns 'negative', 'neutral', 'positive', 'compound'\n",
        "4. Save output into new excel sheet with two sheets, one for quotes and one for non-quotes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bhjOINLHbELR"
      },
      "outputs": [],
      "source": [
        "# run this code if connecting to a Google drive\n",
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JfZEJ-7meKr4"
      },
      "outputs": [],
      "source": [
        "!pip install vaderSentiment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RIRCT71CeIOr"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gW1EacgmeTc7"
      },
      "source": [
        "## Extracting Data from Excel Files"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UdKIjPwKeSwd"
      },
      "outputs": [],
      "source": [
        "# replace with quotes_input.xlsx\n",
        "fp = '/content/drive/My Drive/evaluation_quotes/CBC_qt_output/2023_12_dec_CBC_news_clean_new.xlsx'\n",
        "\n",
        "quotes_df = pd.read_excel(fp, usecols = [\"text_id\", \"text_name\", \"quote\"])\n",
        "\n",
        "non_quotes_df = pd.read_excel(fp, usecols = [\"text_id\", \"text_name\", \"non_quoted_text\"])\n",
        "\n",
        "speakers_df = pd.read_excel(fp, usecols = [\"text_id\", \"text_name\", \"speaker\"])\n",
        "\n",
        "verbs_df = pd.read_excel(fp, usecols = [\"text_id\", \"text_name\", \"verb\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dagWdg12lOI4"
      },
      "outputs": [],
      "source": [
        "# add new columns to dataframes\n",
        "quotes_df['positive'] = pd.Series(dtype='float')\n",
        "quotes_df['negative'] = pd.Series(dtype='float')\n",
        "quotes_df['neutral'] = pd.Series(dtype='float')\n",
        "quotes_df['compound'] = pd.Series(dtype='float')\n",
        "\n",
        "non_quotes_df['positive'] = pd.Series(dtype='float')\n",
        "non_quotes_df['negative'] = pd.Series(dtype='float')\n",
        "non_quotes_df['neutral'] = pd.Series(dtype='float')\n",
        "non_quotes_df['compound'] = pd.Series(dtype='float')\n",
        "\n",
        "speakers_df['positive'] = pd.Series(dtype='float')\n",
        "speakers_df['negative'] = pd.Series(dtype='float')\n",
        "speakers_df['neutral'] = pd.Series(dtype='float')\n",
        "speakers_df['compound'] = pd.Series(dtype='float')\n",
        "\n",
        "verbs_df['positive'] = pd.Series(dtype='float')\n",
        "verbs_df['negative'] = pd.Series(dtype='float')\n",
        "verbs_df['neutral'] = pd.Series(dtype='float')\n",
        "verbs_df['compound'] = pd.Series(dtype='float')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rn8yx9rFhS2T"
      },
      "source": [
        "## Running Quotes through VADER\n",
        "\n",
        "Vader Documentation: https://github.com/cjhutto/vaderSentiment\n",
        "\n",
        "Guide to using Vader: https://medium.com/@rslavanyageetha/vader-a-comprehensive-guide-to-sentiment-analysis-in-python-c4f1868b0d2e\n",
        "\n",
        "Vader sentiment analyzer returns a dictionary of sentiment intensity scores for\n",
        "a particular text input with the following sentiments: negative, neutral,\n",
        "positive, and compound for overall sentiment intensity. The negative, neutral,\n",
        "and positive scores have a value from 0 to 1 and compound scores have a\n",
        "value from -1 to 1, with -1 indicating entirely negative, 0 indicating\n",
        "entirely neutral, and 1 indicating entirely positive.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sncldllAinK1"
      },
      "outputs": [],
      "source": [
        "# helper function to extract scores for each story\n",
        "# args: dataframe, column name of text to be analyzed as a string\n",
        "def get_sentiment_score(df, col):\n",
        "\n",
        "  for index, row in df.iterrows():\n",
        "    analyzer = SentimentIntensityAnalyzer()\n",
        "    score = analyzer.polarity_scores(df[col][index])\n",
        "\n",
        "    df.loc[index, 'positive'] = score['pos']\n",
        "    df.loc[index, 'negative'] = score['neg']\n",
        "    df.loc[index, 'neutral'] = score['neu']\n",
        "    df.loc[index, 'compound'] = score['compound']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0BHvVGmngPKo"
      },
      "outputs": [],
      "source": [
        "# get sentiment scores for all four dataframes\n",
        "get_sentiment_score(quotes_df, 'quote')\n",
        "\n",
        "get_sentiment_score(non_quotes_df, 'non_quoted_text')\n",
        "\n",
        "get_sentiment_score(speakers_df, 'speaker')\n",
        "\n",
        "get_sentiment_score(verbs_df, 'verb')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "Sb1t0s5grHyv",
        "outputId": "fcb277eb-9051-41fe-b1f5-575a1ed12b65"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            text_id                 text_name  \\\n",
              "0  001b0621b7d3f51059f7916f8e7667d4  6578feec58b434ef00050084   \n",
              "1  0047ea8b7faf501b9b6ddb8763f1c8b5  6578876058b434ef0001648e   \n",
              "2  005daa355633d0162360c026c3b67079  65718d7e58b434ef00c88628   \n",
              "3  0065db8f249888574dffa4da85775cd5  6570eb2258b434ef00c361fa   \n",
              "4  0070093b109fa990775e758fd1b3b0cc  6569aaf258b434ef00879ae6   \n",
              "\n",
              "                                               quote  positive  negative  \\\n",
              "0  that reports of Vancouver police denying acces...     0.042     0.152   \n",
              "1  that the federal government will begin a consu...     0.000     0.056   \n",
              "2  he believes airlines are dropping base fares a...     0.061     0.058   \n",
              "3  Moore was working on the top of one of the hom...     0.090     0.026   \n",
              "4  his bid would have cost $3.5 million per year,...     0.102     0.027   \n",
              "\n",
              "   neutral  compound  \n",
              "0    0.806   -0.9327  \n",
              "1    0.944   -0.5994  \n",
              "2    0.881    0.6617  \n",
              "3    0.884    0.9756  \n",
              "4    0.871    0.9371  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a494c970-dccb-4a2d-b33f-4b0eb092056e\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_id</th>\n",
              "      <th>text_name</th>\n",
              "      <th>quote</th>\n",
              "      <th>positive</th>\n",
              "      <th>negative</th>\n",
              "      <th>neutral</th>\n",
              "      <th>compound</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>001b0621b7d3f51059f7916f8e7667d4</td>\n",
              "      <td>6578feec58b434ef00050084</td>\n",
              "      <td>that reports of Vancouver police denying acces...</td>\n",
              "      <td>0.042</td>\n",
              "      <td>0.152</td>\n",
              "      <td>0.806</td>\n",
              "      <td>-0.9327</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0047ea8b7faf501b9b6ddb8763f1c8b5</td>\n",
              "      <td>6578876058b434ef0001648e</td>\n",
              "      <td>that the federal government will begin a consu...</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.056</td>\n",
              "      <td>0.944</td>\n",
              "      <td>-0.5994</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>005daa355633d0162360c026c3b67079</td>\n",
              "      <td>65718d7e58b434ef00c88628</td>\n",
              "      <td>he believes airlines are dropping base fares a...</td>\n",
              "      <td>0.061</td>\n",
              "      <td>0.058</td>\n",
              "      <td>0.881</td>\n",
              "      <td>0.6617</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0065db8f249888574dffa4da85775cd5</td>\n",
              "      <td>6570eb2258b434ef00c361fa</td>\n",
              "      <td>Moore was working on the top of one of the hom...</td>\n",
              "      <td>0.090</td>\n",
              "      <td>0.026</td>\n",
              "      <td>0.884</td>\n",
              "      <td>0.9756</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0070093b109fa990775e758fd1b3b0cc</td>\n",
              "      <td>6569aaf258b434ef00879ae6</td>\n",
              "      <td>his bid would have cost $3.5 million per year,...</td>\n",
              "      <td>0.102</td>\n",
              "      <td>0.027</td>\n",
              "      <td>0.871</td>\n",
              "      <td>0.9371</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a494c970-dccb-4a2d-b33f-4b0eb092056e')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a494c970-dccb-4a2d-b33f-4b0eb092056e button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a494c970-dccb-4a2d-b33f-4b0eb092056e');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-f20e5fc9-8ebe-4c27-aad4-26415a35479b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f20e5fc9-8ebe-4c27-aad4-26415a35479b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-f20e5fc9-8ebe-4c27-aad4-26415a35479b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "quotes_df",
              "summary": "{\n  \"name\": \"quotes_df\",\n  \"rows\": 3520,\n  \"fields\": [\n    {\n      \"column\": \"text_id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"b4021a27d9ca7de33e291929a391242b\",\n          \"042aadbf72e4499a15e4eecf3b7e1e48\",\n          \"1a40c95c83e3955defb5551b7a3beed9\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text_name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"6569bf4758b434ef00884200\",\n          \"658db38f58b434ef00a3c1d1\",\n          \"6572de5f58b434ef00d35177\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3503,\n        \"samples\": [\n          \"Eric Montross, a former North Carolina star and Toronto Raptors big man, has died after a cancer fight\\nMontross died Sunday surrounded by loved ones at his Chapel Hill home\\nand the family knows that the ripples from the generous, thoughtful way that he lived his life will continue in the lives of the many people he touched with his deep and sincere kindness\\n\\\"Eric was a great player and accomplished student, but the impacts he made on our community went way beyond the basketball court\\n\\\"He was a man of faith, a tremendous father, husband and son, and one of the most recognizable ambassadors of the University and Chapel Hill.\\\"\\n\\n\",\n          \"Police officers, firefighters and paramedics responded to a report of a crash near Whiteside just before 6 p.m.,\\nthe driver of the van, a 42-year-old Halifax woman, appeared to have life threatening injuries\\nThe driver of the car, a 28-year-old New Brunswick man, appeared to have life-threatening injuries,\\nboth drivers were removed from the vehicles with the help of fire services\",\n          \"fans should just stop complaining\\nBut weeks later, the 58-year-old Burlington, Ont., resident was dead \\u2014 killed not by the cancer\\n\\\"It was honestly a cruel rollercoaster of emotions\\nBut those tests may be giving some cancer patients and their families a false sense of security;\\nRead more\\n\\nOn average, electric vehicles are less reliable than other cars and trucks\\nElectric vehicles are far less reliable than gasoline-powered cars, trucks and SUVs, mainly because most automakers are still learning how to build a completely new power system\\nElectric vehicles have proved far less reliable, on average, than gasoline-powered cars, trucks and SUVs,\\nEV owners most frequently reported troubles with battery and charging systems and flaws in how the vehicles' body panels and interior parts fit together\\nthat EV manufacturers are still learning to construct completely new power systems, and suggested that, as they do, the overall reliability of electric vehicles should improve\\n\\\"This story is really one of growing pains\\nthat lingering concerns about reliability will likely add to the issues that give many buyers pause when considering a switch to the new technology, joining concerns about higher costs, too few charging stations and long charging times\\nmarketplace@cbc.ca\\n\\nFinding room for carry-on baggage has become 'the Hunger Games' of air travel\\nWhen more passengers are vying for limited space for carry-on luggage, it leads to costly delays\\nthat the box was above the size limits for carry-on baggage\\nAnd it was a really kind of meaningful, sentimental part of this wedding that was about to happen\\nShe and her colleagues spent at least 10 minutes moving people and their carry-on baggage around the plane so that they could make room for the cake box on the floor of a window seat where it wouldn't block anyone's exit in the event of an emergency\\npassengers have become carried away with carry-on baggage, leading to costly delays\\nthis change needs to happen, and that passengers should be allowed to check a bag for free\\nAnd people who want the convenience and control would pay to bring their bags on the plane\\nairlines would likely generate the same or more in baggage revenue\\n\\\"The People vs. The Ticket Giant\\\"\\n\\\"I just feel like we were robbed of time together.\\\"\\n\\\"It's a story of just working out the bugs and the kinks of new technology.\\\"\\n\\\"The basic safety rules and written directives regarding emergency measures for all foreseeable situations...were ignored,\\\"\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"positive\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06460388735924111,\n        \"min\": 0.0,\n        \"max\": 0.767,\n        \"num_unique_values\": 293,\n        \"samples\": [\n          0.016,\n          0.023,\n          0.066\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"negative\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06055901902048814,\n        \"min\": 0.0,\n        \"max\": 0.443,\n        \"num_unique_values\": 283,\n        \"samples\": [\n          0.045,\n          0.199,\n          0.186\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"neutral\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.07206613380825744,\n        \"min\": 0.233,\n        \"max\": 1.0,\n        \"num_unique_values\": 353,\n        \"samples\": [\n          0.918,\n          0.688,\n          0.778\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"compound\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.7760462126061396,\n        \"min\": -0.9996,\n        \"max\": 0.9994,\n        \"num_unique_values\": 2007,\n        \"samples\": [\n          0.106,\n          -0.886,\n          0.8176\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 236
        }
      ],
      "source": [
        "quotes_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "KBRIMdV3tbd6",
        "outputId": "ff082d18-b7ff-474d-8eb6-5449b2fd9ca5"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            text_id                 text_name  \\\n",
              "0  001b0621b7d3f51059f7916f8e7667d4  6578feec58b434ef00050084   \n",
              "1  0047ea8b7faf501b9b6ddb8763f1c8b5  6578876058b434ef0001648e   \n",
              "2  005daa355633d0162360c026c3b67079  65718d7e58b434ef00c88628   \n",
              "3  0065db8f249888574dffa4da85775cd5  6570eb2258b434ef00c361fa   \n",
              "4  0070093b109fa990775e758fd1b3b0cc  6569aaf258b434ef00879ae6   \n",
              "\n",
              "                                     non_quoted_text  positive  negative  \\\n",
              "0  British Columbia's human rights commissioner h...     0.105     0.000   \n",
              "1  The federal government is planning to launch a...     0.000     0.000   \n",
              "2  Can you fly round-trip from Toronto to Lisbon ...     0.063     0.040   \n",
              "3  Jurors in a coroner's inquest on Monday heard ...     0.060     0.054   \n",
              "4  Dr. Todd Young owns virtual health clinic Medi...     0.146     0.026   \n",
              "\n",
              "   neutral  compound  \n",
              "0    0.895    0.6059  \n",
              "1    1.000    0.0000  \n",
              "2    0.897    0.8901  \n",
              "3    0.886    0.4299  \n",
              "4    0.828    0.9935  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d70e8753-eecb-4d2f-ad27-6290f1dbfc6d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_id</th>\n",
              "      <th>text_name</th>\n",
              "      <th>non_quoted_text</th>\n",
              "      <th>positive</th>\n",
              "      <th>negative</th>\n",
              "      <th>neutral</th>\n",
              "      <th>compound</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>001b0621b7d3f51059f7916f8e7667d4</td>\n",
              "      <td>6578feec58b434ef00050084</td>\n",
              "      <td>British Columbia's human rights commissioner h...</td>\n",
              "      <td>0.105</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.895</td>\n",
              "      <td>0.6059</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0047ea8b7faf501b9b6ddb8763f1c8b5</td>\n",
              "      <td>6578876058b434ef0001648e</td>\n",
              "      <td>The federal government is planning to launch a...</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>005daa355633d0162360c026c3b67079</td>\n",
              "      <td>65718d7e58b434ef00c88628</td>\n",
              "      <td>Can you fly round-trip from Toronto to Lisbon ...</td>\n",
              "      <td>0.063</td>\n",
              "      <td>0.040</td>\n",
              "      <td>0.897</td>\n",
              "      <td>0.8901</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0065db8f249888574dffa4da85775cd5</td>\n",
              "      <td>6570eb2258b434ef00c361fa</td>\n",
              "      <td>Jurors in a coroner's inquest on Monday heard ...</td>\n",
              "      <td>0.060</td>\n",
              "      <td>0.054</td>\n",
              "      <td>0.886</td>\n",
              "      <td>0.4299</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0070093b109fa990775e758fd1b3b0cc</td>\n",
              "      <td>6569aaf258b434ef00879ae6</td>\n",
              "      <td>Dr. Todd Young owns virtual health clinic Medi...</td>\n",
              "      <td>0.146</td>\n",
              "      <td>0.026</td>\n",
              "      <td>0.828</td>\n",
              "      <td>0.9935</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d70e8753-eecb-4d2f-ad27-6290f1dbfc6d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-d70e8753-eecb-4d2f-ad27-6290f1dbfc6d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-d70e8753-eecb-4d2f-ad27-6290f1dbfc6d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0b7e59fd-df71-4489-8703-2a6f48098494\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0b7e59fd-df71-4489-8703-2a6f48098494')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0b7e59fd-df71-4489-8703-2a6f48098494 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "non_quotes_df",
              "summary": "{\n  \"name\": \"non_quotes_df\",\n  \"rows\": 3520,\n  \"fields\": [\n    {\n      \"column\": \"text_id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"b4021a27d9ca7de33e291929a391242b\",\n          \"042aadbf72e4499a15e4eecf3b7e1e48\",\n          \"1a40c95c83e3955defb5551b7a3beed9\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text_name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"6569bf4758b434ef00884200\",\n          \"658db38f58b434ef00a3c1d1\",\n          \"6572de5f58b434ef00d35177\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quoted_text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"  .\\n\\n  .\\n\\nThat's now led to the cancellation of Fort Qu'Appelle's annual winter festival. The Fort Winter Festival is typically held over three days in March, with family-friendly events like fireworks, uphill sled drags, dog sledding and horse-drawn wagon rides.\\n\\nBut that isn't in the cards for this coming March.\\n\\n,\\\"   in a Facebook post for the festival.\\n\\n  .\\n\\n\\\"It's sad that they won't be hosting it this year. ,\\\"  , referring to the money and spirit the festival brought to the town.\\n\\nSpring and summer festivals are also in jeopardy due to a lack of volunteers.\\n\\n  , but volunteers are needed for that to continue. (Submitted by Sean Stares)\\n\\nThe Windscape Kite Festival, which has filled Swift Current's skies with colourful kites from around the world for more than a decade, usually runs alongside the Long Day's Night Music Festival \\u2014 but both festivals are now struggling with finances and staffing.\\n\\n  \\n\\n\\\"We just hit a point where we couldn't continue. ,\\\"  .\\n\\n\\n\\n  . ,  .\\n\\nThe more the merrier \\u2014 literally\\n\\nMeanwhile, the Moose Jaw Music Festival \\u2014 which features musical performers of all ages, skill levels and disciplines, including choir and musical theatre \\u2014 is in desperate need of volunteers as well.\\n\\n  . But organizers couldn't even get 20 this past year.\\n\\nA young man performs during the Moose Jaw Music Festival.   . (Submitted by Janie Fries)\\n\\n,  .\\n\\n,\\\"  .\\n\\nShe understands why some people are pulling back from committing to non-work activities.\\n\\n\\\"I think one of the benefits that came out of COVID is that we as a society are much more aware of mental health. ,\\\"  .\\n\\n\\n\\nHowever,  more volunteers would help current festival organizers and volunteers balance their own mental health better, said Fries. With more volunteers, there's less responsibility and demand on time placed on any one person.\\n\\n,  , and she encourages parents of musicians especially to sign up to volunteer.\\n\\nLa Ronge Children's Festival volunteers enjoy the sun and lending a helping hand during the July annual event. (Submitted by Sean Stares)\\n\\nUp north, the La Ronge Children's Festival is also struggling to get both volunteers and committee members.   that pre-pandemic. This past year, there were only three.\\n\\n  .\\n\\n\\\"  . But we need people to be able to help choose programming, writing grants and being the treasurer.\\\"\\n\\n  .\\n\\n  .\\n\\n\\\"The lack of volunteers makes it so it's a stress on the people who do volunteer. ,\\\"  .\\n\\nHaving more volunteers to share the work would likely inspire people to keep volunteering every year.\\n\\n\\\"It's fun if you put in the time. It's valuable and  can  .\\\"\",\n          \"A Montreal man    is seeking permission to launch a class-action lawsuit against two food companies.\\n\\nLaw firm Slater Vecchio LLP filed the application Dec. 12 in Quebec Superior Court alleging that Olivier Archambault got sick after he consumed melons produced by Mexico-based Malichita and distributed in Canada by Arizona-based Trufresh.\\n\\nIn November, the Canadian Food Inspection Agency issued food recall warnings for melons produced by Malichita, and later in the month issued recalls for melons by Rudy brand, which are produced in the same area of Mexico.\\n\\nAs of last week the Public Health Agency of Canada has reported 164 laboratory-confirmed cases and seven deaths from cantaloupe-linked salmonella across eight provinces; 111 of those cases are in Quebec.\\n\\n  .\\n\\n  .\\n\\nThe proposed class action, which has not been authorized or tested in court, is seeking unspecified damages on behalf of individuals in Quebec who purchased and consumed cantaloupes and other fruit in October and November that were the subject of Health Canada recalls.\",\n          \"  .\\n\\nMegan Walker is vice-chair of the board that provides civilian oversight over police. She's also the former director of the London Abused Women's Centre.\\n\\nShe's concerned police might not always release as much information as they can or should in cases of possible intimate-partner violence.   withholding details, including the names of those involved, .\\n\\nWalker's concerns come in the wake of the death of Tiffany Gates and a CBC News report that has Gates' mother asking the London Police Service why details of her death, including her name, have not been released.\\n\\nGates, 30, was found dead in the apartment of her boyfriend on Sept. 7.   . Their names, and the nature of their relationship, only became public her friends and family began to post condolences on social media.\\n\\nFor now,  will only  .\\n\\nTell us what you think! Help shape the future of CBC article pages by taking a quick survey. ,\\\"  . \\n\\nGates's mother Linda Davidson has questioned why police haven't released more information so her daughter's death can be correctly identified as the death of a woman at the hands of her intimate partner.\\n\\nNo standard rules for reporting names of victims\\n\\nAmong Canadian police forces, there is no standard set of rules about how, if, or when police release the names of victims of violent crimes or the people charged with them.\\n\\nA 2019 report commissioned by the Edmonton Police surveyed forces about their polices and found that:\\n\\nMore than one-third (36 per cent) of the responding police services release the names of all homicide victims.\\n\\nMore than one-half (54 per cent) release the names depending on circumstances.\\n\\nOnly seven per cent always withhold victim's names.\\n\\nIn the wake of the report, the Edmonton Police changed their disclosure policy, so the default position will be to name all homicide victims, with some exceptions.\\n\\nYork approach: Case-by-case basis\\n\\nYork Regional Police, for example, take a case-by-case approach on deciding what information they release. In a statement to CBC News,   .\\n\\n,\\\"  . Lisa Moskaluk. ;  also  the name of the man charged with her murder.\\n\\nYork Police didn't disclose the nature of their relationship, though reporters later confirmed the victim and accused were in a relationship.   .\\\". 've  .\\n\\n,  \\n\\n.   a book about the issues surrounding .\\n\\n  .\\n\\nIn cases such as the Tiffany Gates's death \\u2014 where both people involved are dead \\u2014 she doesn't see reason to withold their names.\\n\\n ,\\\"  .\\n\\n  .\\n\\n\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"positive\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.04180939697571297,\n        \"min\": 0.0,\n        \"max\": 0.313,\n        \"num_unique_values\": 210,\n        \"samples\": [\n          0.02,\n          0.132,\n          0.01\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"negative\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.049951429196720624,\n        \"min\": 0.0,\n        \"max\": 0.33,\n        \"num_unique_values\": 231,\n        \"samples\": [\n          0.218,\n          0.047,\n          0.149\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"neutral\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.05464821865641968,\n        \"min\": 0.612,\n        \"max\": 1.0,\n        \"num_unique_values\": 282,\n        \"samples\": [\n          0.963,\n          0.867,\n          0.923\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"compound\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.820569289255619,\n        \"min\": -0.9999,\n        \"max\": 0.9999,\n        \"num_unique_values\": 1945,\n        \"samples\": [\n          0.126,\n          -0.989,\n          0.9622\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 237
        }
      ],
      "source": [
        "non_quotes_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "GWbhOkevswxr",
        "outputId": "7fbf6c76-21af-4bfc-e15d-9d6a7ebe7e11"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            text_id                 text_name  \\\n",
              "0  001b0621b7d3f51059f7916f8e7667d4  6578feec58b434ef00050084   \n",
              "1  0047ea8b7faf501b9b6ddb8763f1c8b5  6578876058b434ef0001648e   \n",
              "2  005daa355633d0162360c026c3b67079  65718d7e58b434ef00c88628   \n",
              "3  0065db8f249888574dffa4da85775cd5  6570eb2258b434ef00c361fa   \n",
              "4  0070093b109fa990775e758fd1b3b0cc  6569aaf258b434ef00879ae6   \n",
              "\n",
              "                                             speaker  positive  negative  \\\n",
              "0  Kasari Govender\\nGovender\\nShe\\nVancouver Poli...     0.000     0.000   \n",
              "1  Housing Minister Sean Fraser\\nFraser\\nFraser\\n...     0.000     0.000   \n",
              "2  Jeff Verman, CEO of Plus Travel Group,\\nVerman...     0.000     0.024   \n",
              "3  Cyr\\nCyr\\nCyr\\nTim Tompkins, owner of TNT Insu...     0.103     0.000   \n",
              "4  Young\\nYoung\\nYoung\\nHe\\nhe\\nspokesperson Trac...     0.000     0.000   \n",
              "\n",
              "   neutral  compound  \n",
              "0    1.000    0.0000  \n",
              "1    1.000    0.0000  \n",
              "2    0.976   -0.2960  \n",
              "3    0.897    0.6808  \n",
              "4    1.000    0.0000  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-45bd87fc-b1dc-4344-93a2-fe1e7b95c7c1\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_id</th>\n",
              "      <th>text_name</th>\n",
              "      <th>speaker</th>\n",
              "      <th>positive</th>\n",
              "      <th>negative</th>\n",
              "      <th>neutral</th>\n",
              "      <th>compound</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>001b0621b7d3f51059f7916f8e7667d4</td>\n",
              "      <td>6578feec58b434ef00050084</td>\n",
              "      <td>Kasari Govender\\nGovender\\nShe\\nVancouver Poli...</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0047ea8b7faf501b9b6ddb8763f1c8b5</td>\n",
              "      <td>6578876058b434ef0001648e</td>\n",
              "      <td>Housing Minister Sean Fraser\\nFraser\\nFraser\\n...</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>005daa355633d0162360c026c3b67079</td>\n",
              "      <td>65718d7e58b434ef00c88628</td>\n",
              "      <td>Jeff Verman, CEO of Plus Travel Group,\\nVerman...</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.024</td>\n",
              "      <td>0.976</td>\n",
              "      <td>-0.2960</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0065db8f249888574dffa4da85775cd5</td>\n",
              "      <td>6570eb2258b434ef00c361fa</td>\n",
              "      <td>Cyr\\nCyr\\nCyr\\nTim Tompkins, owner of TNT Insu...</td>\n",
              "      <td>0.103</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.897</td>\n",
              "      <td>0.6808</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0070093b109fa990775e758fd1b3b0cc</td>\n",
              "      <td>6569aaf258b434ef00879ae6</td>\n",
              "      <td>Young\\nYoung\\nYoung\\nHe\\nhe\\nspokesperson Trac...</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-45bd87fc-b1dc-4344-93a2-fe1e7b95c7c1')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-45bd87fc-b1dc-4344-93a2-fe1e7b95c7c1 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-45bd87fc-b1dc-4344-93a2-fe1e7b95c7c1');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-d492e1b6-0dc0-441a-88cf-4264f621de81\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d492e1b6-0dc0-441a-88cf-4264f621de81')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-d492e1b6-0dc0-441a-88cf-4264f621de81 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "speakers_df",
              "summary": "{\n  \"name\": \"speakers_df\",\n  \"rows\": 3520,\n  \"fields\": [\n    {\n      \"column\": \"text_id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"b4021a27d9ca7de33e291929a391242b\",\n          \"042aadbf72e4499a15e4eecf3b7e1e48\",\n          \"1a40c95c83e3955defb5551b7a3beed9\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text_name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"6569bf4758b434ef00884200\",\n          \"658db38f58b434ef00a3c1d1\",\n          \"6572de5f58b434ef00d35177\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"speaker\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3468,\n        \"samples\": [\n          \"Experts in environmental law\\nA system failure'\\n\\nNigel Bankes, an emeritus professor of law at the University of Calgary who specializes in natural resources and energy law,\\nBankes\\nBankes\\nthe order\\nThe order\\nJason Unger, executive director of Edmonton's Environmental Law Centre,\\nUnger\\nAltaGas\\nAltaGas\\nthe company\\nAlberta Environment\\nThe ministry\\nThe ministry\\nU\\nswered questions\\n\\nUnger\\nHe\\nHe\\nhe\\nUnger\\nhe\\nWe\\n\\n\\n\\n\\n\\n\\n\\n\",\n          \"police\\npolice\\nWatson\\nhe\\nToronto police\\nWatson\\n\\n\",\n          \"The province\\nScott Nauss, the department's senior executive director of safety,\\nCorkum\\nThe province\\nthe department\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"positive\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06995446450024399,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 255,\n        \"samples\": [\n          0.1,\n          0.217,\n          0.174\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"negative\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0548223964766091,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 211,\n        \"samples\": [\n          0.022,\n          0.013,\n          0.172\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"neutral\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.10010639770695579,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 310,\n        \"samples\": [\n          0.728,\n          0.783,\n          0.915\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"compound\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.3191961363764673,\n        \"min\": -0.969,\n        \"max\": 0.9781,\n        \"num_unique_values\": 220,\n        \"samples\": [\n          0.6341,\n          -0.3875,\n          -0.765\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 238
        }
      ],
      "source": [
        "speakers_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "9mIm7m_F0ZGA",
        "outputId": "d0864746-b43d-47b1-b50d-3c296bb5c4e4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            text_id                 text_name  \\\n",
              "0  001b0621b7d3f51059f7916f8e7667d4  6578feec58b434ef00050084   \n",
              "1  0047ea8b7faf501b9b6ddb8763f1c8b5  6578876058b434ef0001648e   \n",
              "2  005daa355633d0162360c026c3b67079  65718d7e58b434ef00c88628   \n",
              "3  0065db8f249888574dffa4da85775cd5  6570eb2258b434ef00c361fa   \n",
              "4  0070093b109fa990775e758fd1b3b0cc  6569aaf258b434ef00879ae6   \n",
              "\n",
              "                                                verb  positive  negative  \\\n",
              "0                       says\\nsays\\nsays\\nsaid\\nsaid     0.000       0.0   \n",
              "1                        announced\\nsaid\\nsaid\\nsays     0.000       0.0   \n",
              "2  said\\nsaid\\nsaid\\nsays\\nsaid\\ntold\\nsays\\nsaid...     0.000       0.0   \n",
              "3  said\\nsaid\\nsaid\\nsaid\\nsaid\\ntestified\\nsaid\\...     0.000       0.0   \n",
              "4  told\\nsaid\\nsays\\nsaid\\nsaid\\nsaid\\nsaid\\nadde...     0.362       0.0   \n",
              "\n",
              "   neutral  compound  \n",
              "0    1.000    0.0000  \n",
              "1    1.000    0.0000  \n",
              "2    1.000    0.0000  \n",
              "3    1.000    0.0000  \n",
              "4    0.638    0.6249  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-30fe745d-5bad-4a3a-aba3-6cc497de5483\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_id</th>\n",
              "      <th>text_name</th>\n",
              "      <th>verb</th>\n",
              "      <th>positive</th>\n",
              "      <th>negative</th>\n",
              "      <th>neutral</th>\n",
              "      <th>compound</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>001b0621b7d3f51059f7916f8e7667d4</td>\n",
              "      <td>6578feec58b434ef00050084</td>\n",
              "      <td>says\\nsays\\nsays\\nsaid\\nsaid</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0047ea8b7faf501b9b6ddb8763f1c8b5</td>\n",
              "      <td>6578876058b434ef0001648e</td>\n",
              "      <td>announced\\nsaid\\nsaid\\nsays</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>005daa355633d0162360c026c3b67079</td>\n",
              "      <td>65718d7e58b434ef00c88628</td>\n",
              "      <td>said\\nsaid\\nsaid\\nsays\\nsaid\\ntold\\nsays\\nsaid...</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0065db8f249888574dffa4da85775cd5</td>\n",
              "      <td>6570eb2258b434ef00c361fa</td>\n",
              "      <td>said\\nsaid\\nsaid\\nsaid\\nsaid\\ntestified\\nsaid\\...</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0070093b109fa990775e758fd1b3b0cc</td>\n",
              "      <td>6569aaf258b434ef00879ae6</td>\n",
              "      <td>told\\nsaid\\nsays\\nsaid\\nsaid\\nsaid\\nsaid\\nadde...</td>\n",
              "      <td>0.362</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.638</td>\n",
              "      <td>0.6249</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-30fe745d-5bad-4a3a-aba3-6cc497de5483')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-30fe745d-5bad-4a3a-aba3-6cc497de5483 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-30fe745d-5bad-4a3a-aba3-6cc497de5483');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-de92a80a-abc6-45af-a8cd-502475fc4974\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-de92a80a-abc6-45af-a8cd-502475fc4974')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-de92a80a-abc6-45af-a8cd-502475fc4974 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "verbs_df",
              "summary": "{\n  \"name\": \"verbs_df\",\n  \"rows\": 3520,\n  \"fields\": [\n    {\n      \"column\": \"text_id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"b4021a27d9ca7de33e291929a391242b\",\n          \"042aadbf72e4499a15e4eecf3b7e1e48\",\n          \"1a40c95c83e3955defb5551b7a3beed9\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text_name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"6569bf4758b434ef00884200\",\n          \"658db38f58b434ef00a3c1d1\",\n          \"6572de5f58b434ef00d35177\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"verb\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3195,\n        \"samples\": [\n          \"think\\ntold\\nsaid\\nsaid\\nsaid\\nsaid\\nsaid\\nsaid\",\n          \"says\\ntold\\nsaid\\nsaid\\nexplains\\n\\n\",\n          \"told\\n\\n\\n\\n\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"positive\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.07494722039126693,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 178,\n        \"samples\": [\n          0.045,\n          0.087,\n          0.249\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"negative\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.07539693101874055,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 188,\n        \"samples\": [\n          0.174,\n          0.592,\n          0.43\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"neutral\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.11153345415148025,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 253,\n        \"samples\": [\n          0.645,\n          0.828,\n          0.851\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"compound\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.18056361066275448,\n        \"min\": -0.9747,\n        \"max\": 0.8555,\n        \"num_unique_values\": 99,\n        \"samples\": [\n          0.5106,\n          0.6369,\n          -0.7184\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 239
        }
      ],
      "source": [
        "verbs_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gYSn8o4Ri4HU"
      },
      "outputs": [],
      "source": [
        "# create a third dataframe to directly compare compound scores between quotes and non quotes\n",
        "scores_all_df = pd.merge(quotes_df[['text_id', 'text_name', 'positive', 'negative', 'compound']],\n",
        "                          non_quotes_df[['text_id', 'text_name', 'positive', 'negative', 'compound']],\n",
        "                          on=['text_id', 'text_name'],\n",
        "                          suffixes=('_quotes', '_non_quotes'))\n",
        "\n",
        "scores_all_df.rename(columns={'positive_quotes': 'quote_pos',\n",
        "                              'negative_quotes': 'quote_neg',\n",
        "                              'compound_quotes': 'quote_comp',\n",
        "                              'positive_non_quotes': 'non_quote_pos',\n",
        "                              'negative_non_quotes': 'non_quote_neg',\n",
        "                              'compound_non_quotes': 'non_quote_comp',\n",
        "                              }, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "7AomSOXD2YMl",
        "outputId": "c6085da8-4ac0-47d9-e945-273e6890cf58"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            text_id                 text_name  quote_pos  \\\n",
              "0  001b0621b7d3f51059f7916f8e7667d4  6578feec58b434ef00050084      0.042   \n",
              "1  0047ea8b7faf501b9b6ddb8763f1c8b5  6578876058b434ef0001648e      0.000   \n",
              "2  005daa355633d0162360c026c3b67079  65718d7e58b434ef00c88628      0.061   \n",
              "3  0065db8f249888574dffa4da85775cd5  6570eb2258b434ef00c361fa      0.090   \n",
              "4  0070093b109fa990775e758fd1b3b0cc  6569aaf258b434ef00879ae6      0.102   \n",
              "\n",
              "   quote_neg  quote_comp  non_quote_pos  non_quote_neg  non_quote_comp  \n",
              "0      0.152     -0.9327          0.105          0.000          0.6059  \n",
              "1      0.056     -0.5994          0.000          0.000          0.0000  \n",
              "2      0.058      0.6617          0.063          0.040          0.8901  \n",
              "3      0.026      0.9756          0.060          0.054          0.4299  \n",
              "4      0.027      0.9371          0.146          0.026          0.9935  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-291de7ac-63d3-487f-8324-402765759edb\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_id</th>\n",
              "      <th>text_name</th>\n",
              "      <th>quote_pos</th>\n",
              "      <th>quote_neg</th>\n",
              "      <th>quote_comp</th>\n",
              "      <th>non_quote_pos</th>\n",
              "      <th>non_quote_neg</th>\n",
              "      <th>non_quote_comp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>001b0621b7d3f51059f7916f8e7667d4</td>\n",
              "      <td>6578feec58b434ef00050084</td>\n",
              "      <td>0.042</td>\n",
              "      <td>0.152</td>\n",
              "      <td>-0.9327</td>\n",
              "      <td>0.105</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.6059</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0047ea8b7faf501b9b6ddb8763f1c8b5</td>\n",
              "      <td>6578876058b434ef0001648e</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.056</td>\n",
              "      <td>-0.5994</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.0000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>005daa355633d0162360c026c3b67079</td>\n",
              "      <td>65718d7e58b434ef00c88628</td>\n",
              "      <td>0.061</td>\n",
              "      <td>0.058</td>\n",
              "      <td>0.6617</td>\n",
              "      <td>0.063</td>\n",
              "      <td>0.040</td>\n",
              "      <td>0.8901</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0065db8f249888574dffa4da85775cd5</td>\n",
              "      <td>6570eb2258b434ef00c361fa</td>\n",
              "      <td>0.090</td>\n",
              "      <td>0.026</td>\n",
              "      <td>0.9756</td>\n",
              "      <td>0.060</td>\n",
              "      <td>0.054</td>\n",
              "      <td>0.4299</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0070093b109fa990775e758fd1b3b0cc</td>\n",
              "      <td>6569aaf258b434ef00879ae6</td>\n",
              "      <td>0.102</td>\n",
              "      <td>0.027</td>\n",
              "      <td>0.9371</td>\n",
              "      <td>0.146</td>\n",
              "      <td>0.026</td>\n",
              "      <td>0.9935</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-291de7ac-63d3-487f-8324-402765759edb')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-291de7ac-63d3-487f-8324-402765759edb button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-291de7ac-63d3-487f-8324-402765759edb');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-8797d3b3-0a81-468a-b8ae-b2574b42fed3\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-8797d3b3-0a81-468a-b8ae-b2574b42fed3')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-8797d3b3-0a81-468a-b8ae-b2574b42fed3 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "scores_all_df",
              "summary": "{\n  \"name\": \"scores_all_df\",\n  \"rows\": 3520,\n  \"fields\": [\n    {\n      \"column\": \"text_id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"b4021a27d9ca7de33e291929a391242b\",\n          \"042aadbf72e4499a15e4eecf3b7e1e48\",\n          \"1a40c95c83e3955defb5551b7a3beed9\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text_name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"6569bf4758b434ef00884200\",\n          \"658db38f58b434ef00a3c1d1\",\n          \"6572de5f58b434ef00d35177\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_pos\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06460388735924111,\n        \"min\": 0.0,\n        \"max\": 0.767,\n        \"num_unique_values\": 293,\n        \"samples\": [\n          0.016,\n          0.023,\n          0.066\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_neg\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06055901902048814,\n        \"min\": 0.0,\n        \"max\": 0.443,\n        \"num_unique_values\": 283,\n        \"samples\": [\n          0.045,\n          0.199,\n          0.186\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_comp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.7760462126061396,\n        \"min\": -0.9996,\n        \"max\": 0.9994,\n        \"num_unique_values\": 2007,\n        \"samples\": [\n          0.106,\n          -0.886,\n          0.8176\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_pos\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.04180939697571297,\n        \"min\": 0.0,\n        \"max\": 0.313,\n        \"num_unique_values\": 210,\n        \"samples\": [\n          0.02,\n          0.132,\n          0.01\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_neg\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.049951429196720624,\n        \"min\": 0.0,\n        \"max\": 0.33,\n        \"num_unique_values\": 231,\n        \"samples\": [\n          0.218,\n          0.047,\n          0.149\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_comp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.820569289255619,\n        \"min\": -0.9999,\n        \"max\": 0.9999,\n        \"num_unique_values\": 1945,\n        \"samples\": [\n          0.126,\n          -0.989,\n          0.9622\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 241
        }
      ],
      "source": [
        "scores_all_df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XJYsNPJFqPqv"
      },
      "source": [
        "### Summary Calculations\n",
        "Add the following columns to the dataframe:\n",
        "* quote_prop_difference: positive score - negative score\n",
        "* quote_prop_direction: if it leans positive (>0), negative (<0), or neutral (=0)\n",
        "* same for non-quotes: non_quote_prop_difference and non_quote_prop_direction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FHqWpRTmqGKK"
      },
      "outputs": [],
      "source": [
        "# calculate score differences for quotes and non_quotes\n",
        "scores_all_df.insert(5, 'quote_prop_difference', scores_all_df['quote_pos'] - scores_all_df['quote_neg'])\n",
        "\n",
        "scores_all_df.insert(9, 'non_quote_prop_difference', scores_all_df['non_quote_pos'] - scores_all_df['non_quote_neg'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "zqjqVCwB25-e",
        "outputId": "fa6347a0-0319-435b-a99b-134be941fd75"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            text_id                 text_name  quote_pos  \\\n",
              "0  001b0621b7d3f51059f7916f8e7667d4  6578feec58b434ef00050084      0.042   \n",
              "1  0047ea8b7faf501b9b6ddb8763f1c8b5  6578876058b434ef0001648e      0.000   \n",
              "2  005daa355633d0162360c026c3b67079  65718d7e58b434ef00c88628      0.061   \n",
              "3  0065db8f249888574dffa4da85775cd5  6570eb2258b434ef00c361fa      0.090   \n",
              "4  0070093b109fa990775e758fd1b3b0cc  6569aaf258b434ef00879ae6      0.102   \n",
              "\n",
              "   quote_neg  quote_comp  quote_prop_difference  non_quote_pos  non_quote_neg  \\\n",
              "0      0.152     -0.9327                 -0.110          0.105          0.000   \n",
              "1      0.056     -0.5994                 -0.056          0.000          0.000   \n",
              "2      0.058      0.6617                  0.003          0.063          0.040   \n",
              "3      0.026      0.9756                  0.064          0.060          0.054   \n",
              "4      0.027      0.9371                  0.075          0.146          0.026   \n",
              "\n",
              "   non_quote_comp  non_quote_prop_difference  \n",
              "0          0.6059                      0.105  \n",
              "1          0.0000                      0.000  \n",
              "2          0.8901                      0.023  \n",
              "3          0.4299                      0.006  \n",
              "4          0.9935                      0.120  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-262ff970-9811-4c93-b161-1c5a0a52baf2\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_id</th>\n",
              "      <th>text_name</th>\n",
              "      <th>quote_pos</th>\n",
              "      <th>quote_neg</th>\n",
              "      <th>quote_comp</th>\n",
              "      <th>quote_prop_difference</th>\n",
              "      <th>non_quote_pos</th>\n",
              "      <th>non_quote_neg</th>\n",
              "      <th>non_quote_comp</th>\n",
              "      <th>non_quote_prop_difference</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>001b0621b7d3f51059f7916f8e7667d4</td>\n",
              "      <td>6578feec58b434ef00050084</td>\n",
              "      <td>0.042</td>\n",
              "      <td>0.152</td>\n",
              "      <td>-0.9327</td>\n",
              "      <td>-0.110</td>\n",
              "      <td>0.105</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.6059</td>\n",
              "      <td>0.105</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0047ea8b7faf501b9b6ddb8763f1c8b5</td>\n",
              "      <td>6578876058b434ef0001648e</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.056</td>\n",
              "      <td>-0.5994</td>\n",
              "      <td>-0.056</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>0.000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>005daa355633d0162360c026c3b67079</td>\n",
              "      <td>65718d7e58b434ef00c88628</td>\n",
              "      <td>0.061</td>\n",
              "      <td>0.058</td>\n",
              "      <td>0.6617</td>\n",
              "      <td>0.003</td>\n",
              "      <td>0.063</td>\n",
              "      <td>0.040</td>\n",
              "      <td>0.8901</td>\n",
              "      <td>0.023</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0065db8f249888574dffa4da85775cd5</td>\n",
              "      <td>6570eb2258b434ef00c361fa</td>\n",
              "      <td>0.090</td>\n",
              "      <td>0.026</td>\n",
              "      <td>0.9756</td>\n",
              "      <td>0.064</td>\n",
              "      <td>0.060</td>\n",
              "      <td>0.054</td>\n",
              "      <td>0.4299</td>\n",
              "      <td>0.006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0070093b109fa990775e758fd1b3b0cc</td>\n",
              "      <td>6569aaf258b434ef00879ae6</td>\n",
              "      <td>0.102</td>\n",
              "      <td>0.027</td>\n",
              "      <td>0.9371</td>\n",
              "      <td>0.075</td>\n",
              "      <td>0.146</td>\n",
              "      <td>0.026</td>\n",
              "      <td>0.9935</td>\n",
              "      <td>0.120</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-262ff970-9811-4c93-b161-1c5a0a52baf2')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-262ff970-9811-4c93-b161-1c5a0a52baf2 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-262ff970-9811-4c93-b161-1c5a0a52baf2');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c573b94f-cb3d-4c50-b3af-3d9ef969095f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c573b94f-cb3d-4c50-b3af-3d9ef969095f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c573b94f-cb3d-4c50-b3af-3d9ef969095f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "scores_all_df",
              "summary": "{\n  \"name\": \"scores_all_df\",\n  \"rows\": 3520,\n  \"fields\": [\n    {\n      \"column\": \"text_id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"b4021a27d9ca7de33e291929a391242b\",\n          \"042aadbf72e4499a15e4eecf3b7e1e48\",\n          \"1a40c95c83e3955defb5551b7a3beed9\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text_name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"6569bf4758b434ef00884200\",\n          \"658db38f58b434ef00a3c1d1\",\n          \"6572de5f58b434ef00d35177\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_pos\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06460388735924111,\n        \"min\": 0.0,\n        \"max\": 0.767,\n        \"num_unique_values\": 293,\n        \"samples\": [\n          0.016,\n          0.023,\n          0.066\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_neg\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06055901902048814,\n        \"min\": 0.0,\n        \"max\": 0.443,\n        \"num_unique_values\": 283,\n        \"samples\": [\n          0.045,\n          0.199,\n          0.186\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_comp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.7760462126061396,\n        \"min\": -0.9996,\n        \"max\": 0.9994,\n        \"num_unique_values\": 2007,\n        \"samples\": [\n          0.106,\n          -0.886,\n          0.8176\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_prop_difference\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.10240706818627447,\n        \"min\": -0.442,\n        \"max\": 0.767,\n        \"num_unique_values\": 1021,\n        \"samples\": [\n          0.07800000000000001,\n          -0.05499999999999999,\n          0.12299999999999998\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_pos\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.04180939697571297,\n        \"min\": 0.0,\n        \"max\": 0.313,\n        \"num_unique_values\": 210,\n        \"samples\": [\n          0.02,\n          0.132,\n          0.01\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_neg\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.049951429196720624,\n        \"min\": 0.0,\n        \"max\": 0.33,\n        \"num_unique_values\": 231,\n        \"samples\": [\n          0.218,\n          0.047,\n          0.149\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_comp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.820569289255619,\n        \"min\": -0.9999,\n        \"max\": 0.9999,\n        \"num_unique_values\": 1945,\n        \"samples\": [\n          0.126,\n          -0.989,\n          0.9622\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_prop_difference\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0741604195743758,\n        \"min\": -0.33,\n        \"max\": 0.313,\n        \"num_unique_values\": 873,\n        \"samples\": [\n          0.10800000000000001,\n          0.13699999999999998,\n          0.107\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 243
        }
      ],
      "source": [
        "scores_all_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ATLP564b27c2"
      },
      "outputs": [],
      "source": [
        "# determine sentiment direction\n",
        "direction = ['positive', 'negative', 'neutral']\n",
        "\n",
        "quote_conds = [\n",
        "    scores_all_df.quote_prop_difference > 0,\n",
        "    scores_all_df.quote_prop_difference < 0,\n",
        "    scores_all_df.quote_prop_difference == 0,\n",
        "]\n",
        "\n",
        "quote_prop_direction = []\n",
        "\n",
        "quote_prop_direction = np.select(quote_conds, direction)\n",
        "\n",
        "scores_all_df.insert(6, 'quote_prop_direction', quote_prop_direction)\n",
        "\n",
        "non_quote_conds = [\n",
        "    scores_all_df.non_quote_prop_difference > 0,\n",
        "    scores_all_df.non_quote_prop_difference < 0,\n",
        "    scores_all_df.non_quote_prop_difference == 0,\n",
        "]\n",
        "\n",
        "scores_all_df['non_quote_prop_direction'] = np.select(non_quote_conds, direction)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        },
        "id": "-NzGWBRF3NRH",
        "outputId": "13cfd640-702b-46ad-85aa-958a3b28ab76"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            text_id                 text_name  quote_pos  \\\n",
              "0  001b0621b7d3f51059f7916f8e7667d4  6578feec58b434ef00050084      0.042   \n",
              "1  0047ea8b7faf501b9b6ddb8763f1c8b5  6578876058b434ef0001648e      0.000   \n",
              "2  005daa355633d0162360c026c3b67079  65718d7e58b434ef00c88628      0.061   \n",
              "3  0065db8f249888574dffa4da85775cd5  6570eb2258b434ef00c361fa      0.090   \n",
              "4  0070093b109fa990775e758fd1b3b0cc  6569aaf258b434ef00879ae6      0.102   \n",
              "\n",
              "   quote_neg  quote_comp  quote_prop_difference quote_prop_direction  \\\n",
              "0      0.152     -0.9327                 -0.110             negative   \n",
              "1      0.056     -0.5994                 -0.056             negative   \n",
              "2      0.058      0.6617                  0.003             positive   \n",
              "3      0.026      0.9756                  0.064             positive   \n",
              "4      0.027      0.9371                  0.075             positive   \n",
              "\n",
              "   non_quote_pos  non_quote_neg  non_quote_comp  non_quote_prop_difference  \\\n",
              "0          0.105          0.000          0.6059                      0.105   \n",
              "1          0.000          0.000          0.0000                      0.000   \n",
              "2          0.063          0.040          0.8901                      0.023   \n",
              "3          0.060          0.054          0.4299                      0.006   \n",
              "4          0.146          0.026          0.9935                      0.120   \n",
              "\n",
              "  non_quote_prop_direction  \n",
              "0                 positive  \n",
              "1                  neutral  \n",
              "2                 positive  \n",
              "3                 positive  \n",
              "4                 positive  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3bfa4a12-2b2d-4a09-8418-77856223725c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_id</th>\n",
              "      <th>text_name</th>\n",
              "      <th>quote_pos</th>\n",
              "      <th>quote_neg</th>\n",
              "      <th>quote_comp</th>\n",
              "      <th>quote_prop_difference</th>\n",
              "      <th>quote_prop_direction</th>\n",
              "      <th>non_quote_pos</th>\n",
              "      <th>non_quote_neg</th>\n",
              "      <th>non_quote_comp</th>\n",
              "      <th>non_quote_prop_difference</th>\n",
              "      <th>non_quote_prop_direction</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>001b0621b7d3f51059f7916f8e7667d4</td>\n",
              "      <td>6578feec58b434ef00050084</td>\n",
              "      <td>0.042</td>\n",
              "      <td>0.152</td>\n",
              "      <td>-0.9327</td>\n",
              "      <td>-0.110</td>\n",
              "      <td>negative</td>\n",
              "      <td>0.105</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.6059</td>\n",
              "      <td>0.105</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0047ea8b7faf501b9b6ddb8763f1c8b5</td>\n",
              "      <td>6578876058b434ef0001648e</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.056</td>\n",
              "      <td>-0.5994</td>\n",
              "      <td>-0.056</td>\n",
              "      <td>negative</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>neutral</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>005daa355633d0162360c026c3b67079</td>\n",
              "      <td>65718d7e58b434ef00c88628</td>\n",
              "      <td>0.061</td>\n",
              "      <td>0.058</td>\n",
              "      <td>0.6617</td>\n",
              "      <td>0.003</td>\n",
              "      <td>positive</td>\n",
              "      <td>0.063</td>\n",
              "      <td>0.040</td>\n",
              "      <td>0.8901</td>\n",
              "      <td>0.023</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0065db8f249888574dffa4da85775cd5</td>\n",
              "      <td>6570eb2258b434ef00c361fa</td>\n",
              "      <td>0.090</td>\n",
              "      <td>0.026</td>\n",
              "      <td>0.9756</td>\n",
              "      <td>0.064</td>\n",
              "      <td>positive</td>\n",
              "      <td>0.060</td>\n",
              "      <td>0.054</td>\n",
              "      <td>0.4299</td>\n",
              "      <td>0.006</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0070093b109fa990775e758fd1b3b0cc</td>\n",
              "      <td>6569aaf258b434ef00879ae6</td>\n",
              "      <td>0.102</td>\n",
              "      <td>0.027</td>\n",
              "      <td>0.9371</td>\n",
              "      <td>0.075</td>\n",
              "      <td>positive</td>\n",
              "      <td>0.146</td>\n",
              "      <td>0.026</td>\n",
              "      <td>0.9935</td>\n",
              "      <td>0.120</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3bfa4a12-2b2d-4a09-8418-77856223725c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-3bfa4a12-2b2d-4a09-8418-77856223725c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-3bfa4a12-2b2d-4a09-8418-77856223725c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-52dcc9ba-faed-4e5f-a5b6-6f5afea2cbab\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-52dcc9ba-faed-4e5f-a5b6-6f5afea2cbab')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-52dcc9ba-faed-4e5f-a5b6-6f5afea2cbab button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "scores_all_df",
              "summary": "{\n  \"name\": \"scores_all_df\",\n  \"rows\": 3520,\n  \"fields\": [\n    {\n      \"column\": \"text_id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"b4021a27d9ca7de33e291929a391242b\",\n          \"042aadbf72e4499a15e4eecf3b7e1e48\",\n          \"1a40c95c83e3955defb5551b7a3beed9\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text_name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"6569bf4758b434ef00884200\",\n          \"658db38f58b434ef00a3c1d1\",\n          \"6572de5f58b434ef00d35177\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_pos\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06460388735924111,\n        \"min\": 0.0,\n        \"max\": 0.767,\n        \"num_unique_values\": 293,\n        \"samples\": [\n          0.016,\n          0.023,\n          0.066\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_neg\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06055901902048814,\n        \"min\": 0.0,\n        \"max\": 0.443,\n        \"num_unique_values\": 283,\n        \"samples\": [\n          0.045,\n          0.199,\n          0.186\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_comp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.7760462126061396,\n        \"min\": -0.9996,\n        \"max\": 0.9994,\n        \"num_unique_values\": 2007,\n        \"samples\": [\n          0.106,\n          -0.886,\n          0.8176\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_prop_difference\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.10240706818627447,\n        \"min\": -0.442,\n        \"max\": 0.767,\n        \"num_unique_values\": 1021,\n        \"samples\": [\n          0.07800000000000001,\n          -0.05499999999999999,\n          0.12299999999999998\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote_prop_direction\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"negative\",\n          \"positive\",\n          \"neutral\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_pos\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.04180939697571297,\n        \"min\": 0.0,\n        \"max\": 0.313,\n        \"num_unique_values\": 210,\n        \"samples\": [\n          0.02,\n          0.132,\n          0.01\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_neg\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.049951429196720624,\n        \"min\": 0.0,\n        \"max\": 0.33,\n        \"num_unique_values\": 231,\n        \"samples\": [\n          0.218,\n          0.047,\n          0.149\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_comp\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.820569289255619,\n        \"min\": -0.9999,\n        \"max\": 0.9999,\n        \"num_unique_values\": 1945,\n        \"samples\": [\n          0.126,\n          -0.989,\n          0.9622\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_prop_difference\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0741604195743758,\n        \"min\": -0.33,\n        \"max\": 0.313,\n        \"num_unique_values\": 873,\n        \"samples\": [\n          0.10800000000000001,\n          0.13699999999999998,\n          0.107\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quote_prop_direction\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"positive\",\n          \"neutral\",\n          \"negative\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 245
        }
      ],
      "source": [
        "scores_all_df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Ghd_HtBy3Qm"
      },
      "source": [
        "## Aggregate Data\n",
        "Add a new sheet (dataframe) called 'summary' with the following columns:\n",
        "* Col 1 values: quote, non_quote, verbs, speakers\n",
        "* Headers:\n",
        "  * **compound_max:** max value of compound scores\n",
        "  * **compound_min:** min value of compound scores\n",
        "  * **compound_avg:** average value of compound scores\n",
        "  * **compound_avg_no_polarity:** average of absolute compound scores\n",
        "  * **pos_comp_count:** number of compound scores that lean positive (exclude everything in the range [-0.05, 0.05])\n",
        "  * **neg_comp_count:** number of compound scores that lean negative\n",
        "  * **avg_prop_diff:** average value of proportion difference; find pos - neg score, then take average of that number\n",
        "  * **pos_prop_dir_count:** number of texts that lean positive from difference\n",
        "  * **neg_prop_dir_count:** number of texts that lean negative from difference\n",
        "  * **avg_pos_prop_diff:** average value of proportion difference between positive differences\n",
        "  * **avg_neg_prop_diff:** average value of proportion difference between negative differences"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r-52KL3P5iwd"
      },
      "outputs": [],
      "source": [
        "# helper function for extracting all the necessary data for each category\n",
        "# where df1 is the original dataframe of each category\n",
        "# and df2 is the scores all dataframe\n",
        "def aggregate_data(df1, df2, category):\n",
        "  row = []\n",
        "\n",
        "  # max/min/avg of compound scores\n",
        "  row.append(df1['compound'].max())\n",
        "  row.append(df1['compound'].min())\n",
        "  row.append(df1['compound'].mean())\n",
        "  row.append(df1['compound'].abs().mean())\n",
        "\n",
        "  # pos and neg comp count\n",
        "  row.append(df1.compound[df1.compound > 0.05].count())\n",
        "  row.append(df1.compound[df1.compound < -0.05].count())\n",
        "\n",
        "  # if category is quotes or non quotes, find the following values\n",
        "  # otherwise append 0s\n",
        "  if category != '':\n",
        "    prop_diff = category + '_prop_difference'\n",
        "    row.append(df2[prop_diff].mean())\n",
        "\n",
        "    # pos and neg direction counts\n",
        "    prop_dir = category + '_prop_direction'\n",
        "    row.append(df2[prop_dir][df2[prop_dir] == 'positive'].count())\n",
        "    row.append(df2[prop_dir][df2[prop_dir] == 'negative'].count())\n",
        "\n",
        "    # pos and neg difference means\n",
        "    row.append(df2[prop_diff][df2[prop_diff] > 0].mean())\n",
        "    row.append(df2[prop_diff][df2[prop_diff] < 0].mean())\n",
        "\n",
        "  else:\n",
        "    row = row + [0, 0, 0, 0, 0]\n",
        "\n",
        "  return row"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "25xmjg1O43ta"
      },
      "outputs": [],
      "source": [
        "# create summary dataframe\n",
        "data_summary = []\n",
        "\n",
        "data_summary.append(aggregate_data(quotes_df, scores_all_df, 'quote'))\n",
        "data_summary.append(aggregate_data(non_quotes_df, scores_all_df, 'non_quote'))\n",
        "data_summary.append(aggregate_data(speakers_df, scores_all_df, ''))\n",
        "data_summary.append(aggregate_data(verbs_df, scores_all_df, ''))\n",
        "\n",
        "summary_df = pd.DataFrame(data_summary)\n",
        "\n",
        "# row and column headers\n",
        "row_labels = ['quote', 'non_quote', 'speaker', 'verb']\n",
        "col_labels = ['compound_max', 'compound_min', 'compound_avg', 'no_polarity_comp_avg',\n",
        "              'pos_comp_count', 'neg_comp_count', 'avg_prop_diff',\n",
        "              'pos_prop_direction_count', 'neg_prop_direction_count',\n",
        "              'avg_pos_prop_difference', 'avg_neg_prop_difference']\n",
        "\n",
        "summary_df.index = row_labels\n",
        "summary_df.columns = col_labels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "jiPbONG1CBL1",
        "outputId": "23207245-eef8-46d9-e743-ab60a2fafc0b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           compound_max  compound_min  compound_avg  no_polarity_comp_avg  \\\n",
              "quote            0.9994       -0.9996      0.257418              0.766441   \n",
              "non_quote        0.9999       -0.9999      0.164688              0.794369   \n",
              "speaker          0.9781       -0.9690      0.066639              0.183614   \n",
              "verb             0.8555       -0.9747     -0.008579              0.070640   \n",
              "\n",
              "           pos_comp_count  neg_comp_count  avg_prop_diff  \\\n",
              "quote                2181            1217       0.027849   \n",
              "non_quote            2039            1415       0.009751   \n",
              "speaker               917             430       0.000000   \n",
              "verb                  317             344       0.000000   \n",
              "\n",
              "           pos_prop_direction_count  neg_prop_direction_count  \\\n",
              "quote                          2216                      1205   \n",
              "non_quote                      2057                      1412   \n",
              "speaker                           0                         0   \n",
              "verb                              0                         0   \n",
              "\n",
              "           avg_pos_prop_difference  avg_neg_prop_difference  \n",
              "quote                     0.085558                -0.075991  \n",
              "non_quote                 0.058409                -0.060783  \n",
              "speaker                   0.000000                 0.000000  \n",
              "verb                      0.000000                 0.000000  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a43f8c3d-2415-455e-ac1e-7845eb2b15d9\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>compound_max</th>\n",
              "      <th>compound_min</th>\n",
              "      <th>compound_avg</th>\n",
              "      <th>no_polarity_comp_avg</th>\n",
              "      <th>pos_comp_count</th>\n",
              "      <th>neg_comp_count</th>\n",
              "      <th>avg_prop_diff</th>\n",
              "      <th>pos_prop_direction_count</th>\n",
              "      <th>neg_prop_direction_count</th>\n",
              "      <th>avg_pos_prop_difference</th>\n",
              "      <th>avg_neg_prop_difference</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>quote</th>\n",
              "      <td>0.9994</td>\n",
              "      <td>-0.9996</td>\n",
              "      <td>0.257418</td>\n",
              "      <td>0.766441</td>\n",
              "      <td>2181</td>\n",
              "      <td>1217</td>\n",
              "      <td>0.027849</td>\n",
              "      <td>2216</td>\n",
              "      <td>1205</td>\n",
              "      <td>0.085558</td>\n",
              "      <td>-0.075991</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>non_quote</th>\n",
              "      <td>0.9999</td>\n",
              "      <td>-0.9999</td>\n",
              "      <td>0.164688</td>\n",
              "      <td>0.794369</td>\n",
              "      <td>2039</td>\n",
              "      <td>1415</td>\n",
              "      <td>0.009751</td>\n",
              "      <td>2057</td>\n",
              "      <td>1412</td>\n",
              "      <td>0.058409</td>\n",
              "      <td>-0.060783</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>speaker</th>\n",
              "      <td>0.9781</td>\n",
              "      <td>-0.9690</td>\n",
              "      <td>0.066639</td>\n",
              "      <td>0.183614</td>\n",
              "      <td>917</td>\n",
              "      <td>430</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>verb</th>\n",
              "      <td>0.8555</td>\n",
              "      <td>-0.9747</td>\n",
              "      <td>-0.008579</td>\n",
              "      <td>0.070640</td>\n",
              "      <td>317</td>\n",
              "      <td>344</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a43f8c3d-2415-455e-ac1e-7845eb2b15d9')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a43f8c3d-2415-455e-ac1e-7845eb2b15d9 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a43f8c3d-2415-455e-ac1e-7845eb2b15d9');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-311697cf-ddda-4c32-8bd6-4de7627cc86b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-311697cf-ddda-4c32-8bd6-4de7627cc86b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-311697cf-ddda-4c32-8bd6-4de7627cc86b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "summary_df",
              "summary": "{\n  \"name\": \"summary_df\",\n  \"rows\": 4,\n  \"fields\": [\n    {\n      \"column\": \"compound_max\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06923300633849915,\n        \"min\": 0.8555,\n        \"max\": 0.9999,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0.9999,\n          0.8555,\n          0.9994\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"compound_min\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.016275748830699027,\n        \"min\": -0.9999,\n        \"max\": -0.969,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          -0.9999,\n          -0.9747,\n          -0.9996\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"compound_avg\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.11584582267713868,\n        \"min\": -0.008579062500000002,\n        \"max\": 0.25741823863636365,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0.16468801136363637,\n          -0.008579062500000002,\n          0.25741823863636365\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"no_polarity_comp_avg\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.38015093769116215,\n        \"min\": 0.07064008522727272,\n        \"max\": 0.7943694886363636,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0.7943694886363636,\n          0.07064008522727272,\n          0.7664414204545454\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pos_comp_count\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 897,\n        \"min\": 317,\n        \"max\": 2181,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          2039,\n          317,\n          2181\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"neg_comp_count\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 543,\n        \"min\": 344,\n        \"max\": 1415,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          1415,\n          344,\n          1217\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"avg_prop_diff\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.01313016206351539,\n        \"min\": 0.0,\n        \"max\": 0.027848863636363637,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.027848863636363637,\n          0.009750568181818183,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pos_prop_direction_count\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1235,\n        \"min\": 0,\n        \"max\": 2216,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          2216,\n          2057,\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"neg_prop_direction_count\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 760,\n        \"min\": 0,\n        \"max\": 1412,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          1205,\n          1412,\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"avg_pos_prop_difference\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.04301229539528878,\n        \"min\": 0.0,\n        \"max\": 0.0855582129963899,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0.0855582129963899,\n          0.05840884783665533,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"avg_neg_prop_difference\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.039968279929526666,\n        \"min\": -0.07599087136929461,\n        \"max\": 0.0,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          -0.07599087136929461,\n          -0.06078257790368272,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 247
        }
      ],
      "source": [
        "summary_df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ICLfCIHhtvOp"
      },
      "source": [
        "## Write Output to New Workbook"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "AH9_9yUeB6G9",
        "outputId": "aaa2fd46-7b74-46bb-bd1f-d2aabb5f01c4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                  text_name                                                url\n",
              "0  6569e97958b434ef00898608  https://subscriptions.cbc.ca/newsletter_static...\n",
              "1  65692bb658b434ef0082ff04  https://www.cbc.ca/news/canada/calgary/gift-ca...\n",
              "2  6569248a58b434ef0082b088  https://www.cbc.ca/news/canada/british-columbi...\n",
              "3  6569252c58b434ef0082c1c3  https://www.cbc.ca/news/canada/toronto/ago-rev...\n",
              "4  656924e558b434ef0082ba26  https://www.cbc.ca/news/canada/nova-scotia/sea..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2b68da76-3f84-4cc6-a4f9-e4c2a0cb31dd\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_name</th>\n",
              "      <th>url</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>6569e97958b434ef00898608</td>\n",
              "      <td>https://subscriptions.cbc.ca/newsletter_static...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>65692bb658b434ef0082ff04</td>\n",
              "      <td>https://www.cbc.ca/news/canada/calgary/gift-ca...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>6569248a58b434ef0082b088</td>\n",
              "      <td>https://www.cbc.ca/news/canada/british-columbi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6569252c58b434ef0082c1c3</td>\n",
              "      <td>https://www.cbc.ca/news/canada/toronto/ago-rev...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>656924e558b434ef0082ba26</td>\n",
              "      <td>https://www.cbc.ca/news/canada/nova-scotia/sea...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2b68da76-3f84-4cc6-a4f9-e4c2a0cb31dd')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2b68da76-3f84-4cc6-a4f9-e4c2a0cb31dd button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2b68da76-3f84-4cc6-a4f9-e4c2a0cb31dd');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-a180c88d-e0b7-486a-8c55-b6a3f7af033f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a180c88d-e0b7-486a-8c55-b6a3f7af033f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-a180c88d-e0b7-486a-8c55-b6a3f7af033f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "metadata_df",
              "summary": "{\n  \"name\": \"metadata_df\",\n  \"rows\": 3760,\n  \"fields\": [\n    {\n      \"column\": \"text_name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3760,\n        \"samples\": [\n          \"65783f9c58b434ef00fed4ba\",\n          \"65856f0e58b434ef006542a4\",\n          \"656aae9158b434ef008fb820\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"url\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3760,\n        \"samples\": [\n          \"https://www.cbc.ca/news/canada/new-brunswick/gibson-street-fredericton-development-council-1.7055994\",\n          \"https://www.cbc.ca/news/canada/kitchener-waterloo/region-of-waterloo-international-airport-holiday-travel-tips-1.7066361\",\n          \"https://www.cbc.ca/news/canada/toronto/metrolinx-technical-briefing-december-transparency-eglinton-crosstown-1.7047254\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 248
        }
      ],
      "source": [
        "# map urls to file names\n",
        "metadata = '/content/drive/My Drive/evaluation_quotes/metadata/2023_12_dec_CBC_news_metadata.xlsx'\n",
        "\n",
        "metadata_df = pd.read_excel(metadata, usecols = [\"filename\", \"url\"])\n",
        "\n",
        "metadata_df.rename(columns={'filename': 'text_name'}, inplace=True)\n",
        "\n",
        "metadata_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-SyGXGjWCo9m"
      },
      "outputs": [],
      "source": [
        "new_quotes_df = quotes_df.merge(metadata_df[['text_name', 'url']], on='text_name', how='left')\n",
        "\n",
        "new_non_quotes_df = non_quotes_df.merge(metadata_df[['text_name', 'url']], on='text_name', how='left')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hxuUPM9oGuT9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "fbd35138-7ea0-4ba9-fffc-37c319326d95"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            text_id                 text_name  \\\n",
              "0  001b0621b7d3f51059f7916f8e7667d4  6578feec58b434ef00050084   \n",
              "1  0047ea8b7faf501b9b6ddb8763f1c8b5  6578876058b434ef0001648e   \n",
              "2  005daa355633d0162360c026c3b67079  65718d7e58b434ef00c88628   \n",
              "3  0065db8f249888574dffa4da85775cd5  6570eb2258b434ef00c361fa   \n",
              "4  0070093b109fa990775e758fd1b3b0cc  6569aaf258b434ef00879ae6   \n",
              "\n",
              "                                               quote  positive  negative  \\\n",
              "0  that reports of Vancouver police denying acces...     0.042     0.152   \n",
              "1  that the federal government will begin a consu...     0.000     0.056   \n",
              "2  he believes airlines are dropping base fares a...     0.061     0.058   \n",
              "3  Moore was working on the top of one of the hom...     0.090     0.026   \n",
              "4  his bid would have cost $3.5 million per year,...     0.102     0.027   \n",
              "\n",
              "   neutral  compound                                                url  \n",
              "0    0.806   -0.9327  https://www.cbc.ca/news/canada/british-columbi...  \n",
              "1    0.944   -0.5994  https://www.cbc.ca/news/politics/canada-home-d...  \n",
              "2    0.881    0.6617  https://www.cbc.ca/news/canada/toronto/air-tra...  \n",
              "3    0.884    0.9756  https://www.cbc.ca/news/canada/new-brunswick/d...  \n",
              "4    0.871    0.9371  https://www.cbc.ca/news/canada/newfoundland-la...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-50ebdf6b-34ab-40b0-afb8-db9c4e3c6182\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_id</th>\n",
              "      <th>text_name</th>\n",
              "      <th>quote</th>\n",
              "      <th>positive</th>\n",
              "      <th>negative</th>\n",
              "      <th>neutral</th>\n",
              "      <th>compound</th>\n",
              "      <th>url</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>001b0621b7d3f51059f7916f8e7667d4</td>\n",
              "      <td>6578feec58b434ef00050084</td>\n",
              "      <td>that reports of Vancouver police denying acces...</td>\n",
              "      <td>0.042</td>\n",
              "      <td>0.152</td>\n",
              "      <td>0.806</td>\n",
              "      <td>-0.9327</td>\n",
              "      <td>https://www.cbc.ca/news/canada/british-columbi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0047ea8b7faf501b9b6ddb8763f1c8b5</td>\n",
              "      <td>6578876058b434ef0001648e</td>\n",
              "      <td>that the federal government will begin a consu...</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.056</td>\n",
              "      <td>0.944</td>\n",
              "      <td>-0.5994</td>\n",
              "      <td>https://www.cbc.ca/news/politics/canada-home-d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>005daa355633d0162360c026c3b67079</td>\n",
              "      <td>65718d7e58b434ef00c88628</td>\n",
              "      <td>he believes airlines are dropping base fares a...</td>\n",
              "      <td>0.061</td>\n",
              "      <td>0.058</td>\n",
              "      <td>0.881</td>\n",
              "      <td>0.6617</td>\n",
              "      <td>https://www.cbc.ca/news/canada/toronto/air-tra...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0065db8f249888574dffa4da85775cd5</td>\n",
              "      <td>6570eb2258b434ef00c361fa</td>\n",
              "      <td>Moore was working on the top of one of the hom...</td>\n",
              "      <td>0.090</td>\n",
              "      <td>0.026</td>\n",
              "      <td>0.884</td>\n",
              "      <td>0.9756</td>\n",
              "      <td>https://www.cbc.ca/news/canada/new-brunswick/d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0070093b109fa990775e758fd1b3b0cc</td>\n",
              "      <td>6569aaf258b434ef00879ae6</td>\n",
              "      <td>his bid would have cost $3.5 million per year,...</td>\n",
              "      <td>0.102</td>\n",
              "      <td>0.027</td>\n",
              "      <td>0.871</td>\n",
              "      <td>0.9371</td>\n",
              "      <td>https://www.cbc.ca/news/canada/newfoundland-la...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-50ebdf6b-34ab-40b0-afb8-db9c4e3c6182')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-50ebdf6b-34ab-40b0-afb8-db9c4e3c6182 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-50ebdf6b-34ab-40b0-afb8-db9c4e3c6182');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7ffa170f-cbc8-4ed6-8d09-994ace193ad5\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7ffa170f-cbc8-4ed6-8d09-994ace193ad5')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7ffa170f-cbc8-4ed6-8d09-994ace193ad5 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "new_quotes_df",
              "summary": "{\n  \"name\": \"new_quotes_df\",\n  \"rows\": 3520,\n  \"fields\": [\n    {\n      \"column\": \"text_id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"b4021a27d9ca7de33e291929a391242b\",\n          \"042aadbf72e4499a15e4eecf3b7e1e48\",\n          \"1a40c95c83e3955defb5551b7a3beed9\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text_name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"6569bf4758b434ef00884200\",\n          \"658db38f58b434ef00a3c1d1\",\n          \"6572de5f58b434ef00d35177\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"quote\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3503,\n        \"samples\": [\n          \"Eric Montross, a former North Carolina star and Toronto Raptors big man, has died after a cancer fight\\nMontross died Sunday surrounded by loved ones at his Chapel Hill home\\nand the family knows that the ripples from the generous, thoughtful way that he lived his life will continue in the lives of the many people he touched with his deep and sincere kindness\\n\\\"Eric was a great player and accomplished student, but the impacts he made on our community went way beyond the basketball court\\n\\\"He was a man of faith, a tremendous father, husband and son, and one of the most recognizable ambassadors of the University and Chapel Hill.\\\"\\n\\n\",\n          \"Police officers, firefighters and paramedics responded to a report of a crash near Whiteside just before 6 p.m.,\\nthe driver of the van, a 42-year-old Halifax woman, appeared to have life threatening injuries\\nThe driver of the car, a 28-year-old New Brunswick man, appeared to have life-threatening injuries,\\nboth drivers were removed from the vehicles with the help of fire services\",\n          \"fans should just stop complaining\\nBut weeks later, the 58-year-old Burlington, Ont., resident was dead \\u2014 killed not by the cancer\\n\\\"It was honestly a cruel rollercoaster of emotions\\nBut those tests may be giving some cancer patients and their families a false sense of security;\\nRead more\\n\\nOn average, electric vehicles are less reliable than other cars and trucks\\nElectric vehicles are far less reliable than gasoline-powered cars, trucks and SUVs, mainly because most automakers are still learning how to build a completely new power system\\nElectric vehicles have proved far less reliable, on average, than gasoline-powered cars, trucks and SUVs,\\nEV owners most frequently reported troubles with battery and charging systems and flaws in how the vehicles' body panels and interior parts fit together\\nthat EV manufacturers are still learning to construct completely new power systems, and suggested that, as they do, the overall reliability of electric vehicles should improve\\n\\\"This story is really one of growing pains\\nthat lingering concerns about reliability will likely add to the issues that give many buyers pause when considering a switch to the new technology, joining concerns about higher costs, too few charging stations and long charging times\\nmarketplace@cbc.ca\\n\\nFinding room for carry-on baggage has become 'the Hunger Games' of air travel\\nWhen more passengers are vying for limited space for carry-on luggage, it leads to costly delays\\nthat the box was above the size limits for carry-on baggage\\nAnd it was a really kind of meaningful, sentimental part of this wedding that was about to happen\\nShe and her colleagues spent at least 10 minutes moving people and their carry-on baggage around the plane so that they could make room for the cake box on the floor of a window seat where it wouldn't block anyone's exit in the event of an emergency\\npassengers have become carried away with carry-on baggage, leading to costly delays\\nthis change needs to happen, and that passengers should be allowed to check a bag for free\\nAnd people who want the convenience and control would pay to bring their bags on the plane\\nairlines would likely generate the same or more in baggage revenue\\n\\\"The People vs. The Ticket Giant\\\"\\n\\\"I just feel like we were robbed of time together.\\\"\\n\\\"It's a story of just working out the bugs and the kinks of new technology.\\\"\\n\\\"The basic safety rules and written directives regarding emergency measures for all foreseeable situations...were ignored,\\\"\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"positive\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06460388735924111,\n        \"min\": 0.0,\n        \"max\": 0.767,\n        \"num_unique_values\": 293,\n        \"samples\": [\n          0.016,\n          0.023,\n          0.066\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"negative\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06055901902048814,\n        \"min\": 0.0,\n        \"max\": 0.443,\n        \"num_unique_values\": 283,\n        \"samples\": [\n          0.045,\n          0.199,\n          0.186\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"neutral\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.07206613380825744,\n        \"min\": 0.233,\n        \"max\": 1.0,\n        \"num_unique_values\": 353,\n        \"samples\": [\n          0.918,\n          0.688,\n          0.778\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"compound\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.7760462126061396,\n        \"min\": -0.9996,\n        \"max\": 0.9994,\n        \"num_unique_values\": 2007,\n        \"samples\": [\n          0.106,\n          -0.886,\n          0.8176\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"url\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"https://www.cbc.ca/news/canada/saskatchewan/festivals-volunteers-pandemic-1.7045858\",\n          \"https://www.cbc.ca/news/canada/montreal/potential-class-action-cantaloupe-salmonella-1.7070363\",\n          \"https://www.cbc.ca/news/canada/london/naming-victims-would-keep-women-safer-police-board-vice-chair-says-1.7052261\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 250
        }
      ],
      "source": [
        "new_quotes_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RafmNDMYG6kF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "23e61a92-42e7-404c-d69e-41f3aa44fad1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            text_id                 text_name  \\\n",
              "0  001b0621b7d3f51059f7916f8e7667d4  6578feec58b434ef00050084   \n",
              "1  0047ea8b7faf501b9b6ddb8763f1c8b5  6578876058b434ef0001648e   \n",
              "2  005daa355633d0162360c026c3b67079  65718d7e58b434ef00c88628   \n",
              "3  0065db8f249888574dffa4da85775cd5  6570eb2258b434ef00c361fa   \n",
              "4  0070093b109fa990775e758fd1b3b0cc  6569aaf258b434ef00879ae6   \n",
              "\n",
              "                                     non_quoted_text  positive  negative  \\\n",
              "0  British Columbia's human rights commissioner h...     0.105     0.000   \n",
              "1  The federal government is planning to launch a...     0.000     0.000   \n",
              "2  Can you fly round-trip from Toronto to Lisbon ...     0.063     0.040   \n",
              "3  Jurors in a coroner's inquest on Monday heard ...     0.060     0.054   \n",
              "4  Dr. Todd Young owns virtual health clinic Medi...     0.146     0.026   \n",
              "\n",
              "   neutral  compound                                                url  \n",
              "0    0.895    0.6059  https://www.cbc.ca/news/canada/british-columbi...  \n",
              "1    1.000    0.0000  https://www.cbc.ca/news/politics/canada-home-d...  \n",
              "2    0.897    0.8901  https://www.cbc.ca/news/canada/toronto/air-tra...  \n",
              "3    0.886    0.4299  https://www.cbc.ca/news/canada/new-brunswick/d...  \n",
              "4    0.828    0.9935  https://www.cbc.ca/news/canada/newfoundland-la...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5fa807db-8833-453e-9c16-1bd8023afa12\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_id</th>\n",
              "      <th>text_name</th>\n",
              "      <th>non_quoted_text</th>\n",
              "      <th>positive</th>\n",
              "      <th>negative</th>\n",
              "      <th>neutral</th>\n",
              "      <th>compound</th>\n",
              "      <th>url</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>001b0621b7d3f51059f7916f8e7667d4</td>\n",
              "      <td>6578feec58b434ef00050084</td>\n",
              "      <td>British Columbia's human rights commissioner h...</td>\n",
              "      <td>0.105</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.895</td>\n",
              "      <td>0.6059</td>\n",
              "      <td>https://www.cbc.ca/news/canada/british-columbi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0047ea8b7faf501b9b6ddb8763f1c8b5</td>\n",
              "      <td>6578876058b434ef0001648e</td>\n",
              "      <td>The federal government is planning to launch a...</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>https://www.cbc.ca/news/politics/canada-home-d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>005daa355633d0162360c026c3b67079</td>\n",
              "      <td>65718d7e58b434ef00c88628</td>\n",
              "      <td>Can you fly round-trip from Toronto to Lisbon ...</td>\n",
              "      <td>0.063</td>\n",
              "      <td>0.040</td>\n",
              "      <td>0.897</td>\n",
              "      <td>0.8901</td>\n",
              "      <td>https://www.cbc.ca/news/canada/toronto/air-tra...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0065db8f249888574dffa4da85775cd5</td>\n",
              "      <td>6570eb2258b434ef00c361fa</td>\n",
              "      <td>Jurors in a coroner's inquest on Monday heard ...</td>\n",
              "      <td>0.060</td>\n",
              "      <td>0.054</td>\n",
              "      <td>0.886</td>\n",
              "      <td>0.4299</td>\n",
              "      <td>https://www.cbc.ca/news/canada/new-brunswick/d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0070093b109fa990775e758fd1b3b0cc</td>\n",
              "      <td>6569aaf258b434ef00879ae6</td>\n",
              "      <td>Dr. Todd Young owns virtual health clinic Medi...</td>\n",
              "      <td>0.146</td>\n",
              "      <td>0.026</td>\n",
              "      <td>0.828</td>\n",
              "      <td>0.9935</td>\n",
              "      <td>https://www.cbc.ca/news/canada/newfoundland-la...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5fa807db-8833-453e-9c16-1bd8023afa12')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-5fa807db-8833-453e-9c16-1bd8023afa12 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-5fa807db-8833-453e-9c16-1bd8023afa12');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-3bfd0b0d-9034-4317-b71e-3e0d4ca6db33\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-3bfd0b0d-9034-4317-b71e-3e0d4ca6db33')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-3bfd0b0d-9034-4317-b71e-3e0d4ca6db33 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "new_non_quotes_df",
              "summary": "{\n  \"name\": \"new_non_quotes_df\",\n  \"rows\": 3520,\n  \"fields\": [\n    {\n      \"column\": \"text_id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"b4021a27d9ca7de33e291929a391242b\",\n          \"042aadbf72e4499a15e4eecf3b7e1e48\",\n          \"1a40c95c83e3955defb5551b7a3beed9\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text_name\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"6569bf4758b434ef00884200\",\n          \"658db38f58b434ef00a3c1d1\",\n          \"6572de5f58b434ef00d35177\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"non_quoted_text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"  .\\n\\n  .\\n\\nThat's now led to the cancellation of Fort Qu'Appelle's annual winter festival. The Fort Winter Festival is typically held over three days in March, with family-friendly events like fireworks, uphill sled drags, dog sledding and horse-drawn wagon rides.\\n\\nBut that isn't in the cards for this coming March.\\n\\n,\\\"   in a Facebook post for the festival.\\n\\n  .\\n\\n\\\"It's sad that they won't be hosting it this year. ,\\\"  , referring to the money and spirit the festival brought to the town.\\n\\nSpring and summer festivals are also in jeopardy due to a lack of volunteers.\\n\\n  , but volunteers are needed for that to continue. (Submitted by Sean Stares)\\n\\nThe Windscape Kite Festival, which has filled Swift Current's skies with colourful kites from around the world for more than a decade, usually runs alongside the Long Day's Night Music Festival \\u2014 but both festivals are now struggling with finances and staffing.\\n\\n  \\n\\n\\\"We just hit a point where we couldn't continue. ,\\\"  .\\n\\n\\n\\n  . ,  .\\n\\nThe more the merrier \\u2014 literally\\n\\nMeanwhile, the Moose Jaw Music Festival \\u2014 which features musical performers of all ages, skill levels and disciplines, including choir and musical theatre \\u2014 is in desperate need of volunteers as well.\\n\\n  . But organizers couldn't even get 20 this past year.\\n\\nA young man performs during the Moose Jaw Music Festival.   . (Submitted by Janie Fries)\\n\\n,  .\\n\\n,\\\"  .\\n\\nShe understands why some people are pulling back from committing to non-work activities.\\n\\n\\\"I think one of the benefits that came out of COVID is that we as a society are much more aware of mental health. ,\\\"  .\\n\\n\\n\\nHowever,  more volunteers would help current festival organizers and volunteers balance their own mental health better, said Fries. With more volunteers, there's less responsibility and demand on time placed on any one person.\\n\\n,  , and she encourages parents of musicians especially to sign up to volunteer.\\n\\nLa Ronge Children's Festival volunteers enjoy the sun and lending a helping hand during the July annual event. (Submitted by Sean Stares)\\n\\nUp north, the La Ronge Children's Festival is also struggling to get both volunteers and committee members.   that pre-pandemic. This past year, there were only three.\\n\\n  .\\n\\n\\\"  . But we need people to be able to help choose programming, writing grants and being the treasurer.\\\"\\n\\n  .\\n\\n  .\\n\\n\\\"The lack of volunteers makes it so it's a stress on the people who do volunteer. ,\\\"  .\\n\\nHaving more volunteers to share the work would likely inspire people to keep volunteering every year.\\n\\n\\\"It's fun if you put in the time. It's valuable and  can  .\\\"\",\n          \"A Montreal man    is seeking permission to launch a class-action lawsuit against two food companies.\\n\\nLaw firm Slater Vecchio LLP filed the application Dec. 12 in Quebec Superior Court alleging that Olivier Archambault got sick after he consumed melons produced by Mexico-based Malichita and distributed in Canada by Arizona-based Trufresh.\\n\\nIn November, the Canadian Food Inspection Agency issued food recall warnings for melons produced by Malichita, and later in the month issued recalls for melons by Rudy brand, which are produced in the same area of Mexico.\\n\\nAs of last week the Public Health Agency of Canada has reported 164 laboratory-confirmed cases and seven deaths from cantaloupe-linked salmonella across eight provinces; 111 of those cases are in Quebec.\\n\\n  .\\n\\n  .\\n\\nThe proposed class action, which has not been authorized or tested in court, is seeking unspecified damages on behalf of individuals in Quebec who purchased and consumed cantaloupes and other fruit in October and November that were the subject of Health Canada recalls.\",\n          \"  .\\n\\nMegan Walker is vice-chair of the board that provides civilian oversight over police. She's also the former director of the London Abused Women's Centre.\\n\\nShe's concerned police might not always release as much information as they can or should in cases of possible intimate-partner violence.   withholding details, including the names of those involved, .\\n\\nWalker's concerns come in the wake of the death of Tiffany Gates and a CBC News report that has Gates' mother asking the London Police Service why details of her death, including her name, have not been released.\\n\\nGates, 30, was found dead in the apartment of her boyfriend on Sept. 7.   . Their names, and the nature of their relationship, only became public her friends and family began to post condolences on social media.\\n\\nFor now,  will only  .\\n\\nTell us what you think! Help shape the future of CBC article pages by taking a quick survey. ,\\\"  . \\n\\nGates's mother Linda Davidson has questioned why police haven't released more information so her daughter's death can be correctly identified as the death of a woman at the hands of her intimate partner.\\n\\nNo standard rules for reporting names of victims\\n\\nAmong Canadian police forces, there is no standard set of rules about how, if, or when police release the names of victims of violent crimes or the people charged with them.\\n\\nA 2019 report commissioned by the Edmonton Police surveyed forces about their polices and found that:\\n\\nMore than one-third (36 per cent) of the responding police services release the names of all homicide victims.\\n\\nMore than one-half (54 per cent) release the names depending on circumstances.\\n\\nOnly seven per cent always withhold victim's names.\\n\\nIn the wake of the report, the Edmonton Police changed their disclosure policy, so the default position will be to name all homicide victims, with some exceptions.\\n\\nYork approach: Case-by-case basis\\n\\nYork Regional Police, for example, take a case-by-case approach on deciding what information they release. In a statement to CBC News,   .\\n\\n,\\\"  . Lisa Moskaluk. ;  also  the name of the man charged with her murder.\\n\\nYork Police didn't disclose the nature of their relationship, though reporters later confirmed the victim and accused were in a relationship.   .\\\". 've  .\\n\\n,  \\n\\n.   a book about the issues surrounding .\\n\\n  .\\n\\nIn cases such as the Tiffany Gates's death \\u2014 where both people involved are dead \\u2014 she doesn't see reason to withold their names.\\n\\n ,\\\"  .\\n\\n  .\\n\\n\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"positive\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.04180939697571297,\n        \"min\": 0.0,\n        \"max\": 0.313,\n        \"num_unique_values\": 210,\n        \"samples\": [\n          0.02,\n          0.132,\n          0.01\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"negative\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.049951429196720624,\n        \"min\": 0.0,\n        \"max\": 0.33,\n        \"num_unique_values\": 231,\n        \"samples\": [\n          0.218,\n          0.047,\n          0.149\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"neutral\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.05464821865641968,\n        \"min\": 0.612,\n        \"max\": 1.0,\n        \"num_unique_values\": 282,\n        \"samples\": [\n          0.963,\n          0.867,\n          0.923\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"compound\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.820569289255619,\n        \"min\": -0.9999,\n        \"max\": 0.9999,\n        \"num_unique_values\": 1945,\n        \"samples\": [\n          0.126,\n          -0.989,\n          0.9622\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"url\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3520,\n        \"samples\": [\n          \"https://www.cbc.ca/news/canada/saskatchewan/festivals-volunteers-pandemic-1.7045858\",\n          \"https://www.cbc.ca/news/canada/montreal/potential-class-action-cantaloupe-salmonella-1.7070363\",\n          \"https://www.cbc.ca/news/canada/london/naming-victims-would-keep-women-safer-police-board-vice-chair-says-1.7052261\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 251
        }
      ],
      "source": [
        "new_non_quotes_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cKjUTq3Uwg9j",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7f3cf873-a54a-444a-f020-d9488daf3e0b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting xlsxwriter\n",
            "  Downloading XlsxWriter-3.2.0-py3-none-any.whl.metadata (2.6 kB)\n",
            "Downloading XlsxWriter-3.2.0-py3-none-any.whl (159 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m159.9/159.9 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: xlsxwriter\n",
            "Successfully installed xlsxwriter-3.2.0\n"
          ]
        }
      ],
      "source": [
        "!pip install xlsxwriter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "knDTwEawj5yb"
      },
      "outputs": [],
      "source": [
        "# replace with 'quotes_sentiment.xlsx'\n",
        "output = '/content/drive/My Drive/evaluation_quotes/CBC_sentiment_output/2023_12_dec_CBC_news_sentiment.xlsx'\n",
        "\n",
        "# create excel writer object to initialize new workbook\n",
        "writer = pd.ExcelWriter(output, engine=\"xlsxwriter\")\n",
        "\n",
        "# write dataframes to different worksheets\n",
        "new_quotes_df.to_excel(writer, sheet_name=\"quotes\", index=False)\n",
        "new_non_quotes_df.to_excel(writer, sheet_name=\"non_quotes\", index=False)\n",
        "speakers_df.to_excel(writer, sheet_name=\"speakers\", index=False)\n",
        "verbs_df.to_excel(writer, sheet_name=\"verbs\", index=False)\n",
        "scores_all_df.to_excel(writer, sheet_name=\"scores_all\", index=False)\n",
        "summary_df.to_excel(writer, sheet_name=\"summary\")\n",
        "\n",
        "# close the excel writer and output file\n",
        "writer.close()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}