logistic_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bf5f776e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package twitter_samples to\n",
      "[nltk_data]     C:\\Users\\rmora\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package twitter_samples is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\rmora\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# run this cell to import nltk\n",
    "import nltk\n",
    "from os import getcwd\n",
    "\n",
    "nltk.download('twitter_samples')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0e754323",
   "metadata": {},
   "outputs": [],
   "source": [
    "filePath = f\"{getcwd()}/../tmp2/\"\n",
    "nltk.data.path.append(filePath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "68a71503",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from nltk.corpus import twitter_samples \n",
    "\n",
    "from utils import process_tweet, build_freqs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2c179bb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# select the set of positive and negative tweets\n",
    "all_positive_tweets = twitter_samples.strings('positive_tweets.json')\n",
    "all_negative_tweets = twitter_samples.strings('negative_tweets.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "401ddc13",
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the data into two pieces, one for training and one for testing (validation set) \n",
    "test_pos = all_positive_tweets[4000:]\n",
    "train_pos = all_positive_tweets[:4000]\n",
    "test_neg = all_negative_tweets[4000:]\n",
    "train_neg = all_negative_tweets[:4000]\n",
    "\n",
    "train_x = train_pos + train_neg \n",
    "test_x = test_pos + test_neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b4d19694",
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine positive and negative labels\n",
    "train_y = np.append(np.ones((len(train_pos), 1)), np.zeros((len(train_neg), 1)), axis=0)\n",
    "test_y = np.append(np.ones((len(test_pos), 1)), np.zeros((len(test_neg), 1)), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7fae2055",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_y.shape = (8000, 1)\n",
      "test_y.shape = (2000, 1)\n"
     ]
    }
   ],
   "source": [
    "# Print the shape train and test sets\n",
    "print(\"train_y.shape = \" + str(train_y.shape))\n",
    "print(\"test_y.shape = \" + str(test_y.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f5fe8fc0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type(freqs) = <class 'dict'>\n",
      "len(freqs) = 11428\n"
     ]
    }
   ],
   "source": [
    "# create frequency dictionary\n",
    "freqs = build_freqs(train_x, train_y)\n",
    "\n",
    "# check the output\n",
    "print(\"type(freqs) = \" + str(type(freqs)))\n",
    "print(\"len(freqs) = \" + str(len(freqs.keys())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "256d9207",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is an example of a positive tweet: \n",
      " #FollowFriday @France_Inte @PKuchly57 @Milipol_Paris for being top engaged members in my community this week :)\n",
      "\n",
      "This is an example of the processed version of the tweet: \n",
      " ['followfriday', 'top', 'engag', 'member', 'commun', 'week', ':)']\n"
     ]
    }
   ],
   "source": [
    "# test the function below\n",
    "print('This is an example of a positive tweet: \\n', train_x[0])\n",
    "print('\\nThis is an example of the processed version of the tweet: \\n', process_tweet(train_x[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "10f578ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z): \n",
    "    '''\n",
    "    Input:\n",
    "        z: is the input (can be a scalar or an array)\n",
    "    Output:\n",
    "        h: the sigmoid of z\n",
    "    '''\n",
    "    \n",
    "    # calculate the sigmoid of z\n",
    "    h = 1 / (1 + np.exp(-z))\n",
    "    \n",
    "    return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "03339e75",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.210340371976294"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# verify that when the model predicts close to 1, but the actual label is 0, the loss is a large positive value\n",
    "-1 * (1 - 0) * np.log(1 - 0.9999) # loss is about 9.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "268f6d43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.210340371976182"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# verify that when the model predicts close to 0 but the actual label is 1, the loss is a large positive value\n",
    "-1 * np.log(0.0001) # loss is about 9.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "34a6eb31",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def gradientDescent(x, y, theta, alpha, num_iters):\n",
    "    '''\n",
    "    Input:\n",
    "        x: matrix of features which is (m,n+1)\n",
    "        y: corresponding labels of the input matrix x, dimensions (m,1)\n",
    "        theta: weight vector of dimension (n+1,1)\n",
    "        alpha: learning rate\n",
    "        num_iters: number of iterations you want to train your model for\n",
    "    Output:\n",
    "        J: the final cost\n",
    "        theta: your final weight vector\n",
    "    Hint: you might want to print the cost to make sure that it is going down.\n",
    "    '''\n",
    "    # get 'm', the number of rows in matrix x\n",
    "    m = x.shape[0]\n",
    "    \n",
    "    for i in range(0, num_iters):\n",
    "        \n",
    "        # get z, the dot product of x and theta\n",
    "        z = np.dot(x, theta)\n",
    "        \n",
    "        # get the sigmoid of z\n",
    "        h = sigmoid(z)\n",
    "        \n",
    "        # calculate the cost function\n",
    "        J = -1. / m * (np.dot((y).transpose(), np.log(h)) + np.dot((1 - y).transpose(), np.log(1 - h)))\n",
    "\n",
    "        # update the weights theta\n",
    "        theta = theta - (alpha / m) * np.dot((x).transpose(), (h - y))\n",
    "        \n",
    "    J = float(J)\n",
    "    return J, theta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8bfdb5d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cost after training is 0.67094970.\n",
      "The resulting vector of weights is [4.1e-07, 0.00035658, 7.309e-05]\n"
     ]
    }
   ],
   "source": [
    "# Check the function\n",
    "# Construct a synthetic test case using numpy PRNG functions\n",
    "np.random.seed(1)\n",
    "# X input is 10 x 3 with ones for the bias terms\n",
    "tmp_X = np.append(np.ones((10, 1)), np.random.rand(10, 2) * 2000, axis=1)\n",
    "# Y Labels are 10 x 1\n",
    "tmp_Y = (np.random.rand(10, 1) > 0.35).astype(float)\n",
    "\n",
    "# Apply gradient descent\n",
    "tmp_J, tmp_theta = gradientDescent(tmp_X, tmp_Y, np.zeros((3, 1)), 1e-8, 700)\n",
    "print(f\"The cost after training is {tmp_J:.8f}.\")\n",
    "print(f\"The resulting vector of weights is {[round(t, 8) for t in np.squeeze(tmp_theta)]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5c8364d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_features(tweet, freqs, process_tweet=process_tweet):\n",
    "    '''\n",
    "    Input: \n",
    "        tweet: a list of words for one tweet\n",
    "        freqs: a dictionary corresponding to the frequencies of each tuple (word, label)\n",
    "    Output: \n",
    "        x: a feature vector of dimension (1,3)\n",
    "    '''\n",
    "    # process_tweet tokenizes, stems, and removes stopwords\n",
    "    word_l = process_tweet(tweet)\n",
    "    \n",
    "    # 3 elements in the form of a 1 x 3 vector\n",
    "    x = np.zeros((1, 3)) \n",
    "    \n",
    "    #bias term is set to 1\n",
    "    x[0,0] = 1 \n",
    "    \n",
    "    # loop through each word in the list of words\n",
    "    for word in word_l:\n",
    "        \n",
    "        # increment the word count for the positive label 1\n",
    "        x[0,1] += freqs.get((word, 1.0), 0)\n",
    "        \n",
    "        # increment the word count for the negative label 0\n",
    "        x[0,2] += freqs.get((word, 0.0), 0)\n",
    "        \n",
    "    assert(x.shape == (1, 3))\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d1e41046",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1.000e+00 3.133e+03 6.100e+01]]\n"
     ]
    }
   ],
   "source": [
    "# Check your function\n",
    "# test 1\n",
    "# test on training data\n",
    "tmp1 = extract_features(train_x[0], freqs)\n",
    "print(tmp1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0e84f3b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "# test 2:\n",
    "# check for when the words are not in the freqs dictionary\n",
    "tmp2 = extract_features('blorb bleeeeb bloooob', freqs)\n",
    "print(tmp2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9e464550",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cost after training is 0.22521260.\n",
      "The resulting vector of weights is [6e-08, 0.0005382, -0.0005583]\n"
     ]
    }
   ],
   "source": [
    "# collect the features 'x' and stack them into a matrix 'X'\n",
    "X = np.zeros((len(train_x), 3))\n",
    "for i in range(len(train_x)):\n",
    "    X[i, :]= extract_features(train_x[i], freqs)\n",
    "\n",
    "# training labels corresponding to X\n",
    "Y = train_y\n",
    "\n",
    "# Apply gradient descent\n",
    "J, theta = gradientDescent(X, Y, np.zeros((3, 1)), 1e-9, 1500)\n",
    "print(f\"The cost after training is {J:.8f}.\")\n",
    "print(f\"The resulting vector of weights is {[round(t, 8) for t in np.squeeze(theta)]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "40cd5ab4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_tweet(tweet, freqs, theta):\n",
    "    '''\n",
    "    Input: \n",
    "        tweet: a string\n",
    "        freqs: a dictionary corresponding to the frequencies of each tuple (word, label)\n",
    "        theta: (3,1) vector of weights\n",
    "    Output: \n",
    "        y_pred: the probability of a tweet being positive or negative\n",
    "    '''\n",
    "    # extract the features of the tweet and store it into x\n",
    "    x = extract_features(tweet, freqs)\n",
    "    \n",
    "    # make the prediction using x and theta\n",
    "    y_pred = sigmoid(np.dot(x, theta))\n",
    "    \n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "310fda1c",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I am happy -> 0.519275\n",
      "I am bad -> 0.494347\n",
      "this movie should have been great. -> 0.515980\n",
      "great -> 0.516065\n",
      "great great -> 0.532097\n",
      "great great great -> 0.548063\n",
      "great great great great -> 0.563930\n"
     ]
    }
   ],
   "source": [
    "# Run this cell to test your function\n",
    "for tweet in ['I am happy', 'I am bad', 'this movie should have been great.', 'great', 'great great', 'great great great', 'great great great great']:\n",
    "    print( '%s -> %f' % (tweet, predict_tweet(tweet, freqs, theta)))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9165fb9b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.83110766]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Feel free to check the sentiment of your own tweet below\n",
    "my_tweet = 'I am learning :)'\n",
    "predict_tweet(my_tweet, freqs, theta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "732b6ce3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_logistic_regression(test_x, test_y, freqs, theta, predict_tweet=predict_tweet):\n",
    "    \"\"\"\n",
    "    Input: \n",
    "        test_x: a list of tweets\n",
    "        test_y: (m, 1) vector with the corresponding labels for the list of tweets\n",
    "        freqs: a dictionary with the frequency of each pair (or tuple)\n",
    "        theta: weight vector of dimension (3, 1)\n",
    "    Output: \n",
    "        accuracy: (# of tweets classified correctly) / (total # of tweets)\n",
    "    \"\"\"\n",
    "    \n",
    "    # the list for storing predictions\n",
    "    y_hat = []\n",
    "    \n",
    "    for tweet in test_x:\n",
    "        # get the label prediction for the tweet\n",
    "        y_pred = predict_tweet(tweet, freqs, theta)\n",
    "        \n",
    "        if y_pred > 0.5:\n",
    "            # append 1.0 to the list\n",
    "            y_hat.append(1.0)\n",
    "        else:\n",
    "            # append 0 to the list\n",
    "            y_hat.append(0.0)\n",
    "\n",
    "    # With the above implementation, y_hat is a list, but test_y is (m,1) array\n",
    "    # convert both to one-dimensional arrays in order to compare them using the '==' operator\n",
    "    accuracy = (y_hat == np.squeeze(test_y)).sum()/len(test_x)\n",
    "\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "fd9f3d08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic regression model's accuracy = 0.9950\n"
     ]
    }
   ],
   "source": [
    "tmp_accuracy = test_logistic_regression(test_x, test_y, freqs, theta)\n",
    "print(f\"Logistic regression model's accuracy = {tmp_accuracy:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0e81d8c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label Predicted Tweet\n",
      "THE TWEET IS: @MarkBreech Not sure it would be good thing 4 my bottom daring 2 say 2 Miss B but Im gonna be so stubborn on mouth soaping ! #NotHavingit :p\n",
      "THE PROCESSED TWEET IS: ['sure', 'would', 'good', 'thing', '4', 'bottom', 'dare', '2', 'say', '2', 'miss', 'b', 'im', 'gonna', 'stubborn', 'mouth', 'soap', 'nothavingit', ':p']\n",
      "1\t0.48942982\tb'sure would good thing 4 bottom dare 2 say 2 miss b im gonna stubborn mouth soap nothavingit :p'\n",
      "THE TWEET IS: I'm playing Brain Dots : ) #BrainDots\n",
      "http://t.co/UGQzOx0huu\n",
      "THE PROCESSED TWEET IS: [\"i'm\", 'play', 'brain', 'dot', 'braindot']\n",
      "1\t0.48418982\tb\"i'm play brain dot braindot\"\n",
      "THE TWEET IS: I'm playing Brain Dots : ) #BrainDots http://t.co/aOKldo3GMj http://t.co/xWCM9qyRG5\n",
      "THE PROCESSED TWEET IS: [\"i'm\", 'play', 'brain', 'dot', 'braindot']\n",
      "1\t0.48418982\tb\"i'm play brain dot braindot\"\n",
      "THE TWEET IS: I'm playing Brain Dots : ) #BrainDots http://t.co/R2JBO8iNww http://t.co/ow5BBwdEMY\n",
      "THE PROCESSED TWEET IS: [\"i'm\", 'play', 'brain', 'dot', 'braindot']\n",
      "1\t0.48418982\tb\"i'm play brain dot braindot\"\n",
      "THE TWEET IS: off to the park to get some sunlight : )\n",
      "THE PROCESSED TWEET IS: ['park', 'get', 'sunlight']\n",
      "1\t0.49636406\tb'park get sunlight'\n",
      "THE TWEET IS: @msarosh Uff Itna Miss karhy thy ap :p\n",
      "THE PROCESSED TWEET IS: ['uff', 'itna', 'miss', 'karhi', 'thi', 'ap', ':p']\n",
      "1\t0.48250522\tb'uff itna miss karhi thi ap :p'\n",
      "THE TWEET IS: @phenomyoutube u probs had more fun with david than me : (\n",
      "THE PROCESSED TWEET IS: ['u', 'prob', 'fun', 'david']\n",
      "0\t0.50988296\tb'u prob fun david'\n",
      "THE TWEET IS: pats jay : (\n",
      "THE PROCESSED TWEET IS: ['pat', 'jay']\n",
      "0\t0.50040366\tb'pat jay'\n",
      "THE TWEET IS: my beloved grandmother : ( https://t.co/wt4oXq5xCf\n",
      "THE PROCESSED TWEET IS: ['belov', 'grandmoth']\n",
      "0\t0.50000002\tb'belov grandmoth'\n",
      "THE TWEET IS: Sr. Financial Analyst - Expedia, Inc.: (#Bellevue, WA) http://t.co/ktknMhvwCI #Finance #ExpediaJobs #Job #Jobs #Hiring\n",
      "THE PROCESSED TWEET IS: ['sr', 'financi', 'analyst', 'expedia', 'inc', 'bellevu', 'wa', 'financ', 'expediajob', 'job', 'job', 'hire']\n",
      "0\t0.50648699\tb'sr financi analyst expedia inc bellevu wa financ expediajob job job hire'\n"
     ]
    }
   ],
   "source": [
    "print('Label Predicted Tweet')\n",
    "for x,y in zip(test_x,test_y):\n",
    "    y_hat = predict_tweet(x, freqs, theta)\n",
    "\n",
    "    if np.abs(y - (y_hat > 0.5)) > 0:\n",
    "        print('THE TWEET IS:', x)\n",
    "        print('THE PROCESSED TWEET IS:', process_tweet(x))\n",
    "        print('%d\\t%0.8f\\t%s' % (y, y_hat, ' '.join(process_tweet(x)).encode('ascii', 'ignore')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a7e0f799",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ridicul', 'hot', 'day', \"i'v\", 'felt', 'hot', 'sweati', 'day']\n",
      "[[0.51494155]]\n",
      "Positive sentiment\n"
     ]
    }
   ],
   "source": [
    "my_tweet = 'This is a ridiculously hot day! I\\'ve felt hot and sweaty all day!'\n",
    "print(process_tweet(my_tweet))\n",
    "y_hat = predict_tweet(my_tweet, freqs, theta)\n",
    "print(y_hat)\n",
    "if y_hat > 0.5:\n",
    "    print('Positive sentiment')\n",
    "else: \n",
    "    print('Negative sentiment')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.5 ('venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  },
  "vscode": {
   "interpreter": {
    "hash": "c0f1c366766a836eae421a9ddb90e0d4d927ea01deb72fbd9113cea0ab1240cb"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}