classifying-fake-news-with-bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-05-03T22:34:08.767623Z",
     "iopub.status.busy": "2022-05-03T22:34:08.767275Z",
     "iopub.status.idle": "2022-05-03T22:34:09.433524Z",
     "shell.execute_reply": "2022-05-03T22:34:09.432670Z",
     "shell.execute_reply.started": "2022-05-03T22:34:08.767572Z"
    }
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "\n",
    "real_data = pd.read_csv('/kaggle/input/fakenew/Tweet_train.csv')\n",
    "test2_data = pd.read_csv('/kaggle/input/fakenew/Tweet_test.csv') \n",
    "\n",
    "real_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T22:34:13.409453Z",
     "iopub.status.busy": "2022-05-03T22:34:13.409068Z",
     "iopub.status.idle": "2022-05-03T22:34:13.419789Z",
     "shell.execute_reply": "2022-05-03T22:34:13.418754Z",
     "shell.execute_reply.started": "2022-05-03T22:34:13.409398Z"
    }
   },
   "outputs": [],
   "source": [
    "test2_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I will then add a column to each of the dataset telling if the row is fake new or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T21:01:30.561434Z",
     "iopub.status.busy": "2022-05-03T21:01:30.560671Z",
     "iopub.status.idle": "2022-05-03T21:01:30.578662Z",
     "shell.execute_reply": "2022-05-03T21:01:30.577714Z",
     "shell.execute_reply.started": "2022-05-03T21:01:30.561371Z"
    }
   },
   "outputs": [],
   "source": [
    "real_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now I will put both of the dataset into the same dataframe to use it more easily"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T22:38:51.925434Z",
     "iopub.status.busy": "2022-05-03T22:38:51.925084Z",
     "iopub.status.idle": "2022-05-03T22:38:52.847688Z",
     "shell.execute_reply": "2022-05-03T22:38:52.846811Z",
     "shell.execute_reply.started": "2022-05-03T22:38:51.925386Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "\n",
    "data = real_data\n",
    "\n",
    "# Shuffle the data\n",
    "data = shuffle(data).reset_index(drop=True)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now for the fun part\n",
    "I will split the data into three sets: training, validation and testing sets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-28T02:46:52.973949Z",
     "iopub.status.busy": "2022-04-28T02:46:52.973577Z",
     "iopub.status.idle": "2022-04-28T02:46:52.989635Z",
     "shell.execute_reply": "2022-04-28T02:46:52.988688Z",
     "shell.execute_reply.started": "2022-04-28T02:46:52.973894Z"
    }
   },
   "source": [
    "train_data, validate_data, test_data = np.split(data.sample(frac=1), [int(.6*len(data)), int(.8*len(data))])\n",
    "\n",
    "train_data = train_data.reset_index(drop=True)\n",
    "validate_data = validate_data.reset_index(drop=True)\n",
    "test_data = test_data.reset_index(drop=True)\n",
    "\n",
    "\n",
    "print(\"Size of training set: {}\".format(len(train_data)))\n",
    "print(\"Size of validation set: {}\".format(len(validate_data)))\n",
    "print(\"Size of testing set: {}\".format(len(test_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T22:38:56.319692Z",
     "iopub.status.busy": "2022-05-03T22:38:56.319344Z",
     "iopub.status.idle": "2022-05-03T22:38:56.445257Z",
     "shell.execute_reply": "2022-05-03T22:38:56.444509Z",
     "shell.execute_reply.started": "2022-05-03T22:38:56.319644Z"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "def text_cleaning(text):\n",
    "    text = re.sub(\"[^a-zA-Z]\", \" \", text) # removing punctuation\n",
    "    return text\n",
    "\n",
    "data['text content'] = data['text content'].apply(text_cleaning)\n",
    "\n",
    "train_data, validate_data, test_data = np.split(data.sample(frac=1), [int(.9*len(data)), int(1*len(data))])\n",
    "\n",
    "train_data = train_data.reset_index(drop=True)\n",
    "validate_data = validate_data.reset_index(drop=True)\n",
    "test_data = test_data.reset_index(drop=True)\n",
    "\n",
    "print(\"Size of training set: {}\".format(len(train_data)))\n",
    "print(\"Size of validation set: {}\".format(len(validate_data)))\n",
    "print(\"Size of testing set: {}\".format(len(test_data)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Installing the required libraries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T22:39:00.106983Z",
     "iopub.status.busy": "2022-05-03T22:39:00.106598Z",
     "iopub.status.idle": "2022-05-03T22:39:47.914103Z",
     "shell.execute_reply": "2022-05-03T22:39:47.913024Z",
     "shell.execute_reply.started": "2022-05-03T22:39:00.106930Z"
    }
   },
   "outputs": [],
   "source": [
    "conda install -y pytorch torchvision cudatoolkit=10.1 -c pytorch\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T22:39:47.916676Z",
     "iopub.status.busy": "2022-05-03T22:39:47.916326Z",
     "iopub.status.idle": "2022-05-03T22:39:55.087195Z",
     "shell.execute_reply": "2022-05-03T22:39:55.086372Z",
     "shell.execute_reply.started": "2022-05-03T22:39:47.916624Z"
    }
   },
   "outputs": [],
   "source": [
    "pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T22:40:57.361092Z",
     "iopub.status.busy": "2022-05-03T22:40:57.360691Z",
     "iopub.status.idle": "2022-05-03T22:41:01.276154Z",
     "shell.execute_reply": "2022-05-03T22:41:01.275342Z",
     "shell.execute_reply.started": "2022-05-03T22:40:57.361038Z"
    }
   },
   "outputs": [],
   "source": [
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\")\n",
    "model.config.num_labels = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I am going to freeze all the pretrained portion of the network and add a new section at the end of the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T22:41:01.396670Z",
     "iopub.status.busy": "2022-05-03T22:41:01.396071Z",
     "iopub.status.idle": "2022-05-03T22:41:01.409809Z",
     "shell.execute_reply": "2022-05-03T22:41:01.408959Z",
     "shell.execute_reply.started": "2022-05-03T22:41:01.396460Z"
    }
   },
   "outputs": [],
   "source": [
    "# Freeze the pre trained parameters\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "# Add three new layers at the end of the network\n",
    "model.classifier = nn.Sequential(\n",
    "    nn.Linear(768, 256),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(256, 64),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(64, 2),\n",
    "    nn.Softmax(dim=1)\n",
    ")\n",
    "\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T22:41:04.204810Z",
     "iopub.status.busy": "2022-05-03T22:41:04.204253Z",
     "iopub.status.idle": "2022-05-03T22:41:04.209985Z",
     "shell.execute_reply": "2022-05-03T22:41:04.209040Z",
     "shell.execute_reply.started": "2022-05-03T22:41:04.204730Z"
    }
   },
   "outputs": [],
   "source": [
    "criterion = nn.MSELoss().to(device)\n",
    "optimizer = optim.SGD(model.classifier.parameters(), lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T22:41:05.896131Z",
     "iopub.status.busy": "2022-05-03T22:41:05.895676Z",
     "iopub.status.idle": "2022-05-03T22:41:05.902825Z",
     "shell.execute_reply": "2022-05-03T22:41:05.901933Z",
     "shell.execute_reply.started": "2022-05-03T22:41:05.896080Z"
    }
   },
   "outputs": [],
   "source": [
    "def preprocess_text(text):\n",
    "    parts = []\n",
    "\n",
    "    text_len = len(text.split(' '))\n",
    "    delta = 300\n",
    "    max_parts = 5\n",
    "    nb_cuts = int(text_len / delta)\n",
    "    nb_cuts = min(nb_cuts, max_parts)\n",
    "    \n",
    "    for i in range(nb_cuts + 1):\n",
    "        text_part = ' '.join(text.split(' ')[i * delta: (i + 1) * delta])\n",
    "        parts.append(tokenizer.encode(text_part, return_tensors=\"pt\", max_length=500).to(device))\n",
    "\n",
    "    return parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T22:41:08.393117Z",
     "iopub.status.busy": "2022-05-03T22:41:08.392726Z",
     "iopub.status.idle": "2022-05-03T22:43:07.175328Z",
     "shell.execute_reply": "2022-05-03T22:43:07.174473Z",
     "shell.execute_reply.started": "2022-05-03T22:41:08.393067Z"
    }
   },
   "outputs": [],
   "source": [
    "print_every = 300\n",
    "\n",
    "total_loss = 0\n",
    "all_losses = []\n",
    "\n",
    "CUDA_LAUNCH_BLOCKING=1\n",
    "\n",
    "model.train()\n",
    "\n",
    "for idx, row in train_data.iterrows():\n",
    "    text_parts = preprocess_text(str(row['text content']))\n",
    "    label = torch.tensor([row['is_misinfo']]).long().to(device)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    overall_output = torch.zeros((1, 2)).float().to(device)\n",
    "    for part in text_parts:\n",
    "        if len(part) > 0:\n",
    "            try:\n",
    "                input = part.reshape(-1)[:512].reshape(1, -1)\n",
    "                # print(input.shape)\n",
    "                overall_output += model(input, labels=label)[1].float().to(device)\n",
    "            except Exception as e:\n",
    "                print(str(e))\n",
    "\n",
    "#     overall_output /= len(text_parts)\n",
    "    overall_output = F.softmax(overall_output[0], dim=-1)\n",
    "\n",
    "    if label == 0:\n",
    "        label = torch.tensor([1.0, 0.0]).float().to(device)\n",
    "    elif label == 1:\n",
    "        label = torch.tensor([0.0, 1.0]).float().to(device)\n",
    "\n",
    "    # print(overall_output, label)\n",
    "\n",
    "    loss = criterion(overall_output, label)\n",
    "    total_loss += loss.item()\n",
    "    \n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    if idx % print_every == 0 and idx > 0:\n",
    "        average_loss = total_loss / print_every\n",
    "        print(\"{}/{}. Average loss: {}\".format(idx, len(train_data), average_loss))\n",
    "        all_losses.append(average_loss)\n",
    "        total_loss = 0\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T22:43:07.177227Z",
     "iopub.status.busy": "2022-05-03T22:43:07.176869Z",
     "iopub.status.idle": "2022-05-03T22:43:07.885290Z",
     "shell.execute_reply": "2022-05-03T22:43:07.884019Z",
     "shell.execute_reply.started": "2022-05-03T22:43:07.177180Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "test2_data = pd.read_csv('/kaggle/input/fakenew/Tweet_test.csv') \n",
    "%matplotlib inline\n",
    "torch.save(model.state_dict(), \"model_after_train.pt\")\n",
    "\n",
    "plt.plot(all_losses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now I will test the accuracy of the model on the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T21:20:35.504658Z",
     "iopub.status.busy": "2022-05-03T21:20:35.504271Z",
     "iopub.status.idle": "2022-05-03T21:21:02.787271Z",
     "shell.execute_reply": "2022-05-03T21:21:02.785647Z",
     "shell.execute_reply.started": "2022-05-03T21:20:35.504606Z"
    }
   },
   "outputs": [],
   "source": [
    "total = len(test2_data)\n",
    "number_right = 0\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    for idx, row in test2_data.iterrows():\n",
    "        text_parts = preprocess_text(str(row['text content']))\n",
    "        label = torch.tensor([row['is_misinfo']]).float().to(device)\n",
    "        \n",
    "        overall_output = torch.zeros((1,2)).to(device)\n",
    "        try:\n",
    "            for part in text_parts:\n",
    "                if len(part) > 0:\n",
    "                    overall_output += model(part.reshape(1, -1))[0]\n",
    "        except RuntimeError:\n",
    "            print(\"GPU out of memory, skipping this entry.\")\n",
    "            continue\n",
    "            \n",
    "        overall_output = F.softmax(overall_output[0], dim=-1)\n",
    "            \n",
    "        result = overall_output.max(0)[1].float().item()\n",
    " \n",
    "        if result == label.item():\n",
    "            number_right += 1\n",
    "            \n",
    "        if idx % print_every == 0 and idx > 0:\n",
    "            print(\"{}/{}. Current accuracy: {}\".format(idx, total, number_right / idx))\n",
    "            \n",
    "print(\"Accuracy on test data: {}\".format(number_right / total))\n",
    "            \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-03T21:05:40.296970Z",
     "iopub.status.busy": "2022-05-03T21:05:40.296586Z",
     "iopub.status.idle": "2022-05-03T21:05:40.305355Z",
     "shell.execute_reply": "2022-05-03T21:05:40.304127Z",
     "shell.execute_reply.started": "2022-05-03T21:05:40.296918Z"
    }
   },
   "outputs": [],
   "source": [
    "def test(text):\n",
    "    text_parts = preprocess_text(text)\n",
    "    overall_output = torch.zeros((1,2)).to(device)\n",
    "    try:\n",
    "        for part in text_parts:\n",
    "            if len(part) > 0:\n",
    "                overall_output += model(part.reshape(1, -1))[0]\n",
    "    except RuntimeError:\n",
    "        print(\"GPU out of memory, skipping this entry.\")\n",
    "\n",
    "    overall_output = F.softmax(overall_output[0], dim=-1)\n",
    "\n",
    "    value, result = overall_output.max(0)\n",
    "\n",
    "    term = \"fake\"\n",
    "    if result.item() == 0:\n",
    "        term = \"real\"\n",
    "\n",
    "    print(\"{} at {}%\".format(term, value.item() * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fake1 = \"\"\"\n",
    "Americans to fund killing babies in abortion that she has been caught trying to add taxpayer financing of abortions to the bill to combat the Coronavirus and provide economic stimulus to the nation as it deals with the COVD-19 outbreak.\n",
    "Nancy Pelosi has a long history of promoting abortion and her first act after becoming Speaker in 2019 was pushing legislation to use tax money for abortions. So it’s no surprise she is trying to exploit the Coronavirus pandemic to push abortion funding again.\n",
    "As The Daily Caller reports: House Speaker Nancy Pelosi sought to include a potential way to guarantee federal funding for abortion into the coronavirus economic stimulus plan, according to multiple senior White House officials.\n",
    "Speaking to the Daily Caller, those officials alleged that while negotiating the stimulus with U.S. Treasury Secretary Steve Mnuchin, Pelosi tried to lobby for “several” provisions that stalled bipartisan commitment to the effort. One was a mandate for up to $1 billion to reimburse laboratory claims, which White House officials say would set a precedent of health spending without protections outlined in the Hyde Amendment.\n",
    "LifeNews depends on the support of readers like you to combat the pro-abortion media. Please donate now.\n",
    "“A New mandatory funding stream that does not have Hyde protections would be unprecedented,” one White House official explained. “Under the guise of protecting people, Speaker Pelosi is working to make sure taxpayer dollars are spent covering abortion—which is not only backwards, but goes against historical norms.”\n",
    "A second White House official referred to the provision as a “slush fund” and yet another questioned “what the Hyde Amendment and abortion have to do with protecting Americans from coronavirus?”\n",
    "Americans should insist to their members of Congress that we need a clean bill that provides aggressive action to help patients and spur the economy. Killing babies with our tax dollars is not the answer to the coronavirus and the situation should not be exploited for political gain.\n",
    "\"\"\"\n",
    "\n",
    "true1 = \"\"\"\n",
    "Price spikes, however, would cause demand to wither and some expensive avocados might be leftover, and stores might try to ration avocados, he added.\n",
    "\"Exactly what the retail strategy would be in this case, I’m not sure. But we would have vastly fewer avocados,\" Sumner said.\n",
    "Just how fast avocados would disappear, if at all, would depend on whether the Trump administration enacts a full or partial border closure. White House economic adviser Larry Kudlow told CNBC he’s looking for ways to keep some commerce flowing.\n",
    "\"We are looking at different options, particularly if you can keep those freight lanes, the truck lanes, open,\" he said this week.  \n",
    "Ben Holtz owns Rocky H Ranch, a 70-acre family-run avocado farm in northern San Diego County. He agreed avocados would run out within weeks.\n",
    "\"Mexico is the big player today. California is not. You shut down the border and California can’t produce to meet the demand,\" Holtz said. \"There will be people without their guacamole.\"\n",
    "While Mexico’s avocado harvest is year-round, California’s is limited to April through July. Growers in the state have picked only about 3 percent of what’s expected to be a much smaller crop of about 175 million pounds this year, Holtz said. A heat wave last summer reduced the crop size.\n",
    "California’s avocado harvest has averaged approximately 300 million pounds in recent years, according to data from the California Avocado Commission. By contrast, the U.S. has imported more than 1.5 billion pounds of avocados from Mexico annually. Representatives from the commission did not respond to requests for this article.\n",
    "Altogether, the U.S. received 43 percent of its fruit and vegetable imports from Mexico in 2016, according to the U.S. Department of Agriculture.\n",
    "Also affecting this year’s avocado supply, a California avocado company in March recalled shipments to six states last month after fears the fruit might be contaminated with a bacterium that can cause health risks.\n",
    "Until the early 2000s, California was the nation’s leading supplier of avocados, Holtz said. Mexico gradually overtook the state and now dominates sales in the U.S.\n",
    "\"It’s a very big possibility,\" Holtz said of avocado shortages. \"Three weeks would dry up the Mexican inventory. California alone consumes more avocados than are grown in our state. Cold storage supply chain is basically three weeks or less of inventory. Most of the time it’s seven days.\"\n",
    "A spokeswoman for the California Restaurant Association said \"we haven’t heard concerns from restaurants, it doesn’t mean they aren’t worried.\" A national grocers association said it will \"continue to closely monitor any developments\" at the border, but did not have information about the potential impact on avocados.\n",
    "\"\"\"\n",
    "\n",
    "test(fake1)\n",
    "test(true1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}