TradeBotProject_PyTorch-Sentiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bc983035-b226-4bc2-a91c-c91202bf9911",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Your are a MASTER Python Developer capble of accurate and concise code analysis and 'bug' correction.\n",
    "# I will provide the specifications of the computer in use.\n",
    "# I will provide you with the current code I am working with.\n",
    "# I will provide the specifications of the computer in use.\n",
    "# You will assess the code, the associated '#comments', and the error message.\n",
    "# You will identify the specific block or blocks of code, based on the provide code block labeling, i.e. 'BLOCK FOUR' or 'BLOCK EIGHT', that is/are responsible for the error.\n",
    "# You will provide in a code window the identified block or blocks of code with appropriate corrections\n",
    "#\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1c04b0dd-6052-4253-9001-7b898b132b4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba817d91-e074-44cf-ba17-76b95d2df638",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7e5a364-a221-489a-a3c2-2fd3059fefcd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d96ac569-2f31-4ecc-b149-c98ce2bc27c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "\n",
    "# Imports\n",
    "import sys\n",
    "import os\n",
    "import platform\n",
    "from os.path import exists\n",
    "from watermark import watermark\n",
    "\n",
    "import torch\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d0255d17-3aff-4387-a480-c057f7ee0503",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: mps\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "\n",
    "# Set the device\n",
    "device = \"mps\" if torch.backends.mps.is_available() else \"cpu\"\n",
    "print(f\"Using device: {device}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7fa77c0e-b166-46c6-b2e5-5d3f8799c228",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python Platform: macOS-13.0.1-arm64-arm-64bit\n",
      "Python 3.9.15 (main, Nov 24 2022, 08:28:41) \n",
      "[Clang 14.0.6 ]\n",
      "Last updated: 2023-05-08T12:39:22.646488-05:00\n",
      "\n",
      "Python implementation: CPython\n",
      "Python version       : 3.9.15\n",
      "IPython version      : 8.6.0\n",
      "\n",
      "Compiler    : Clang 14.0.6 \n",
      "OS          : Darwin\n",
      "Release     : 22.1.0\n",
      "Machine     : arm64\n",
      "Processor   : arm\n",
      "CPU cores   : 10\n",
      "Architecture: 64bit\n",
      "\n",
      "torch   : 1.13.0\n",
      "sys     : 3.9.15 (main, Nov 24 2022, 08:28:41) \n",
      "[Clang 14.0.6 ]\n",
      "pandas  : 1.5.2\n",
      "platform: 1.0.8\n",
      "\n",
      "PyTorch version: 1.13.0\n",
      "Is MPS (Metal Performance Shader) built? True\n",
      "Is MPS available? True\n"
     ]
    }
   ],
   "source": [
    "# Report Technologies\n",
    "print(f'Python Platform: {platform.platform()}')\n",
    "print(f'Python {sys.version}')\n",
    "print(watermark())\n",
    "print(watermark(iversions=True, globals_=globals()))\n",
    "\n",
    "# Check PyTorch has access to MPS (Metal Performance Shader, Apple's GPU architecture)\n",
    "print(f\"PyTorch version: {torch.__version__}\")\n",
    "print(f\"Is MPS (Metal Performance Shader) built? {torch.backends.mps.is_built()}\")\n",
    "print(f\"Is MPS available? {torch.backends.mps.is_available()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d7273495-70a5-44ed-bb31-7650221d854a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n    Downloading wikipedia edits for BTC  '"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using wikipedia BTC page edit history as proxy for public interest and sentiment\n",
    "# - additional potential sources for 'sentiment analysis' => tweets, google trends\n",
    "\"\"\"\n",
    "    Downloading wikipedia edits for BTC  \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f8cb2a89-1555-4d40-828f-639ba9667d81",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK ONE\n",
    "#\n",
    "\n",
    "import mwclient  # Module for interacting with MediaWiki API.\n",
    "import time  # Module for handling time-related operations.\n",
    "\n",
    "# Initialize a MediaWiki client site which is a class that enables work with a specific wiki site,\n",
    "# in this case, the English (en) version of Wikipedia.\n",
    "site = mwclient.Site('en.wikipedia.org')  # Replace 'en' with the appropriate language code for other sites.\n",
    "\n",
    "# Specify which page to use by creating a Page object using the page's title.\n",
    "page = site.pages['Bitcoin']  # Replace 'Bitcoin' with the title of alternatively desired Wikipedia page.\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6744238d-0fa7-4517-b015-4d4891641be7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK TWO\n",
    "#\n",
    "\n",
    "# Obtain from Wikipedia the list of revisions for the specified page.\n",
    "revs = list(page.revisions())  # Returns a list of dictionaries containing information about each revision of the page.\n",
    "                              # Each dictionary contains keys such as 'user', 'comment', 'timestamp', etc. that provide\n",
    "                              # details about the revision. The list is sorted in reverse chronological order by default.\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c152be04-92d6-4afe-a2d9-08b41f519230",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('revid', 1151233254),\n",
       "             ('parentid', 1149274508),\n",
       "             ('minor', ''),\n",
       "             ('user', 'Rodw'),\n",
       "             ('timestamp',\n",
       "              time.struct_time(tm_year=2023, tm_mon=4, tm_mday=22, tm_hour=18, tm_min=46, tm_sec=9, tm_wday=5, tm_yday=112, tm_isdst=-1)),\n",
       "             ('comment',\n",
       "              'Disambiguating links to [[Central American University]] (link changed to [[Central American University (San Salvador)]]) using [[User:Qwertyytrewqqwerty/DisamAssist|DisamAssist]].')])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#\n",
    "# BLOCK\n",
    "#\n",
    "\n",
    "# Look at the first set of revisions for the specified page.\n",
    "# Outputs an ordered dictionary, which is a Python class that functions as a hybrid-like list and dictionary combined.\n",
    "revs[0]  # Returns a dictionary containing information about the first revision of the page.\n",
    "         # The dictionary contains keys such as 'user', 'comment', 'timestamp', etc. that provide details about the revision.\n",
    "         # The specific information returned depends on the MediaWiki API and the parameters used to query the revisions.\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ab4dc6f8-25eb-492a-94f8-bff5be8841d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK THREE\n",
    "#\n",
    "\n",
    "# Sort the list of revisions for the specified page in ascending order based on their timestamps.\n",
    "# The sorted() function is used here with a key parameter, which takes a function that returns a value to sort by.\n",
    "# In this case, the lambda function returns the timestamp value from each revision dictionary.\n",
    "revs = sorted(revs, key=lambda rev: rev['timestamp'])  # Returns the same list of revision dictionaries, but sorted\n",
    "                                                       # in ascending order by their timestamps.\n",
    "    \n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aa58db6a-76ba-47e5-9ddf-0685baed5215",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('revid', 275832581),\n",
       "             ('parentid', 0),\n",
       "             ('user', 'Pratyeka'),\n",
       "             ('timestamp',\n",
       "              time.struct_time(tm_year=2009, tm_mon=3, tm_mday=8, tm_hour=16, tm_min=41, tm_sec=7, tm_wday=6, tm_yday=67, tm_isdst=-1)),\n",
       "             ('comment', 'creation (stub)')])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Look at the first revision in the sorted/reordered list of revisions for the specified page.\n",
    "# Outputs an ordered dictionary, which is a Python class that functions as a hybrid-like list and dictionary combined.\n",
    "revs[0]  \n",
    "# Returns a dictionary containing information about the first revision of the page in the sorted list."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f0851b7-876b-4eef-bd0f-a75b7f88f58e",
   "metadata": {},
   "source": [
    "# Identifying the 'sentiment'of the wikipedia page edits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "97df7925-ecb3-4c52-88d3-0fbf4d3f1b95",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is a test string-12345-of-67890\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# BLOCK FOUR\n",
    "#\n",
    "\n",
    "# import the regular expressions module\n",
    "import re\n",
    "\n",
    "# modified pattern string\n",
    "fixed_pattern = \"(.*?)-\\\\d{5}-of-\\\\d{5}\"\n",
    "\n",
    "# create a regular expression object using the fixed pattern string\n",
    "re_obj = re.compile(fixed_pattern)\n",
    "\n",
    "# use the regular expression object to search for matches in a string\n",
    "text = \"This is a test string-12345-of-67890.\"\n",
    "match = re_obj.search(text)\n",
    "\n",
    "print(match.group(0))\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5cea7962-2ad7-49fb-a194-72e476fa7929",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK FIVE\n",
    "#\n",
    "\n",
    "# This library provides for pipelines to 'deep learning' models; requires TensorFlow 2.0 or PyTorch\n",
    "# AutoModelForSequenceClassification provides a pre-trained sequence classification model\n",
    "# AutoTokenizer is used to automatically select the appropriate tokenizer for a given pre-trained model. \n",
    "# The tokenizer is responsible for preprocessing raw text data into a format understood by the deep learning model.\n",
    "# The pipeline function is used for running pre-trained models.\n",
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline\n",
    "\n",
    "\n",
    "# Load the pre-trained model and tokenizer\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"distilbert-base-uncased-finetuned-sst-2-english\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
    "\n",
    "# Create a sentiment analysis pipeline using the loaded model and tokenizer\n",
    "sentiment_pipeline = pipeline('sentiment-analysis',\n",
    "                              model=model,            # use the loaded model\n",
    "                              tokenizer=tokenizer,    # use the tokenizer\n",
    "                              framework='pt',         # set the deep learning framework to PyTorch\n",
    "                              max_length=256,         # set the maximum input length to a smaller value (e.g., 256)\n",
    "                              truncation=True)\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c67f9d20-b448-45dd-a188-6b708a5aea96",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK SIX\n",
    "#\n",
    "\n",
    "from typing import List, Dict, Any\n",
    "# from transformers import pipeline\n",
    "\n",
    "def find_sentiment(texts: List[str]) -> List[float]:\n",
    "    \"\"\"\n",
    "    This function takes in a list of strings of text and returns a list of sentiment scores between -1 and 1,\n",
    "    where negative values indicate negative sentiment and positive values indicate positive sentiment.\n",
    "    \"\"\"\n",
    "    # Load the sentiment analysis model\n",
    "    # sentiment_pipeline = pipeline(\"sentiment-analysis\")\n",
    "    \n",
    "    scores: List[float] = []\n",
    "    for text in texts:\n",
    "        # Run a list of texts through the sentiment analysis model and get a list of sentiment scores, \n",
    "        # each score is a dictionary containing the sentiment label ('POSITIVE' or 'NEGATIVE') and score (a float between 0 and 1)\n",
    "        sents: List[Dict[str, Any]] = sentiment_pipeline([text[:250]])  # run the text through the sentiment analysis model and get the result\n",
    "        \n",
    "        # Extract the sentiment score from the dictionary\n",
    "        score = sents[0]['score']\n",
    "        if sents[0]['label'] == 'NEGATIVE':\n",
    "            score = -score\n",
    "        \n",
    "        # Append the sentiment score to the list of scores\n",
    "        scores.append(score)\n",
    "        \n",
    "    return scores\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "804633b8-d20a-43f8-842a-35b44d22b3b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.9994040727615356, -0.9994834661483765, -0.9756968021392822]\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# BLOCK SEVEN\n",
    "#\n",
    "\n",
    "# Test the sentiment analysis model\n",
    "# Define a list of text strings to analyze\n",
    "texts = [\n",
    "    \"This is a positive sentence.\",\n",
    "    \"This is a negative sentence.\",\n",
    "    \"This is a neutral sentence.\",\n",
    "]\n",
    "\n",
    "# Call the find_sentiment function to analyze the texts\n",
    "sentiment_scores = find_sentiment(texts)\n",
    "\n",
    "# Print the sentiment scores\n",
    "print(sentiment_scores)\n",
    "\n",
    "#\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b787a34-9f43-490a-bbb9-bf30ee99de9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK EIGHT\n",
    "#\n",
    "\n",
    "# Define the 'edits' dictionary\n",
    "edits: Dict[str, Dict[str, Any]] = {}\n",
    "\n",
    "# Iterate through the revisions in the sorted list 'revs'\n",
    "for rev in revs:\n",
    "    # Get the date of the revision using the 'timestamp' attribute\n",
    "    date: str = time.strftime('%Y-%m-%d', rev['timestamp'])\n",
    "    \n",
    "    # Check if the date is not in the 'edits' dictionary\n",
    "    if date not in edits:\n",
    "        # If the date is not in the 'edits' dictionary, create a new dictionary with the following attributes:\n",
    "        # - 'sentiments': a list to store the sentiment analysis scores\n",
    "        # - 'edt_count': an integer to store the number of edits made on the date\n",
    "        edits[date]: Dict[str, Any] = {'sentiments': [], 'edt_count': 0}\n",
    "        \n",
    "    # Increment the 'edt_count' by 1 for the current date\n",
    "    edits[date]['edt_count'] += 1\n",
    "        \n",
    "    # Get the comment for the revision, if it exists\n",
    "    comment: str = rev.get('comment', '')\n",
    "    \n",
    "    # Call the 'find_sentiment' function to get the sentiment score for the comment\n",
    "    sentiment_score: float = find_sentiment(comment)\n",
    "    \n",
    "    # Append the sentiment score to the 'sentiments' list for the current date\n",
    "    edits[date]['sentiments'].append(sentiment_score)\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "675eb5cd-7854-46d8-b63d-abfb0ad058e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK NINE\n",
    "#\n",
    "\n",
    "from statistics import mean  # import mean function from statistics module\n",
    "\n",
    "# iterate through each key in the edits dictionary\n",
    "for key in edits:\n",
    "    \n",
    "    # check if there are any sentiments present for the current key\n",
    "    if len(edits[key]['sentiments']) > 0:\n",
    "        \n",
    "        # calculate the mean sentiment for the current key\n",
    "        edits[key]['sentiment'] = mean(edits[key]['sentiments'])\n",
    "        \n",
    "        # calculate the negative sentiment score as a ratio of the total sentiment scores\n",
    "        edits[key]['percent_neg_sentiment'] = len([s for s in edits[key]['sentiments'] if s < 0]) / len(edits[key]['sentiments'])\n",
    "                                           \n",
    "    # if there are no sentiments present for the current key\n",
    "    else:\n",
    "        \n",
    "        # set the sentiment score to 0\n",
    "        edits[key]['sentiment'] = 0\n",
    "                                           \n",
    "        # set the negative sentiment score to 0\n",
    "        edits[key]['neg_sentiment'] = 0\n",
    "\n",
    "        # remove the 'sentiments' key from the current key in the 'edits' dictionary\n",
    "        del edits[key]['sentiments']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad977b9e-68a1-431f-9197-1b9ce8d0522f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK SIX\n",
    "#\n",
    "\n",
    "# Provides for pipeline to 'deep learning' models; requires TensorFlow 2.0 or PyTorch\n",
    "# from transformers import pipeline  # import the transformers library, which includes the pipeline function for running pre-trained models\n",
    "\n",
    "# initialize the Deep Learning sentiment analysis model using the pipeline function from transformers\n",
    "# reduced maximum input length to a smaller value (e.g., 256)\n",
    "# switch to PyTorch by setting the framework parameter to \"pt\"\n",
    "# sentiment_pipeline = pipeline(\"sentiment-analysis\",\n",
    "#                               model=\"distilbert-base-uncased-finetuned-sst-2-english\",\n",
    "#                               tokenizer=\"distilbert-base-uncased\",\n",
    "#                               framework=\"pt\")\n",
    "\n",
    "\n",
    "# sentiment_pipeline = pipeline(\"sentiment-analysis\",\n",
    "#                               model=\"distilbert-base-uncased-finetuned-sst-2-english\",\n",
    "#                               tokenizer=\"distilbert-base-uncased\",\n",
    "#                               framework=\"pt\",\n",
    "#                               max_length=256)\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98ff11ae-b395-4192-be66-0ebcef44a5a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "\n",
    "# # Set the model and tokenizer names\n",
    "# model_name = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "# tokenizer_name = \"distilbert-base-uncased\"\n",
    "\n",
    "# # Load the pre-trained model and tokenizer\n",
    "# model = AutoModelForSequenceClassification.from_pretrained(model_name)\n",
    "# tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)\n",
    "\n",
    "# # Create a sentiment analysis pipeline using the loaded model and tokenizer\n",
    "# sentiment_pipeline = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5042f8a2-ee67-47df-86d3-0b0d6406d90e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2fb8a9f-951e-4c10-9a46-d88217c860c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# BLOCK SEVEN\n",
    "#\n",
    "\n",
    "# from typing import Dict, List, Tuple, Any  # import List and Tuple types from the typing module for type hinting\n",
    "\n",
    "# def find_sentiment(text: List[str]) -> List[float]:\n",
    "#     \"\"\"\n",
    "#     This function takes in a list of strings of text and returns a list of sentiment scores between -1 and 1,\n",
    "#     where negative values indicate negative sentiment and positive values indicate positive sentiment.\n",
    "#     \"\"\"\n",
    "    \n",
    "#     # Run a list of texts through the sentiment analysis model and get a list of sentiment scores, \n",
    "#     # each score is a dictionary containing the sentiment label ('POSITIVE' or 'NEGATIVE') and score (a float between 0 and 1)\n",
    "\n",
    "#     sents: List[Dict[str, Any]] = sentiment_pipeline([text[:250]])[0]  # run the text through the sentiment analysis model and get the result\n",
    "#     scores: List[float] = []\n",
    "#     for sent in sents:\n",
    "#         score: float = sent['score']\n",
    "#         if sent['label'] == 'NEGATIVE':\n",
    "#             score *= -1\n",
    "#         scores.append(score)\n",
    "#     return scores\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc478fdf-6a7d-4b86-9e68-2173a523045f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88c7dc46-608e-47f5-947c-7fe75d3565b2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cdf99d1-6d1b-4698-a452-08bcb97b8415",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dee5f8f-3fba-4648-afd8-e83d33caf31d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27a2ae57-9668-43d9-b3b7-fca96af11cec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bc3b94e-0bca-4d23-9457-1490ade3302e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0ec8626-858c-4331-9206-cece2657d0c1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34993c10-5815-4a0d-a70b-2709ebbdc3e0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17855c01-072b-4887-9159-b17c4b967cf5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14cb7847-af67-4cfb-899c-95c113a5130b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97b27b0d-f82a-4fbc-a273-e18c89d1dcac",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#\n",
    "\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff345703-09e6-468a-9e91-23811dd3f3ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "#BLOCK ALPHA\n",
    "# Define the 'edits' dictionary\n",
    "edits: Dict[str, Dict[str, Any]] = {}\n",
    "\n",
    "# Iterate through the revisions in the sorted list 'revs'\n",
    "for rev in revs:\n",
    "    # Get the date of the revision using the 'timestamp' attribute\n",
    "    date: str = time.strftime('%Y-%m-%d', rev['timestamp'])\n",
    "    \n",
    "    # Check if the date is not in the 'edits' dictionary\n",
    "    if date not in edits:\n",
    "        # If the date is not in the 'edits' dictionary, create a new dictionary with the following attributes:\n",
    "        # - 'sentiments': a list to store the sentiment analysis scores\n",
    "        # - 'edt_count': an integer to store the number of edits made on the date\n",
    "        edits[date]: Dict[str, Any] = {'sentiments': [], 'edt_count': 0, 'sentiment': 0, 'percent_neg_sentiment': 0}\n",
    "        \n",
    "    # Increment the 'edt_count' by 1 for the current date\n",
    "    edits[date]['edt_count'] += 1\n",
    "        \n",
    "    # Get the comment for the revision, if it exists\n",
    "    comment: str = rev.get('comment', '')\n",
    "    \n",
    "    # Call the 'find_sentiment' function to get the sentiment score for the comment\n",
    "    sentiment_score: float = find_sentiment(comment)\n",
    "    \n",
    "    # Append the sentiment score to the 'sentiments' list for the current date\n",
    "    edits[date]['sentiments'].append(sentiment_score)\n",
    "\n",
    "#BLOCK BETA\n",
    "from statistics import mean  # import mean function from statistics module\n",
    "\n",
    "\n",
    "# iterate through each key in the edits dictionary\n",
    "for key in edits:\n",
    "    \n",
    "    # calculate the mean sentiment for the current key\n",
    "    if 'sentiments' in edits[key] and len(edits[key]['sentiments']) > 0:\n",
    "        edits[key]['sentiment'] = mean(edits[key]['sentiments'])\n",
    "        edits[key]['percent_neg_sentiment'] = len([s for s in edits[key]['sentiments'] if s < 0]) / len(edits[key]['sentiments'])\n",
    "        \n",
    "    # if there are no sentiments present for the current key\n",
    "    else:\n",
    "        edits[key]['sentiment'] = 0\n",
    "        edits[key]['percent_neg_sentiment'] = 0\n",
    "        \n",
    "        # remove the 'sentiments' key from the current key in the 'edits' dictionary\n",
    "        if 'sentiments' in edits[key]:\n",
    "            del edits[key]['sentiments']\n",
    "\n",
    "        \n",
    "        \n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0981022a-5568-48b1-a58d-2708f9d299be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the 'edits' dictionary\n",
    "edits: Dict[str, Dict[str, Any]] = {}\n",
    "\n",
    "# Iterate through the revisions in the sorted list 'revs'\n",
    "for rev in revs:\n",
    "    # Get the date of the revision using the 'timestamp' attribute\n",
    "    date: str = time.strftime('%Y-%m-%d', rev['timestamp'])\n",
    "    \n",
    "    # Check if the date is not in the 'edits' dictionary\n",
    "    if date not in edits:\n",
    "        # If the date is not in the 'edits' dictionary, create a new dictionary with the following attributes:\n",
    "        # - 'sentiments': a list to store the sentiment analysis scores\n",
    "        # - 'edt_count': an integer to store the number of edits made on the date\n",
    "        edits[date]: Dict[str, Any] = {'sentiments': [], 'edt_count': 0}\n",
    "        \n",
    "    # Increment the 'edt_count' by 1 for the current date\n",
    "    edits[date]['edt_count'] += 1\n",
    "        \n",
    "    # Get the comment for the revision, if it exists\n",
    "    comment: str = rev.get('comment', '')\n",
    "    \n",
    "    # Call the 'find_sentiment' function to get the sentiment score for the comment\n",
    "    sentiment_score: float = find_sentiment(comment)\n",
    "    \n",
    "    # Append the sentiment score to the 'sentiments' list for the current date\n",
    "    edits[date]['sentiments'].append(sentiment_score)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aac1652-3fe0-46df-a424-fce1cc2bdcfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from statistics import mean  # import mean function from statistics module\n",
    "\n",
    "# iterate through each key in the edits dictionary\n",
    "for key in edits:\n",
    "    \n",
    "    # check if there are any sentiments present for the current key\n",
    "    if len(edits[key]['sentiments']) > 0:\n",
    "        \n",
    "        # calculate the mean sentiment for the current key\n",
    "        edits[key]['sentiment'] = mean(edits[key]['sentiments'])\n",
    "        \n",
    "        # calculate the negative sentiment score as a ratio of the total sentiment scores\n",
    "        edits[key]['percent_neg_sentiment'] = len([s for s in edits[key]['sentiments'] if s < 0]) / len(edits[key]['sentiments'])\n",
    "                                           \n",
    "    # if there are no sentiments present for the current key\n",
    "    else:\n",
    "        \n",
    "        # set the sentiment score to 0\n",
    "        edits[key]['sentiment'] = 0\n",
    "                                           \n",
    "        # set the negative sentiment score to 0\n",
    "        edits[key]['neg_sentiment'] = 0\n",
    "\n",
    "        # remove the 'sentiments' key from the current key in the 'edits' dictionary\n",
    "        del edits[key]['sentiments']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1deb3dfa-5c23-4dbb-8b22-5e5500b12126",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'edits' is Key value Pair/s dictionary, each key => date\n",
    "# there are 3 key values; 'edt_count', 'sentiment', and 'neg_sentiment'\n",
    "# 'edt_count' = number of times BTC wikipedia page comments was edited on a given day\n",
    "# 'sentiment' = average sentiment for that day\n",
    "# 'percent_neg_sentiment' = percentage of edits that express sentiment negativity on that given day\n",
    "\n",
    "# edits"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31bec3fe-f6bb-4c01-af27-61ffa816623e",
   "metadata": {},
   "source": [
    "# generate a dataframe with sentiment data\n",
    "transform -\n",
    "structure -\n",
    "clean-up dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40c1842c-69c9-4e87-a9aa-453eb4d1b999",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import pandas module for creating and manipulating dataframes\n",
    "import pandas as pd\n",
    "\n",
    "# Create a dataframe 'edits_df' from the dictionary 'edits' with index orientation set to 'index'\n",
    "# takes in list of dictionaries, orient on 'index' ensures that each dictionary is a seperate row of the dataframe\n",
    "BTC_edits_df = pd.DataFrame.from_dict(edits, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb5ab62f-3dd3-40d7-ad31-a9ab52b69d01",
   "metadata": {},
   "outputs": [],
   "source": [
    "BTC_edits_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93f8a177-d6c1-4ab9-8534-332b3d2c9e37",
   "metadata": {},
   "source": [
    "# Addressing the problem of gaps in dates of activity in the dataframe\n",
    "there are data gaps within the dataframe\n",
    "\n",
    "i.e. asset or security is traded more often than days of comments or sentiment generating activity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01bacee2-c78d-48fd-a2b7-3861ce7283a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from DateTime import DateTime\n",
    "\n",
    "dates = pd.date_range(start= '2009-03-08', end= datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb0a53fe-a104-43bf-ab11-4c70542a9c05",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9 (pytorch)",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}