25-Benchmarking-Facebook-Scraping.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"25-Facebook-Scraping.ipynb","private_outputs":true,"provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyOktWDNTqV879oFULzKLRw5"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# Benchmarking time required to scrape FB for possible configs"],"metadata":{"id":"sYLbz4T9F-K3"}},{"cell_type":"code","source":["import math\n","\n","def getPostTime(x):\n","    return math.ceil(x / 5) * 10\n","\n","def getCommentTime(x):\n","    return x \n","\n","def getReplyTime(x):\n","    return math.ceil(x / 8) * 10"],"metadata":{"id":"QAHL6NE67Nzb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def convertTime(t):\n","    unit = \"secs\"\n","    f = 12 * 28 * 24 * 60 * 60\n","    if t > f:\n","        t /= f\n","        unit = \"years\"\n","    f /= 12\n","    if t > f:\n","        t /= f\n","        unit = \"months\"\n","    f /= 28\n","    if t > f:\n","        t /= f\n","        unit = \"days\"\n","    f /= 24\n","    if t > f:\n","        t /= f\n","        unit = \"hrs\"\n","    f /= 60\n","    if t > f:\n","        t /= f\n","        unit = \"mins\"\n","    assert t <= 60\n","    return t, unit"],"metadata":{"id":"05Z8edgV-Gmp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["s = []\n","\n","for postCount in [100, 500, 1000, 2000]:\n","    for commentCount in [100, 500, 1000, 5000]:\n","        for replyCount in [0, 10, 50, 100]:\n","            timeTaken = 0\n","            timeTaken += getPostTime(postCount)\n","            timeTaken += postCount * getCommentTime(commentCount)\n","            timeTaken += postCount * commentCount * getReplyTime(replyCount)\n","            timeProcessed, unit = convertTime(timeTaken)\n","            val = f\"{int(timeProcessed * 100) / 100} {unit}\"\n","            postCount_ = str(postCount)\n","            commentCount_= str(commentCount)\n","            replyCount_= str(replyCount)\n","            dump = f\"{postCount_:12}\\t{commentCount_:12}\\t{replyCount_:12}\\t{val}\"\n","            s.append((timeTaken, dump))"],"metadata":{"id":"H8wXVJ9R9X7T"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["s = sorted(s)"],"metadata":{"id":"3ks6l2Kb_ozg"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(f\"{'#Post':12}\\t{'#Comments':12}\\t{'#Replies':12}\\t{'Time':12}\")\n","for x in s:\n","    print(x[1])"],"metadata":{"id":"Tm2fwF8SCD6L"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import pickle as pkl\n","from pprint import pprint\n","from scraper import Session"],"metadata":{"id":"wciAlT0bGdLI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["with open(\"credentials.txt\", \"r\") as fp:\n","    text = fp.readlines()\n","    EMAIL = text[0].strip().split()[1]\n","    PASSWORD = text[1].strip().split()[1]"],"metadata":{"id":"H4grgjswGp76"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!wget -O /tmp/chromedriver.zip https://chromedriver.storage.googleapis.com/2.33/chromedriver_linux64.zip "],"metadata":{"id":"cthzDwCdHlyB"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!unzip /tmp/chromedriver.zip chromedriver -d /usr/local/bin/"],"metadata":{"id":"F2GuUB8tHpbM"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!apt-get install -y chromium-browser"],"metadata":{"id":"u9GDzb5oIKRN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["sess = Session(\n","    credentials=(EMAIL, \n","                 PASSWORD), \n","    chromeDriverPath=\"chromedriver\"\n",")"],"metadata":{"id":"_6JVWXF3G-1v"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!chromedriver"],"metadata":{"id":"Jv3GN6OHHyrW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!apt-get install libgconf-2-4"],"metadata":{"id":"U0F4vApZIbTR"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[""],"metadata":{"id":"94AaAggxJCcc"},"execution_count":null,"outputs":[]}]}