data_collection_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle \n",
    "from random import randint\n",
    "with open('pagesPerProjectStronglyRelated20200325.pickle','rb') as f:\n",
    "    pagesPerProject = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tables_in_wikishared</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bounce_records</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cx_corpora</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cx_lists</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cx_suggestions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cx_translations</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cx_translators</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>echo_unread_wikis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>reading_list</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>reading_list_entry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>reading_list_project</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>urlshortcodes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>wikimedia_editor_tasks_counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>wikimedia_editor_tasks_edit_streak</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>wikimedia_editor_tasks_keys</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>wikimedia_editor_tasks_targets_passed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Tables_in_wikishared\n",
       "0                          bounce_records\n",
       "1                              cx_corpora\n",
       "2                                cx_lists\n",
       "3                          cx_suggestions\n",
       "4                         cx_translations\n",
       "5                          cx_translators\n",
       "6                       echo_unread_wikis\n",
       "7                            reading_list\n",
       "8                      reading_list_entry\n",
       "9                    reading_list_project\n",
       "10                          urlshortcodes\n",
       "11          wikimedia_editor_tasks_counts\n",
       "12     wikimedia_editor_tasks_edit_streak\n",
       "13            wikimedia_editor_tasks_keys\n",
       "14  wikimedia_editor_tasks_targets_passed"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import wmfdata as wmf\n",
    "wmf.mariadb.run(\"\"\"\n",
    "SHOW tables\n",
    "\"\"\", \"wikishared\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Total number of edits to strongly-related COVID articles globally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Function to count revisions\n",
    "# Modified Start Date to Dec 01, 2019\n",
    "\n",
    "import mwapi\n",
    "\n",
    "def countRevisions(page_name,project,date):\n",
    "    \"\"\"\n",
    "    page_name: str, article title, ex: 'COVID-19'\n",
    "    project: str, project id, ex: 'es.wikipedia'\n",
    "    date: timestamp, counting from given day example '2019-12-01T00:00:00Z'\n",
    "    \n",
    "    \"\"\"\n",
    "    counter = 0\n",
    "    session = mwapi.Session(\"https://%s.org\" % project, user_agent=\"dsaez@wikimedia.org - COVID-19 research\")\n",
    "    for response_doc in session.get(action='query', prop='revisions', titles=page_name, \n",
    "                                    rvprop=['ids', 'timestamp'], rvlimit=100, rvdir=\"newer\", \n",
    "                                    formatversion=2, rvstart=date, continuation=True):\n",
    "        for rev_doc in response_doc['query']['pages'][0]['revisions']:\n",
    "            rev_id = rev_doc['revid']\n",
    "            timestamp = rev_doc['timestamp']\n",
    "            counter += 1\n",
    "    return counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#there are some errors/warnings. Don't worry, we are getting a lower-bound\n",
    "results = []\n",
    "startDate = '2020-01-01T00:00:00Z'\n",
    "for project, pages in pagesPerProject.items():\n",
    "    #print(project)\n",
    "    for page in pages:\n",
    "        try:\n",
    "            c = countRevisions(page,project,startDate)\n",
    "            results.append([project,page,c])\n",
    "        except:\n",
    "            #print('error in %s %s' % (page,project))\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame(results)\n",
    "df.rename(columns={0:'project',1:'article',2:'edits'},inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results\n",
    "##### statistics from Dec 1st 2019 to March 30th, 2020\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of projects 146\n",
      "Total number of Articles 2558\n",
      "Total number of edits 299572\n",
      "Avg Edits per Day 3841.0\n",
      "Avg Edits per hour 160.0\n"
     ]
    }
   ],
   "source": [
    "print('Total number of projects %s' % len(df.project.unique()))\n",
    "print('Total number of Articles %s' % df.shape[0])\n",
    "print('Total number of edits %s' % df.edits.sum())\n",
    "avgPerDay = round(df.edits.sum()/(31+28+19))\n",
    "print('Avg Edits per Day %s' % avgPerDay )\n",
    "avgPerHour = round(df.edits.sum()/(24*(31+28+19)))\n",
    "print('Avg Edits per hour %s' % avgPerHour )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Edits From March 1st to March 30, 2020\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_Mar2020 = []\n",
    "startDate = '2020-03-01T00:00:00Z'\n",
    "for project, pages in pagesPerProject.items():\n",
    "   # print(project)\n",
    "    for page in pages:\n",
    "        try:\n",
    "            c = countRevisions(page,project,startDate)\n",
    "            results.append([project,page,c])\n",
    "        except:\n",
    "            #print('error in %s %s' % (page,project))\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame(results_Mar2020)\n",
    "df.rename(columns={0:'project',1:'article',2:'edits'},inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Total number of projects %s' % len(df.project.unique()))\n",
    "print('Total number of Articles %s' % df.shape[0])\n",
    "print('Total number of edits %s' % df.edits.sum())\n",
    "avgPerDay = round(df.edits.sum()/(19))\n",
    "print('Avg Edits per Day %s' % avgPerDay )\n",
    "avgPerHour = round(df.edits.sum()/(24*(19)))\n",
    "print('Avg Edits per hour %s' % avgPerHour )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Edits From Feb 1st to Feb 29, 2020\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_Mar2020 = []\n",
    "startDate = '2020-02-01T00:00:00Z'\n",
    "for project, pages in pagesPerProject.items():\n",
    "   # print(project)\n",
    "    for page in pages:\n",
    "        try:\n",
    "            c = countRevisions(page,project,startDate)\n",
    "            results.append([project,page,c])\n",
    "        except:\n",
    "            #print('error in %s %s' % (page,project))\n",
    "            pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Edits From Jan 1st to Jan 31, 2020\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Edits From dec 1st to Dec 31, 2019\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}