04-sampling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime as dt\n",
    "import re\n",
    "import json\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# An arbitrary integer to use as a sampling seed so the sampling is deterministic and can be re-run\n",
    "RANDOM_STATE = 13440\n",
    "\n",
    "frame = pd.read_parquet(\"secrets/sampling-frame.parquet\")\n",
    "strata_targets = pd.read_table(\"definitions/strata-sample-targets.tsv\").set_index(\"project_group\")\n",
    "strata_targets = strata_targets.astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set sample sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stratify sample\n",
    "strata_sizes = frame.groupby([\"project_group\", \"edit_bin\"]).size().unstack()\n",
    "\n",
    "# `strata_targets` has string column names, but `strata_sizes` has categoricals\n",
    "strata_targets.columns = strata_sizes.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The targets must not be larger than the strata, or the sampling will hit an error\n",
    "excessive_targets = (strata_targets > strata_sizes)\n",
    "assert excessive_targets.any(axis=None) == False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Staggered targets:** In 2022, we are exploring a survey distribution mechanism which is likely to improve the response rate. As the sampling target calculation is based on the response rate of a particular strata, so that we meet the required sample size eventually, a higher response rate would require less number of editors to be targeted. For the initial distribution of the survey, for the three large wikis, Wikimedia Commons, German Wikipedia and English Wikipedia, we will target 75% of actual targets. In the follow-up sample, we will be calculating the targets based on the response rate of initial sample of 2022, rather than 2021. This will also help in preventing saturating communities from survey participation calls."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>edit_bin</th>\n",
       "      <th>10-29</th>\n",
       "      <th>30-149</th>\n",
       "      <th>150-599</th>\n",
       "      <th>600-1199</th>\n",
       "      <th>1200+</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>project_group</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>arwiki</th>\n",
       "      <td>28</td>\n",
       "      <td>60</td>\n",
       "      <td>56</td>\n",
       "      <td>34</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asia_wps</th>\n",
       "      <td>12</td>\n",
       "      <td>20</td>\n",
       "      <td>82</td>\n",
       "      <td>49</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cee_wps</th>\n",
       "      <td>119</td>\n",
       "      <td>528</td>\n",
       "      <td>614</td>\n",
       "      <td>299</td>\n",
       "      <td>946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>commons</th>\n",
       "      <td>97</td>\n",
       "      <td>367</td>\n",
       "      <td>402</td>\n",
       "      <td>194</td>\n",
       "      <td>978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dewiki</th>\n",
       "      <td>53</td>\n",
       "      <td>381</td>\n",
       "      <td>474</td>\n",
       "      <td>255</td>\n",
       "      <td>482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enwiki</th>\n",
       "      <td>443</td>\n",
       "      <td>1035</td>\n",
       "      <td>641</td>\n",
       "      <td>273</td>\n",
       "      <td>477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eswiki</th>\n",
       "      <td>68</td>\n",
       "      <td>328</td>\n",
       "      <td>370</td>\n",
       "      <td>151</td>\n",
       "      <td>358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>frwiki</th>\n",
       "      <td>103</td>\n",
       "      <td>569</td>\n",
       "      <td>589</td>\n",
       "      <td>223</td>\n",
       "      <td>641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>itwiki</th>\n",
       "      <td>44</td>\n",
       "      <td>241</td>\n",
       "      <td>252</td>\n",
       "      <td>104</td>\n",
       "      <td>328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jawiki</th>\n",
       "      <td>74</td>\n",
       "      <td>534</td>\n",
       "      <td>437</td>\n",
       "      <td>141</td>\n",
       "      <td>227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kowiki</th>\n",
       "      <td>8</td>\n",
       "      <td>40</td>\n",
       "      <td>50</td>\n",
       "      <td>17</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>malay_wps</th>\n",
       "      <td>17</td>\n",
       "      <td>77</td>\n",
       "      <td>113</td>\n",
       "      <td>47</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>meaf_wps</th>\n",
       "      <td>94</td>\n",
       "      <td>375</td>\n",
       "      <td>405</td>\n",
       "      <td>198</td>\n",
       "      <td>582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metawiki</th>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>34</td>\n",
       "      <td>32</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlwiki</th>\n",
       "      <td>24</td>\n",
       "      <td>109</td>\n",
       "      <td>116</td>\n",
       "      <td>64</td>\n",
       "      <td>168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>20</td>\n",
       "      <td>140</td>\n",
       "      <td>180</td>\n",
       "      <td>106</td>\n",
       "      <td>415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ptwiki</th>\n",
       "      <td>48</td>\n",
       "      <td>149</td>\n",
       "      <td>140</td>\n",
       "      <td>56</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ruwiki</th>\n",
       "      <td>70</td>\n",
       "      <td>370</td>\n",
       "      <td>365</td>\n",
       "      <td>192</td>\n",
       "      <td>460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sasia_wps</th>\n",
       "      <td>19</td>\n",
       "      <td>70</td>\n",
       "      <td>79</td>\n",
       "      <td>61</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>viwiki</th>\n",
       "      <td>11</td>\n",
       "      <td>57</td>\n",
       "      <td>51</td>\n",
       "      <td>28</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weur_wps</th>\n",
       "      <td>36</td>\n",
       "      <td>255</td>\n",
       "      <td>321</td>\n",
       "      <td>159</td>\n",
       "      <td>559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikidata</th>\n",
       "      <td>23</td>\n",
       "      <td>92</td>\n",
       "      <td>114</td>\n",
       "      <td>69</td>\n",
       "      <td>288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zhwiki</th>\n",
       "      <td>63</td>\n",
       "      <td>319</td>\n",
       "      <td>331</td>\n",
       "      <td>147</td>\n",
       "      <td>337</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "edit_bin       10-29  30-149  150-599  600-1199  1200+\n",
       "project_group                                         \n",
       "arwiki            28      60       56        34     98\n",
       "asia_wps          12      20       82        49    132\n",
       "cee_wps          119     528      614       299    946\n",
       "commons           97     367      402       194    978\n",
       "dewiki            53     381      474       255    482\n",
       "enwiki           443    1035      641       273    477\n",
       "eswiki            68     328      370       151    358\n",
       "frwiki           103     569      589       223    641\n",
       "itwiki            44     241      252       104    328\n",
       "jawiki            74     534      437       141    227\n",
       "kowiki             8      40       50        17     81\n",
       "malay_wps         17      77      113        47    110\n",
       "meaf_wps          94     375      405       198    582\n",
       "metawiki           2       9       34        32     69\n",
       "nlwiki            24     109      116        64    168\n",
       "other             20     140      180       106    415\n",
       "ptwiki            48     149      140        56    164\n",
       "ruwiki            70     370      365       192    460\n",
       "sasia_wps         19      70       79        61    175\n",
       "viwiki            11      57       51        28     64\n",
       "weur_wps          36     255      321       159    559\n",
       "wikidata          23      92      114        69    288\n",
       "zhwiki            63     319      331       147    337"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stagger_wikis = ['commons', 'dewiki', 'enwiki']\n",
    "staggered_strata_targets = strata_targets.copy()\n",
    "staggered_strata_targets.loc[stagger_wikis, :] = staggered_strata_targets.loc[stagger_wikis, :]*0.75\n",
    "staggered_strata_targets = staggered_strata_targets.apply(np.ceil).astype(int)\n",
    "staggered_strata_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set target to 0 for any strata with fewer than 10 users,\n",
    "# to prevent cases where de-anonymization of data would be too easy\n",
    "strata_sizes = staggered_strata_targets.where(strata_sizes >= 10, other=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>project_group</th>\n",
       "      <th>edit_bin</th>\n",
       "      <th>sample_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>arwiki</td>\n",
       "      <td>10-29</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>asia_wps</td>\n",
       "      <td>10-29</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cee_wps</td>\n",
       "      <td>10-29</td>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>commons</td>\n",
       "      <td>10-29</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dewiki</td>\n",
       "      <td>10-29</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  project_group edit_bin  sample_size\n",
       "0        arwiki    10-29           28\n",
       "1      asia_wps    10-29           12\n",
       "2       cee_wps    10-29          119\n",
       "3       commons    10-29           97\n",
       "4        dewiki    10-29           53"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strata = (\n",
    "  strata_sizes\n",
    "  .unstack()\n",
    "  .reset_index()\n",
    "  .rename({0: \"sample_size\"}, axis=1)\n",
    "  # Ensure columns are ordered in the expected way\n",
    "  [[\"project_group\", \"edit_bin\", \"sample_size\"]]\n",
    ")\n",
    "\n",
    "strata.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pull samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_stratum(project_group, edit_bin, sample_size):\n",
    "    \"\"\"\n",
    "    returns the sampled users as a data frame\n",
    "    \"\"\"\n",
    "    stratum = frame.query(\"project_group == @project_group & edit_bin == @edit_bin\")\n",
    "    sample = stratum.sample(n=sample_size, random_state=RANDOM_STATE)\n",
    "    return sample\n",
    "\n",
    "f = []\n",
    "\n",
    "for row in strata.itertuples(index=False):\n",
    "    f.append(sample_stratum(*row))\n",
    "    \n",
    "sampled_users = pd.concat(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save samples table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampled_users.to_csv(\"secrets/sampled_users.tsv\", sep=\"\\t\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}