05-followup-sampling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime as dt\n",
    "import re\n",
    "import json\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# An arbitrary integer to use as a sampling seed so the sampling is deterministic and can be re-run\n",
    "RANDOM_STATE = 788319\n",
    "\n",
    "frame = pd.read_parquet(\"secrets/sampling-frame.parquet\")\n",
    "previous_sample = pd.read_table(\"secrets/sampled_users.tsv\").set_index(\"user_name\")\n",
    "original_strata_targets = pd.read_table('definitions/strata-sample-targets.tsv').set_index('project_group')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The sampling targets (both for actual and follow-up) are dependent on the response rates of the respective wikis. For enwiki, dewiki, and commonswiki, we only sample 75% of the actual target in anticipation of higher response rates (due to a new distribution mechanism). The follow-up sample is only required for enwiki, where we will sampling the remainder of 25% of the actual sampling targets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>10-29</th>\n",
       "      <th>30-149</th>\n",
       "      <th>150-599</th>\n",
       "      <th>600-1199</th>\n",
       "      <th>1200+</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>project_group</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>arwiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asia_wps</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cee_wps</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>commons</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dewiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>enwiki</th>\n",
       "      <td>148</td>\n",
       "      <td>345</td>\n",
       "      <td>214</td>\n",
       "      <td>91</td>\n",
       "      <td>159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eswiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>frwiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>itwiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jawiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kowiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>malay_wps</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>meaf_wps</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metawiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlwiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ptwiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ruwiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sasia_wps</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>viwiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weur_wps</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wikidata</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zhwiki</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               10-29  30-149  150-599  600-1199  1200+\n",
       "project_group                                         \n",
       "arwiki             0       0        0         0      0\n",
       "asia_wps           0       0        0         0      0\n",
       "cee_wps            0       0        0         0      0\n",
       "commons            0       0        0         0      0\n",
       "dewiki             0       0        0         0      0\n",
       "enwiki           148     345      214        91    159\n",
       "eswiki             0       0        0         0      0\n",
       "frwiki             0       0        0         0      0\n",
       "itwiki             0       0        0         0      0\n",
       "jawiki             0       0        0         0      0\n",
       "kowiki             0       0        0         0      0\n",
       "malay_wps          0       0        0         0      0\n",
       "meaf_wps           0       0        0         0      0\n",
       "metawiki           0       0        0         0      0\n",
       "nlwiki             0       0        0         0      0\n",
       "other              0       0        0         0      0\n",
       "ptwiki             0       0        0         0      0\n",
       "ruwiki             0       0        0         0      0\n",
       "sasia_wps          0       0        0         0      0\n",
       "viwiki             0       0        0         0      0\n",
       "weur_wps           0       0        0         0      0\n",
       "wikidata           0       0        0         0      0\n",
       "zhwiki             0       0        0         0      0"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strata_targets = original_strata_targets.copy()\n",
    "strata_targets.iloc[:] = 0.\n",
    "strata_targets.loc['enwiki', :] = round(original_strata_targets.loc['enwiki', :] * 0.25)\n",
    "strata_targets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove previously sampled users from frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame = frame.drop(previous_sample.index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set sample sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stratify sample\n",
    "strata_sizes = frame.groupby([\"project_group\", \"edit_bin\"]).size().unstack()\n",
    "\n",
    "# `strata_targets` has string column names, but `strata_sizes` has categoricals\n",
    "strata_targets.index = strata_sizes.index\n",
    "strata_targets.columns = strata_sizes.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "enwiki (10-29) capped at 147\n"
     ]
    }
   ],
   "source": [
    "for i in strata_targets.index:\n",
    "    for col in strata_targets.columns:\n",
    "        if strata_targets.loc[i, col] > strata_sizes.loc[i, col]:\n",
    "            strata_targets.loc[i, col] = strata_sizes.loc[i, col]\n",
    "            print(f'{i} ({col}) capped at {strata_sizes.loc[i, col]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The targets must not be larger than the strata, or the sampling will hit an error\n",
    "excessive_targets = (strata_targets > strata_sizes)\n",
    "assert excessive_targets.any(axis=None) == False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set target to 0 for any strata with fewer than 10 users,\n",
    "# to prevent cases where de-anonymization of data would be too easy\n",
    "strata_sizes = strata_targets.where(strata_sizes >= 10, other=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>project_group</th>\n",
       "      <th>edit_bin</th>\n",
       "      <th>sample_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>arwiki</td>\n",
       "      <td>10-29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>asia_wps</td>\n",
       "      <td>10-29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cee_wps</td>\n",
       "      <td>10-29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>commons</td>\n",
       "      <td>10-29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dewiki</td>\n",
       "      <td>10-29</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  project_group edit_bin  sample_size\n",
       "0        arwiki    10-29            0\n",
       "1      asia_wps    10-29            0\n",
       "2       cee_wps    10-29            0\n",
       "3       commons    10-29            0\n",
       "4        dewiki    10-29            0"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strata = (\n",
    "  strata_sizes\n",
    "  .unstack()\n",
    "  .reset_index()\n",
    "  .rename({0: \"sample_size\"}, axis=1)\n",
    "  # Ensure columns are ordered in the expected way\n",
    "  [[\"project_group\", \"edit_bin\", \"sample_size\"]]\n",
    ")\n",
    "\n",
    "strata.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pull samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_stratum(project_group, edit_bin, sample_size):\n",
    "    \"\"\"\n",
    "    returns the sampled users as a data frame\n",
    "    \"\"\"\n",
    "    stratum = frame.query(\"project_group == @project_group & edit_bin == @edit_bin\")\n",
    "    sample = stratum.sample(n=sample_size, random_state=RANDOM_STATE)\n",
    "    return sample\n",
    "\n",
    "f = []\n",
    "\n",
    "for row in strata.itertuples(index=False):\n",
    "    f.append(sample_stratum(*row))\n",
    "    \n",
    "sampled_users = pd.concat(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 956 entries, Jerzyewaz to Bungle\n",
      "Data columns (total 5 columns):\n",
      " #   Column         Non-Null Count  Dtype   \n",
      "---  ------         --------------  -----   \n",
      " 0   email          956 non-null    object  \n",
      " 1   home_project   956 non-null    category\n",
      " 2   global_edits   956 non-null    float64 \n",
      " 3   project_group  956 non-null    category\n",
      " 4   edit_bin       956 non-null    category\n",
      "dtypes: category(3), float64(1), object(1)\n",
      "memory usage: 46.4+ KB\n"
     ]
    }
   ],
   "source": [
    "sampled_users.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save samples table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampled_users.to_csv(\"secrets/followup-sampled-users.tsv\", sep=\"\\t\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}