main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Installing required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter(action='ignore')\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "# !pip install statsmodels\n",
    "import statsmodels.stats.api as sms\n",
    "from scipy.stats import ttest_1samp, shapiro, levene, ttest_ind, mannwhitneyu, \\\n",
    "    pearsonr, spearmanr, kendalltau, f_oneway, kruskal\n",
    "from statsmodels.stats.proportion import proportions_ztest\n",
    "\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', 10)\n",
    "pd.set_option('display.width', 500)\n",
    "pd.set_option('display.float_format', lambda x: '%.5f' % x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userid</th>\n",
       "      <th>version</th>\n",
       "      <th>sum_gamerounds</th>\n",
       "      <th>retention_1</th>\n",
       "      <th>retention_7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>116</td>\n",
       "      <td>gate_30</td>\n",
       "      <td>3</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>337</td>\n",
       "      <td>gate_30</td>\n",
       "      <td>38</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>377</td>\n",
       "      <td>gate_40</td>\n",
       "      <td>165</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>483</td>\n",
       "      <td>gate_40</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>488</td>\n",
       "      <td>gate_40</td>\n",
       "      <td>179</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userid  version  sum_gamerounds  retention_1  retention_7\n",
       "0     116  gate_30               3        False        False\n",
       "1     337  gate_30              38         True        False\n",
       "2     377  gate_40             165         True        False\n",
       "3     483  gate_40               1        False        False\n",
       "4     488  gate_40             179         True         True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = pd.read_csv(\"Data/cookie_cats.csv\")\n",
    "df = df1.copy()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset Overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userid</th>\n",
       "      <th>version</th>\n",
       "      <th>sum_gamerounds</th>\n",
       "      <th>retention_1</th>\n",
       "      <th>retention_7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>90184</th>\n",
       "      <td>9999441</td>\n",
       "      <td>gate_40</td>\n",
       "      <td>97</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90185</th>\n",
       "      <td>9999479</td>\n",
       "      <td>gate_40</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90186</th>\n",
       "      <td>9999710</td>\n",
       "      <td>gate_30</td>\n",
       "      <td>28</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90187</th>\n",
       "      <td>9999768</td>\n",
       "      <td>gate_40</td>\n",
       "      <td>51</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90188</th>\n",
       "      <td>9999861</td>\n",
       "      <td>gate_40</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        userid  version  sum_gamerounds  retention_1  retention_7\n",
       "90184  9999441  gate_40              97         True        False\n",
       "90185  9999479  gate_40              30        False        False\n",
       "90186  9999710  gate_30              28         True        False\n",
       "90187  9999768  gate_40              51         True        False\n",
       "90188  9999861  gate_40              16        False        False"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90189, 5)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 90189 entries, 0 to 90188\n",
      "Data columns (total 5 columns):\n",
      " #   Column          Non-Null Count  Dtype \n",
      "---  ------          --------------  ----- \n",
      " 0   userid          90189 non-null  int64 \n",
      " 1   version         90189 non-null  object\n",
      " 2   sum_gamerounds  90189 non-null  int64 \n",
      " 3   retention_1     90189 non-null  bool  \n",
      " 4   retention_7     90189 non-null  bool  \n",
      "dtypes: bool(2), int64(2), object(1)\n",
      "memory usage: 2.2+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>userid</th>\n",
       "      <td>90189.00000</td>\n",
       "      <td>4998412.23450</td>\n",
       "      <td>2883285.60770</td>\n",
       "      <td>116.00000</td>\n",
       "      <td>2512230.00000</td>\n",
       "      <td>4995815.00000</td>\n",
       "      <td>7496452.00000</td>\n",
       "      <td>9999861.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum_gamerounds</th>\n",
       "      <td>90189.00000</td>\n",
       "      <td>51.87246</td>\n",
       "      <td>195.05086</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>5.00000</td>\n",
       "      <td>16.00000</td>\n",
       "      <td>51.00000</td>\n",
       "      <td>49854.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     count          mean           std       min           25%           50%           75%           max\n",
       "userid         90189.00000 4998412.23450 2883285.60770 116.00000 2512230.00000 4995815.00000 7496452.00000 9999861.00000\n",
       "sum_gamerounds 90189.00000      51.87246     195.05086   0.00000       5.00000      16.00000      51.00000   49854.00000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userid            0\n",
       "version           0\n",
       "sum_gamerounds    0\n",
       "retention_1       0\n",
       "retention_7       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Descriptive Statistics\n",
    "Our two groups are 'gate30' and 'gate40' since we will do the A/B test according to our business problem. Accordingly, let's get to know our data more closely."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userid</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>version</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gate_30</th>\n",
       "      <td>44700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gate_40</th>\n",
       "      <td>45489</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         userid\n",
       "version        \n",
       "gate_30   44700\n",
       "gate_40   45489"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of unique players in groups A and B.\n",
    "\n",
    "df.groupby(\"version\")[[\"userid\"]].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">sum_gamerounds</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>max</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>version</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gate_30</th>\n",
       "      <td>52.45626</td>\n",
       "      <td>49854</td>\n",
       "      <td>44700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gate_40</th>\n",
       "      <td>51.29878</td>\n",
       "      <td>2640</td>\n",
       "      <td>45489</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        sum_gamerounds              \n",
       "                  mean    max  count\n",
       "version                             \n",
       "gate_30       52.45626  49854  44700\n",
       "gate_40       51.29878   2640  45489"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(\"version\").agg({\"sum_gamerounds\" : [\"mean\",\"max\",\"count\"]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='version', ylabel='count'>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAGwCAYAAAC0HlECAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAp0klEQVR4nO3de3RU5aH38d8kkEm4TGi4JESCYGEBUS6LcJujomDKqJGDCh7QqClEuqCBCqlc0mIQvGCxClgQtFYjPbBAysEL0QAnSGRBBAzEcpFUPfRAD+QCkgwESCDZ7x9t9ssY1IcQmEn4ftaatZy9nzzzzKy1zZc9eyYOy7IsAQAA4AcF+XsBAAAADQHRBAAAYIBoAgAAMEA0AQAAGCCaAAAADBBNAAAABogmAAAAA038vYDGorq6WkePHlXLli3lcDj8vRwAAGDAsiydOnVK0dHRCgr64XNJRFM9OXr0qGJiYvy9DAAAUAdHjhxRhw4dfnAM0VRPWrZsKemfL7rL5fLzagAAgAmv16uYmBj79/gPIZrqSc1bci6Xi2gCAKCBMbm0hgvBAQAADBBNAAAABogmAAAAA0QTAACAAaIJAADAANEEAABggGgCAAAwQDQBAAAYIJoAAAAMEE0AAAAGiCYAAAADRBMAAIABogkAAMAA0QQAAGCAaAIAADDQxN8LAAD8U9y05f5eAhBw8l563N9LsHGmCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADPDpuQZmXd7X/l4CEHAeiOvi7yUAuA5wpgkAAMAA0QQAAGCAaAIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADBBNAAAABogmAAAAA0QTAACAAaIJAADAANEEAABggGgCAAAwQDQBAAAYIJoAAAAMEE0AAAAGiCYAAAADRBMAAIABogkAAMAA0QQAAGCAaAIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgIGAiaYXX3xRDodDU6ZMsbedO3dOKSkpat26tVq0aKGRI0eqqKjI5+cOHz6shIQENWvWTO3atdO0adN04cIFnzFbtmxR37595XQ61aVLF2VkZNR6/CVLlqhTp04KDQ3VwIEDtXPnzqvxNAEAQAMVENG0a9cuvf766+rVq5fP9qlTp+rDDz/UmjVrlJOTo6NHj+rBBx+091dVVSkhIUGVlZXavn273nnnHWVkZCg9Pd0ec+jQISUkJGjIkCHKz8/XlClT9MQTT2jDhg32mNWrVys1NVWzZ8/W7t271bt3b3k8HhUXF1/9Jw8AABoEv0fT6dOnlZiYqD/+8Y/6yU9+Ym8vKyvTn/70J73yyisaOnSo4uLi9Pbbb2v79u367LPPJEkbN27UgQMH9J//+Z/q06eP7rnnHj377LNasmSJKisrJUnLli1T586d9fLLL6tHjx6aNGmSRo0apQULFtiP9corr2j8+PEaO3asYmNjtWzZMjVr1kxvvfXW9667oqJCXq/X5wYAABovv0dTSkqKEhISFB8f77M9Ly9P58+f99nevXt3dezYUbm5uZKk3Nxc9ezZU5GRkfYYj8cjr9er/fv322O+O7fH47HnqKysVF5ens+YoKAgxcfH22MuZd68eQoPD7dvMTExdXwFAABAQ+DXaFq1apV2796tefPm1dpXWFiokJAQtWrVymd7ZGSkCgsL7TEXB1PN/pp9PzTG6/Xq7NmzOn78uKqqqi45pmaOS0lLS1NZWZl9O3LkiNmTBgAADVITfz3wkSNH9OSTT2rTpk0KDQ311zLqzOl0yul0+nsZAADgGvHbmaa8vDwVFxerb9++atKkiZo0aaKcnBy9+uqratKkiSIjI1VZWanS0lKfnysqKlJUVJQkKSoqqtan6Wru/9gYl8ulsLAwtWnTRsHBwZccUzMHAACA36Lprrvu0t69e5Wfn2/f+vXrp8TERPu/mzZtquzsbPtnCgoKdPjwYbndbkmS2+3W3r17fT7ltmnTJrlcLsXGxtpjLp6jZkzNHCEhIYqLi/MZU11drezsbHsMAACA396ea9mypW655Rafbc2bN1fr1q3t7cnJyUpNTVVERIRcLpcmT54st9utQYMGSZKGDRum2NhYPfbYY5o/f74KCws1a9YspaSk2G+dTZgwQYsXL9b06dM1btw4bd68We+++64yMzPtx01NTVVSUpL69eunAQMGaOHChSovL9fYsWOv0asBAAACnd+iycSCBQsUFBSkkSNHqqKiQh6PR6+99pq9Pzg4WOvXr9fEiRPldrvVvHlzJSUlae7cufaYzp07KzMzU1OnTtWiRYvUoUMHvfnmm/J4PPaY0aNHq6SkROnp6SosLFSfPn2UlZVV6+JwAABw/XJYlmX5exGNgdfrVXh4uMrKyuRyua7a46zL+/qqzQ00VA/EdfH3EupF3LTl/l4CEHDyXnr8qs5/Ob+//f49TQAAAA0B0QQAAGCAaAIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADBBNAAAABogmAAAAA0QTAACAAaIJAADAANEEAABggGgCAAAwQDQBAAAYIJoAAAAMEE0AAAAGiCYAAAADRBMAAIABogkAAMAA0QQAAGCAaAIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADBBNAAAABogmAAAAA0QTAACAAaIJAADAANEEAABggGgCAAAwQDQBAAAYIJoAAAAMEE0AAAAGiCYAAAADRBMAAIABogkAAMAA0QQAAGCAaAIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADBBNAAAABogmAAAAA0QTAACAAaIJAADAANEEAABgwK/RtHTpUvXq1Usul0sul0tut1sff/yxvf/cuXNKSUlR69at1aJFC40cOVJFRUU+cxw+fFgJCQlq1qyZ2rVrp2nTpunChQs+Y7Zs2aK+ffvK6XSqS5cuysjIqLWWJUuWqFOnTgoNDdXAgQO1c+fOq/KcAQBAw+TXaOrQoYNefPFF5eXl6fPPP9fQoUM1YsQI7d+/X5I0depUffjhh1qzZo1ycnJ09OhRPfjgg/bPV1VVKSEhQZWVldq+fbveeecdZWRkKD093R5z6NAhJSQkaMiQIcrPz9eUKVP0xBNPaMOGDfaY1atXKzU1VbNnz9bu3bvVu3dveTweFRcXX7sXAwAABDSHZVmWvxdxsYiICL300ksaNWqU2rZtq5UrV2rUqFGSpIMHD6pHjx7Kzc3VoEGD9PHHH+u+++7T0aNHFRkZKUlatmyZZsyYoZKSEoWEhGjGjBnKzMzUvn377McYM2aMSktLlZWVJUkaOHCg+vfvr8WLF0uSqqurFRMTo8mTJ2vmzJlG6/Z6vQoPD1dZWZlcLld9viQ+1uV9fdXmBhqqB+K6+HsJ9SJu2nJ/LwEIOHkvPX5V57+c398Bc01TVVWVVq1apfLycrndbuXl5en8+fOKj4+3x3Tv3l0dO3ZUbm6uJCk3N1c9e/a0g0mSPB6PvF6vfbYqNzfXZ46aMTVzVFZWKi8vz2dMUFCQ4uPj7TGXUlFRIa/X63MDAACNl9+jae/evWrRooWcTqcmTJigdevWKTY2VoWFhQoJCVGrVq18xkdGRqqwsFCSVFhY6BNMNftr9v3QGK/Xq7Nnz+r48eOqqqq65JiaOS5l3rx5Cg8Pt28xMTF1ev4AAKBh8Hs0devWTfn5+dqxY4cmTpyopKQkHThwwN/L+lFpaWkqKyuzb0eOHPH3kgAAwFXUxN8LCAkJUZcu/7weIS4uTrt27dKiRYs0evRoVVZWqrS01OdsU1FRkaKioiRJUVFRtT7lVvPpuovHfPcTd0VFRXK5XAoLC1NwcLCCg4MvOaZmjktxOp1yOp11e9IAAKDB8fuZpu+qrq5WRUWF4uLi1LRpU2VnZ9v7CgoKdPjwYbndbkmS2+3W3r17fT7ltmnTJrlcLsXGxtpjLp6jZkzNHCEhIYqLi/MZU11drezsbHsMAACAX880paWl6Z577lHHjh116tQprVy5Ulu2bNGGDRsUHh6u5ORkpaamKiIiQi6XS5MnT5bb7dagQYMkScOGDVNsbKwee+wxzZ8/X4WFhZo1a5ZSUlLss0ATJkzQ4sWLNX36dI0bN06bN2/Wu+++q8zMTHsdqampSkpKUr9+/TRgwAAtXLhQ5eXlGjt2rF9eFwAAEHj8Gk3FxcV6/PHHdezYMYWHh6tXr17asGGDfvazn0mSFixYoKCgII0cOVIVFRXyeDx67bXX7J8PDg7W+vXrNXHiRLndbjVv3lxJSUmaO3euPaZz587KzMzU1KlTtWjRInXo0EFvvvmmPB6PPWb06NEqKSlRenq6CgsL1adPH2VlZdW6OBwAAFy/Au57mhoqvqcJ8B++pwlovPieJgAAgAaGaAIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADBBNAAAABogmAAAAA0QTAACAAaIJAADAANEEAABggGgCAAAwQDQBAAAYIJoAAAAMEE0AAAAGiCYAAAADRBMAAICBOkXT0KFDVVpaWmu71+vV0KFDr3RNAAAAAadO0bRlyxZVVlbW2n7u3Dlt3br1ihcFAAAQaJpczuC//vWv9n8fOHBAhYWF9v2qqiplZWXphhtuqL/VAQAABIjLiqY+ffrI4XDI4XBc8m24sLAw/eEPf6i3xQEAAASKy4qmQ4cOybIs3XTTTdq5c6fatm1r7wsJCVG7du0UHBxc74sEAADwt8uKphtvvFGSVF1dfVUWAwAAEKguK5ou9tVXX+mTTz5RcXFxrYhKT0+/4oUBAAAEkjpF0x//+EdNnDhRbdq0UVRUlBwOh73P4XAQTQAAoNGpUzQ999xzev755zVjxoz6Xg8AAEBAqtP3NJ08eVIPPfRQfa8FAAAgYNUpmh566CFt3LixvtcCAAAQsOr09lyXLl309NNP67PPPlPPnj3VtGlTn/2/+tWv6mVxAAAAgaJO0fTGG2+oRYsWysnJUU5Ojs8+h8NBNAEAgEanTtF06NCh+l4HAABAQKvTNU0AAADXmzqdaRo3btwP7n/rrbfqtBgAAIBAVadoOnnypM/98+fPa9++fSotLb3kH/IFAABo6OoUTevWrau1rbq6WhMnTtRPf/rTK14UAABAoKm3a5qCgoKUmpqqBQsW1NeUAAAAAaNeLwT/5ptvdOHChfqcEgAAICDU6e251NRUn/uWZenYsWPKzMxUUlJSvSwMAAAgkNQpmvbs2eNzPygoSG3bttXLL7/8o5+sAwAAaIjqFE2ffPJJfa8DAAAgoNUpmmqUlJSooKBAktStWze1bdu2XhYFAAAQaOp0IXh5ebnGjRun9u3ba/DgwRo8eLCio6OVnJysM2fO1PcaAQAA/K5O0ZSamqqcnBx9+OGHKi0tVWlpqd5//33l5OTo17/+dX2vEQAAwO/q9Pbc2rVr9Ze//EV33nmnve3ee+9VWFiY/uM//kNLly6tr/UBAAAEhDqdaTpz5owiIyNrbW/Xrh1vzwEAgEapTtHkdrs1e/ZsnTt3zt529uxZzZkzR263u94WBwAAECjq9PbcwoULdffdd6tDhw7q3bu3JOmLL76Q0+nUxo0b63WBAAAAgaBO0dSzZ0999dVXWrFihQ4ePChJevjhh5WYmKiwsLB6XSAAAEAgqFM0zZs3T5GRkRo/frzP9rfeekslJSWaMWNGvSwOAAAgUNTpmqbXX39d3bt3r7X95ptv1rJly654UQAAAIGmTtFUWFio9u3b19retm1bHTt27IoXBQAAEGjqFE0xMTHatm1bre3btm1TdHT0FS8KAAAg0NTpmqbx48drypQpOn/+vIYOHSpJys7O1vTp0/lGcAAA0CjVKZqmTZumEydO6Je//KUqKyslSaGhoZoxY4bS0tLqdYEAAACBoE7R5HA49Lvf/U5PP/20vvzyS4WFhalr165yOp31vT4AAICAUKdoqtGiRQv179+/vtYCAAAQsOp0ITgAAMD1hmgCAAAwQDQBAAAYIJoAAAAM+DWa5s2bp/79+6tly5Zq166d7r//fhUUFPiMOXfunFJSUtS6dWu1aNFCI0eOVFFRkc+Yw4cPKyEhQc2aNVO7du00bdo0XbhwwWfMli1b1LdvXzmdTnXp0kUZGRm11rNkyRJ16tRJoaGhGjhwoHbu3FnvzxkAADRMfo2mnJwcpaSk6LPPPtOmTZt0/vx5DRs2TOXl5faYqVOn6sMPP9SaNWuUk5Ojo0eP6sEHH7T3V1VVKSEhQZWVldq+fbveeecdZWRkKD093R5z6NAhJSQkaMiQIcrPz9eUKVP0xBNPaMOGDfaY1atXKzU1VbNnz9bu3bvVu3dveTweFRcXX5sXAwAABDSHZVmWvxdRo6SkRO3atVNOTo4GDx6ssrIytW3bVitXrtSoUaMkSQcPHlSPHj2Um5urQYMG6eOPP9Z9992no0ePKjIyUpK0bNkyzZgxQyUlJQoJCdGMGTOUmZmpffv22Y81ZswYlZaWKisrS5I0cOBA9e/fX4sXL5YkVVdXKyYmRpMnT9bMmTN/dO1er1fh4eEqKyuTy+Wq75fGti7v66s2N9BQPRDXxd9LqBdx05b7ewlAwMl76fGrOv/l/P4OqGuaysrKJEkRERGSpLy8PJ0/f17x8fH2mO7du6tjx47Kzc2VJOXm5qpnz552MEmSx+OR1+vV/v377TEXz1EzpmaOyspK5eXl+YwJCgpSfHy8Pea7Kioq5PV6fW4AAKDxCphoqq6u1pQpU3TrrbfqlltukSQVFhYqJCRErVq18hkbGRmpwsJCe8zFwVSzv2bfD43xer06e/asjh8/rqqqqkuOqZnju+bNm6fw8HD7FhMTU7cnDgAAGoSAiaaUlBTt27dPq1at8vdSjKSlpamsrMy+HTlyxN9LAgAAV9EV/RmV+jJp0iStX79en376qTp06GBvj4qKUmVlpUpLS33ONhUVFSkqKsoe891PudV8uu7iMd/9xF1RUZFcLpfCwsIUHBys4ODgS46pmeO7nE4nf2sPAIDriF/PNFmWpUmTJmndunXavHmzOnfu7LM/Li5OTZs2VXZ2tr2toKBAhw8fltvtliS53W7t3bvX51NumzZtksvlUmxsrD3m4jlqxtTMERISori4OJ8x1dXVys7OtscAAIDrm1/PNKWkpGjlypV6//331bJlS/v6ofDwcIWFhSk8PFzJyclKTU1VRESEXC6XJk+eLLfbrUGDBkmShg0bptjYWD322GOaP3++CgsLNWvWLKWkpNhngiZMmKDFixdr+vTpGjdunDZv3qx3331XmZmZ9lpSU1OVlJSkfv36acCAAVq4cKHKy8s1duzYa//CAACAgOPXaFq6dKkk6c477/TZ/vbbb+vnP/+5JGnBggUKCgrSyJEjVVFRIY/Ho9dee80eGxwcrPXr12vixIlyu91q3ry5kpKSNHfuXHtM586dlZmZqalTp2rRokXq0KGD3nzzTXk8HnvM6NGjVVJSovT0dBUWFqpPnz7KysqqdXE4AAC4PgXU9zQ1ZHxPE+A/fE8T0HjxPU0AAAANDNEEAABggGgCAAAwQDQBAAAYIJoAAAAMEE0AAAAGiCYAAAADRBMAAIABogkAAMAA0QQAAGCAaAIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADBBNAAAABogmAAAAA0QTAACAAaIJAADAANEEAABggGgCAAAwQDQBAAAYIJoAAAAMEE0AAAAGiCYAAAADRBMAAIABogkAAMAA0QQAAGCAaAIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADBBNAAAABogmAAAAA0QTAACAAaIJAADAANEEAABggGgCAAAwQDQBAAAYIJoAAAAMEE0AAAAGiCYAAAADRBMAAIABogkAAMAA0QQAAGCAaAIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMODXaPr00081fPhwRUdHy+Fw6L333vPZb1mW0tPT1b59e4WFhSk+Pl5fffWVz5hvv/1WiYmJcrlcatWqlZKTk3X69GmfMX/96191++23KzQ0VDExMZo/f36ttaxZs0bdu3dXaGioevbsqY8++qjeny8AAGi4/BpN5eXl6t27t5YsWXLJ/fPnz9err76qZcuWaceOHWrevLk8Ho/OnTtnj0lMTNT+/fu1adMmrV+/Xp9++ql+8Ytf2Pu9Xq+GDRumG2+8UXl5eXrppZf0zDPP6I033rDHbN++XQ8//LCSk5O1Z88e3X///br//vu1b9++q/fkAQBAg+KwLMvy9yIkyeFwaN26dbr//vsl/fMsU3R0tH7961/rqaeekiSVlZUpMjJSGRkZGjNmjL788kvFxsZq165d6tevnyQpKytL9957r/7xj38oOjpaS5cu1W9/+1sVFhYqJCREkjRz5ky99957OnjwoCRp9OjRKi8v1/r16+31DBo0SH369NGyZcsuud6KigpVVFTY971er2JiYlRWViaXy1Xvr0+NdXlfX7W5gYbqgbgu/l5CvYibttzfSwACTt5Lj1/V+b1er8LDw41+fwfsNU2HDh1SYWGh4uPj7W3h4eEaOHCgcnNzJUm5ublq1aqVHUySFB8fr6CgIO3YscMeM3jwYDuYJMnj8aigoEAnT560x1z8ODVjah7nUubNm6fw8HD7FhMTc+VPGgAABKyAjabCwkJJUmRkpM/2yMhIe19hYaHatWvns79JkyaKiIjwGXOpOS5+jO8bU7P/UtLS0lRWVmbfjhw5crlPEQAANCBN/L2AhsrpdMrpdPp7GQAA4BoJ2DNNUVFRkqSioiKf7UVFRfa+qKgoFRcX++y/cOGCvv32W58xl5rj4sf4vjE1+wEAAAI2mjp37qyoqChlZ2fb27xer3bs2CG32y1JcrvdKi0tVV5enj1m8+bNqq6u1sCBA+0xn376qc6fP2+P2bRpk7p166af/OQn9piLH6dmTM3jAAAA+DWaTp8+rfz8fOXn50v658Xf+fn5Onz4sBwOh6ZMmaLnnntOH3zwgfbu3avHH39c0dHR9ifsevToobvvvlvjx4/Xzp07tW3bNk2aNEljxoxRdHS0JOmRRx5RSEiIkpOTtX//fq1evVqLFi1SamqqvY4nn3xSWVlZevnll3Xw4EE988wz+vzzzzVp0qRr/ZIAAIAA5ddrmj7//HMNGTLEvl8TMklJScrIyND06dNVXl6uX/ziFyotLdVtt92mrKwshYaG2j+zYsUKTZo0SXfddZeCgoI0cuRIvfrqq/b+8PBwbdy4USkpKYqLi1ObNm2Unp7u811O//Zv/6aVK1dq1qxZ+s1vfqOuXbvqvffe0y233HINXgUAANAQBMz3NDV0l/M9D1eC72kCauN7moDGi+9pAgAAaGCIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADBBNAAAABogmAAAAA0QTAACAAaIJAADAANEEAABggGgCAAAwQDQBAAAYIJoAAAAMEE0AAAAGiCYAAAADRBMAAIABogkAAMAA0QQAAGCAaAIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADBBNAAAABogmAAAAA0QTAACAAaIJAADAANEEAABggGgCAAAwQDQBAAAYIJoAAAAMEE0AAAAGiCYAAAADRBMAAIABogkAAMAA0QQAAGCAaAIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADBBNAAAABogmAAAAA0QTAACAAaIJAADAANEEAABggGgCAAAwQDQBAAAYIJoAAAAMEE0AAAAGiCYAAAADRNN3LFmyRJ06dVJoaKgGDhyonTt3+ntJAAAgABBNF1m9erVSU1M1e/Zs7d69W71795bH41FxcbG/lwYAAPyMaLrIK6+8ovHjx2vs2LGKjY3VsmXL1KxZM7311lv+XhoAAPCzJv5eQKCorKxUXl6e0tLS7G1BQUGKj49Xbm5urfEVFRWqqKiw75eVlUmSvF7vVV3nmdOnrur8QEN0tY+7a6Wq4qy/lwAEnKt9fNfMb1nWj44lmv7l+PHjqqqqUmRkpM/2yMhIHTx4sNb4efPmac6cObW2x8TEXLU1AgBwvQn/w4Rr8jinTp1SeHj4D44hmuooLS1Nqamp9v3q6mp9++23at26tRwOhx9XhmvB6/UqJiZGR44ckcvl8vdyANQjju/ri2VZOnXqlKKjo390LNH0L23atFFwcLCKiop8thcVFSkqKqrWeKfTKafT6bOtVatWV3OJCEAul4v/qQKNFMf39ePHzjDV4ELwfwkJCVFcXJyys7PtbdXV1crOzpbb7fbjygAAQCDgTNNFUlNTlZSUpH79+mnAgAFauHChysvLNXbsWH8vDQAA+BnRdJHRo0erpKRE6enpKiwsVJ8+fZSVlVXr4nDA6XRq9uzZtd6iBdDwcXzj+zgsk8/YAQAAXOe4pgkAAMAA0QQAAGCAaAIAADBANAEAABggmoDv2LJlixwOh0pLS+ttzhMnTujuu+9WdHS0nE6nYmJiNGnSpFp/U2nLli3q27evnE6nunTpooyMjHpbA4Crc3xf7MSJE+rQocMlH4Pju+EjmoBrICgoSCNGjNAHH3ygv/3tb8rIyNB///d/a8KE//83lQ4dOqSEhAQNGTJE+fn5mjJlip544glt2LDBjysHcDmSk5PVq1evWts5vhsJC2hkvF6v9cgjj1jNmjWzoqKirFdeecW64447rCeffNKyLMtavny5FRcXZ7Vo0cKKjIy0Hn74YauoqMiyLMs6dOiQJcnnlpSUZFmWZVVVVVkvvPCC1alTJys0NNTq1auXtWbNmjqvc9GiRVaHDh3s+9OnT7duvvlmnzGjR4+2PB5PnR8DaGwC+fh+7bXXrDvuuMPKzs62JFknT56093F8Nw6caUKjk5qaqm3btumDDz7Qpk2btHXrVu3evdvef/78eT377LP64osv9N577+nvf/+7fv7zn0uSYmJitHbtWklSQUGBjh07pkWLFkmS5s2bp+XLl2vZsmXav3+/pk6dqkcffVQ5OTmXvcajR4/qv/7rv3THHXfY23JzcxUfH+8zzuPxKDc397LnBxqrQD2+Dxw4oLlz52r58uUKCqr9q5Xju5Hwd7UB9cnr9VpNmzb1+RdiaWmp1axZM/tfot+1a9cuS5J16tQpy7Is65NPPqn1r8Rz585ZzZo1s7Zv3+7zs8nJydbDDz9svL4xY8ZYYWFhliRr+PDh1tmzZ+19Xbt2tV544QWf8ZmZmZYk68yZM8aPATRWgXp8nzt3zurVq5f15z//+Xsfg+O7ceBMExqV//mf/9H58+c1YMAAe1t4eLi6detm38/Ly9Pw4cPVsWNHtWzZ0j7bc/jw4e+d9+uvv9aZM2f0s5/9TC1atLBvy5cv1zfffGO8vgULFmj37t16//339c033yg1NbUOzxK4PgXq8Z2WlqYePXro0UcfvYJnh4aAvz2H60p5ebk8Ho88Ho9WrFihtm3b6vDhw/J4PKqsrPzenzt9+rQkKTMzUzfccIPPvsv5+1RRUVGKiopS9+7dFRERodtvv11PP/202rdvr6ioKBUVFfmMLyoqksvlUlhY2GU8S+D65K/je/Pmzdq7d6/+8pe/SJKsf/11sjZt2ui3v/2t5syZw/HdSBBNaFRuuukmNW3aVLt27VLHjh0lSWVlZfrb3/6mwYMH6+DBgzpx4oRefPFFxcTESJI+//xznzlCQkIkSVVVVfa22NhYOZ1OHT582Oc6pCtRXV0tSaqoqJAkud1uffTRRz5jNm3aJLfbXS+PBzR0gXp8r127VmfPnrXv79q1S+PGjdPWrVv105/+VBLHd2NBNKFRadmypZKSkjRt2jRFRESoXbt2mj17toKCguRwONSxY0eFhIToD3/4gyZMmKB9+/bp2Wef9ZnjxhtvlMPh0Pr163XvvfcqLCxMLVu21FNPPaWpU6equrpat912m8rKyrRt2za5XC4lJSX94Lo++ugjFRUVqX///mrRooX279+vadOm6dZbb1WnTp0kSRMmTNDixYs1ffp0jRs3Tps3b9a7776rzMzMq/VyAQ1KoB7fNWFU4/jx45KkHj16qFWrVpI4vhsNf19UBdS3S30kecCAAdbMmTMty7KslStXWp06dbKcTqfldrutDz74wJJk7dmzx55j7ty5VlRUlOVwOOyPJFdXV1sLFy60unXrZjVt2tRq27at5fF4rJycnB9d0+bNmy23222Fh4dboaGhVteuXa0ZM2b4XChqWf+8gLRPnz5WSEiIddNNN1lvv/12Pb0qQOMQiMf3d13qQvCa7RzfDZvDsv715ivQSJWXl+uGG27Qyy+/rOTkZH8vB0A94vjGtcTbc2h09uzZo4MHD2rAgAEqKyvT3LlzJUkjRozw88oAXCmOb/gTXzmARun3v/+9evfurfj4eJWXl2vr1q1q06bNVXu8CRMm+HxU+eLbxX8qBcCV4/iGv/D2HFAPiouLa/3x3Roul0vt2rW7xisCUF84vlGDaAIAADDA23MAAAAGiCYAAAADRBMAAIABogkAAMAA0QQAdfD3v/9dDodD+fn5/l4KgGuET88BQB1UVVWppKREbdq0UZMmfE8wcD0gmgDgEs6fP6+mTZv6exkAAghvzwFo8N544w1FR0erurraZ/uIESM0btw4SdL777+vvn37KjQ0VDfddJPmzJmjCxcu2GMdDoeWLl2qf//3f1fz5s31/PPP6+TJk0pMTFTbtm0VFhamrl276u2335Z06bfncnJyNGDAADmdTrVv314zZ870eYw777xTv/rVrzR9+nRFREQoKipKzzzzzNV7YQDUK6IJQIP30EMP6cSJE/rkk0/sbd9++62ysrKUmJiorVu36vHHH9eTTz6pAwcO6PXXX1dGRoaef/55n3meeeYZPfDAA9q7d6/GjRunp59+WgcOHNDHH3+sL7/8UkuXLv3eP9fxf//3f7r33nvVv39/ffHFF1q6dKn+9Kc/6bnnnvMZ984776h58+basWOH5s+fr7lz52rTpk31/6IAqH8WADQCI0aMsMaNG2fff/31163o6GirqqrKuuuuu6wXXnjBZ/yf//xnq3379vZ9SdaUKVN8xgwfPtwaO3bsJR/v0KFDliRrz549lmVZ1m9+8xurW7duVnV1tT1myZIlVosWLayqqirLsizrjjvusG677Tafefr372/NmDHj8p8wgGuOM00AGoXExEStXbtWFRUVkqQVK1ZozJgxCgoK0hdffKG5c+f6/KHV8ePH69ixYzpz5ow9R79+/XzmnDhxolatWqU+ffpo+vTp2r59+/c+/pdffim32y2Hw2Fvu/XWW3X69Gn94x//sLf16tXL5+fat2+v4uLiK3ruAK4NoglAozB8+HBZlqXMzEwdOXJEW7duVWJioiTp9OnTmjNnjvLz8+3b3r179dVXXyk0NNSeo3nz5j5z3nPPPfrf//1fTZ06VUePHtVdd92lp5566orW+d2Lyx0OR61rsQAEJj4nC6BRCA0N1YMPPqgVK1bo66+/Vrdu3dS3b19JUt++fVVQUKAuXbpc9rxt27ZVUlKSkpKSdPvtt2vatGn6/e9/X2tcjx49tHbtWlmWZZ9t2rZtm1q2bKkOHTpc2ZMDEBCIJgCNRmJiou677z7t379fjz76qL09PT1d9913nzp27KhRo0bZb9nt27ev1oXaF0tPT1dcXJxuvvlmVVRUaP369erRo8clx/7yl7/UwoULNXnyZE2aNEkFBQWaPXu2UlNTFRTESX2gMeBIBtBoDB06VBERESooKNAjjzxib/d4PFq/fr02btyo/v37a9CgQVqwYIFuvPHGH5wvJCREaWlp6tWrlwYPHqzg4GCtWrXqkmNvuOEGffTRR9q5c6d69+6tCRMmKDk5WbNmzarX5wjAf/hySwAAAAOcaQIAADBANAEAABggmgAAAAwQTQAAAAaIJgAAAANEEwAAgAGiCQAAwADRBAAAYIBoAgAAMEA0AQAAGCCaAAAADPw/99s72GDjAFkAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.groupby(\"version\")[\"sum_gamerounds\"].count()\n",
    "sns.countplot(x=\"version\",palette=\"Paired\",data=df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='version'>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAHaCAYAAACgkCtLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhJElEQVR4nO3de5CV9X3H8c8isCC4i6CyIAvB8YKYoiPedjSaKEo1WqyYMQYbotSOilRBa6STeBsNmbQTbGa8NTVqrMZWjTFqNDEkIdrgJXgZYyL1FkGRFbXsAtYF4fSPTHeyAQ0Ly++w7us1cybZ5zzn2e86+bnvPOd5ztZUKpVKAAAK6VXtAQCAnkV8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIrqXe0B/tT69euzdOnS7LDDDqmpqan2OADAJqhUKlm5cmWGDx+eXr0++tzGNhcfS5cuTWNjY7XHAAA2w5IlSzJixIiP3Gebi48ddtghyR+Gr6urq/I0AMCmaG1tTWNjY/vv8Y+yzcXH/7/VUldXJz4AoJvZlEsmXHAKABQlPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARfWu9gD8kZqaak9ASZVKtScAqApnPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFudUWoAA30vcsbqT/aM58AABFiQ8AoCjxAQAUJT4AgKLEBwBQVKfi47LLLktNTU2Hx5gxY9qff//99zN9+vQMGTIkAwcOzOTJk9Pc3NzlQwMA3Venz3zss88+efPNN9sfjz76aPtzM2fOzH333Zc777wz8+fPz9KlS3PSSSd16cAAQPfW6c/56N27dxoaGjbY3tLSkhtvvDG33357jjzyyCTJTTfdlL333juPPfZYDjnkkI0er62tLW1tbe1ft7a2dnYkAKAb6fSZjxdffDHDhw/PbrvtlilTpmTx4sVJkoULF2bt2rWZMGFC+75jxozJyJEjs2DBgg893pw5c1JfX9/+aGxs3IwfAwDoLjoVHwcffHBuvvnmPPTQQ7nuuuvy6quv5lOf+lRWrlyZZcuWpW/fvhk0aFCH1wwdOjTLli370GPOnj07LS0t7Y8lS5Zs1g8CAHQPnXrb5dhjj23/7+PGjcvBBx+cUaNG5T//8z/Tv3//zRqgtrY2tbW1m/VaAKD72aJbbQcNGpQ999wzL730UhoaGrJmzZqsWLGiwz7Nzc0bvUYEAOiZtig+Vq1alZdffjnDhg3L+PHj06dPn8ybN6/9+UWLFmXx4sVpamra4kEBgI+HTr3tcuGFF+aEE07IqFGjsnTp0lx66aXZbrvtcuqpp6a+vj7Tpk3LrFmzMnjw4NTV1WXGjBlpamr60DtdAICep1Px8frrr+fUU0/NO++8k5133jmHHXZYHnvssey8885Jkrlz56ZXr16ZPHly2traMnHixFx77bVbZXAAoHuqqVQqlWoP8cdaW1tTX1+flpaW1NXVVXucsmpqqj0BJW1bS4+tzOruWXri6u7M729/2wUAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUJT4AACKEh8AQFHiAwAoSnwAAEWJDwCgKPEBABQlPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUJT4AACKEh8AQFHiAwAoSnwAAEWJDwCgKPEBABQlPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUVsUH1//+tdTU1OT888/v33b+++/n+nTp2fIkCEZOHBgJk+enObm5i2dEwD4mNjs+HjyySdzww03ZNy4cR22z5w5M/fdd1/uvPPOzJ8/P0uXLs1JJ520xYMCAB8PmxUfq1atypQpU/Ltb387O+64Y/v2lpaW3HjjjfnmN7+ZI488MuPHj89NN92UX/3qV3nsscc2eqy2tra0trZ2eAAAH1+bFR/Tp0/PZz/72UyYMKHD9oULF2bt2rUdto8ZMyYjR47MggULNnqsOXPmpL6+vv3R2Ni4OSMBAN1Ep+PjjjvuyFNPPZU5c+Zs8NyyZcvSt2/fDBo0qMP2oUOHZtmyZRs93uzZs9PS0tL+WLJkSWdHAgC6kd6d2XnJkiU577zz8vDDD6dfv35dMkBtbW1qa2u75FgAwLavU2c+Fi5cmLfeeiv7779/evfund69e2f+/Pn51re+ld69e2fo0KFZs2ZNVqxY0eF1zc3NaWho6Mq5AYBuqlNnPo466qg899xzHbadfvrpGTNmTL785S+nsbExffr0ybx58zJ58uQkyaJFi7J48eI0NTV13dQAQLfVqfjYYYcd8slPfrLDtgEDBmTIkCHt26dNm5ZZs2Zl8ODBqaury4wZM9LU1JRDDjmk66YGALqtTsXHppg7d2569eqVyZMnp62tLRMnTsy1117b1d8GAOimaiqVSqXaQ/yx1tbW1NfXp6WlJXV1ddUep6yammpPQEnb1tJjK7O6e5aeuLo78/vb33YBAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUJT4AACKEh8AQFHiAwAoSnwAAEWJDwCgKPEBABQlPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUJT4AACKEh8AQFHiAwAoSnwAAEWJDwCgKPEBABQlPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUFSn4uO6667LuHHjUldXl7q6ujQ1NeXBBx9sf/7999/P9OnTM2TIkAwcODCTJ09Oc3Nzlw8NAHRfnYqPESNG5Otf/3oWLlyYX//61znyyCMzadKkPP/880mSmTNn5r777sudd96Z+fPnZ+nSpTnppJO2yuAAQPdUU6lUKltygMGDB+ef/umfcvLJJ2fnnXfO7bffnpNPPjlJ8sILL2TvvffOggULcsghh2z09W1tbWlra2v/urW1NY2NjWlpaUldXd2WjNb91NRUewJK2rKlRzdjdfcsPXF1t7a2pr6+fpN+f2/2NR/r1q3LHXfckdWrV6epqSkLFy7M2rVrM2HChPZ9xowZk5EjR2bBggUfepw5c+akvr6+/dHY2Li5IwEA3UCn4+O5557LwIEDU1tbm7POOiv33HNPxo4dm2XLlqVv374ZNGhQh/2HDh2aZcuWfejxZs+enZaWlvbHkiVLOv1DAADdR+/OvmCvvfbKM888k5aWltx1112ZOnVq5s+fv9kD1NbWpra2drNfDwB0L52Oj759+2b33XdPkowfPz5PPvlk/uVf/iWnnHJK1qxZkxUrVnQ4+9Hc3JyGhoYuGxgA6N62+HM+1q9fn7a2towfPz59+vTJvHnz2p9btGhRFi9enKampi39NgDAx0SnznzMnj07xx57bEaOHJmVK1fm9ttvzy9+8Yv8+Mc/Tn19faZNm5ZZs2Zl8ODBqaury4wZM9LU1PShd7oAAD1Pp+Ljrbfeyhe/+MW8+eabqa+vz7hx4/LjH/84Rx99dJJk7ty56dWrVyZPnpy2trZMnDgx11577VYZHADonrb4cz66WmfuE/7Y8TkfPcu2tfTYyqzunqUnru4in/MBALA5xAcAUJT4AACKEh8AQFHiAwAoSnwAAEWJDwCgKPEBABQlPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUJT4AACKEh8AQFHiAwAoSnwAAEWJDwCgKPEBABQlPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUJT4AACKEh8AQFHiAwAoSnwAAEWJDwCgKPEBABTVqfiYM2dODjzwwOywww7ZZZddcuKJJ2bRokUd9nn//fczffr0DBkyJAMHDszkyZPT3NzcpUMDAN1Xp+Jj/vz5mT59eh577LE8/PDDWbt2bY455pisXr26fZ+ZM2fmvvvuy5133pn58+dn6dKlOemkk7p8cACge6qpVCqVzX3x8uXLs8suu2T+/Pk5/PDD09LSkp133jm33357Tj755CTJCy+8kL333jsLFizIIYcc8meP2dramvr6+rS0tKSurm5zR+ueamqqPQElbf7SoxuyunuWnri6O/P7e4uu+WhpaUmSDB48OEmycOHCrF27NhMmTGjfZ8yYMRk5cmQWLFiw0WO0tbWltbW1wwMA+Pja7PhYv359zj///Bx66KH55Cc/mSRZtmxZ+vbtm0GDBnXYd+jQoVm2bNlGjzNnzpzU19e3PxobGzd3JACgG9js+Jg+fXp+85vf5I477tiiAWbPnp2Wlpb2x5IlS7boeADAtq335rzo3HPPzf33359f/vKXGTFiRPv2hoaGrFmzJitWrOhw9qO5uTkNDQ0bPVZtbW1qa2s3ZwwAoBvq1JmPSqWSc889N/fcc09+9rOfZfTo0R2eHz9+fPr06ZN58+a1b1u0aFEWL16cpqamrpkYAOjWOnXmY/r06bn99ttz7733Zocddmi/jqO+vj79+/dPfX19pk2bllmzZmXw4MGpq6vLjBkz0tTUtEl3ugAAH3+dutW25kNuBb3pppvypS99KckfPmTsggsuyPe+9720tbVl4sSJufbaaz/0bZc/5VZbegy32vYoVnfP0hNXd2d+f2/R53xsDeKDHmPbWnpsZVZ3z9ITV3exz/kAAOgs8QEAFCU+AICixAcAUJT4AACKEh8AQFHiAwAoSnwAAEWJDwCgKPEBABQlPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUJT4AACKEh8AQFHiAwAoSnwAAEWJDwCgKPEBABQlPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUJT4AACKEh8AQFHiAwAoSnwAAEV1Oj5++ctf5oQTTsjw4cNTU1OTH/zgBx2er1QqueSSSzJs2LD0798/EyZMyIsvvthV8wIA3Vyn42P16tXZd999c80112z0+W984xv51re+leuvvz6PP/54BgwYkIkTJ+b999/f4mEBgO6vd2dfcOyxx+bYY4/d6HOVSiVXX311vvKVr2TSpElJku9+97sZOnRofvCDH+Tzn//8lk0LAHR7XXrNx6uvvpply5ZlwoQJ7dvq6+tz8MEHZ8GCBRt9TVtbW1pbWzs8AICPry6Nj2XLliVJhg4d2mH70KFD25/7U3PmzEl9fX37o7GxsStHAgC2MVW/22X27NlpaWlpfyxZsqTaIwEAW1GXxkdDQ0OSpLm5ucP25ubm9uf+VG1tberq6jo8AICPry6Nj9GjR6ehoSHz5s1r39ba2prHH388TU1NXfmtAIBuqtN3u6xatSovvfRS+9evvvpqnnnmmQwePDgjR47M+eefnyuvvDJ77LFHRo8ena9+9asZPnx4TjzxxK6cGwDopjodH7/+9a/zmc98pv3rWbNmJUmmTp2am2++ORdddFFWr16dv/u7v8uKFSty2GGH5aGHHkq/fv26bmoAoNuqqVQqlWoP8cdaW1tTX1+flpaWnnf9R01NtSegpG1r6bGVWd09S09c3Z35/V31u10AgJ5FfAAARYkPAKAo8QEAFCU+AICixAcAUJT4AACKEh8AQFHiAwAoSnwAAEWJDwCgKPEBABQlPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUJT4AACKEh8AQFHiAwAoSnwAAEWJDwCgKPEBABQlPgCAosQHAFCU+AAAihIfAEBR4gMAKEp8AABFiQ8AoCjxAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUJT4AACKEh8AQFFbLT6uueaafOITn0i/fv1y8MEH54knntha3woA6Ea2Snz8x3/8R2bNmpVLL700Tz31VPbdd99MnDgxb7311tb4dgBAN7JV4uOb3/xmzjzzzJx++ukZO3Zsrr/++my//fb5zne+szW+HQDQjfTu6gOuWbMmCxcuzOzZs9u39erVKxMmTMiCBQs22L+trS1tbW3tX7e0tCRJWltbu3o02Lb43zh8bPXE1f3/v7crlcqf3bfL4+Ptt9/OunXrMnTo0A7bhw4dmhdeeGGD/efMmZPLL798g+2NjY1dPRpsW+rrqz0BsJX05NW9cuXK1P+Zf791eXx01uzZszNr1qz2r9evX5933303Q4YMSU1NTRUno4TW1tY0NjZmyZIlqaurq/Y4QBeyvnuWSqWSlStXZvjw4X923y6Pj5122inbbbddmpubO2xvbm5OQ0PDBvvX1tamtra2w7ZBgwZ19Vhs4+rq6vzLCT6mrO+e48+d8fh/XX7Bad++fTN+/PjMmzevfdv69eszb968NDU1dfW3AwC6ma3ytsusWbMyderUHHDAATnooINy9dVXZ/Xq1Tn99NO3xrcDALqRrRIfp5xySpYvX55LLrkky5Yty3777ZeHHnpog4tQoba2NpdeeukGb70B3Z/1zYepqWzKPTEAAF3E33YBAIoSHwBAUeIDAChKfAAARYkPAKAo8UHV/OkfFQQ+PqxvPor4oKiHH344xx13XHbcccdsv/322X777bPjjjvmuOOOy09/+tNqjwdsAeubTeVzPijmlltuyd/+7d/m5JNPzsSJE9s/dK65uTk/+clPctddd+XGG2/M3/zN31R5UqCzrG86Q3xQzJ577pnzzjsv06dP3+jz1157bebOnZsXX3yx8GTAlrK+6QzxQTH9+vXLs88+m7322mujzy9atCj77bdf/vd//7fwZMCWsr7pDNd8UMw+++yTG2+88UOf/853vpOxY8cWnAjoKtY3neHMB8X84he/yPHHH5/ddtstEyZM6PCe8Lx58/LKK6/kgQceyOGHH17lSYHOsr7pDPFBUb///e9z3XXX5bHHHsuyZcuSJA0NDWlqaspZZ52VT3ziE9UdENhs1jebSnwAAEW55oOqa25uzuLFi6s9BrAVXH755Xn77berPQbbGPFBMStXrsxpp52WUaNGZerUqVmzZk2mT5+eYcOGZfTo0TniiCPS2tpa7TGBzdDa2rrBo6WlJVdddVVeeeWV9m2QeNuFgmbMmJGf/vSnOeecc/L9738/9fX1efnll3P99ddn3bp1Ofvss3PiiSfmqquuqvaoQCdtt912G91eqVRSU1PT/p/r1q0rPBnbIvFBMSNHjswtt9ySz3zmM1m6dGlGjBiRH/7whzn++OOTJA888EAuuOCCvPDCC1WeFOisESNGZL/99ssFF1yQXr3+cFK9UqlkwoQJ+bd/+7eMHj06SXLEEUdUc0y2EeKDYvr165cXX3wxjY2NSZIBAwbk6aefzp577pkkee211zJ27NisXr26mmMCm+Hdd9/NtGnT0tLSkltvvTW77rprkqRPnz559tlnfcYHHbjmg2KGDBmS5cuXt389adKkDBo0qP3rVatWpba2tgqTAVtq8ODBueeee/K5z30uBx10UL73ve9VeyS2Yb2rPQA9x7hx4/Lkk09m//33T5LcfvvtHZ5/8skns/fee1djNKCLnH322TniiCPyhS98Iffdd1+1x2Eb5W0Xinn33XfTq1evDmc7/tiDDz6Y/v3759Of/nTRuYCut2bNmlx88cX5+c9/nu9///vt13xAIj7Yhp1zzjm54oorstNOO1V7FKCLWd89m/hgm1VXV5dnnnkmu+22W7VHAbqY9d2zueCUbZYuho8v67tnEx8AQFHiAwAoSnwAAEWJDwCgKPHBNuu0005LXV1dtccAtgLru2dzqy1V8cgjj+SGG27Iyy+/nLvuuiu77rprbr311owePTqHHXZYtccDtoD1zZ/jzAfF3X333Zk4cWL69++fp59+Om1tbUmSlpaWfO1rX6vydMCWsL7ZFOKD4q688spcf/31+fa3v50+ffq0bz/00EPz1FNPVXEyYEtZ32wK8UFxixYtyuGHH77B9vr6+qxYsaL8QECXsb7ZFOKD4hoaGvLSSy9tsP3RRx/1UcvQzVnfbArxQXFnnnlmzjvvvDz++OOpqanJ0qVLc9ttt+XCCy/M2WefXe3xgC1gfbMpeld7AHqeiy++OOvXr89RRx2V9957L4cffnhqa2tz4YUXZsaMGdUeD9gC1jebwq22VM2aNWvy0ksvZdWqVRk7dmwGDhxY7ZGALmJ981G87UJxZ5xxRlauXJm+fftm7NixOeiggzJw4MCsXr06Z5xxRrXHA7aA9c2mcOaD4rbbbru8+eab2WWXXTpsf/vtt9PQ0JAPPvigSpMBW8r6ZlO45oNiWltbU6lUUqlUsnLlyvTr16/9uXXr1uVHP/rRBv/CAroH65vOEB8UM2jQoNTU1KSmpiZ77rnnBs/X1NTk8ssvr8JkwJayvukMb7tQzPz581OpVHLkkUfm7rvvzuDBg9uf69u3b0aNGpXhw4dXcUJgc1nfdIb4oLjXXnstjY2N6dXL9c7wcWN9synEB1Xz3nvvZfHixVmzZk2H7ePGjavSREBXsb75KK75oLjly5fn9NNPz4MPPrjR59etW1d4IqCrWN9sCufFKO7888/PihUr8vjjj6d///556KGHcsstt2SPPfbID3/4w2qPB2wB65tN4cwHxf3sZz/LvffemwMOOCC9evXKqFGjcvTRR6euri5z5szJZz/72WqPCGwm65tN4cwHxa1evbr9fv8dd9wxy5cvT5L8xV/8RZ566qlqjgZsIeubTSE+KG6vvfbKokWLkiT77rtvbrjhhrzxxhu5/vrrM2zYsCpPB2wJ65tN4W4Xivv3f//3fPDBB/nSl76UhQsX5i//8i/zzjvvpG/fvrnllltyyimnVHtEYDNZ32wK8UHVvffee3nhhRcycuTI7LTTTtUeB+hC1jcb44JTips1a9ZGt9fU1KRfv37ZfffdM2nSpA6fkAh0D9Y3m8KZD4r7zGc+k6eeeirr1q3LXnvtlST57//+72y33XYZM2ZMFi1alJqamjz66KMZO3ZslacFOsP6ZlO44JTiJk2alAkTJmTp0qVZuHBhFi5cmNdffz1HH310Tj311Lzxxhs5/PDDM3PmzGqPCnSS9c2mcOaD4nbdddc8/PDDG/y/nueffz7HHHNM3njjjTz11FM55phj8vbbb1dpSmBzWN9sCmc+KK6lpSVvvfXWBtuXL1+e1tbWJH/489x/+jchgG2f9c2mEB8UN2nSpJxxxhm555578vrrr+f111/PPffck2nTpuXEE09MkjzxxBPZc889qzso0GnWN5vC2y4Ut2rVqsycOTPf/e5388EHHyRJevfunalTp2bu3LkZMGBAnnnmmSTJfvvtV71BgU6zvtkU4oOqWbVqVV555ZUkyW677ZaBAwdWeSKgq1jffBTxAQAU5ZoPAKAo8QEAFCU+AICixAcAUJT4AKrq97//fWpqatpvvwQ+/tztAlTVunXrsnz58uy0007p3dsf2oaeQHwAW9XatWvTp0+fao8BbEO87QK0+9d//dcMHz4869ev77D9/z8yO0nuvffe7L///unXr1922223XH755e2fZJkkNTU1ue666/JXf/VXGTBgQK666qr8z//8T6ZMmZKdd945/fv3zx577JGbbropycbfdpk/f34OOuig1NbWZtiwYbn44os7fI9Pf/rT+fu///tcdNFFGTx4cBoaGnLZZZdtvX8wQJcSH0C7z33uc3nnnXfy85//vH3bu+++m4ceeihTpkzJI488ki9+8Ys577zz8tvf/jY33HBDbr755lx11VUdjnPZZZflr//6r/Pcc8/ljDPOyFe/+tX89re/zYMPPpjf/e53ue6667LTTjttdIY33ngjxx13XA488MA8++yzue6663LjjTfmyiuv7LDfLbfckgEDBuTxxx/PN77xjVxxxRV5+OGHu/4fCtD1KgB/ZNKkSZUzzjij/esbbrihMnz48Mq6desqRx11VOVrX/tah/1vvfXWyrBhw9q/TlI5//zzO+xzwgknVE4//fSNfr9XX321kqTy9NNPVyqVSuUf//EfK3vttVdl/fr17ftcc801lYEDB1bWrVtXqVQqlSOOOKJy2GGHdTjOgQceWPnyl7/c+R8YKM6ZD6CDKVOm5O67705bW1uS5LbbbsvnP//59OrVK88++2yuuOKKDBw4sP1x5pln5s0338x7773XfowDDjigwzHPPvvs3HHHHdlvv/1y0UUX5Ve/+tWHfv/f/e53aWpqSk1NTfu2Qw89NKtWrcrrr7/evm3cuHEdXjds2LCN/il3YNsjPoAOTjjhhFQqlTzwwANZsmRJHnnkkUyZMiXJH/5Y2OWXX55nnnmm/fHcc8/lxRdfTL9+/dqPMWDAgA7HPPbYY/Paa69l5syZWbp0aY466qhceOGFWzTnn17EWlNTs8G1KsC2yX1tQAf9+vXLSSedlNtuuy0vvfRS9tprr+y///5Jkv333z+LFi3K7rvv3unj7rzzzpk6dWqmTp2aT33qU/mHf/iH/PM///MG++299965++67U6lU2s9+/Nd//Vd22GGHjBgxYst+OGCbID6ADUyZMiXHH398nn/++Zx22mnt2y+55JIcf/zxGTlyZE4++eT2t2J+85vfbHBB6B+75JJLMn78+Oyzzz5pa2vL/fffn7333nuj+55zzjm5+uqrM2PGjJx77rlZtGhRLr300syaNSu9ejlZCx8HVjKwgSOPPDKDBw/OokWL8oUvfKF9+8SJE3P//ffnJz/5SQ488MAccsghmTt3bkaNGvWRx+vbt29mz56dcePG5fDDD892222XO+64Y6P77rrrrvnRj36UJ554Ivvuu2/OOuusTJs2LV/5yle69GcEqseHjAEARTnzAQAUJT4AgKLEBwBQlPgAAIoSHwBAUeIDAChKfAAARYkPAKAo8QEAFCU+AICixAcAUNT/AfOzF4+8LmOpAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.groupby(\"version\")[\"sum_gamerounds\"].mean().plot(kind=\"bar\", color=[ \"red\", \"cyan\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we interpret the results, the average number of game rounds seems to be more for 'gate30'. But what if it happened by chance?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A/B Testing\n",
    "<Img src=\"Images/AB.png\">\n",
    "\n",
    "### What is A/B Testing?\n",
    "A/B testing, also known as split testing, is a method of comparing two versions of a web page or app against each other to determine which one performs better. It is a common practice used in marketing, product development, and user experience optimization to make data-driven decisions and improve overall performance.\n",
    "\n",
    "### Why is A/B Testing done?\n",
    "The primary purpose of A/B testing is to identify which variation (A or B) of a web page or app produces better results in terms of a specific key performance indicator (KPI). This could be anything from click-through rates, conversion rates, user engagement, or any other measurable metric that aligns with the organization's goals. By comparing two versions under controlled conditions, businesses can gather objective data and make informed decisions to improve their digital assets and achieve better outcomes.\n",
    "\n",
    "### A/B Test Phases\n",
    "To apply the A/B Testing for the dataset we should have follow the fundamental steps according to literature of the A/B Testing. These steps can be seen below:\n",
    "\n",
    "- Hypothesis\n",
    "- Assumptions Control\n",
    "- Apply of the Hyphothesis\n",
    "\n",
    "##### 1. Hypothesis\n",
    "The hypothesis stage of A/B testing is the stage where the underlying hypothesis of the test is determined and the purpose of the experiment is clearly defined.\n",
    "<br>\n",
    "According to the literature, the A/B test has two hypotheses, H0 and H1. Let us explain these hypotheses.\n",
    "<br>\n",
    "<B>H0 : null hypothesis\n",
    "<br>\n",
    "H1 : alternative hypothesis</B>\n",
    "<br>\n",
    "The hypothesis we want to test is whether H1 is true with probability.\n",
    "<br>\n",
    "\n",
    "<b>Attention!</b>\n",
    "\n",
    "“H0 hypothesis is rejected” or “H0 hypothesis cannot be rejected”. Therefore, an interpretation such as \"H0 hypothesis is accepted\" or \"H1 hypothesis is accepted\" cannot be made.\n",
    "\n",
    "H0: \"There is no statistically significant difference between 'gate30' and 'gate40'. \"\n",
    "##### 2.Assumptions Control\n",
    "<b>2.1 Normal Distribution</b>\n",
    "<br>\n",
    "The normality assumption is based on the assumption that the data have a normal distribution. The normal distribution is a type of distribution in which the data are distributed in the form of a bell curve, symmetrical with its mean and variability with its standard deviation.\n",
    "<br><br>\n",
    "The assumption of normality in A/B tests is especially important so that parametric tests (for example, independent two-sample t-test) can produce accurate results. Parametric tests work on the assumption that the data fit a normal distribution, and the validity of this assumption increases the reliability of the results.\n",
    "<br><br>\n",
    "Various methods can be used to evaluate whether the normality assumption is valid. The Shapiro-Wilk Test is also a statistical method that tests the conformity of the data to the normal distribution. The H0 hypothesis states that the data have a normal distribution.\n",
    "<br><br>\n",
    "If the data do not satisfy the assumption of normality, alternative methods such as non-parametric tests (for example, Mann-Whitney U test) or transformation of the data can be used. Non-parametric tests are statistical methods that can work with data that do not satisfy the assumption of normality.\n",
    "\n",
    "##### H0 : The assumption of normal distribution is provided.\n",
    "##### H1 : The assumption of normal distribution is not provided.\n",
    "##### p-value < 0.05 => H0 -> Reject\n",
    "##### p-value > 0.05 => H0 -> Cannot Reject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Stat = 0.0881, p-value = 0.0000\n"
     ]
    }
   ],
   "source": [
    "# gate30\n",
    "\n",
    "test_stat, pvalue = shapiro(df.loc[df[\"version\"] == \"gate_30\", \"sum_gamerounds\"])\n",
    "print(\"Test Stat = %.4f, p-value = %.4f\" % (test_stat, pvalue))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Stat = 0.4826, p-value = 0.0000\n"
     ]
    }
   ],
   "source": [
    "# gate40\n",
    "\n",
    "test_stat, pvalue = shapiro(df.loc[df[\"version\"] == \"gate_40\", \"sum_gamerounds\"])\n",
    "print(\"Test Stat = %.4f, p-value = %.4f\" % (test_stat, pvalue))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In both cases, p-value = 0. The normality assumption was rejected because it was less than 0.05."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>2.2 Variance Homogeneity Assumption</b>\n",
    "\n",
    "<br>In statistical methods such as A/B testing, the assumption of homogeneity of variance is based on the assumption that data distributions between two or more groups have the same or similar variance. This assumption refers to the situation where the data of the groups show homogeneous and similar variability.\n",
    "<br><br>\n",
    "Homogeneity of variance is especially important in parametric tests such as ANOVA (Analysis of Variance) and independent two-sample t-test. These tests produce more reliable results when the data conform to the assumption of homogeneity of variance. If the assumption of homogeneity of variance is not valid, alternative test methods or transformations can be used.\n",
    "<br><br>\n",
    "Various methods can be used to control and evaluate variance homogeneity. Levene's test is a statistical test that evaluates whether the variances of the groups are homogeneous. The H0 hypothesis states that the variances of the groups are homogeneous.\n",
    "<br><br>\n",
    "##### H0 : The variance is homogeneous.\n",
    "##### H1 : The variance is not homogeneous.\n",
    "##### p-value < 0.05 => H0 -> Reject\n",
    "##### p-value > 0.05 => H0 -> Cannot Reject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Stat = 0.5292, p-value = 0.4669\n"
     ]
    }
   ],
   "source": [
    "test_stat, pvalue = levene(df.loc[df[\"version\"] == \"gate_30\", \"sum_gamerounds\"],\n",
    "                            df.loc[df[\"version\"] == \"gate_40\", \"sum_gamerounds\"])\n",
    "print(\"Test Stat = %.4f, p-value = %.4f\" % (test_stat, pvalue))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# H0 cannot be rejected because the p-value is greater than 0.05."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 3. Apply of the Hyphothesis\n",
    "<br><br>\n",
    "If the normality assumption is rejected for the A/B test, but the variance homogeneity assumption is satisfied, then it is recommended to use non-parametric test methods.\n",
    "<br><br>\n",
    "Non-parametric tests are statistical tests used when data do not satisfy the normality assumption or do not comply with other assumptions.\n",
    "<br><br>\n",
    "The <b>Mann-Whitney U Test</b> is a test used to compare two independent groups. If the homogeneity of variance assumption is not met, Welch's t-test can be used instead.\n",
    "<br><br>\n",
    "##### p-value < 0.05 => H0 -> Reject\n",
    "##### p-value > 0.05 => H0 -> Cannot Reject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Stat = 1024331250.5000, p-value = 0.0502\n"
     ]
    }
   ],
   "source": [
    "test_stat, pvalue = mannwhitneyu(df.loc[df[\"version\"] == \"gate_30\", \"sum_gamerounds\"],\n",
    "                            df.loc[df[\"version\"] == \"gate_40\", \"sum_gamerounds\"])\n",
    "print(\"Test Stat = %.4f, p-value = %.4f\" % (test_stat, pvalue))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# H0 cannot be rejected because the p-value is greater than 0.05.\n",
    "# That is, there is no statistically significant difference between 'gate30' and 'gate40' with 95% confidence. Therefore, the result came about by chance."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Summary\n",
    "In this notebook work, we tried to understand the business problem for Cookie Cats Games. The company was undecided about where to place the door in the game. We decided to analyze how the location of this door affects the continuation of their earnings. First of all, we got to know the dataset. Afterwards, we made a short analysis between the variables. At the end of this analysis, gate30 seemed to be more advantageous. But we needed to prove it statistically. We formed our H0 hypothesis and said 'there is no statistically significant difference between these two doors'. We then used the shapiro test for the assumption of normality and the levene test for homogeneity of variance. We used the non-parametric Mann-Whitney U Test as the normality assumption was rejected. As a result, we can say that there is no statistically significant difference between these two doors with 95% confidence. Therefore, the company can continue to use gate30."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ABTest",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}