Data Scientist Challenge 3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from datetime import timedelta\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner_item_data = pd.read_csv('learner_item_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>created_at</th>\n",
       "      <th>trainer_item_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6953977</td>\n",
       "      <td>2017-01-16 10:54:43.386</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6953977</td>\n",
       "      <td>2017-01-16 10:56:25.717</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6953977</td>\n",
       "      <td>2017-01-16 10:55:50.642</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6953977</td>\n",
       "      <td>2017-01-16 10:54:19.796</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6953977</td>\n",
       "      <td>2017-01-16 10:55:10.654</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      uuid              created_at  trainer_item_id\n",
       "0  6953977 2017-01-16 10:54:43.386                0\n",
       "1  6953977 2017-01-16 10:56:25.717                1\n",
       "2  6953977 2017-01-16 10:55:50.642                2\n",
       "3  6953977 2017-01-16 10:54:19.796                3\n",
       "4  6953977 2017-01-16 10:55:10.654                1"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner_item_data['created_at'] = pd.to_datetime(learner_item_data['created_at'])\n",
    "learner_item_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_groups = pd.read_csv('test_groups.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_group</th>\n",
       "      <th>uuid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>control</td>\n",
       "      <td>6953977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test</td>\n",
       "      <td>1607983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>control</td>\n",
       "      <td>6658627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>control</td>\n",
       "      <td>4848596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>test</td>\n",
       "      <td>1653336</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  test_group     uuid\n",
       "0    control  6953977\n",
       "1       test  1607983\n",
       "2    control  6658627\n",
       "3    control  4848596\n",
       "4       test  1653336"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_groups.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Group User Sessions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define treshold value as one hour\n",
    "T = timedelta(seconds=3600)\n",
    "\n",
    "# add a column containing previous timestamp\n",
    "learner_item_data_sessions =  pd.concat([learner_item_data.sort_values(['uuid','created_at']),\n",
    "                       learner_item_data.sort_values(['uuid','created_at']).groupby('uuid')['created_at'].transform(lambda x:x.shift(1))]\n",
    "                      ,axis=1)\n",
    "learner_item_data_sessions.columns = ['uuid','created_at', 'trainer_item_id','prev_created_at']\n",
    "\n",
    "# create the new session column\n",
    "learner_item_data_sessions['new_session'] = ((learner_item_data_sessions['created_at'] - learner_item_data_sessions['prev_created_at'])>T).astype(int)\n",
    "\n",
    "# create the session_id\n",
    "learner_item_data_sessions['increment'] = learner_item_data_sessions.sort_values(['uuid','created_at']).groupby(\"uuid\")['new_session'].cumsum()\n",
    "learner_item_data_sessions['session_id'] = learner_item_data_sessions['uuid'].astype(str) + '_' + learner_item_data_sessions['increment'].astype(str)\n",
    "\n",
    "# sorting DF\n",
    "learner_item_data_sessions = learner_item_data_sessions.sort_values(['uuid','created_at'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2a. What is the distribution (histogram) of session lengths?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "session_length = learner_item_data_sessions.groupby(['session_id'])['created_at'].max()-learner_item_data_sessions.groupby(['session_id'])['created_at'].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Text(0, 0.5, 'Frequency'), Text(0.5, 0, 'Session Duration in Seconds')]"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGpdJREFUeJzt3Xm4ZVV55/HvT0BRMQxS0shgoaI2Ri3LEjBOqBERB0zHKGpiaRPLtNhqbFtx6ODE02KiOGtQiaAi4BSJjcECwamboSpAMShS4gCIgBaCOKDg23/sdanDterW2cU599xb9f08z3nO2mtP79nn3vvevdbea6eqkCRpWHeadACSpPnFxCFJ6sXEIUnqxcQhSerFxCFJ6sXEIUnqxcQhSerFxCFJ6sXEIUnqZctJBzAOO+64Yy1cuHDSYUjSvLJy5cqfVdWCDS23SSaOhQsXsmLFikmHIUnzSpIfDbOcTVWSpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReNsk7x++o48/+8VDLPX+f3ccciSTNPZ5xSJJ6MXFIknoxcUiSehlr4kjywyQXJjk/yYpWt0OS5Ukua+/bt/okeV+S1UlWJVk8sJ2lbfnLkiwdZ8ySpJnNxhnHE6pqUVUtadOHAadX1Z7A6W0a4KnAnu21DPgwdIkGOBzYB9gbOHwq2UiSZt8kmqoOAo5t5WOBZw3UH1eds4DtkuwMPAVYXlVrqup6YDlwwGwHLUnqjDtxFPDVJCuTLGt1O1XV1a38U2CnVt4FuGJg3Stb3frqJUkTMO77OB5TVVcluRewPMl3B2dWVSWpUeyoJaZlALvv7v0VkjQuYz3jqKqr2vu1wBfp+iiuaU1QtPdr2+JXAbsNrL5rq1tf/fR9HV1VS6pqyYIFG3xkriRpI40tcSS5e5J7TJWB/YGLgJOBqSujlgJfauWTgRe2q6v2BW5oTVqnAvsn2b51iu/f6iRJEzDOpqqdgC8mmdrP8VX170nOBU5KcgjwI+A5bflTgAOB1cCvgRcDVNWaJG8Dzm3LvbWq1owxbknSDMaWOKrqcuBh66j/OfCkddQXcOh6tnUMcMyoY5Qk9eed45KkXkwckqReTBySpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReTBySpF5MHJKkXsaeOJJskeS8JF9u03skOTvJ6iQnJrlzq79Lm17d5i8c2MbrW/2lSZ4y7pglSes3G2ccrwS+MzB9JHBUVd0fuB44pNUfAlzf6o9qy5FkL+Bg4MHAAcCHkmwxC3FLktZhrIkjya7A04CPtekATwQ+1xY5FnhWKx/Upmnzn9SWPwg4oapurqofAKuBvccZtyRp/cZ9xvEe4LXAH9r0PYFfVNUtbfpKYJdW3gW4AqDNv6Etf1v9Ota5TZJlSVYkWXHdddeN+nNIkpqxJY4kTweuraqV49rHoKo6uqqWVNWSBQsWzMYuJWmztOUYt/1o4JlJDgS2Bv4EeC+wXZIt21nFrsBVbfmrgN2AK5NsCWwL/HygfsrgOpKkWTa2M46qen1V7VpVC+k6t79WVS8AzgCe3RZbCnyplU9u07T5X6uqavUHt6uu9gD2BM4ZV9ySpJmN84xjfV4HnJDk7cB5wMdb/ceBTyZZDayhSzZU1cVJTgIuAW4BDq2qW2c/bEkSzFLiqKozgTNb+XLWcVVUVf0W+Kv1rH8EcMT4IpQkDcs7xyVJvZg4JEm9TKKPY5Nx/Nk/Hmq55++z+5gjkaTZ4xmHJKkXE4ckqRcThySpFxOHJKkXE4ckqRcThySpFxOHJKkXE4ckqRcThySpFxOHJKkXE4ckqRcThySpFxOHJKkXE4ckqRcThySpFxOHJKkXE4ckqRcThySpl6ESR5KHjDsQSdL8MOwZx4eSnJPkZUm2HWtEkqQ5bajEUVWPBV4A7AasTHJ8kiePNTJJ0pw0dB9HVV0GvAl4HfB44H1Jvpvkv4wrOEnS3DNsH8dDkxwFfAd4IvCMqvrPrXzUGOOTJM0xWw653PuBjwFvqKrfTFVW1U+SvGkskUmS5qRhE8fTgN9U1a0ASe4EbF1Vv66qT44tOknSnDNsH8dpwF0Hpu/W6iRJm5lhE8fWVXXT1EQr3208IUmS5rJhE8evkiyemkjyCOA3MywvSdpEDZs4XgV8Nsk3k3wLOBF4+UwrJNm63TR4QZKLk7yl1e+R5Owkq5OcmOTOrf4ubXp1m79wYFuvb/WXJnnKxnxQSdJoDNU5XlXnJnkQ8MBWdWlV/X4Dq90MPLGqbkqyFfCtJF8BXg0cVVUnJPkIcAjw4fZ+fVXdP8nBwJHAc5PsBRwMPBi4N3BakgdMddRLkmZXn0EOHwk8FFgMPC/JC2dauDpT/SJbtVfR3fvxuVZ/LPCsVj6oTdPmPylJWv0JVXVzVf0AWA3s3SNuSdIIDXXGkeSTwP2A84Gp//QLOG4D620BrATuD3wQ+D7wi6q6pS1yJbBLK+8CXAFQVbckuQG4Z6s/a2Czg+sM7msZsAxg9913H+ZjSZI2wrD3cSwB9qqq6rPx1py0KMl2wBeBB/WMr8++jgaOBliyZEmvOCVJwxu2qeoi4D9t7E6q6hfAGcCjgO2STCWsXYGrWvkqukEUafO3BX4+WL+OdSRJs2zYxLEjcEmSU5OcPPWaaYUkC9qZBknuCjyZbqyrM4Bnt8WWAl9q5ZPbNG3+19oZzsnAwe2qqz2APYFzhoxbkjRiwzZVvXkjtr0zcGzr57gTcFJVfTnJJcAJSd4OnAd8vC3/ceCTSVYDa+iupKKqLk5yEnAJcAtwqFdUSdLkDHs57teT3AfYs6pOS3I3YIsNrLMKePg66i9nHVdFVdVvgb9az7aOAI4YJlZJ0ngNO6z6S+gukf3nVrUL8K/jCkqSNHcN28dxKPBo4Ea47aFO9xpXUJKkuWvYxHFzVf1uaqJd9eQlr5K0GRo2cXw9yRuAu7ZnjX8W+LfxhSVJmquGTRyHAdcBFwIvBU6he/64JGkzM+xVVX8APtpekqTN2LBjVf2AdfRpVNV9Rx6RJGlO6zNW1ZSt6e632GH04UiS5rphm6p+Pq3qPUlWAv8w+pA2Pcef/eOhlnv+Po7qK2nuG7apavHA5J3ozkCGPVuRJG1Chv3j/66B8i3AD4HnjDwaSdKcN2xT1RPGHYgkaX4Ytqnq1TPNr6p3jyYcSdJc1+eqqkfSPRsD4Bl0z8S4bBxBSZLmrmETx67A4qr6JUCSNwP/p6r+elyBSZLmpmGHHNkJ+N3A9O9anSRpMzPsGcdxwDlJvtimnwUcO56QJElz2bBXVR2R5CvAY1vVi6vqvPGFJUmaq4ZtqgK4G3BjVb0XuDLJHmOKSZI0hw376NjDgdcBr29VWwGfGldQkqS5a9gzjr8Angn8CqCqfgLcY1xBSZLmrmETx++qqmhDqye5+/hCkiTNZcMmjpOS/DOwXZKXAKfhQ50kabM07FVV/9SeNX4j8EDgH6pq+VgjkyTNSRtMHEm2AE5rAx2aLCRpM7fBpqqquhX4Q5JtZyEeSdIcN+yd4zcBFyZZTruyCqCqXjGWqCRJc9awieML7SVJ2szNmDiS7F5VP64qx6WSJAEb7uP416lCks+PORZJ0jywocSRgfJ9+2w4yW5JzkhySZKLk7yy1e+QZHmSy9r79q0+Sd6XZHWSVUkWD2xraVv+siRL+8QhSRqtDSWOWk95GLcA/6Oq9gL2BQ5NshdwGHB6Ve0JnN6mAZ4K7Nley4APQ5dogMOBfYC9gcOnko0kafZtKHE8LMmNSX4JPLSVb0zyyyQ3zrRiVV1dVf/Ryr8EvgPsAhzE2md5HEv3bA9a/XHVOYvuLvWdgacAy6tqTVVdT3cvyQEb8VklSSMwY+d4VW0xip0kWQg8HDgb2Kmqrm6zfsraJwnuAlwxsNqVrW599ZKkCejzPI6NkmQb4PPAq6rqdmcpgwMnjmA/y5KsSLLiuuuuG8UmJUnrMNbEkWQruqTx6aqaug/kmtYERXu/ttVfBew2sPqurW599bdTVUdX1ZKqWrJgwYLRfhBJ0m3GljiSBPg48J2qevfArJOBqSujlgJfGqh/Ybu6al/ghtakdSqwf5LtW6f4/q1OkjQBw945vjEeDfwN3VAl57e6NwDvoBum/RDgR8Bz2rxTgAOB1cCvgRcDVNWaJG8Dzm3LvbWq1owxbknSDNJ1M2xalixZUitWrNjo9Y8/+8cjjGb0nr/P7pMOQdImKMnKqlqyoeXG3jkuSdq0mDgkSb2YOCRJvZg4JEm9mDgkSb2YOCRJvZg4JEm9mDgkSb2YOCRJvZg4JEm9mDgkSb2YOCRJvZg4JEm9mDgkSb2YOCRJvZg4JEm9jPMJgBqTYR805QOfJI2DZxySpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReTBySpF5MHJKkXkwckqReTBySpF4ccmQT5tAkksbBMw5JUi8mDklSLyYOSVIvY0scSY5Jcm2SiwbqdkiyPMll7X37Vp8k70uyOsmqJIsH1lnalr8sydJxxStJGs44zzg+ARwwre4w4PSq2hM4vU0DPBXYs72WAR+GLtEAhwP7AHsDh08lG0nSZIztqqqq+kaShdOqDwL2a+VjgTOB17X646qqgLOSbJdk57bs8qpaA5BkOV0y+sy44t4cDXv1FXgFlqTZ7+PYqaqubuWfAju18i7AFQPLXdnq1lcvSZqQiXWOt7OLGtX2kixLsiLJiuuuu25Um5UkTTPbieOa1gRFe7+21V8F7Daw3K6tbn31f6Sqjq6qJVW1ZMGCBSMPXJLUme3EcTIwdWXUUuBLA/UvbFdX7Qvc0Jq0TgX2T7J96xTfv9VJkiZkbJ3jST5D17m9Y5Ir6a6OegdwUpJDgB8Bz2mLnwIcCKwGfg28GKCq1iR5G3BuW+6tUx3lkqTJGOdVVc9bz6wnrWPZAg5dz3aOAY4ZYWiSpDvAO8clSb04Oq4kzQNzabRrzzgkSb2YOCRJvdhUpV7m0umypMnwjEOS1IuJQ5LUi01VGgubtKRNl2cckqReTBySpF5MHJKkXkwckqRe7BzXRNmJLs0/nnFIknoxcUiSejFxSJJ6MXFIknoxcUiSevGqKs0Lw159NSyv0pI2nmcckqReTBySpF5sqtJmyRsPpY1n4pBm0KdvxSSjzYVNVZKkXjzjkEbEK7+0ufCMQ5LUi2cc0hxlB77mKhOHtJkwEWlUTBzSPDfqvhVpQ0wckm5nHInIs5hNi4lD0thN6qzIhDUe8yZxJDkAeC+wBfCxqnrHhEOSNMfNh4Q1H5sa50XiSLIF8EHgycCVwLlJTq6qSyYbmST9sfmYDPqYL/dx7A2srqrLq+p3wAnAQROOSZI2S/MlcewCXDEwfWWrkyTNsnnRVDWMJMuAZW3ypiSX3oHN7Qj87I5HNXLG1Y9x9WNc/czJuF5wx+K6zzALzZfEcRWw28D0rq3uNlV1NHD0KHaWZEVVLRnFtkbJuPoxrn6Mq5/NOa750lR1LrBnkj2S3Bk4GDh5wjFJ0mZpXpxxVNUtSV4OnEp3Oe4xVXXxhMOSpM3SvEgcAFV1CnDKLO1uJE1eY2Bc/RhXP8bVz2YbV6pq3PuQJG1C5ksfhyRpjjBxDEhyQJJLk6xOctgE9v/DJBcmOT/Jila3Q5LlSS5r79u3+iR5X4t1VZLFI4zjmCTXJrlooK53HEmWtuUvS7J0THG9OclV7Zidn+TAgXmvb3FdmuQpA/Uj/Z6T7JbkjCSXJLk4yStb/USP2QxxTfSYJdk6yTlJLmhxvaXV75Hk7LaPE9uFMCS5S5te3eYv3FC8I47rE0l+MHC8FrX6WfvZb9vcIsl5Sb7cpid3vKrKV9dctwXwfeC+wJ2BC4C9ZjmGHwI7Tqt7J3BYKx8GHNnKBwJfAQLsC5w9wjgeBywGLtrYOIAdgMvb+/atvP0Y4noz8Jp1LLtX+w7vAuzRvtstxvE9AzsDi1v5HsD32v4nesxmiGuix6x97m1aeSvg7HYcTgIObvUfAf5bK78M+EgrHwycOFO8Y4jrE8Cz17H8rP3st+2+Gjge+HKbntjx8oxjrbk6rMlBwLGtfCzwrIH646pzFrBdkp1HscOq+gaw5g7G8RRgeVWtqarrgeXAAWOIa30OAk6oqpur6gfAarrveOTfc1VdXVX/0cq/BL5DN7LBRI/ZDHGtz6wcs/a5b2qTW7VXAU8EPtfqpx+vqeP4OeBJSTJDvKOOa31m7Wc/ya7A04CPtekwweNl4lhrLgxrUsBXk6xMdyc8wE5VdXUr/xTYqZVnO96+ccxmfC9vTQXHTDUHTSqu1izwcLr/VufMMZsWF0z4mLVml/OBa+n+sH4f+EVV3bKOfdy2/zb/BuCesxFXVU0dryPa8ToqyV2mxzVt/+P4Ht8DvBb4Q5u+JxM8XiaOueUxVbUYeCpwaJLHDc6s7nxz4pfBzZU4mg8D9wMWAVcD75pUIEm2AT4PvKqqbhycN8ljto64Jn7MqurWqlpENwrE3sCDZjuGdZkeV5I/BV5PF98j6ZqfXjebMSV5OnBtVa2czf3OxMSx1gaHNRm3qrqqvV8LfJHuF+qaqSao9n5tW3y24+0bx6zEV1XXtF/2PwAfZe2p96zGlWQruj/On66qL7TqiR+zdcU1V45Zi+UXwBnAo+iaeqbuLRvcx237b/O3BX4+S3Ed0Jr8qqpuBv6F2T9ejwaemeSHdM2ET6R7NtHkjtfGdIxsii+6myEvp+s0muoAfPAs7v/uwD0Gyv+Xrl30H7l9B+s7W/lp3L5j7pwRx7OQ23dC94qD7j+zH9B1Dm7fyjuMIa6dB8p/T9eGC/Bgbt8ReDldJ+/Iv+f22Y8D3jOtfqLHbIa4JnrMgAXAdq18V+CbwNOBz3L7zt6XtfKh3L6z96SZ4h1DXDsPHM/3AO+YxM9+2/Z+rO0cn9jxGtkfmk3hRXeVxPfo2lvfOMv7vm/7Ui8ALp7aP13b5OnAZcBpUz+A7Yf1gy3WC4ElI4zlM3RNGL+nawc9ZGPiAP4rXQfcauDFY4rrk22/q+jGLxv8o/jGFtelwFPH9T0Dj6FrhloFnN9eB076mM0Q10SPGfBQ4Ly2/4uAfxj4HTinffbPAndp9Vu36dVt/n03FO+I4/paO14XAZ9i7ZVXs/azP7Dd/VibOCZ2vLxzXJLUi30ckqReTBySpF5MHJKkXkwckqReTBySpF5MHBqbJG9so4yuaqOK7jOi7d47yec2vORQ2zqzjRS6Ksl3k3wgyXaj2Hbb/n5J/mxg+u+SvHAE2+19DJI8vY2uekG6EXNfekfj2BhJbtrwUprLvBxXY5HkUcC7gf2q6uYkOwJ3rqqfTDi020lyJt1IsSvasNT/m+56/Mf32MaWtXbMoOnz3gzcVFX/NIp4N1a7g/xHwN5VdWUbb2lhVV06gVhuqqptZnu/Gh3PODQuOwM/q26YBqrqZ1NJI8kjkny9DeZ46sCwHK9o/wmvSnJCq3v8wHMQzktyjyQL057Jke4ZCv+S7jkm5yV5Qqt/UZIvJPn39kyEd24o4OpGfn0tsHuShw3up23zNS0RTJ2pvCfdc1NemeQZ7dkH5yU5LclObWDBvwP+vsX/2HTPwnhN28aiJGe1z/vFrH1ex5lJjkz3bIjvJXns9FinHYNhPus96O4A/3n7rDdPJY0kC5J8Psm57fXoVr/NwLFdleQvW/3zWt1FSY4ciOmmJEe0M5qzkuzU6vdI8v/aOm8fWH7nJN9ox+aidX1OzU0mDo3LV4Hd2h++DyV5PNz2n+/76Z5v8AjgGOCIts5hwMOr6qF0f3ABXgMcWt3Ac48FfjNtP4fSjSH4EOB5wLFJtm7zFgHPBR4CPDfJbmxAVd1Kd/f+MIPu3bmqllTVu4BvAftW1cPpxhN6bVX9kG4oiKOqalFVfXPa+scBr2uf90Lg8IF5W1bV3sCrptWvz4yftarW0N0l/qMkn0nygiRTv//vbTE+EvhL2tDdwP8Cbqiqh7QYv5bk3sCRdOMlLQIemWRqOO+7A2dV1cOAbwAvGdj+h9t3NDVaMMDzgVPbd/swujvbNQ+YODQW1T3X4BHAMuA64MQkLwIeCPwpsDzd8NVvohtsDbqhHj6d5K+BqaafbwPvTvIKunGEpjcJPYZuGAiq6rt0zTEPaPNOr6obquq3wCXAfYYMP0Mud+JAeVfg1CQXAv+Tblyg9e8g2Zbu83y9VR1L96CqKVMDJa6kG59rQzb4Wavqb4En0Q1D8Rq6pA3w58AH2vdxMvAn6UbU/XO6ITWm1r+eboTYM6vquvZdfHog7t8BX15H3I+mGy4GuuFOppwLvLidxT2kumeGaB4wcWhsqhuB9cyqOhx4Od1/swEubv+BL2r/ze7fVnka3R+qxcC5re/gHcDf0g069+0kfYbfvnmgfCtdU82MkmxB91/7d+iS1+DvyNbTFv/VQPn9wAfaf9UvXceyfU3FPlTcDPlZq+rCqjoKeDLd9wHdZ9x34DvZpdY+0KiP39faTtPpMfxRZ2p1D+Z6HN0IrZ/ICC4a0OwwcWgskjwwyZ4DVYvozgYuBRak6zwnyVZJHtyaTXarqjPonnewLbBNkvu1P3ZH0v2HOj1xfBN4QdvWA4Dd2z42Juat6DrHr6iqVcA1wL2S3DNdZ/LTZ1h9W9YOUb10oP6XdP0Lt1NVNwDXD7Tr/w3w9enLjUrrr9hvoGrq+4CuWfG/Dyy7qBWX0zUFTtVvT3e28vgkO7Yk+7wh4v423Sit0L6rtr37ANdU1UfpmscWr2NdzUEmDo3LNnT9DZckWUV71nXrgH42cGSSC+jatf+MbvjuT7WmnvOA91X3TIRXtY7TVXSj4n5l2n4+BNyprXci8KKpDvkePt22fxFdO/1BAFX1e+CtdH8slwPfnWEbbwY+m2Ql8LOB+n8D/mKqc3zaOkuBf2z7XtT2NS4BXpvu0uPzgbcAL2rzXgEsaR3gl7C2f+ntwPbt+F8APKG6JxoeRvesiguAlVX1pQ3s+5V0Dya7kNs/cW4/4IIk59H1z7z3jn5IzQ4vx5Uk9eIZhySpFxOHJKkXE4ckqRcThySpFxOHJKkXE4ckqRcThySpFxOHJKmX/w/frRoGQn0QfwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# due to a lot of rare values and a very skewed distribution with a long tail, I capped the bigger sessions than 4000\n",
    "session_length_capped = np.where(session_length.dt.seconds > 4000, 4000, session_length.dt.seconds)\n",
    "h = sns.distplot(session_length_capped, bins=30, kde=False)\n",
    "h.set(xlabel='Session Duration in Seconds', ylabel='Frequency')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2b. How many people do several sessions a day? How many sessions? "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Assumptions: I'm accounting only for the days users logged in and did at least an item, so it's not the overall average, including all days since their sign up (which I don't have access to) but only the amount of users that when logged in, always do only one session or users that at least more than once, had different sessions in the same day."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner_item_data_sessions['date'] = learner_item_data_sessions['created_at'].dt.date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_sessions_per_user = learner_item_data_sessions.groupby(['uuid'])['session_id'].nunique()\n",
    "total_days_per_user = learner_item_data_sessions.groupby(['uuid'])['date'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_sessions_per_day = pd.DataFrame(total_sessions_per_user/total_days_per_user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "850"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of people who at least for one day had more than one session\n",
    "avg_sessions_per_day[avg_sessions_per_day[0]>1].count()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.3624654135962855"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# avg number of sessions for those who have more than one session per day\n",
    "avg_sessions_per_day[avg_sessions_per_day[0]>1].mean()[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_df = learner_item_data_sessions.merge(test_groups, on='uuid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>created_at</th>\n",
       "      <th>trainer_item_id</th>\n",
       "      <th>prev_created_at</th>\n",
       "      <th>new_session</th>\n",
       "      <th>increment</th>\n",
       "      <th>session_id</th>\n",
       "      <th>date</th>\n",
       "      <th>test_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2990</td>\n",
       "      <td>2016-12-25 11:27:22.905</td>\n",
       "      <td>2</td>\n",
       "      <td>NaT</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2990_0</td>\n",
       "      <td>2016-12-25</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2990</td>\n",
       "      <td>2016-12-25 11:27:28.985</td>\n",
       "      <td>0</td>\n",
       "      <td>2016-12-25 11:27:22.905</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2990_0</td>\n",
       "      <td>2016-12-25</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2990</td>\n",
       "      <td>2016-12-25 11:27:33.603</td>\n",
       "      <td>3</td>\n",
       "      <td>2016-12-25 11:27:28.985</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2990_0</td>\n",
       "      <td>2016-12-25</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2990</td>\n",
       "      <td>2016-12-25 11:27:38.010</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-12-25 11:27:33.603</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2990_0</td>\n",
       "      <td>2016-12-25</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2990</td>\n",
       "      <td>2016-12-28 09:40:57.271</td>\n",
       "      <td>604</td>\n",
       "      <td>2016-12-25 11:27:38.010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2990_1</td>\n",
       "      <td>2016-12-28</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uuid              created_at  trainer_item_id         prev_created_at  \\\n",
       "0  2990 2016-12-25 11:27:22.905                2                     NaT   \n",
       "1  2990 2016-12-25 11:27:28.985                0 2016-12-25 11:27:22.905   \n",
       "2  2990 2016-12-25 11:27:33.603                3 2016-12-25 11:27:28.985   \n",
       "3  2990 2016-12-25 11:27:38.010                1 2016-12-25 11:27:33.603   \n",
       "4  2990 2016-12-28 09:40:57.271              604 2016-12-25 11:27:38.010   \n",
       "\n",
       "   new_session  increment session_id        date test_group  \n",
       "0            0          0     2990_0  2016-12-25    control  \n",
       "1            0          0     2990_0  2016-12-25    control  \n",
       "2            0          0     2990_0  2016-12-25    control  \n",
       "3            0          0     2990_0  2016-12-25    control  \n",
       "4            1          1     2990_1  2016-12-28    control  "
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_sessions_per_user = merged_df.groupby(['test_group','uuid'])['session_id'].nunique()\n",
    "total_days_per_user = merged_df.groupby(['test_group','uuid'])['date'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_sessions_per_day = pd.DataFrame(total_sessions_per_user/total_days_per_user).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "control_group_proportion = avg_sessions_per_day[(avg_sessions_per_day[0]>1) & (avg_sessions_per_day['test_group'] == 'control')]['uuid'].count()/avg_sessions_per_day[avg_sessions_per_day['test_group'] == 'control']['uuid'].count()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1132513344163379"
      ]
     },
     "execution_count": 261,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "control_group_proportion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_group_proportion = avg_sessions_per_day[(avg_sessions_per_day[0]>1) & (avg_sessions_per_day['test_group'] == 'test')]['uuid'].count()/avg_sessions_per_day[avg_sessions_per_day['test_group'] == 'test']['uuid'].count()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.11884438608010506"
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_group_proportion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [],
   "source": [
    "session_length_df = pd.DataFrame(session_length).reset_index()\n",
    "session_length_df['uuid'] = session_length_df['session_id'].apply(lambda x: x.split('_')[0]).astype(int)\n",
    "session_length_df = session_length_df.merge(test_groups, on='uuid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test_group\n",
       "control    422\n",
       "test       414\n",
       "Name: session_length, dtype: int64"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session_length_df['session_length'] = session_length_df['created_at'].dt.seconds\n",
    "session_length_df.groupby('test_group')['session_length'].median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>uuid</th>\n",
       "      <th>test_group</th>\n",
       "      <th>session_length</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test_group</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>control</th>\n",
       "      <td>14201</td>\n",
       "      <td>13832</td>\n",
       "      <td>4309</td>\n",
       "      <td>1</td>\n",
       "      <td>3102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>9730</td>\n",
       "      <td>9479</td>\n",
       "      <td>3046</td>\n",
       "      <td>1</td>\n",
       "      <td>2672</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            session_id  created_at  uuid  test_group  session_length\n",
       "test_group                                                          \n",
       "control          14201       13832  4309           1            3102\n",
       "test              9730        9479  3046           1            2672"
      ]
     },
     "execution_count": 304,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session_length_df.groupby('test_group').nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3a. Is there a difference between the experiment groups in any of the two activity metrics?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pvalue(con_conv, test_conv, con_size, test_size):\n",
    "    lift = -abs(test_conv - con_conv)\n",
    "    \n",
    "    scale_one = con_conv * (1-con_conv) * (1/ con_size)\n",
    "    scale_two = test_conv * (1-test_conv) * (1/ test_size)\n",
    "    scale_val = (scale_one + scale_two) ** 0.5\n",
    "    \n",
    "    p_value = 2 * stats.norm.cdf(lift, loc=0, scale = scale_val)  \n",
    "    return p_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.46148601597730854"
      ]
     },
     "execution_count": 299,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# multiple sessions p-value\n",
    "con_conv = 0.1132513344163379\n",
    "test_conv = 0.11884438608010506\n",
    "con_size = 4309\n",
    "test_size = 3046\n",
    "get_pvalue(con_conv,test_conv,con_size,test_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x2488c4860>"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAELCAYAAAASrNdlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xt8XHWd//HXd+6TTO5p06Zpm7T0QksplNJKuRUKcvGCIiyCKBTd/rwt4qqLPnRXVl1XWFwrgoD+AEVxuYqyCCjUKkWktIXeKE2b3tu0TdI098xkLt/9Y05LC71k0plMJnk/H4/zyJkzZ875zMnknTPfc873GGstIiKSG1zZLkBERHpPoS0ikkMU2iIiOUShLSKSQxTaIiI5RKEtIpJDFNoiIjlEoS0ikkMU2iIiOcSTyszl5eW2uro6Q6WIiAxOK1asaLLWDkvHslIK7erqapYvX56O9YqIDBnGmG3pWpaaR0REcohCW0Qkhyi0RURySEpt2iIytEWjUXbu3Ek4HM52KQNSIBCgqqoKr9ebsXUotEWk13bu3ElBQQHV1dUYY7JdzoBirWXfvn3s3LmTmpqajK1HzSMi0mvhcJiysjIF9hEYYygrK8v4txCFtoikRIF9dP2xbRTaIiI5RG3aItJnv1m6Pa3Lu272mF7Nt2fPHm655RaWLVtGcXExFRUVLFy4kIkTJ6a0voULF7JgwQLy8vJSet3cuXO58847mTlzZkqvS4fUQrurCZY/lKFSjmDm/P5bl4jkBGstH/3oR7nhhht49NFHAVi1ahV79+7tU2hff/31RwzteDyO2+1OS83ppOYREckpixcvxuv18tnPfvbgtOnTp3POOefwta99jVNOOYVp06bx2GOPAfCXv/yFuXPnctVVVzF58mQ+8YlPYK3lrrvuor6+ngsuuIALLrgAgFAoxFe+8hWmT5/O3//+dxYtWsTpp5/OtGnTuOmmm4hEIll5z4dSaItITlm7di1nnHHGe6b/9re/ZeXKlaxatYqXXnqJr33ta+zevRuAN998k4ULF7Ju3To2b97M3/72N26++WYqKytZvHgxixcvBqCzs5PZs2ezatUqZs6cyY033shjjz3GmjVriMVi3Hvvvf36Xo9EoS0ig8Irr7zCtddei9vtpqKigvPPP59ly5YBMGvWLKqqqnC5XJx22mls3br1iMtwu9187GMfA6C2tpaampqDTS433HADL7/8cr+8l2NRaItITpk6dSorVqxI6TV+v//guNvtJhaLHXG+QCAwINuxD6XQFpGccuGFFxKJRPjZz352cNrq1aspLi7mscceIx6P09jYyMsvv8ysWbOOuayCggLa29uP+NykSZPYunUrdXV1APzqV7/i/PPPT98b6SOd8icifdbbU/TSyRjD008/zS233MLtt99OIBCgurqahQsX0tHRwfTp0zHGcMcddzBixAjWr19/1GUtWLCASy+99GDb9qECgQAPPfQQV199NbFYjDPPPPOwg5/ZYqy1vZ555pRqu/zhb2ewnHevUKf8iQwkb7/9NieffHK2yxjQjrSNjDErrLVpOalbzSMiIjlEoS0ikkMU2iIiOUShLSKSQxTaIiI5RKEtIpJDdJ62iPRdunv9PM5pvvv27WPevHlAsntWt9vNsGHDAHj99dfx+Xy9Ws2DDz7I5ZdfzogRI06s3ixQaItIzigrK2PlypUA3HbbbYRCIb761a+mvJwHH3yQGTNmKLRFRLLll7/8Jffccw89PT3MmTOHu+++m0Qiwfz581m5ciXWWhYsWEBFRQUrV67kmmuuIRgMprSHPhAotEUk561du5ann36aV199FY/Hw4IFC3j00UcZP348TU1NrFmzBoCWlhaKi4v5yU9+wt13381pp52W5cpTp9AWkZz30ksvsWzZsoO3/+ru7mb06NFccskl1NbWcvPNN/OBD3yA97///Vmu9MQptEUk51lruemmm/jud7/7nudWr17N888/zz333MNTTz11WO+AuUin/IlIzrvooot4/PHHaWpqApJnmWzfvp3GxkastVx99dV85zvf4Y033gCO3SXrQKc9bRHpuwHSE+e0adP49re/zUUXXUQikcDr9XLffffhdrv59Kc/jbUWYwy33347APPnz+czn/lMTh6IVNesItJr6pr1+NQ1q4iIHKTQFhHJIQptEUlJKk2qQ01/bBuFtoj0WiAQYN++fQruI7DWsm/fPgKBQEbXo7NHRKTXqqqq2LlzJ42NjdkuZUAKBAJUVVVldB0KbRHpNa/XS01NTbbLGNLUPCIikkMU2iIiOUShLSKSQxTaIiI5RKEtIpJDFNoiIjlEoS0ikkMU2iIiOUShLSKSQxTaIiI5RKEtIpJDFNoiIjlEoS0ikkMU2iIiOUShLSKSQxTaIiI5RKEtIpJDFNoiIjlEoS0ikkNSukdkZyTO0i3NmarlPTbFt6d9mdfNHpP2ZYqI9BftaYuI5BCFtohIDlFoi4jkEIW2iEgOUWiLiOQQhbaISA5RaIuI5BCFtohIDlFoi4jkEIW2iEgOUWiLiOQQhbaISA5RaIuI5BCFtohIDlFoi4jkEIW2iEgOUWiLiOQQhbaISA5Je2ibRIxQ53bcse50L1pEZMhL6R6RvVHRvIyxe18EIOwrpSNYSUdwFJ3BSjoDI7GutK9SRGTISHuClrSvp9tXRmPxdELd9RR2bqO8dS0ACVx0BSroDI6iI1hJZ3AU3f4yMGqlERHpjbSGtjvWTUHXTnYNO4fdw845ON0bbSPUXU9+9y5C3fWUta6hYv9yAOIunxPgyT3yjuAoot7CdJYlIjJopDW0izvqMFhaQhMOmx71FrLfW8j+wsnJCdYS6NlHqHsX+d31hLp3MWLfa7hsAoth4+h/YH/hpHSWJiIyKKQ5tDcSdefRGRx17BmNIewvJ+wvp6l4enJSIkZeeC/jdz1NZeMS9hdMTGdpIiKDQvoak22C4vY6WgomgDGpv9zloTNvFHvK3kcoXE+oe2faShMRGSzSFtoFXTvwJMInvIfcVHwqMVeAEfuWpqkyEZHBI22hXdy+kYRx0ZY/7oSWk3D5aCiZQWnb2+R116epOhGRwSF9od2xkfa8scTd/hNe1p7SM7EY/G88QHs4mobqREQGh7SEtr9nP3mRxmR79gnYE/byRH05n9twOs/FZzGn9VleWbctHSWKiAwKaTl7pLh9IwD7Q6m3Z7fHXPx9fyFL9hWyoTMPg2VKQRfNoTMp2v8a43Y9Q8fUmwn5dSWliEiaQnsD3b4yIv7SXs0fTRjeaA3x8r5C3mwLEbeGqkCE60Y1cHZpG+W+GNhi9kSn8qm2F/j+pmuZN2VkOkoVEclpJxzarngPhV3b2Ft65jHnS1io7QiypLmI1/YX0Bl3U+yJcemw/ZxX1srYYOTwMwWNYdP4T3H2qlvxbvkzPROvxefR5e4iMrSdcGgXdW7GZeNHbc+uD/tYsq+QJc2FNPb48LsSzCpu59yyVk4p6MJ9jFO6t4+4mFPX3cl14T/wxPZLOWtc2YmWKyKS0044tIvbNxJz+WnPG3NwWlvUzd+cdupNXUEMlmmFnfxDZROzitsJuG2vlm1dXjZVX8t5G+/iro1vkqiZh6sPF+6IiAwWJxba1lLcsZHW0Hgi1sOK/SFebi5iVWs+cQxjg2Gur2rg7JI2Sn2xPq2ibszVTK27jyt7nmFN/ZlMG1V0QiWLiOSyEwrtYPdufLEOngqfyY9WnUR3wk2pN8oHKpo5t6yNMcHICRfY4ytm66gPc+XOZ3hkwyZOqTwdo71tERmi+hTaTT0e/tRYwsnNqznFGB7smM2sknbOK21jSkEXrjRnam319UzY+STnt/+BLfsmMq48lN4ViIjkiD6F9vc3jmZ32McXg2+yxzOGO6bsxe/qXTt1X7QVjGdX2Vnc0PQiN9V+nHHlJ3YRj4hIrkr5HLqGiJddYT+frdzIBLuNeMm4jAb2ARtqPkmF2c/4pkXsbQtnfH0iIgNRyqH9VnseABe4VgL0W7/Xu8vPpiWvms94nueVDY39sk4RkYGmT6Fd6IkxPrKeiLeQbv/wTNT1XsbFhurrOdW1GdeuZbR1qyMpERl6+hTa00OtFHVuoiU0sU83POirLaM+RNhTwI3u53h1075+W6+IyECRUmhHE4bmqJfL/GtxJ6K0FJyUqbqOKO7JY/Poq7jUvYydW9cTicb7df0iItmWUmh3J5Kzz2YVceOhNb8mI0Udy4ax12Iw/IP9I8u2Nvf7+kVEsim10I67KPP2MKprPW35NViXN1N1HVVXcCQ7RlzEJzyLeaNuJ/FE5s9cEREZKFIK7XDcxcX5mwlEW2jJ4t3Sa6uvp4BOLoouZvXOlqzVISLS31IK7TiGSzxvAJzwXWpORFPxdJqKTmGB9wVe2dCAtdrbFpGhIeWzR06NraEzUEGPtzAT9fSOMdRWf5Ix7GZy51I2NnRkrxYRkX6UUmj7TZzS8HZaQtm/jHz7iIvp8g9jge8FlmzUxTYiMjSkFNolrk4MNqvt2QdYl5cNY6/jLNbgblrPrpbubJckIpJxKYV2AV1E3Xl0BCszVU9K6kZfRczl5zPeP2pvW0SGhNSaR2w4eQDSDIx7Nfb4itky6kN8xLWE+l072N/Vk+2SREQyKqX0NTYxINqzD1U79np8RPm4+8/8ra4p2+WIiGRUirvMhtbQuMxU0kdtBePZXT6Hm3wvsWprI909urRdRAav1M7TdvmIuwOZqqXP1ldfT2mimYvsqyzdoo6kRGTwSi20B2BgQ7Kv7db8aj4f+BOvbmoiGk9kuyQRkYxIMbT9marjxBgXtWOvZ1Kijkk961i5Q5e2i8jglFJoJ8wJ3bw9o7aM+hARTwG3BP/Ako1NJHRpu4gMQgPj3L00iHvyWF9zA+cmljOmcw21e9qzXZKISNoNmtAGqK3+JN2+Ur7hf4KXNzRkuxwRkbQbVKEd8+Tx1vgFnMlbjG55ne37OrNdkohIWg2q0AaoG3017YGR3Op9jJd113YRGWQGXWgn3D7WTvg808xmxjYsoqk9ku2SRETSZtCFNsDWUR9if14NX/E+zqsb92a7HBGRtBmUoW2NmzWTbuYkU8/YXc/QEYlluyQRkbQYlKENsLNiHntCU7jZ/RTL6nZnuxwRkbQYtKGNMbx18i2MMk3UbH2MnpgubReR3Dd4QxvYW34WWwtnssA8zdotO7NdjojICRvUoQ1QO/XLlJs2xm16mHhCl7aLSG4b9KG9r/hU1hWey/WJZ9iyfXu2yxEROSGDPrQBNk37MiET5qQNP8eqIykRyWEDt9s+YPz2J9K2rLWBM7iy+zne+P09nFGVf+SZZs5P2/pERDJhSOxpA4RHnY2bBO3r/pTtUkRE+mzIhLYNFPF6YA7nRJawZY9uSSYiuWnIhDZAvGo2PXhpWfPHbJciItInQyq0/YE8lhfO4/TI6zQ11Ge7HBGRlA2p0AYYN/08Wmw+Laufz3YpIiIpG3KhXVUc4C/5l3FSeA1dDZuzXY6ISEqGXGgDnHTqHPbaYlpXPwc6b1tEcsiQDO1Tyl08G/gwI8N1RPe+ne1yRER6bUiGNsD4KTPZlhhO59rnwKoHQBHJDUM2tM+vTPA/3ispDu/E1q/MdjkiIr0yZEPbGDhp8qm8nRhN97rnIRHPdkkiIsc1ZEMb4MNje3jAdRV5kUbYsTTb5YiIHNeQDm2fCyZPmMSKxAQ63/4TRLuzXZKIyDEN6dAGmD8xzF+KryQ/1sLa392Z7XJERI5pyIe228AXzh7JSvcpjFn7U5aveD3bJYmIHNWQD22AgBvGzbmShMtD2TOf4q1NusONiAxMCm1HYVEpiasfZrRpoPXXn2RbY2u2SxIReQ+F9iFKp1xA89z/ZI5dyWv3f5HG9ki2SxIROYxC+12Gz/1/NEy5kWtiz/DIfd+jPRzNdkkiIgcptI9g+Md+SPOIs/l8xz389wMPE4npwhsRGRgU2kfi9lB6wyNEQlV8oeE2vvfIH0kk1BugiGSfQvtogiUU3PgkBd4E1276Oj94ZjlW3biKSJYptI9l2ER81/yCya4dnL7iG9z3l7psVyQiQ5xC+zjMhIvh4u9wmXsZPYu+zxPLd2S7JBEZwhTaveCa80Xip17Hlzy/5a9P/5w/r9+b7ZJEZIhSaPeGMbg/vJD4qFn80Hsfdz/yFG9s35/tqkRkCFJo95bHj/vaR/AUlHOf506+9tCfqGvoyHZVIjLEKLRTERqO+7pHKXd38SN7J//4wCvsaQ1nuyoRGUIU2qkaeSquK+/nVDZwS/gebnhgKa1dumpSRPqHQrsvplwBc7/BFeZlLtj/GP/48HLCUV01KSKZp9Duq/P+BaZcwa3u35C3fRFfevRN4rpqUkQyTKHdVy4XfORezIhT+FnevdSte4N//f1aXTUpIhml0D4Rvnz4+P/g8wd5svDHvLR0FT9etDHbVYnIIObJdgEDyvKH+va6066j+LX7+GPev3LNolsZ1ryCT4zrxVklM+f3bX0iMmRpTzsdSmowc/6JYm+M3/lv4/mV23hhly/bVYnIIKTQTpeiKsw5XyYQKuYXvjtYvGwVSxu92a5KRAaZIdc8snRLc0aX7676JDXbn+T2rvu559WPsv2kMxiT13PEeTfF+3YD4etmjzmREkUkh2lPO83i7gCbxl7LjoLT+YL7afybXqBJF02KSJootDPAutzUj/4ga0vm8WHXK1TUPUFX5Mh72yIiqVBoZ4oxdFaezculVzOdDVTXPYwNt2W7KhHJcQrtDPOPPJlnyz9NmW1hwqaH6GxtynZJIpLDFNr9oHLECJ4e9jki1sPsHQ+wetteehIm22WJSA5SaPeTyRX51I6fT4NrODe1/4xX121hY0N7tssSkRyj0O5HecE8miZ9gl3BSdxqHqZi6X/y2OtbaQ+ra1cR6R2Fdj9LuH3sHncV9cUz+aznf/nHhu9zz0trWbplHwl1NiUixzHkLq4ZEIyLHZWXsbfiPD5Y+9+Mdbdw48qbeWPbCD5y+ihGFgWzXaGIDFDa084WY3h73HyWnHYnU+wmXs6/lbmdL/DTxRt4bs1uIjHdVEFE3kuhnWU7Rl7C83Mep6NwAt913c8fQt+ncdMbLHxpI+vqdV63iBxOoT0AtBWMZ9Hsh/j7tO9RzW6e93+Tr5pf89TSWn712jZaunQ1pYgkKbQHCmPYUnUFz573DJurPsInE7/nldDXGdO4mIUvbWTJxkbdzkxEFNoDTY+vmNen3caf3vcw7kAh93t+yC+DP2L12jX89C91bG/uynaJIpJFCu0BqqnkdJ4/+3HemPQVzrSrWBy8lasjv+WBv9byzafX0Nqtc7tFhiKF9gBmXV7Wj7uRZ899hr3DzuKf+TWLQv/GhmUvMu+Hf+X3K3fpRsIiQ4xCOwd0BUey5Iy7+OuMuyjzRHjC9+98330/3350CZ968HW2NnVmu0QR6ScK7Ryyq+ICnj33d3D2l7g4+mdeK7iVMdt/x/sX/pW7Fm3Uud0iQ4BJ5ev1yeNH21/84JYMliO9MbumFNrqYc0TsH8L2zxjWdh1OevyZnLbjDBnDT9Ge7fuAC/S74wxK6y1M9OxLF3GnqsKK2HOP8HOZYzdtIgfxe5lf6yAX786j18VnccFNXlcNipCyKs2b5HBRKGdy4wLRs+GqlnQtIHCLUv4YsPvSXQ+w/OrzuSzKy+mfGQNV46NcHZFD2514S2S8xTag4ExMGwS7mGToGsfrq2vcOm2pXwwvpT1DWN4qP4SvuWdzWVjLFdWtTNpREG2KxaRPlJoDzZ5ZZgpV+CZdBnsWsHELUu4vf3ndJrf8JvNc/n0jzsprjyJK0+v4sOnVVIe8me7YhFJgQ5E5qDZNaW9n9laaN4MW5dg96wGa1nqncVPOi/kNaYxd+JwrpxRxbyThxPwujNXtMgQpgOR0nvGQNl4KBuP6W6Bng7et+IXvM+3lKbAWB7ecSH/WXsa3/CP5IPTK/nYjFHMGFOCMWoAFxmItKedg1La0363mfMhGoZ1v4Ol90P9GwDs9Y3hufA0Xoqdyt7iGXxoRg1XzhjF6NK8NFUtMnSlc09boT3EbBpz9WGPCzq3Utn4CpWNSxjWvBxPooduArwSn8rixGnUFb6PUdUTmTaqKK3NJ9fNHpO2ZYkMdGoekbRpz6+mNr+a2urrcce6qGheRmXjEmY3LOHi8ArofoDat6p4ee1pvFUyB3/N2dSMKMHtUvOJSDYotOWguCeP+uHnUz/8fJhiKezcwsiGJZTt/ivz217A0/YsHSsDLGUam0vm0FRxLvnDqynN9+FSG7hIv1BoDzHjtz+R0vwxTx57R19GY/xCQh1biTdvZXrXBua1LIOWH7N7fSnrbDW7PKNp94/E5lcwrMBHVTCCx7y3OUZEToxCW3ol4fbTVjQJiiaxyb6f7eFGaNmOv2s3U3t2cUHiTVzdFrphT2MJbyWq2eEZze5dLjrLphEqr2JEYQCfR32UiZwIhbakzhiiweEQHE4U6AB2xiMEuveQ6GjA27mHqT31XJBYiavlGWiBvXXFrE3UsNk7gYbQyfymfQ7VNScxtbKIoqA32+9IJGcotCUt4m4/naGxEBpLN9AG7IhH2F84meC+tYSa1zK5Yz1zo0/ibkvAK7B/SYg6O5K93tFEisbhHzGJsrFTqZlwCsNLi7L9lkQGJIW2ZEzC7aepbCaUJc90ehtwx7ooaa9lTt4Oure/RUVjLeM7VlHc/GdoBtZB/DnDDlNBc3AMkaJxUDaB4MjJlIyZSkXlGLweXbkpQ5dCWzLqaAc+Q+WlhCbPgsmzkhOiYTpaG9nd2ERHSxN0NFAY3sPErpUE9/TAW8nZ2m2QOjOCZk8FkUA55JURKCiluLiUkSUhis+er6s5ZVBTaMvA4A0QKh/NhPLRh02OJxLsaWmjeV8T4dZGTGcDwfBeJsTqKGtfirvDQkNy3oj1sPXFO9jnraQzbxSxorG4y2oIjTyJ8tGTqBw+XAdCJecptCUrlm5pTu0FnlIoK4WySfQArcBWG8eG2wh3dZAIt+KJtOCxUcqiu5nQVktRWwfsAFYmF9FsQ9SbCho9I2jxjaQrWEkkVIUtHI0tGk0gVJjS+ea6qlOyQaEtOcsaNwRLCARLgOQe+qYxV7PFed7T04pp2YZp2UagfQf53TsoDtczIbaZ4V1L8XXFYN87y2u2IXab4TS6htPsHUGbfySdeZVE8kcRKxhNIFRCYdCrvXXJKvU9IkOTtdieTrq7Ooh2t+PuaSMQbaEgvp/i+H6Gs48APYe9pM3msdMOo4ESWl1FxPylxMsm4C6sxFdWRUH5aMrKR1BRHKA8349Ll/qLQ32PiJwoYzD+EHn+EJSMODi52xl2W4sn3oUJtxINt0OkHW9PK8FoC+PiTRQmNlEcaYd6koMjbL3stSUsp5RWTzmd/uFE8yugoBJP8Si8RSMIFA0nVFhCcb6P4qCPoqCXgNelA6jSKwptkSMxhpgnH0L5EEpOijpDG7AbmD22ECZeQry1no7G7XQ27aCneSe2bTcjO/cwLryFwu7X8XX3QNPhi49aN/spoNkWsJUQLRTQ5Skm7C2mx1dCLFACwTJcoTLcoXJ8hcMJhQoPBn1xnpeiPC8Ffo/CfohRaIv00dJtbbDtXac0equgrArKnMfW4o6H8cWSe+uJaDc22o2JdeOJdeONd1ER76Q6sZ88u45QpBNXxEL7e9fXY910EKTDBmknj3qCdNogXa48wq48etz5ycETIurOJ+oNEfPkE/OGSHhDWG8eeEPgz8fj9fOxM0aR5/OQ7/OQ73fjcautPhcotEUyyRjiniDdniAEhh/2VORI8zsh74134Yl34Yl14Yp1k4h2QzSMjfcQ8RbhjnZSEu+kIt6BP76XQKKLvFgX/ljPURZ8uKh10/Wany4CNFk/2/HTbYL0mAA97iAxZ4h78rGePKwviPHl4fLm4/IF8QTy8fjz8QTy8Qby8QXy8QXzCeYX4A+GyAv4yfN51OyTAQptkYHECfm4J8g7u+tH1nmkl9s4rngP7kQEdyKCKx4hHuvBxqIk4lFsPArOUBIAYhFMLEIo0UNhPII7sR9vYg/eWARfTwQ/EbzEUn4bPdZNGD9t+IjgJ248xIyXhMtL3OUj4fJhXV6s2wduH3h8GLcP4/Hj8vgxXj9urw+PN4DL68Xj8eH2+PF4vXh8PjweH16fH6/Ph9vtA7cXXB7npxfcnuRyDy7fD24/eHzJn25v8lZ8OUihLTKIWONOhj7B487b2MtlGhvHlYgmBxvFxqJE43ES8Rhx56eNxyARhXgMEjGMM687EcVlY8ll2BiuRBxXIobbRvAQw2PjeIjhJYaHGD6ieInjI4rfpP7PIhUx4yFuPCTcQazb+Ufi9pFw+8Htw3r84A5gPX6MNwgeP8YbwOUJ4PIFMN4gLl8AtzeI2xfE5Q0k/zl4Aof8k3CGNFJoi8gxWeMm7nYTdweSE3zJH4beBYh1hsTx5rPQYw2RuItwwkU4DtGEJRaHWCJBPJ4glrDEbYJEIkEibhkW8hGNJ4jF48QTCeLxOPF4gkQiTiIRxyZikIg7/0iSgysRw2Xf+SeRHGL4jPPTeew3rfhpxE80OZjkzwA9yX8uJt7nbXoiFNoiMiAYA35j8bviFHK0QDSA2xkOcDlDauIWoglDT8JFjzXErKE7YWh3xmPWEEscMn7YtGQXC9g4xOMYm/xnYGxycCeS3y6M8y0D/ivl+o5GoS0iQ5LbgNttCbjTtcdsAK8zZI7O8RERySEpXcZujGkHajNXTtqU857LGQacXKgRVGe6qc70ypU6J1lrC9KxoFSbR2rTdf18Jhljlg/0OnOhRlCd6aY60yuX6kzXstQ8IiKSQxTaIiI5JNXQ/llGqki/XKgzF2oE1ZluqjO9hlydKR2IFBGR7FLziIhIDulVaBtjLjXG1Bpj6owxX890Ub2oZ6sxZo0xZuWBo7LGmFJjzIvGmI3OzxJnujHG3OXUvtoYMyODdT1ojGkwxqw9ZFrKdRljbnDm32iMuaGf6rzNGLPL2aYrjTGXH/LcN5w6a40xlxwyPWOfC2PMaGPMYmPMOmPMW8aYLznTB9T2PEadA238vQ9fAAAHs0lEQVR7BowxrxtjVjl1/rszvcYYs9RZ52PGGJ8z3e88rnOerz5e/Rmu8xfGmC2HbM/TnOnZ/DtyG2PeNMY86zzun21prT3mQPJ60U3AOJK9DqwCphzvdZkcgK1A+bum3QF83Rn/OnC7M3458DzJy5XeByzNYF3nATOAtX2tCygFNjs/S5zxkn6o8zbgq0eYd4rzO/cDNc5n4cB1xBn7XAAjgRnOeAGwwallQG3PY9Q50LanAULOuBdY6mynx4GPO9PvAz7njH8euM8Z/zjw2LHq74c6fwFcdYT5s/l39M/Ab4Bnncf9si17s6c9C6iz1m621vYAjwJX9OJ1/e0K4JfO+C+Bjxwy/WGb9BpQbIwZmYkCrLUvA+++zXiqdV0CvGitbbbW7gdeBC7thzqP5grgUWttxFq7Bagj+ZnI6OfCWrvbWvuGM94OvA2MYoBtz2PUeTTZ2p7WWtvhPDxwrbUFLgSedKa/e3se2M5PAvOMMeYY9We6zqPJyu/dGFMFfAD4/85jQz9ty96E9ihgxyGPd3LsD2V/sMCfjDErjDELnGkV1trdzvgeoMIZz3b9qdaVzXq/6HzFfPBAs8Mx6um3Op2vk6eT3OsasNvzXXXCANueztf5lUADyRDbBLRYaw/0gXroOg/W4zzfSrKD736v01p7YHv+h7M9f2SMOdDfaba250LgX3in88Iy+mlb5uqByHOstTOAy4AvGGPOO/RJm/zuMeBOixmodTnuBcYDp5G8BeIPs1tOkjEmBDwF3GKtbTv0uYG0PY9Q54DbntbauLX2NKCK5B7d5CyXdETvrtMYcwrwDZL1nkmyyePWbNVnjPkg0GCtXZGN9fcmtHcBow95XOVMyxpr7S7nZwPwNMkP4N4DzR7OzwZn9mzXn2pdWanXWrvX+WNJAD/nna9pWavTGOMlGYSPWGt/60wecNvzSHUOxO15gLW2BVgMnEWyOeFAdxaHrvNgPc7zRcC+LNV5qdMMZa21EeAhsrs9zwY+bIzZSrIZ60Lgx/TXtuxFY7uHZCN+De8cIJnam4b6TAxAPlBwyPirJNuq/ovDD1Dd4Yx/gMMPVLye4fqqOfwAX0p1kdyL2ELy4EmJM17aD3WOPGT8yyTb2gCmcvjBks0kD5pl9HPhbJeHgYXvmj6gtucx6hxo23MYUOyMB4ElwAeBJzj84NnnnfEvcPjBs8ePVX8/1DnykO29EPjBAPk7mss7ByL7ZVv2trDLSR4V3wR8M91vPMWNNM55o6uAtw7UQ7KNaBGwEXjpwC/I+WXe49S+BpiZwdr+h+RX4SjJ9qlP96Uu4CaSByXqgPn9VOevnDpWA89weOh806mzFrisPz4XwDkkmz5WAyud4fKBtj2PUedA256nAm869awF/u2Qv6fXnW3zBOB3pgecx3XO8+OOV3+G6/yzsz3XAr/mnTNMsvZ35KxjLu+Edr9sS10RKSKSQ3L1QKSIyJCk0BYRySEKbRGRHKLQFhHJIQptEZEcotAWEckhCm0ZMIwxlcaYJ48/Z0rL3GqMKU/zMquNMdcd8vhGY8zd6VyHyNEotGXAsNbWW2uvynYdvVANXHe8mUQyQaEtfWKMyTfG/MHprH6tMeYaY8wZxpi/Or0v/vGQPkJuNsmbBKw2xjzqTDv/kA7t3zTGFDh7sGud5wPGmIdM8mYXbxpjLnCm32iM+a0x5gWnc/s7Uqj5eqeD/ZXGmPuNMW5neocx5j+c9/KaMabCmT7eebzGGPM9Y8yBLkN/AJzrLOfLzrTKvtQkkrJMXNqpYfAPwMeAnx/yuIhkPzDDnMfXAA864/W8c0nvgX4l/hc42xkPkex7oxqnPxTgK4e8fjKwneTlwDeS7KOhyHm8DRh9jDq3AuXAyc46vc70nwKfcsYt8CFn/A7gW874s8C1zvhngQ5nfC7OpcvO45Rq0qDhRAbtaUtfrQEuNsbcbow5l2RvZacALzp9IX+LZK9lkOxH4hFjzPXAgf6G/wb8tzHmZpJBHjt88ZxDso8JrLXrSQbhROe5RdbaVmttGFgHjO1FvfOAM4BlTn3zSPYVAdBDMqABVpD85wHJXvCecMZ/c5zl96UmkZR5jj+LyHtZazeY5P34Lge+R7JDn7estWcdYfYPkLzF2YeAbxpjpllrf2CM+YPz+r8598cL93L1kUPG4/Tuc2yAX1prv3GE56LW2gOd8PR2eemoSSRl2tOWPjHGVAJd1tpfk+wudTYwzBhzlvO81xgz1RjjItlUsJhkx/VFQMgYM95au8ZaezuwjPd2yL8E+ISzrInAGJI9ofXVIuAqY8xwZ5mlxpjj7Q2/RrIZCJJdah7QTvJ+kCL9TnsD0lfTgP8yxiRIdvH6OZJNH3cZY4pIfrYWkuxq9NfONAPcZa1tMcZ81zm4mCDZxe7zJG+Se8BPgXuNMWuc5d5orY0kb62XOmvtOmPMt0jeps7l1PwFks0uR3OLU/s3gRdI3iYKks09cWPMKpI3nN3fp6JE+kBds4ochTEmD+i21lpjzMdJHpQciDe1liFEe9oiR3cGcLdJ7t63kOxUXySrtKctg4IxZinJ2zYd6pPW2jXZqEckUxTaIiI5RGePiIjkEIW2iEgOUWiLiOQQhbaISA5RaIuI5JD/A9yjotBP5HfbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# comparison plot of the distributions of session length of both control and test group\n",
    "fig, ax = plt.subplots()\n",
    "sns.distplot(session_length_df[session_length_df['test_group']=='control']['session_length'], ax=ax, label='Control')\n",
    "sns.distplot(session_length_df[session_length_df['test_group']=='test']['session_length'], ax=ax, label='Test')\n",
    "ax.set_xlim([0, 4000])\n",
    "ax.set_yticks([])\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Statistics=68928962.000, p=0.381\n",
      "Same distribution (fail to reject H0)\n"
     ]
    }
   ],
   "source": [
    "# since we are using the median and the distribution is not normal, Mann Whitney U test \n",
    "# is a better choice of measurement for median time per session differences\n",
    "from scipy.stats import mannwhitneyu\n",
    "\n",
    "test = session_length_df[session_length_df['test_group']=='test']['created_at'].dt.seconds\n",
    "control = session_length_df[session_length_df['test_group']=='control']['created_at'].dt.seconds\n",
    "# compare samples\n",
    "stat, p = mannwhitneyu(control, test)\n",
    "print('Statistics=%.3f, p=%.3f' % (stat, p))\n",
    "# interpret\n",
    "alpha = 0.05\n",
    "if p > alpha:\n",
    "\tprint('Same distribution (fail to reject H0)')\n",
    "else:\n",
    "\tprint('Different distribution (reject H0)')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3b. If there is a difference, how big is it?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is no statistical significant difference between both groups averages on both sessions and users statistics."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3c. What would you recommend the Product Manager to do next?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I'd recommend him to not make the change since there's no proven benefit in either statistics we tracked so unless there's any other metric of business relevance, he should try another iteration."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}