create-counts-buys.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d6af3d98",
   "metadata": {
    "papermill": {
     "duration": 0.006832,
     "end_time": "2023-03-17T22:10:33.574260",
     "exception": false,
     "start_time": "2023-03-17T22:10:33.567428",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Calculations for buys\n",
    "Some calculations, that are used later to create features for buys (this means, for both carts and orders models) are performed in this notebook. Those are the calculations, that do not need information about cart/order candidates and thus can be done separately.\n",
    "Those calculations include:\n",
    "* conversion rate - means conversion from click to either cart or order\n",
    "* conversion to carts - conversion from either clicks, previously carted aids or previously ordered aids to carts\n",
    "* conversion to orders - conversion from either clicks, carts or previously ordered aids to new orders\n",
    "* average per aid clicks before buy\n",
    "* daily total carts/orders per aid\n",
    "* average w2vec similarity between the last one aid in session and 5 aids before it"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53b4612a",
   "metadata": {
    "papermill": {
     "duration": 0.005138,
     "end_time": "2023-03-17T22:10:33.584947",
     "exception": false,
     "start_time": "2023-03-17T22:10:33.579809",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5b3eda8e",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-17T22:10:33.598047Z",
     "iopub.status.busy": "2023-03-17T22:10:33.597641Z",
     "iopub.status.idle": "2023-03-17T22:10:34.752371Z",
     "shell.execute_reply": "2023-03-17T22:10:34.751134Z"
    },
    "papermill": {
     "duration": 1.165604,
     "end_time": "2023-03-17T22:10:34.756310",
     "exception": false,
     "start_time": "2023-03-17T22:10:33.590706",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "from gensim.models import Word2Vec\n",
    "from pandarallel import pandarallel\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bdceca0f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:10:34.774657Z",
     "iopub.status.busy": "2023-03-17T22:10:34.774037Z",
     "iopub.status.idle": "2023-03-17T22:10:34.783610Z",
     "shell.execute_reply": "2023-03-17T22:10:34.782664Z"
    },
    "papermill": {
     "duration": 0.020906,
     "end_time": "2023-03-17T22:10:34.785718",
     "exception": false,
     "start_time": "2023-03-17T22:10:34.764812",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Count how often users buy a product given they clicked to view it\n",
    "def count_conversion(train_data):\n",
    "    df = pd.read_parquet(train_data)\n",
    "    df_events = df.groupby('aid')['session'].nunique()\n",
    "    df_events.name = 'aid_counts'\n",
    "    df = df.loc[df['type'] > 0]\n",
    "    df_buys = df.groupby('aid')['session'].nunique()\n",
    "    df_buys.name = 'aid_buys'\n",
    "    df = pd.merge(df_events, df_buys, how='left', left_index=True, right_index=True)\n",
    "    df['aid_buys'] = df['aid_buys'].fillna(0)\n",
    "    df['conv'] = ((2 + df['aid_buys'])/(18 + df['aid_counts'])).astype(np.float32)\n",
    "    df = df.reset_index()\n",
    "    df['aid'] = df['aid'].astype(np.int32)\n",
    "    df = df[['aid', 'conv']]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "323063a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:10:34.798771Z",
     "iopub.status.busy": "2023-03-17T22:10:34.798159Z",
     "iopub.status.idle": "2023-03-17T22:10:34.809500Z",
     "shell.execute_reply": "2023-03-17T22:10:34.808449Z"
    },
    "papermill": {
     "duration": 0.02036,
     "end_time": "2023-03-17T22:10:34.811755",
     "exception": false,
     "start_time": "2023-03-17T22:10:34.791395",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Count how often users order a product if they clicked, carted or ordered it before\n",
    "def count_2order_conversion(train_data):\n",
    "    df = pd.read_parquet(train_data)\n",
    "    df_events = df.groupby('aid')['session'].nunique()\n",
    "    df_events.name = 'aid_counts'\n",
    "    df_carts = df.loc[df['type'] == 1]\n",
    "    df_carts = df_carts.groupby('aid')['session'].nunique()\n",
    "    df_carts.name = 'aid_carts'\n",
    "    df = df.loc[df['type'] == 2]\n",
    "    df_orders = df.groupby('aid')['session'].nunique()\n",
    "    df_orders.name = 'aid_orders'\n",
    "    df_double_orders = df.groupby(['aid', 'session']).size()\n",
    "    df_double_orders = df_double_orders.loc[df_double_orders > 1]\n",
    "    df_double_orders = df_double_orders.reset_index()\n",
    "    df_double_orders = df_double_orders.groupby('aid')['session'].nunique()\n",
    "    df_double_orders.name = 'double_orders'\n",
    "    \n",
    "    df = pd.merge(df_events, df_carts, how='left', left_index=True, right_index=True)\n",
    "    df = pd.merge(df, df_orders, how='left', left_index=True, right_index=True)\n",
    "    df = pd.merge(df, df_double_orders, how='left', left_index=True, right_index=True)\n",
    "    \n",
    "    df['aid_carts'] =  df['aid_carts'].fillna(0)\n",
    "    df['aid_orders'] =  df['aid_orders'].fillna(0)\n",
    "    df['double_orders'] =  df['double_orders'].fillna(0)\n",
    "    \n",
    "    df['click2order_conv'] = ((1 + df['aid_orders'])/(31 + df['aid_counts'])).astype(np.float32)\n",
    "    df['cart2order_conv'] = ((7 + df['aid_orders'])/(28 + df['aid_carts'])).astype(np.float32)\n",
    "    df['order2order_conv'] = ((1 + df['double_orders'])/(18 + df['aid_orders'])).astype(np.float32)\n",
    "    df = df.reset_index()\n",
    "    \n",
    "    df = df[['aid', 'click2order_conv', 'cart2order_conv', 'order2order_conv']]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "466c0bfe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:10:34.824744Z",
     "iopub.status.busy": "2023-03-17T22:10:34.824414Z",
     "iopub.status.idle": "2023-03-17T22:10:34.839695Z",
     "shell.execute_reply": "2023-03-17T22:10:34.838937Z"
    },
    "papermill": {
     "duration": 0.024222,
     "end_time": "2023-03-17T22:10:34.841737",
     "exception": false,
     "start_time": "2023-03-17T22:10:34.817515",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Count how often users put a product in a cart if they clicked, carted or ordered it before.\n",
    "def count_2cart_conversion(train_data):\n",
    "    df = pd.read_parquet(train_data)\n",
    "    df_events = df.groupby('aid')['session'].nunique()\n",
    "    df_events.name = 'aid_counts'\n",
    "    df_carts = df.loc[df['type'] == 1]\n",
    "    df_carts = df_carts.groupby('aid')['session'].nunique()\n",
    "    df_carts.name = 'aid_carts'\n",
    "    df_orders = df.loc[df['type'] == 2]\n",
    "    df_orders = df_orders.groupby('aid')['session'].nunique()\n",
    "    df_orders.name = 'aid_orders'\n",
    "    df_double_carts = df.loc[df['type'] == 1]\n",
    "    df_double_carts = df_double_carts.groupby(['aid', 'session']).size()\n",
    "    df_double_carts = df_double_carts.loc[df_double_carts > 1].reset_index()\n",
    "    df_double_carts = df_double_carts.groupby('aid')['session'].nunique()\n",
    "    df_double_carts.name = 'double_carts'\n",
    "\n",
    "    df = df.loc[df['type'] > 0]\n",
    "    df_session_orders = df.loc[df['type'] == 2]\n",
    "    df_session_orders = df_session_orders.groupby(['aid', 'session']).agg(ts_min=('ts', np.min))\n",
    "    df_session_carts = df.loc[df['type'] == 1]\n",
    "    df_session_carts = df_session_carts.groupby(['aid', 'session']).agg(ts_max=('ts', np.max))\n",
    "    df_session_orders = pd.merge(df_session_orders, df_session_carts, how='left',\n",
    "                             left_index=True, right_index=True)\n",
    "    df_session_orders = df_session_orders.loc[df_session_orders['ts_min'] < df_session_orders['ts_max']]\n",
    "    df_session_orders = df_session_orders.reset_index()\n",
    "    df_session_orders = df_session_orders.groupby('aid')['session'].nunique()\n",
    "    df_session_orders.name = 'carts_after_orders'\n",
    "    \n",
    "    df = pd.merge(df_events, df_carts, how='left', left_index=True, right_index=True)\n",
    "    df = pd.merge(df, df_orders, how='left', left_index=True, right_index=True)\n",
    "    df = pd.merge(df, df_double_carts, how='left', left_index=True, right_index=True)\n",
    "    df = pd.merge(df, df_session_orders, how='left', left_index=True, right_index=True)\n",
    "\n",
    "    df['aid_carts'] =  df['aid_carts'].fillna(0)\n",
    "    df['aid_orders'] =  df['aid_orders'].fillna(0)\n",
    "    df['double_carts'] =  df['double_carts'].fillna(0)\n",
    "    df['carts_after_orders'] =  df['carts_after_orders'].fillna(0)\n",
    "    \n",
    "    df['click2cart_conv'] = ((2 + df['aid_carts'])/(18 + df['aid_counts'])).astype(np.float32)\n",
    "    df['order2cart_conv'] = ((1 + df['carts_after_orders'])/(50 + df['aid_orders'])).astype(np.float32)\n",
    "    df['cart2cart_conv'] = ((2 + df['double_carts'])/(20 + df['aid_carts'])).astype(np.float32)\n",
    "    df = df.reset_index()\n",
    "    \n",
    "    df = df[['aid', 'click2cart_conv', 'order2cart_conv', 'cart2cart_conv']]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e91f74df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:10:34.854619Z",
     "iopub.status.busy": "2023-03-17T22:10:34.854328Z",
     "iopub.status.idle": "2023-03-17T22:10:34.865016Z",
     "shell.execute_reply": "2023-03-17T22:10:34.863967Z"
    },
    "papermill": {
     "duration": 0.019561,
     "end_time": "2023-03-17T22:10:34.867116",
     "exception": false,
     "start_time": "2023-03-17T22:10:34.847555",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Average number of clicks on an item before buy.\n",
    "def clicks_before_buy(train_data):\n",
    "    df_click_data = pd.read_parquet(train_data)\n",
    "    df_click_data['n'] = df_click_data.groupby('session').cumcount().astype(np.int16)\n",
    "    df_buy_data = df_click_data.loc[df_click_data['type'] > 0]\n",
    "    df_buy_data = df_buy_data.groupby(['session','aid']).agg({'n':'min'})\n",
    "    df_buy_data =  df_buy_data.rename(columns={'n':'first_buy_n'})\n",
    "    df_click_data = pd.merge(df_click_data, df_buy_data, how='inner', on=['session','aid'])\n",
    "    del df_buy_data\n",
    "    gc.collect()\n",
    "    df_clicks_before_buy = df_click_data.loc[df_click_data['n'] <= df_click_data['first_buy_n']]\n",
    "    df_clicks_before_buy = df_clicks_before_buy.groupby(['session','aid']).size()\n",
    "    df_clicks_before_buy = df_clicks_before_buy.reset_index()\n",
    "    df_clicks_before_buy = df_clicks_before_buy.rename(columns={0 :'clicks_before_buy'})\n",
    "    df_clicks_before_buy['clicks_before_buy'] = df_clicks_before_buy['clicks_before_buy'].clip(0,30)\n",
    "    \n",
    "    session_max = df_clicks_before_buy['session'].max()\n",
    "    aids = df_clicks_before_buy['aid'].unique()\n",
    "    df_add1 = pd.DataFrame({'session': session_max + 1, 'aid': aids})\n",
    "    df_add1['clicks_before_buy'] = 2\n",
    "    df_add2 = df_add1.copy()\n",
    "    df_add2['session'] = session_max + 2\n",
    "    df_clicks_before_buy = pd.concat([df_clicks_before_buy, df_add1, df_add2])\n",
    "    \n",
    "    df_clicks_before_buy = df_clicks_before_buy.groupby(['aid']).agg({'clicks_before_buy':'mean'})\n",
    "    return df_clicks_before_buy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7359b2eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:10:34.880613Z",
     "iopub.status.busy": "2023-03-17T22:10:34.880237Z",
     "iopub.status.idle": "2023-03-17T22:10:34.885767Z",
     "shell.execute_reply": "2023-03-17T22:10:34.884668Z"
    },
    "papermill": {
     "duration": 0.01493,
     "end_time": "2023-03-17T22:10:34.888152",
     "exception": false,
     "start_time": "2023-03-17T22:10:34.873222",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Function to calculate mean w2vec similarity between the last aid and up to 4 previous aids.\n",
    "def agg_mean(x, w2v_model):\n",
    "    similarities = []\n",
    "    for item in x.aid:\n",
    "        similarities.append(w2v_model.wv.similarity(item, x.aid_last))\n",
    "    return np.mean(similarities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d553ae0d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:10:34.903226Z",
     "iopub.status.busy": "2023-03-17T22:10:34.902858Z",
     "iopub.status.idle": "2023-03-17T22:10:34.910119Z",
     "shell.execute_reply": "2023-03-17T22:10:34.909183Z"
    },
    "papermill": {
     "duration": 0.016472,
     "end_time": "2023-03-17T22:10:34.912130",
     "exception": false,
     "start_time": "2023-03-17T22:10:34.895658",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load cross-validation or test sessions and select the last aid and up to 4 previous aids.\n",
    "def prepare_sessions_4_w2vec(sessions_path):\n",
    "    df_sessions = pd.read_parquet(sessions_path)\n",
    "    df_sessions = df_sessions.sort_values(['session','ts'],ascending=[True,False])\n",
    "    df_sessions['n'] = df_sessions.groupby('session').cumcount().astype(np.int16)\n",
    "    df_sessions_last = df_sessions.loc[df_sessions['n'] == 0]\n",
    "    df_sessions_last = df_sessions_last[['session', 'aid']].rename(columns={'aid':'aid_last'})\n",
    "    df_sessions = df_sessions.loc[(df_sessions['n'] < 5) & (df_sessions['n'] > 0)]\n",
    "    df_sessions = (df_sessions.groupby('session').agg({'aid': lambda x: x.tolist()}))\n",
    "    df_sessions = pd.merge(df_sessions, df_sessions_last, how='inner', left_index=True, right_on='session')\n",
    "    return df_sessions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "206295ad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:10:34.925094Z",
     "iopub.status.busy": "2023-03-17T22:10:34.924747Z",
     "iopub.status.idle": "2023-03-17T22:10:34.929756Z",
     "shell.execute_reply": "2023-03-17T22:10:34.929128Z"
    },
    "papermill": {
     "duration": 0.013717,
     "end_time": "2023-03-17T22:10:34.931734",
     "exception": false,
     "start_time": "2023-03-17T22:10:34.918017",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Function to manage the workflow for the w2vec session similarity.\n",
    "def calculate_w2vec_history_feature(sessions_path, w2v_model_path):\n",
    "    w2v_model = Word2Vec.load(w2v_model_path)\n",
    "    df_sessions = prepare_sessions_4_w2vec(sessions_path)\n",
    "    pandarallel.initialize(nb_workers=4)\n",
    "    df_sessions['history_mean']  = df_sessions.parallel_apply(\n",
    "        lambda x: agg_mean(x, w2v_model), axis=1\n",
    "    )\n",
    "    df_sessions = df_sessions[['session', 'history_mean']]\n",
    "    return df_sessions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afc41d30",
   "metadata": {
    "papermill": {
     "duration": 0.005789,
     "end_time": "2023-03-17T22:10:34.943456",
     "exception": false,
     "start_time": "2023-03-17T22:10:34.937667",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Counts for cross-validation datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f812a111",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:10:34.956808Z",
     "iopub.status.busy": "2023-03-17T22:10:34.956438Z",
     "iopub.status.idle": "2023-03-17T22:10:34.960878Z",
     "shell.execute_reply": "2023-03-17T22:10:34.960147Z"
    },
    "papermill": {
     "duration": 0.013439,
     "end_time": "2023-03-17T22:10:34.962714",
     "exception": false,
     "start_time": "2023-03-17T22:10:34.949275",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Paths for trunked_sessions and history (full sessions).\n",
    "trunked_sessions_path = '/kaggle/input/otto-prepare-cv/cv_inputs.parquet'\n",
    "trunked_sessions_path2 = '/kaggle/input/otto-prepare-cv/cv_inputs2.parquet'\n",
    "click_data_cv = '/kaggle/input/otto-prepare-cv/cv_train.parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "88aec50a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:10:34.975904Z",
     "iopub.status.busy": "2023-03-17T22:10:34.975385Z",
     "iopub.status.idle": "2023-03-17T22:14:33.092813Z",
     "shell.execute_reply": "2023-03-17T22:14:33.091955Z"
    },
    "papermill": {
     "duration": 238.132778,
     "end_time": "2023-03-17T22:14:33.101258",
     "exception": false,
     "start_time": "2023-03-17T22:10:34.968480",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the history w2vec similarity for both cross-validation datasets.\n",
    "w2vec_path_cv = '/kaggle/input/otto-word2vec-exp/word2vec_cv_exp.wordvectors'\n",
    "\n",
    "df_history_cv1 = calculate_w2vec_history_feature(trunked_sessions_path, w2vec_path_cv)\n",
    "df_history_cv1.to_parquet('history_change_cv1.parquet')\n",
    "\n",
    "df_history_cv2 = calculate_w2vec_history_feature(trunked_sessions_path2, w2vec_path_cv)\n",
    "df_history_cv2.to_parquet('history_change_cv2.parquet')\n",
    "\n",
    "del df_history_cv1, df_history_cv2\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2fcc7566",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:14:33.116918Z",
     "iopub.status.busy": "2023-03-17T22:14:33.115868Z",
     "iopub.status.idle": "2023-03-17T22:15:07.535620Z",
     "shell.execute_reply": "2023-03-17T22:15:07.534726Z"
    },
    "papermill": {
     "duration": 34.430086,
     "end_time": "2023-03-17T22:15:07.538101",
     "exception": false,
     "start_time": "2023-03-17T22:14:33.108015",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate average daily carts per aid for both cross-validation datasets and for the last week of full sessions.\n",
    "df_daily_counts_train = otto_common.create_average_daily_counts(click_data_cv, False, 1)\n",
    "df_daily_counts_train.to_parquet('daily_counts_carts_train.parquet')\n",
    "\n",
    "df_daily_counts_cv1 = otto_common.create_average_daily_counts(trunked_sessions_path, True, 1)\n",
    "df_daily_counts_cv1.to_parquet('daily_counts_carts_cv1.parquet')\n",
    "\n",
    "df_daily_counts_cv2 = otto_common.create_average_daily_counts(trunked_sessions_path2, True, 1)\n",
    "df_daily_counts_cv2.to_parquet('daily_counts_carts_cv2.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "bc2b64ed",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:15:07.553101Z",
     "iopub.status.busy": "2023-03-17T22:15:07.552403Z",
     "iopub.status.idle": "2023-03-17T22:15:18.706780Z",
     "shell.execute_reply": "2023-03-17T22:15:18.705639Z"
    },
    "papermill": {
     "duration": 11.1645,
     "end_time": "2023-03-17T22:15:18.709464",
     "exception": false,
     "start_time": "2023-03-17T22:15:07.544964",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate average daily orders per aid for both cross-validation datasets and for last week of full sessions.\n",
    "df_daily_counts_train = otto_common.create_average_daily_counts(click_data_cv, False, 2)\n",
    "df_daily_counts_train.to_parquet('daily_counts_orders_train.parquet')\n",
    "\n",
    "df_daily_counts_cv1 = otto_common.create_average_daily_counts(trunked_sessions_path, True, 2)\n",
    "df_daily_counts_cv1.to_parquet('daily_counts_orders_cv1.parquet')\n",
    "\n",
    "df_daily_counts_cv2 = otto_common.create_average_daily_counts(trunked_sessions_path2, True, 2)\n",
    "df_daily_counts_cv2.to_parquet('daily_counts_orders_cv2.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e1636cef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:15:18.723797Z",
     "iopub.status.busy": "2023-03-17T22:15:18.723486Z",
     "iopub.status.idle": "2023-03-17T22:17:16.741969Z",
     "shell.execute_reply": "2023-03-17T22:17:16.740601Z"
    },
    "papermill": {
     "duration": 118.029654,
     "end_time": "2023-03-17T22:17:16.745378",
     "exception": false,
     "start_time": "2023-03-17T22:15:18.715724",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate average clicks before buy per aid.\n",
    "df_clicks_before_buy = clicks_before_buy(click_data_cv)\n",
    "df_clicks_before_buy.to_parquet('clicks_before_buy_cv.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "79fc2a33",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:17:16.760833Z",
     "iopub.status.busy": "2023-03-17T22:17:16.760141Z",
     "iopub.status.idle": "2023-03-17T22:18:28.731181Z",
     "shell.execute_reply": "2023-03-17T22:18:28.730025Z"
    },
    "papermill": {
     "duration": 71.981106,
     "end_time": "2023-03-17T22:18:28.734033",
     "exception": false,
     "start_time": "2023-03-17T22:17:16.752927",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate general conversion rate per aid.\n",
    "df_conversions = count_conversion(click_data_cv)\n",
    "df_conversions.to_parquet('conversions_cv.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6b43d622",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:18:28.748917Z",
     "iopub.status.busy": "2023-03-17T22:18:28.748562Z",
     "iopub.status.idle": "2023-03-17T22:19:44.088271Z",
     "shell.execute_reply": "2023-03-17T22:19:44.087045Z"
    },
    "papermill": {
     "duration": 75.350572,
     "end_time": "2023-03-17T22:19:44.091165",
     "exception": false,
     "start_time": "2023-03-17T22:18:28.740593",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate click2order, cart2order and order2order conversion rates.\n",
    "df_2order_conversions = count_2order_conversion(click_data_cv)\n",
    "df_2order_conversions.to_parquet('2order_conversions_cv.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6fcb2127",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:19:44.106741Z",
     "iopub.status.busy": "2023-03-17T22:19:44.106337Z",
     "iopub.status.idle": "2023-03-17T22:21:38.864199Z",
     "shell.execute_reply": "2023-03-17T22:21:38.862944Z"
    },
    "papermill": {
     "duration": 114.768881,
     "end_time": "2023-03-17T22:21:38.867085",
     "exception": false,
     "start_time": "2023-03-17T22:19:44.098204",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate click2cart, cart2cart and order2cart conversion rates.\n",
    "df_2cart_conversions = count_2cart_conversion(click_data_cv)\n",
    "df_2cart_conversions.to_parquet('2cart_conversions_cv.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "34511fdd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:21:38.883088Z",
     "iopub.status.busy": "2023-03-17T22:21:38.882658Z",
     "iopub.status.idle": "2023-03-17T22:21:38.998631Z",
     "shell.execute_reply": "2023-03-17T22:21:38.997456Z"
    },
    "papermill": {
     "duration": 0.125831,
     "end_time": "2023-03-17T22:21:39.000637",
     "exception": false,
     "start_time": "2023-03-17T22:21:38.874806",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df_daily_counts_train, df_daily_counts_cv1, df_daily_counts_cv2, df_conversions, df_2order_conversions\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f09f82b7",
   "metadata": {
    "papermill": {
     "duration": 0.006647,
     "end_time": "2023-03-17T22:21:39.014243",
     "exception": false,
     "start_time": "2023-03-17T22:21:39.007596",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Counts for test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d95230c6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:21:39.028551Z",
     "iopub.status.busy": "2023-03-17T22:21:39.028197Z",
     "iopub.status.idle": "2023-03-17T22:21:39.032859Z",
     "shell.execute_reply": "2023-03-17T22:21:39.031905Z"
    },
    "papermill": {
     "duration": 0.014383,
     "end_time": "2023-03-17T22:21:39.034798",
     "exception": false,
     "start_time": "2023-03-17T22:21:39.020415",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "trunked_sessions_path = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "click_data_test = '/kaggle/input/otto-prepare-cv/train_full.parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "94b9284e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:21:39.049054Z",
     "iopub.status.busy": "2023-03-17T22:21:39.048721Z",
     "iopub.status.idle": "2023-03-17T22:23:41.578723Z",
     "shell.execute_reply": "2023-03-17T22:23:41.577456Z"
    },
    "papermill": {
     "duration": 122.547064,
     "end_time": "2023-03-17T22:23:41.588071",
     "exception": false,
     "start_time": "2023-03-17T22:21:39.041007",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the history w2vec similarity for test dataset.\n",
    "w2vec_path_test = '/kaggle/input/otto-word2vec-exp/word2vec_test_exp.wordvectors'\n",
    "\n",
    "df_history_test = calculate_w2vec_history_feature(trunked_sessions_path, w2vec_path_test)\n",
    "df_history_test.to_parquet('history_change_test.parquet')\n",
    "\n",
    "del df_history_test\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c5e2a48a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:23:41.603468Z",
     "iopub.status.busy": "2023-03-17T22:23:41.603088Z",
     "iopub.status.idle": "2023-03-17T22:24:24.115118Z",
     "shell.execute_reply": "2023-03-17T22:24:24.112614Z"
    },
    "papermill": {
     "duration": 42.524597,
     "end_time": "2023-03-17T22:24:24.119563",
     "exception": false,
     "start_time": "2023-03-17T22:23:41.594966",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate average daily carts per aid for test dataset and for the last week of full sessions.\n",
    "df_daily_counts_test_full = otto_common.create_average_daily_counts(click_data_test, False, 1)\n",
    "df_daily_counts_test_full.to_parquet('daily_counts_carts_test_full.parquet')\n",
    "\n",
    "df_daily_counts_test_trunked =otto_common.create_average_daily_counts(trunked_sessions_path, True, 1)\n",
    "df_daily_counts_test_trunked.to_parquet('daily_counts_carts_test_trunked.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "df4fb9d1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:24:24.147401Z",
     "iopub.status.busy": "2023-03-17T22:24:24.146680Z",
     "iopub.status.idle": "2023-03-17T22:24:39.586205Z",
     "shell.execute_reply": "2023-03-17T22:24:39.584729Z"
    },
    "papermill": {
     "duration": 15.45763,
     "end_time": "2023-03-17T22:24:39.588906",
     "exception": false,
     "start_time": "2023-03-17T22:24:24.131276",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate average daily orders per aid for test dataset and for the last week of full sessions.\n",
    "df_daily_counts_test_full = otto_common.create_average_daily_counts(click_data_test, False, 2)\n",
    "df_daily_counts_test_full.to_parquet('daily_counts_orders_test_full.parquet')\n",
    "\n",
    "df_daily_counts_test_trunked = otto_common.create_average_daily_counts(trunked_sessions_path, True, 2)\n",
    "df_daily_counts_test_trunked.to_parquet('daily_counts_orders_test_trunked.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3a4b1989",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:24:39.604694Z",
     "iopub.status.busy": "2023-03-17T22:24:39.604309Z",
     "iopub.status.idle": "2023-03-17T22:27:18.952451Z",
     "shell.execute_reply": "2023-03-17T22:27:18.950994Z"
    },
    "papermill": {
     "duration": 159.361404,
     "end_time": "2023-03-17T22:27:18.957415",
     "exception": false,
     "start_time": "2023-03-17T22:24:39.596011",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate average clicks before buy per aid.\n",
    "df_clicks_before_buy = clicks_before_buy(click_data_test)\n",
    "df_clicks_before_buy.to_parquet('clicks_before_buy_test.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "582c838f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:27:18.977795Z",
     "iopub.status.busy": "2023-03-17T22:27:18.977397Z",
     "iopub.status.idle": "2023-03-17T22:27:19.119153Z",
     "shell.execute_reply": "2023-03-17T22:27:19.117984Z"
    },
    "papermill": {
     "duration": 0.154754,
     "end_time": "2023-03-17T22:27:19.122196",
     "exception": false,
     "start_time": "2023-03-17T22:27:18.967442",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df_daily_counts_test_full, df_daily_counts_test_trunked\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6975d8d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:27:19.139235Z",
     "iopub.status.busy": "2023-03-17T22:27:19.138355Z",
     "iopub.status.idle": "2023-03-17T22:28:55.270452Z",
     "shell.execute_reply": "2023-03-17T22:28:55.269448Z"
    },
    "papermill": {
     "duration": 96.144687,
     "end_time": "2023-03-17T22:28:55.274359",
     "exception": false,
     "start_time": "2023-03-17T22:27:19.129672",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate general conversion rate per aid.\n",
    "df_conversions = count_conversion(click_data_test)\n",
    "df_conversions.to_parquet('conversions_test.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3e97aa37",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:28:55.295068Z",
     "iopub.status.busy": "2023-03-17T22:28:55.294213Z",
     "iopub.status.idle": "2023-03-17T22:30:34.649434Z",
     "shell.execute_reply": "2023-03-17T22:30:34.648005Z"
    },
    "papermill": {
     "duration": 99.368011,
     "end_time": "2023-03-17T22:30:34.652773",
     "exception": false,
     "start_time": "2023-03-17T22:28:55.284762",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate click2order, cart2order and order2order conversion rates.\n",
    "df_2order_conversions = count_2order_conversion(click_data_test)\n",
    "df_2order_conversions.to_parquet('2order_conversions_test.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "23ca1189",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:30:34.671114Z",
     "iopub.status.busy": "2023-03-17T22:30:34.670650Z",
     "iopub.status.idle": "2023-03-17T22:33:18.313994Z",
     "shell.execute_reply": "2023-03-17T22:33:18.312642Z"
    },
    "papermill": {
     "duration": 163.655664,
     "end_time": "2023-03-17T22:33:18.317095",
     "exception": false,
     "start_time": "2023-03-17T22:30:34.661431",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculate click2cart, cart2cart and order2cart conversion rates.\n",
    "df_2cart_conversions = count_2cart_conversion(click_data_test)\n",
    "df_2cart_conversions.to_parquet('2cart_conversions_test.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1374.580274,
   "end_time": "2023-03-17T22:33:19.360197",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-17T22:10:24.779923",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}