otto-click2buy-buy2buy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "002ceb28",
   "metadata": {
    "papermill": {
     "duration": 0.005657,
     "end_time": "2023-03-24T11:39:34.953245",
     "exception": false,
     "start_time": "2023-03-24T11:39:34.947588",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Click2buy and buy2buy co-visitation matrixes\n",
    "This notebook prepares click-to-buy and buy-to-buy co-visitation matrixes. It is used to generate candidates and calculate features for the carts and orders models.\n",
    "There are three different outputs both for the cross-validation and for test datasets. There is full version for both matrixes and a version of click-to-buy matrix limited to top 70 records, which is used for candidate generation.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76fdcdf1",
   "metadata": {
    "papermill": {
     "duration": 0.004076,
     "end_time": "2023-03-24T11:39:34.961898",
     "exception": false,
     "start_time": "2023-03-24T11:39:34.957822",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9c8a4d4b",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-24T11:39:34.973057Z",
     "iopub.status.busy": "2023-03-24T11:39:34.972322Z",
     "iopub.status.idle": "2023-03-24T11:39:35.012505Z",
     "shell.execute_reply": "2023-03-24T11:39:35.011127Z"
    },
    "papermill": {
     "duration": 0.049307,
     "end_time": "2023-03-24T11:39:35.015515",
     "exception": false,
     "start_time": "2023-03-24T11:39:34.966208",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "  \n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "759f8b31",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T11:39:35.026451Z",
     "iopub.status.busy": "2023-03-24T11:39:35.026026Z",
     "iopub.status.idle": "2023-03-24T11:39:35.036469Z",
     "shell.execute_reply": "2023-03-24T11:39:35.035332Z"
    },
    "papermill": {
     "duration": 0.018799,
     "end_time": "2023-03-24T11:39:35.038938",
     "exception": false,
     "start_time": "2023-03-24T11:39:35.020139",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class CalculateCovisitationMatrixClick2Buy(otto_common.CalculateCovisitationMatrix):\n",
    "    '''\n",
    "    Child class with logic specific to this type of covisitation matrix.\n",
    "    It counts all aids that were either carted or ordered within certain time after some aid was clicked.\n",
    "    The weight value is calculated in a way that makes pairs of events with smaller time difference more important.\n",
    "    '''\n",
    "\n",
    "    def calculate_weights(self, df_i):\n",
    "        df_i = df_i.sort_values(['session','ts'],ascending=[True,False])\n",
    "        df_i = df_i.reset_index(drop=True)\n",
    "        df_i = df_i.merge(df_i,on='session')\n",
    "        df_i = df_i.loc[ (df_i.ts_y >= df_i.ts_x) & (df_i.type_y > 0)]\n",
    "        df_i['time_diff'] = df_i.ts_y - df_i.ts_x\n",
    "        df_i = df_i.loc[ (df_i.time_diff).abs() < 10 * 60 * 60]\n",
    "        df_i = df_i.groupby(['session','aid_x', 'aid_y']).agg({'time_diff':'min'}).reset_index()\n",
    "        df_i['wgt'] = (60 * 60)/(60 * 60 + df_i.time_diff)\n",
    "        df_i = df_i[['aid_x','aid_y','wgt']]\n",
    "        df_i = df_i.loc[df_i['aid_x'].isin(self.aid_list)]\n",
    "        df_i.wgt = df_i.wgt.astype('float32')\n",
    "        df_i = self.groupby_reset_and_reduce(df_i)\n",
    "        return df_i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "af87adbc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T11:39:35.050022Z",
     "iopub.status.busy": "2023-03-24T11:39:35.049579Z",
     "iopub.status.idle": "2023-03-24T11:39:35.058554Z",
     "shell.execute_reply": "2023-03-24T11:39:35.057367Z"
    },
    "papermill": {
     "duration": 0.017463,
     "end_time": "2023-03-24T11:39:35.060916",
     "exception": false,
     "start_time": "2023-03-24T11:39:35.043453",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class CalculateCovisitationMatrixBuy2Buy(otto_common.CalculateCovisitationMatrix):\n",
    "    '''\n",
    "    Child class with logic specific to this type of covisitation matrix.\n",
    "    It counts all aids that were either carted or ordered within certain time after some aid was either carted or ordered.\n",
    "    The weight value is the same for each pair of events.\n",
    "    '''\n",
    "\n",
    "    def calculate_weights(self, df_i):\n",
    "        df_i = df_i.loc[df_i['type'] > 0]\n",
    "        df_i = df_i.sort_values(['session','ts'],ascending=[True,False])\n",
    "        df_i = df_i.reset_index(drop=True)\n",
    "        df_i = df_i.merge(df_i,on='session')\n",
    "        df_i = df_i.loc[ ((df_i.ts_x - df_i.ts_y).abs()< 5 * 24 * 60 * 60)]\n",
    "        df_i = df_i[['session', 'aid_x', 'aid_y']].drop_duplicates(['session', 'aid_x', 'aid_y']) \n",
    "        df_i['wgt'] = 1\n",
    "        df_i = df_i[['aid_x','aid_y','wgt']]\n",
    "        df_i = df_i.loc[df_i['aid_x'].isin(self.aid_list)]\n",
    "        df_i.wgt = df_i.wgt.astype('float32')\n",
    "        df_i = self.groupby_reset_and_reduce(df_i)\n",
    "        return df_i"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aaa35e02",
   "metadata": {
    "papermill": {
     "duration": 0.004183,
     "end_time": "2023-03-24T11:39:35.069719",
     "exception": false,
     "start_time": "2023-03-24T11:39:35.065536",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Co-visitation matrices for cross-validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fffb4694",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T11:39:35.080844Z",
     "iopub.status.busy": "2023-03-24T11:39:35.080348Z",
     "iopub.status.idle": "2023-03-24T11:39:41.825736Z",
     "shell.execute_reply": "2023-03-24T11:39:41.824588Z"
    },
    "papermill": {
     "duration": 6.75439,
     "end_time": "2023-03-24T11:39:41.828440",
     "exception": false,
     "start_time": "2023-03-24T11:39:35.074050",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Make a list of AIDs, that appear in cross-validation sets.\n",
    "# aid_x is filtered to only include aids from the list, while aid_y is not filtered\n",
    "# This reduces size of resulting matrix.\n",
    "trunked_sessions_path = '/kaggle/input/otto-prepare-cv/cv_inputs.parquet'\n",
    "trunked_sessions_path2 = '/kaggle/input/otto-prepare-cv/cv_inputs2.parquet'\n",
    "cv_list = otto_common.build_aid_list(trunked_sessions_path, trunked_sessions_path2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2342f967",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T11:39:41.841511Z",
     "iopub.status.busy": "2023-03-24T11:39:41.841064Z",
     "iopub.status.idle": "2023-03-24T12:24:52.247017Z",
     "shell.execute_reply": "2023-03-24T12:24:52.245590Z"
    },
    "papermill": {
     "duration": 2710.417294,
     "end_time": "2023-03-24T12:24:52.250871",
     "exception": false,
     "start_time": "2023-03-24T11:39:41.833577",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n"
     ]
    }
   ],
   "source": [
    "# Input parameters and click2buy matrix calculation for cross-validation dataset.\n",
    "click_data_cv = '/kaggle/input/otto-prepare-cv/cv_train.parquet'\n",
    "n_splits = 200\n",
    "n_splits_concat = 30\n",
    "top_n = 70 # This parameter is both for cv and test\n",
    "\n",
    "covisitation_click2buy_cv = CalculateCovisitationMatrixClick2Buy(n_splits, n_splits_concat, cv_list)\n",
    "df_matrix_cv = covisitation_click2buy_cv.generate_covisitation_matrix(click_data_cv)\n",
    "df_matrix_cv.to_parquet('matrix_click2buy_cv.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d8a1808e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T12:24:52.286881Z",
     "iopub.status.busy": "2023-03-24T12:24:52.286440Z",
     "iopub.status.idle": "2023-03-24T12:26:07.064254Z",
     "shell.execute_reply": "2023-03-24T12:26:07.063143Z"
    },
    "papermill": {
     "duration": 74.799666,
     "end_time": "2023-03-24T12:26:07.067185",
     "exception": false,
     "start_time": "2023-03-24T12:24:52.267519",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "571.5 MB\n"
     ]
    }
   ],
   "source": [
    "# Reduce click2buy matrix to top 70, check dataframe size and export to file.\n",
    "df_matrix_cv = covisitation_click2buy_cv.get_top_n(df_matrix_cv, top_n)\n",
    "size = df_matrix_cv.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_matrix_cv.to_parquet('matrix_click2buy_cv_top_n.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bd9a93be",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T12:26:07.103348Z",
     "iopub.status.busy": "2023-03-24T12:26:07.102276Z",
     "iopub.status.idle": "2023-03-24T12:26:07.246532Z",
     "shell.execute_reply": "2023-03-24T12:26:07.245172Z"
    },
    "papermill": {
     "duration": 0.165127,
     "end_time": "2023-03-24T12:26:07.249266",
     "exception": false,
     "start_time": "2023-03-24T12:26:07.084139",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df_matrix_cv, covisitation_click2buy_cv\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4e726105",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T12:26:07.284682Z",
     "iopub.status.busy": "2023-03-24T12:26:07.284264Z",
     "iopub.status.idle": "2023-03-24T12:33:21.496845Z",
     "shell.execute_reply": "2023-03-24T12:33:21.495814Z"
    },
    "papermill": {
     "duration": 434.233567,
     "end_time": "2023-03-24T12:33:21.499538",
     "exception": false,
     "start_time": "2023-03-24T12:26:07.265971",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n"
     ]
    }
   ],
   "source": [
    "# Input parameters and buy2buy matrix calculation for cross-validation dataset.\n",
    "# As about 90% of all events in history are clicks, so it is possible to use much lower values for number of splits and speed up the calculation of \n",
    "# buy2buy matrix.\n",
    "\n",
    "n_splits = 50\n",
    "n_splits_concat = 5\n",
    "\n",
    "covisitation_buy2buy_cv = CalculateCovisitationMatrixBuy2Buy(n_splits, n_splits_concat, cv_list)\n",
    "df_matrix_b2b_cv = covisitation_buy2buy_cv.generate_covisitation_matrix(click_data_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "55755db6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T12:33:21.540096Z",
     "iopub.status.busy": "2023-03-24T12:33:21.539639Z",
     "iopub.status.idle": "2023-03-24T12:33:26.213057Z",
     "shell.execute_reply": "2023-03-24T12:33:26.211101Z"
    },
    "papermill": {
     "duration": 4.696474,
     "end_time": "2023-03-24T12:33:26.215618",
     "exception": false,
     "start_time": "2023-03-24T12:33:21.519144",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.2 GB\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check dataframe size and export to file\n",
    "size = df_matrix_b2b_cv.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_matrix_b2b_cv.to_parquet('matrix_buy2buy_cv.parquet')\n",
    "\n",
    "del df_matrix_b2b_cv, cv_list, covisitation_buy2buy_cv\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63663935",
   "metadata": {
    "papermill": {
     "duration": 0.020003,
     "end_time": "2023-03-24T12:33:26.255356",
     "exception": false,
     "start_time": "2023-03-24T12:33:26.235353",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Co-visitation matrices for test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4bbfcba0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T12:33:26.296365Z",
     "iopub.status.busy": "2023-03-24T12:33:26.295983Z",
     "iopub.status.idle": "2023-03-24T12:33:29.267094Z",
     "shell.execute_reply": "2023-03-24T12:33:29.265864Z"
    },
    "papermill": {
     "duration": 2.994453,
     "end_time": "2023-03-24T12:33:29.269749",
     "exception": false,
     "start_time": "2023-03-24T12:33:26.275296",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Make a list of AIDs, that appear in test dataset.\n",
    "trunked_sessions_path = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "full_aid_list = otto_common.build_aid_list(trunked_sessions_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7bb1ebf6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T12:33:29.310597Z",
     "iopub.status.busy": "2023-03-24T12:33:29.309971Z",
     "iopub.status.idle": "2023-03-24T14:02:32.494149Z",
     "shell.execute_reply": "2023-03-24T14:02:32.492786Z"
    },
    "papermill": {
     "duration": 5343.208661,
     "end_time": "2023-03-24T14:02:32.497878",
     "exception": false,
     "start_time": "2023-03-24T12:33:29.289217",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n",
      "200\n",
      "201\n",
      "202\n",
      "203\n",
      "204\n",
      "205\n",
      "206\n",
      "207\n",
      "208\n",
      "209\n",
      "210\n",
      "211\n",
      "212\n",
      "213\n",
      "214\n",
      "215\n",
      "216\n",
      "217\n",
      "218\n",
      "219\n",
      "220\n",
      "221\n",
      "222\n",
      "223\n",
      "224\n",
      "225\n",
      "226\n",
      "227\n",
      "228\n",
      "229\n",
      "230\n",
      "231\n",
      "232\n",
      "233\n",
      "234\n",
      "235\n",
      "236\n",
      "237\n",
      "238\n",
      "239\n",
      "240\n",
      "241\n",
      "242\n",
      "243\n",
      "244\n",
      "245\n",
      "246\n",
      "247\n",
      "248\n",
      "249\n",
      "250\n",
      "251\n",
      "252\n",
      "253\n",
      "254\n",
      "255\n",
      "256\n",
      "257\n",
      "258\n",
      "259\n",
      "260\n",
      "261\n",
      "262\n",
      "263\n",
      "264\n",
      "265\n",
      "266\n",
      "267\n",
      "268\n",
      "269\n",
      "270\n",
      "271\n",
      "272\n",
      "273\n",
      "274\n",
      "275\n",
      "276\n",
      "277\n",
      "278\n",
      "279\n",
      "280\n",
      "281\n",
      "282\n",
      "283\n",
      "284\n",
      "285\n",
      "286\n",
      "287\n",
      "288\n",
      "289\n",
      "290\n",
      "291\n",
      "292\n",
      "293\n",
      "294\n",
      "295\n",
      "296\n",
      "297\n",
      "298\n",
      "299\n",
      "300\n",
      "301\n",
      "302\n",
      "303\n",
      "304\n",
      "305\n",
      "306\n",
      "307\n",
      "308\n",
      "309\n",
      "310\n",
      "311\n",
      "312\n",
      "313\n",
      "314\n",
      "315\n",
      "316\n",
      "317\n",
      "318\n",
      "319\n",
      "320\n",
      "321\n",
      "322\n",
      "323\n",
      "324\n",
      "325\n",
      "326\n",
      "327\n",
      "328\n",
      "329\n",
      "330\n",
      "331\n",
      "332\n",
      "333\n",
      "334\n",
      "335\n",
      "336\n",
      "337\n",
      "338\n",
      "339\n",
      "340\n",
      "341\n",
      "342\n",
      "343\n",
      "344\n",
      "345\n",
      "346\n",
      "347\n",
      "348\n",
      "349\n",
      "350\n",
      "351\n",
      "352\n",
      "353\n",
      "354\n",
      "355\n",
      "356\n",
      "357\n",
      "358\n",
      "359\n",
      "360\n",
      "361\n",
      "362\n",
      "363\n",
      "364\n",
      "365\n",
      "366\n",
      "367\n",
      "368\n",
      "369\n",
      "370\n",
      "371\n",
      "372\n",
      "373\n",
      "374\n",
      "375\n",
      "376\n",
      "377\n",
      "378\n",
      "379\n",
      "380\n",
      "381\n",
      "382\n",
      "383\n",
      "384\n",
      "385\n",
      "386\n",
      "387\n",
      "388\n",
      "389\n",
      "390\n",
      "391\n",
      "392\n",
      "393\n",
      "394\n",
      "395\n",
      "396\n",
      "397\n",
      "398\n",
      "399\n"
     ]
    }
   ],
   "source": [
    "# Input parameters and click2buy matrix calculation for test dataset.\n",
    "click_data_full = '/kaggle/input/otto-prepare-cv/train_full.parquet'\n",
    "n_splits = 400\n",
    "n_splits_concat = 40\n",
    "\n",
    "covisitation_click2buy_full = CalculateCovisitationMatrixClick2Buy(n_splits, n_splits_concat, full_aid_list)\n",
    "df_matrix_full = covisitation_click2buy_full.generate_covisitation_matrix(click_data_full)\n",
    "df_matrix_full.to_parquet('matrix_click2buy_test.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ceadf0dc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T14:02:32.589865Z",
     "iopub.status.busy": "2023-03-24T14:02:32.589326Z",
     "iopub.status.idle": "2023-03-24T14:04:09.754207Z",
     "shell.execute_reply": "2023-03-24T14:04:09.752780Z"
    },
    "papermill": {
     "duration": 97.214173,
     "end_time": "2023-03-24T14:04:09.756801",
     "exception": false,
     "start_time": "2023-03-24T14:02:32.542628",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "584.9 MB\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reduce click2buy matrix to top 70, check dataframe size and export to file.\n",
    "df_matrix_full = covisitation_click2buy_full.get_top_n(df_matrix_full, top_n)\n",
    "size = df_matrix_full.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_matrix_full.to_parquet('matrix_click2buy_test_top_n.parquet')\n",
    "\n",
    "del df_matrix_full, covisitation_click2buy_full\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "56ec65c8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T14:04:09.847274Z",
     "iopub.status.busy": "2023-03-24T14:04:09.846778Z",
     "iopub.status.idle": "2023-03-24T14:15:51.771842Z",
     "shell.execute_reply": "2023-03-24T14:15:51.770506Z"
    },
    "papermill": {
     "duration": 701.974075,
     "end_time": "2023-03-24T14:15:51.775111",
     "exception": false,
     "start_time": "2023-03-24T14:04:09.801036",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n"
     ]
    }
   ],
   "source": [
    "# Input parameters and buy2buy matrix calculation for test dataset.\n",
    "n_splits = 70\n",
    "n_splits_concat = 10\n",
    "\n",
    "covisitation_buy2buy_full = CalculateCovisitationMatrixBuy2Buy(n_splits, n_splits_concat, full_aid_list)\n",
    "df_matrix_b2b_full = covisitation_buy2buy_full.generate_covisitation_matrix(click_data_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a4583e9a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-24T14:15:51.875322Z",
     "iopub.status.busy": "2023-03-24T14:15:51.874548Z",
     "iopub.status.idle": "2023-03-24T14:15:57.693131Z",
     "shell.execute_reply": "2023-03-24T14:15:57.691775Z"
    },
    "papermill": {
     "duration": 5.872109,
     "end_time": "2023-03-24T14:15:57.696366",
     "exception": false,
     "start_time": "2023-03-24T14:15:51.824257",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.5 GB\n"
     ]
    }
   ],
   "source": [
    "# Check dataframe size and export to file.\n",
    "size = df_matrix_b2b_full.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_matrix_b2b_full.to_parquet('matrix_buy2buy_test.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 9393.879528,
   "end_time": "2023-03-24T14:15:58.687978",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-24T11:39:24.808450",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}