final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "material-monkey",
   "metadata": {},
   "source": [
    "### Loading Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "genetic-native",
   "metadata": {},
   "outputs": [],
   "source": [
    "#loading libraries\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "from datetime import datetime\n",
    "from f1optimization_faron import get_best_prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "decimal-wonder",
   "metadata": {},
   "source": [
    "### Final function to return product recommendation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "detected-timber",
   "metadata": {},
   "outputs": [],
   "source": [
    "def final(X = None):\n",
    "    \n",
    "    \"\"\"\n",
    "    Function returns the predicted product recommendations if its existing user, else returns \n",
    "    most frequently purchased product at that hour and on that day.\n",
    "    \n",
    "    user_id           : provided\n",
    "    order_hour_of_day : calculated by seeing current time\n",
    "    order_dow         : calulated by seeing current day of week\n",
    "    \n",
    "    \"\"\"\n",
    "\n",
    "    #get current time\n",
    "    start_time = datetime.now()\n",
    "\n",
    "    # datetime object containing current date and time\n",
    "    now = datetime.now()\n",
    "\n",
    "    # dd/mm/YY H:M:S\n",
    "    dt_string = now.strftime(\"%d/%m/%Y %H:%M:%S\")\n",
    "    today = int(dt_string.split(\"/\")[0])\n",
    "\n",
    "    #get data from user end\n",
    "    user_id = int(X['user_id']) #user_id\n",
    "    order_hour_of_day = int(dt_string.split(\" \")[1].split(\":\")[0]) #current date\n",
    "    order_dow = datetime.today().weekday() #current day of week\n",
    "\n",
    "    \"\"\"\n",
    "    check if the user is new or existing,\n",
    "    this file contains user's last order date\n",
    "    \n",
    "    Handling: since in our model days since prior order is a feature, which cannot be calculated without user's last order date.\n",
    "              assuming that this model is deployed on 21-march-2021, all users last order are taken as 21-march-2021.\n",
    "              But this assumption likely introduces new bugs.\n",
    "              Ex- user A , never made a purchase with days_prior_order = 3, if he makes he does this in future , the below handles,\n",
    "              such exceptions\n",
    "    \"\"\"\n",
    "    \n",
    "    ulp = pd.read_pickle(\"user_last_purchase.pkl\")\n",
    "    \n",
    "    #if user is new -> cold start -> recommend products which are most frequently purchased based on time and day of week\n",
    "    if user_id not in ulp['user_id'].values:\n",
    "\n",
    "        #get top 10 products based on hour of day and day of week\n",
    "        top= pd.read_pickle('top10_products.pkl')\n",
    "        top_products = top[(top['order_dow']==order_dow) & (top['order_hour_of_day']==order_hour_of_day)]['product_name'].values.tolist()\n",
    "        top_products = {i: value for i,value in enumerate(top_products)}\n",
    "        #paths = get_image_paths(top5_products)\n",
    "        predictions={}\n",
    "        predictions['top'] =  top_products\n",
    "\n",
    "        del ulp, top,now, today, dt_string, order_dow, order_hour_of_day\n",
    "\n",
    "        end_time = datetime.now()\n",
    "        difference = end_time - start_time\n",
    "        #print(\"Total Time : {} seconds\".format(difference))\n",
    "        time = \"{}\".format(difference)\n",
    "        \n",
    "        return predictions,time\n",
    "    \n",
    "    #if user is existing one\n",
    "    user_last_order_date = ulp[ulp['user_id']==user_id]['date'].values.tolist()[0]\n",
    "\n",
    "    days_since_prior_order = today - int(user_last_order_date.split('-')[-1])\n",
    "    del ulp, now, today, dt_string, user_last_order_date\n",
    "    \n",
    "    #build features\n",
    "    hour_rate = pd.read_pickle(\"hour_reorder_rate.pkl\")\n",
    "    day_rate = pd.read_pickle(\"day_reorder_rate.pkl\")\n",
    "    p_days_rate = pd.read_pickle(\"p_days_since_prior_order_reorder_rate.pkl\")\n",
    "    u_days_rate = pd.read_pickle(\"u_days_since_prior_order_reorder_rate.pkl\")\n",
    "    up_days_rate = pd.read_pickle(\"days_since_prior_reorder_rate.pkl\")\n",
    "    merged_up_features = pd.read_pickle(\"merged_user_product_features.pkl\")\n",
    "\n",
    "    #collect features based on hour of day , day of week, and user id\n",
    "    featurized_data = merged_up_features[merged_up_features['user_id']==user_id]\n",
    "    hour_r = hour_rate[hour_rate['order_hour_of_day']==order_hour_of_day]\n",
    "    day_r = day_rate[day_rate['order_dow'] == order_dow]\n",
    "    p_days = p_days_rate[p_days_rate['days_since_prior_order']==days_since_prior_order]\n",
    "    u_days = u_days_rate[(u_days_rate['user_id']==user_id) & (u_days_rate['days_since_prior_order']==days_since_prior_order)]\n",
    "    \n",
    "    \"\"\"\n",
    "    if user never made a purchase in the gap of (days_since_prior_order) between 2 orders, make a new dataframe, \n",
    "    with the given number of days , and with reorder_rate = 0.0\n",
    "    \"\"\"\n",
    "    \n",
    "    if p_days.empty:\n",
    "        #handle\n",
    "        p_days = pd.DataFrame(columns = p_days.columns)\n",
    "        products_x = pd.read_pickle('product_mappings.pkl')\n",
    "        p_days['product_id'] = products_x['product_id']\n",
    "        p_days['days_since_prior_order'] = days_since_prior_order\n",
    "        p_days['p_days_since_prior_order_reorder_rate']=0.0\n",
    "    \n",
    "    up_days = up_days_rate[(up_days_rate['user_id']==user_id) & (up_days_rate['days_since_prior_order']==days_since_prior_order)]\n",
    "\n",
    "    if up_days.empty:\n",
    "        #handle\n",
    "        up_days = pd.DataFrame(columns = up_days_rate.columns)\n",
    "        products_x = pd.read_pickle('product_mappings.pkl')\n",
    "        up_days['product_id'] = products_x['product_id']\n",
    "        up_days['user_id'] = user_id\n",
    "        up_days['days_since_prior_order'] = days_since_prior_order\n",
    "        up_days['days_since_prior_reorder_rate']=0\n",
    "        del products_x\n",
    "    \n",
    "    if u_days.empty:\n",
    "        #handle\n",
    "        u_days = pd.DataFrame(columns = u_days.columns)\n",
    "        df2 = {'user_id': user_id, 'days_since_prior_order': days_since_prior_order, 'u_days_since_prior_order_reorder_rate': 0}\n",
    "        u_days = u_days.append(df2, ignore_index = True)\n",
    "        del df2\n",
    "\n",
    "    up_days = up_days_rate[(up_days_rate['user_id']==user_id) & (up_days_rate['days_since_prior_order']==days_since_prior_order)]\n",
    "    \n",
    "    \"\"\"\n",
    "    if user never made a purchase in the gap of (days_since_prior_order) between 2 orders, make a new dataframe, \n",
    "    with the given number of days , and with reorder_rate = 0.0\n",
    "    \"\"\"\n",
    "    if up_days.empty:\n",
    "        #handle\n",
    "        up_days = pd.DataFrame(columns = up_days_rate.columns)\n",
    "        products_x = pd.read_pickle('product_mappings.pkl')\n",
    "        up_days['product_id'] = products_x['product_id']\n",
    "        up_days['user_id'] = user_id\n",
    "        up_days['days_since_prior_order'] = days_since_prior_order\n",
    "        up_days['days_since_prior_reorder_rate']=0\n",
    "        del products_x\n",
    "\n",
    "\n",
    "    #print(up_days_rate[up_days_rate['user_id']==user_id])\n",
    "    #print(u_days_rate[u_days_rate['user_id']==user_id])\n",
    "    #print(day_rate)\n",
    "    del merged_up_features, hour_rate, day_rate, p_days_rate, u_days_rate, up_days_rate\n",
    "\n",
    "    featurized_data = pd.merge(featurized_data, up_days, on = ['user_id', 'product_id'])\n",
    "\n",
    "    featurized_data = pd.merge(featurized_data, hour_r, on = 'product_id')\n",
    "    featurized_data = pd.merge(featurized_data, day_r, on = 'product_id')\n",
    "    featurized_data = pd.merge(featurized_data, p_days, on = ['product_id','days_since_prior_order'])\n",
    "    featurized_data = pd.merge(featurized_data, u_days, on = ['user_id','days_since_prior_order'])\n",
    "    featurized_data = featurized_data[['user_id', 'product_id', 'u_p_order_rate', 'u_p_reorder_rate',\\\n",
    "                                       'u_p_avg_position', 'u_p_orders_since_last', 'max_streak', 'user_reorder_rate',\\\n",
    "                                       'user_unique_products', 'user_total_products', 'user_avg_cart_size', \\\n",
    "                                       'user_avg_days_between_orders', 'user_reordered_products_ratio', \\\n",
    "                                       'product_reorder_rate', 'avg_pos_incart', 'p_reduced_feat_1', 'p_reduced_feat_2', \\\n",
    "                                       'p_reduced_feat_3', 'aisle_id', 'department_id', 'aisle_reorder_rate', \\\n",
    "                                       'dept_reorder_rate', 'order_dow', 'order_hour_of_day', 'days_since_prior_order',\\\n",
    "                                       'hour_reorder_rate', 'day_reorder_rate', 'p_days_since_prior_order_reorder_rate',\\\n",
    "                                       'u_days_since_prior_order_reorder_rate', 'days_since_prior_reorder_rate']]\n",
    "\n",
    "    del up_days,u_days,p_days,day_r,hour_r\n",
    "\n",
    "    #model\n",
    "    with open(\"catboost_v3.pkl\", \"rb\") as f:\n",
    "        model = pickle.load(f)\n",
    "    #model = pickle.load(open(\"catboost_v3.pkl\", \"rb\"))\n",
    "    \n",
    "    data = featurized_data.drop(['user_id', 'product_id'], axis = 1)\n",
    "    ypred = model.predict_proba(data)\n",
    "    ypred = ypred[:,-1] #get probabilities of class 1\n",
    "    del data,model\n",
    "\n",
    "    #run faron's optimization code to get most probable set of products which might be reordered\n",
    "    \n",
    "    #set showThreshold = True , while debugging to print the threshold\n",
    "    recommended_products = get_best_prediction(featurized_data['product_id'].tolist(), ypred.tolist(), None, showThreshold = True)\n",
    "    recommended_products = recommended_products.replace(\"None\", \"\")\n",
    "    recommended_products = list(map(int, recommended_products.split()))\n",
    "    products_x = pd.read_pickle('product_mappings.pkl')\n",
    "    recommended_products = products_x.loc[products_x['product_id'].isin(recommended_products)]['product_name'].values.tolist()\n",
    "    recommended_products = {i: value for i,value in enumerate(recommended_products)}\n",
    "\n",
    "    predictions= {}\n",
    "    predictions['recommend'] = recommended_products\n",
    "\n",
    "    end_time = datetime.now()\n",
    "    difference = end_time - start_time\n",
    "    #print(\"Total Time : {} seconds\".format(difference))\n",
    "    time = \"{}\".format(difference)\n",
    "\n",
    "    del featurized_data, products_x\n",
    "    return predictions, time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "thousand-identity",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold : P(X) > 0.3746\n",
      "Maximum F1 : 0.5942\n",
      "\n",
      "====================\n",
      "Recommended products\n",
      "====================\n",
      "Organic Whole Kernel Sweet Corn No Salt Added\n",
      "Semi-Soft Cheese, Ripened\n",
      "Organic Red Onion\n",
      "Red Raspberries\n",
      "Snow Peas\n",
      "Large Organic Omega3 Brown Eggs\n",
      "Organic Soba\n",
      "Plain Whole Milk Yogurt\n",
      "Organic Blackberries\n",
      "Uncured Genoa Salami\n",
      "Organic Grape Tomatoes\n",
      "Whole Vitamin D Milk\n"
     ]
    }
   ],
   "source": [
    "#existing user\n",
    "\n",
    "X = {}\n",
    "X['user_id'] = 5\n",
    "recommended_products = final(X)\n",
    "\n",
    "print()\n",
    "print(\"=\"*20)\n",
    "print(\"Recommended products\")\n",
    "print(\"=\"*20)\n",
    "for _,value in recommended_products[0]['recommend'].items():\n",
    "    print(value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "heard-screening",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold : P(X) > 0.5853\n",
      "Maximum F1 : 0.7680\n",
      "\n",
      "====================\n",
      "Recommended products\n",
      "====================\n",
      "Imported Mineral Water\n",
      "Organic Bunny Fruit Snacks Berry Patch\n",
      "Chocolate Peppermint Stick Bar\n",
      "Organic Sticks Low Moisture Part Skim Mozzarella String Cheese\n",
      "Goldfish Cheddar Baked Snack Crackers\n",
      "Organics Chocolate Milk with DHA\n",
      "Original Whipped Cream Cheese\n",
      "Organic Uncured Sliced Black Forest Ham\n",
      "Pirate's Booty Aged White Cheddar Baked Rice and Corn Puffs\n",
      "Plain Mini Bagels\n",
      "Grape White/Green Seedless\n"
     ]
    }
   ],
   "source": [
    "#existing user\n",
    "\n",
    "X = {}\n",
    "X['user_id'] = 55\n",
    "recommended_products = final(X)\n",
    "\n",
    "print()\n",
    "print(\"=\"*20)\n",
    "print(\"Recommended products\")\n",
    "print(\"=\"*20)\n",
    "for _,value in recommended_products[0]['recommend'].items():\n",
    "    print(value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "liable-double",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recommended products\n",
      "====================\n",
      "Banana\n",
      "Bag of Organic Bananas\n",
      "Organic Strawberries\n",
      "Organic Baby Spinach\n",
      "Organic Hass Avocado\n",
      "Organic Avocado\n",
      "Strawberries\n",
      "Organic Whole Milk\n",
      "Large Lemon\n",
      "Limes\n",
      "Time taken : 0:00:00.010989\n"
     ]
    }
   ],
   "source": [
    "#new user\n",
    "\n",
    "X = {}\n",
    "X['user_id'] = 225000\n",
    "recommended_products = final(X)\n",
    "print(\"Recommended products\")\n",
    "print(\"=\"*20)\n",
    "for _,value in recommended_products[0]['top'].items():\n",
    "    print(value)\n",
    "print(\"Time taken :\", recommended_products[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caring-delta",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}