TEST_Basic_Calculations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import Amazon_Movie_Parser as amp\n",
    "import Graph_Amazon_Movies as gam\n",
    "# save numpy array as csv file\n",
    "from numpy import savetxt\n",
    "# load numpy array from csv file\n",
    "from numpy import loadtxt\n",
    "import networkx as nx\n",
    "import datetime\n",
    "import Computations as cmp\n",
    "from networkx.algorithms import bipartite\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[12 14 30]\n",
      "[[12 12 12]\n",
      " [14 14 14]\n",
      " [30 30 30]]\n",
      "[[0.16666667 0.33333333 0.5       ]\n",
      " [0.07142857 0.35714286 0.57142857]\n",
      " [0.66666667 0.13333333 0.2       ]]\n"
     ]
    }
   ],
   "source": [
    "# Normalization test\n",
    "tt = np.array([[2, 4, 6], [1, 5, 8], [20, 4, 6]])\n",
    "#print(tt.mean(axis=1))\n",
    "#tt_norm=tt/tt.mean(axis=1)\n",
    "tt_sum = tt.sum(axis=1)\n",
    "print(tt.sum(axis=1))\n",
    "print(np.tile(tt_sum, (3, 1)).T)\n",
    "# divide each element in the row with the sum of the row so the sum will be equal to 1\n",
    "tt_norm=tt/np.tile(tt_sum, (3, 1)).T\n",
    "print(tt_norm)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2, 6]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt =[2, 4, 6, 8, 9, 11]\n",
    "# random.choices \n",
    "random.sample(tt, k=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 0. 1. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "R=np.zeros(5)\n",
    "R[2]=1\n",
    "print(R)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmp1 = cmp.GraphComp()\n",
    "#Normalize_Matrix(self, arr_to_normalize, dim = 'row'):\n",
    "tt = np.array([[2, 4, 6], [1, 5, 8], [20, 4, 6]])\n",
    "tt_norm = cmp1.Normalize_Matrix(tt, dim = 'row')\n",
    "print(tt_norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#row_sums = a.sum(axis=1)\n",
    "#new_matrix = a / row_sums[:, numpy.newaxis]\n",
    "\n",
    "tt = np.array([[2, 4, 6], [1, 5, 8], [20, 4, 6]])\n",
    "tt_row_sums = tt.sum(axis=1)\n",
    "\n",
    "new_matrix = tt / tt_row_sums[:, np.newaxis]\n",
    "print(new_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Normalize_Array(arr_to_normalize, axis=0):\n",
    "    if axis == 1:\n",
    "        arr_sum_per_row = arr_to_normalize.sum(axis=1)\n",
    "        arr_norm = arr_to_normalize/np.tile(arr_sum_per_row, (arr_sum_per_row.shape[0], 1)).T\n",
    "    elif axis == 0:\n",
    "        arr_sum_per_row = arr_to_normalize.sum(axis=0)\n",
    "        arr_norm = arr_to_normalize/np.tile(arr_sum_per_row, (arr_sum_per_row.shape[1], 1)).T\n",
    "    return arr_norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lst_test = [('A141HP4LYPW', 'B003AI2VGA', '3.0'),\n",
    "     ('B000063W1R', 'A141HP4LYPW', '4.0'),\n",
    "     ('6304286961', 'A141HP4LYPW', '5.0'),\n",
    "     ('A141HP4LYPW', '5556167281', '5.0')]\n",
    "\n",
    "# create a sample graph with edges\n",
    "def create_sample_graph(lst_test):\n",
    "    '''\n",
    "    lst_test = [('A141HP4LYPW', 'B003AI2VGA', '3.0'),\n",
    "     ('B000063W1R', 'A141HP4LYPW', '4.0'),\n",
    "     ('6304286961', 'A141HP4LYPW', '5.0'),\n",
    "     ('A141HP4LYPW', '5556167281', '5.0')]\n",
    "    '''\n",
    "    print(lst_test)\n",
    "\n",
    "    df_movies = amp.Reorganize_Edges_DataFrame(lst_test)\n",
    "    \n",
    "    return df_movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculation time-Crete graph from the movies file: 0:00:00.006679\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "shape too large to be a matrix.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-b8ea264b7877>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;31m#print(P)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mgrpComp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNormalize_Matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'row'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/Masters/KTH_DASE-ICT Innovation_Data Science/Data Mining/Project/test_graph/KTH-ID2211-Data-Mining-master/Computations.py\u001b[0m in \u001b[0;36mNormalize_Matrix\u001b[0;34m(self, arr_to_normalize, dim)\u001b[0m\n\u001b[1;32m    111\u001b[0m         \u001b[0marr_row_sums\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marr_to_normalize\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m         \u001b[0marr_norm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marr_to_normalize\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0marr_row_sums\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnewaxis\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0marr_norm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/numpy/matrixlib/defmatrix.py\u001b[0m in \u001b[0;36m__array_finalize__\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    180\u001b[0m                 \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 182\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"shape too large to be a matrix.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    183\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m             \u001b[0mnewshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: shape too large to be a matrix."
     ]
    }
   ],
   "source": [
    "grpComp = cmp.GraphComp()\n",
    "\n",
    "grp = gam.Graph_Amazon()\n",
    "file_name='data/movies.txt'; n_movies=140; prs_out='dictionary'\n",
    "\n",
    "max_connected_gr_amazon_movies = grpComp.Create_Bipartite_Giant_Component(grp, file_name, n_movies, prs_out)\n",
    "\n",
    "P = nx.to_numpy_matrix(max_connected_gr_amazon_movies)\n",
    "#print(P)\n",
    "\n",
    "grpComp.Normalize_Matrix(P, dim = 'row')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.    1.    0.    ... 0.    0.    0.   ]\n",
      " [0.15  0.    0.15  ... 0.    0.    0.   ]\n",
      " [0.    0.375 0.    ... 0.    0.    0.   ]\n",
      " ...\n",
      " [0.    0.    0.    ... 0.    0.    0.   ]\n",
      " [0.    0.    0.    ... 0.    0.    0.   ]\n",
      " [0.    0.    0.    ... 0.    0.    0.   ]]\n"
     ]
    }
   ],
   "source": [
    "tt_row_sums = P.sum(axis=1)\n",
    "\n",
    "new_matrix = P / tt_row_sums[np.newaxis][0]\n",
    "print(new_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(77, 1)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt_row_sums[np.newaxis][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lst_test = [('A141HP4LYPW', 'B003AI2VGA', '3.0'),\n",
    "     ('B000063W1R', 'A141HP4LYPW', '4.0'),\n",
    "     ('6304286961', 'A141HP4LYPW', '5.0'),\n",
    "     ('A141HP4LYPW', '5556167281', '5.0')]\n",
    "\n",
    "grp1 = grp.Graph_Amazon()\n",
    "grp_test=grp1.Create_Graph_From_List_WITH_Weight(lst_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grp_test.edges.data('weight', default=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grp_test.nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(grp_test.nodes)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grp_test.has_edge('A141HP4LYPW', 'B003AI2VGAx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdata=grp_test.get_edge_data('A141HP4LYPW', 'B003AI2VGA')\n",
    "print(gdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdata['weight']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grp_test.get_edge_data('A141HP4LYPW', 'B003AI2VGA')['weight']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(grp_test.get_edge_data('A141HP4LYPW', 'B003AI2VGAx'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this gives the first n nodes as movies and the rest as users\n",
    "# it other words top_nodes (node group with fewer nodes in it) are listed first\n",
    "P = nx.to_numpy_matrix(grp_test)  ## corresponds to M in the notes\n",
    "P_norm = Normalize_Array(P)\n",
    "\n",
    "n_nodes = P.shape[0]   ## P = nxn matrix\n",
    "R = np.diag(np.ones(n_nodes))\n",
    "R_zero = R.copy()\n",
    "beta = 0.15\n",
    "print(P_norm)\n",
    "print(R)\n",
    "print(R_zero)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# by doing the below conversion, we are just considering the links between the nodes, we ignore the weights\n",
    "# we try to understand if considering the weights will benefit us or not\n",
    "P[P>0]=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Normalize_Array(P)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "R_P = R.copy()\n",
    "R_R = R.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "R_P = (1-beta)*np.dot(P_norm, R_P) + beta*R_zero\n",
    "print(R_P)\n",
    "\n",
    "# since there is no direction, using P or R as \n",
    "R_R = (1-beta)*np.dot(R_R, P_norm) + beta*R_zero\n",
    "print(R_R)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "R_R[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "llt=np.array(list(R_R[3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.argsort(llt)[0][0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_nodes=np.argsort(llt)\n",
    "\n",
    "user_similarity = []\n",
    "\n",
    "# we will not consider the user himself or movies as similar nodes\n",
    "for idx in sorted_nodes[0][0]:\n",
    "    if idx != 3 and idx > 0:\n",
    "        user_similarity.append(idx)\n",
    "        \n",
    "user_similarity_top2 = user_similarity[-2:]\n",
    "\n",
    "print(user_similarity)\n",
    "print(user_similarity_top2)\n",
    "print(list(grp_test.nodes)[user_similarity_top2[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Create_Bipartite_Giant_Component_VAL(grp, file_name='data/movies.txt', start_index=40000, n_movies=5000, prs_out='dictionary'):\n",
    "    start_time = datetime.datetime.now()\n",
    "    # we can save train and validation data in files so that we don't need to read movies.txt each time\n",
    "\n",
    "    #max_connected_gr_amazon_movies = grp.Create_Bipartite_VALIDATION(file_name, n_movies, prs_out, start_index)\n",
    "    max_connected_gr_amazon_movies = grp.Create_Bipartite_VALIDATION(file_name, start_index, n_movies, prs_out)\n",
    "\n",
    "    #bottom_nodes, top_nodes = bipartite.sets(max_connected_gr_amazon_movies)\n",
    "    end_time = datetime.datetime.now()\n",
    "    print('Calculation time-Crete graph from the movies file: {}'.format(end_time-start_time))\n",
    "    \n",
    "    return max_connected_gr_amazon_movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grp_v = gam.Graph_Amazon()\n",
    "file_name_v='data/movies.txt'; start_index_v=40000; n_movies_v=5000; prs_out_v='dictionary'\n",
    "\n",
    "max_connected_gr_amazon_movies_VAL = \\\n",
    "Create_Bipartite_Giant_Component_VAL(grp_v, file_name_v, start_index_v, n_movies_v, prs_out_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_connected_gr_amazon_movies_VAL.edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "file_name = 'data/movies.txt'; start_index = 4; n_movies = 5; prs_out = 'screen'\n",
    "\n",
    "movie_dict_old = amp.AmazonMovies(n_movies, file_name, prs_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "file_name = 'data/movies.txt'; start_index = 200; n_movies = 100; prs_out = 'screen'\n",
    "\n",
    "movie_dict = amp.AmazonMovies_VALIDATION(n_movies, file_name, prs_out, start_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x='abcdaaaaaaaaaaaaaaa'\n",
    "if 'abcz' in x:\n",
    "    print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import Amazon_Movie_Parser as amp\n",
    "file_name = 'data/movies.txt'; n_movies=50000; prs_out = 'screen'\n",
    "userID = 'ADFH2AD4HML73';   movieID = 'B00005952Q'\n",
    "amp.Filter_AmazonMovies(n_movies, userID, movieID, file_name, prs_out)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Amazon_movies.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}