common.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "some common code for ctr model\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn.utils.rnn import pad_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FirstOrder(nn.Module):\n",
    "    def __init__(self, params):\n",
    "        super(FirstOrder, self).__init__()\n",
    "        # parse params\n",
    "        self.device = params['device']\n",
    "        self.feature_size = params['feature_size']\n",
    "        \n",
    "        weights_first_order = torch.empty(self.feature_size, 1, \n",
    "                                          dtype=torch.float32, device=self.device,\n",
    "                                          requires_grad=True)\n",
    "        nn.init.normal_(weights_first_order)\n",
    "        self.weights_first_order = nn.Parameter(weights_first_order)\n",
    "        \n",
    "    def forward(self, feature_values, feature_idx):  \n",
    "        weights_first_order = self.weights_first_order[feature_idx, :]\n",
    "        first_order = torch.mul(feature_values, weights_first_order.squeeze())\n",
    "        return first_order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SecondOrder(nn.Module):\n",
    "    def __init__(self, params, get_embeddings=False):\n",
    "        super(SecondOrder, self).__init__()\n",
    "        # parse params\n",
    "        self.device = params['device']\n",
    "        self.feature_size = params['feature_size']\n",
    "        self.embedding_size = params['embedding_size']\n",
    "        self.get_embeddings = get_embeddings\n",
    "        \n",
    "        feature_embeddings = torch.empty(self.feature_size, self.embedding_size, \n",
    "                              dtype=torch.float32, device=self.device, \n",
    "                              requires_grad=True)\n",
    "        nn.init.normal_(feature_embeddings)\n",
    "        self.feature_embeddings = nn.Parameter(feature_embeddings)\n",
    "        \n",
    "    def forward(self, feature_values, feature_idx):  \n",
    "        embeddings = self.feature_embeddings[feature_idx, :]\n",
    "        ## second order\n",
    "        temp1 = torch.pow(torch.einsum('bf,bfk->bk', (feature_values, embeddings)), 2)\n",
    "        temp2 = torch.einsum('bf,bfk->bk', (torch.pow(feature_values, 2), torch.pow(embeddings, 2)))\n",
    "        second_order = temp1-temp2\n",
    "        if self.get_embeddings:\n",
    "            return second_order, embeddings\n",
    "        else:\n",
    "            return second_order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FirstOrderMutiHot(nn.Module):\n",
    "    '''support muti-hot feature for fm\n",
    "    \n",
    "    '''\n",
    "    def __init__(self, params):\n",
    "        super(FirstOrderMutiHot, self).__init__()\n",
    "        # parse params\n",
    "        self.device = params['device']\n",
    "        self.feature_size = params['feature_size']\n",
    "        self.field_size = params['field_size']\n",
    "        self.fea_name = params['fea_name']\n",
    "        self.max_len = params['max_len'] \n",
    "    \n",
    "        weights_first_order = torch.empty(self.feature_size+2, 1, \n",
    "                                          dtype=torch.float32, device=self.device,\n",
    "                                          requires_grad=True)\n",
    "        nn.init.normal_(weights_first_order)\n",
    "        self.weights_first_order = nn.Parameter(weights_first_order)\n",
    "        \n",
    "    def forward(self, feature_values, feature_idx):\n",
    "        batch_size = feature_values.shape[0]\n",
    "        \n",
    "        # feature index and value padding\n",
    "        feature_idx_concat, feature_values_concat = [], []\n",
    "        for t in self.fea_name:\n",
    "            feature_idx_concat = feature_idx_concat + list(feature_idx[t])\n",
    "            feature_values_concat = feature_values_concat + list(feature_values[t])\n",
    "\n",
    "        seqLen = torch.tensor(list(map(len, feature_idx_concat)), dtype=torch.float32, device=self.device)\n",
    "        seqLen = torch.transpose(seqLen.reshape(self.field_size, batch_size), 0, 1)\n",
    "        feature_idx_padded = pad_sequence(feature_idx_concat, batch_first=True, padding_value=self.feature_size)[:, 0:self.max_len].to(self.device)\n",
    "        feature_values_padded = pad_sequence(feature_values_concat, batch_first=True, padding_value=0)[:, 0:self.max_len].to(self.device)\n",
    "        \n",
    "        # first_order\n",
    "        weights_first_order = self.weights_first_order[feature_idx_padded]\n",
    "        first_order = torch.mul(feature_values_padded, weights_first_order.squeeze())\n",
    "        first_order = first_order.reshape(self.field_size, batch_size, -1)\n",
    "        first_order = torch.transpose(first_order, 0, 1)\n",
    "        first_order = first_order.sum(dim=2)\n",
    "        first_order = first_order / seqLen\n",
    "           \n",
    "        return first_order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SecondOrderMutiHot(nn.Module):\n",
    "    '''support muti-hot feature for fm\n",
    "    \n",
    "    '''\n",
    "    def __init__(self, params, get_embeddings=False):\n",
    "        super(SecondOrderMutiHot, self).__init__()\n",
    "        # parse params\n",
    "        self.device = params['device']\n",
    "        self.feature_size = params['feature_size']\n",
    "        self.field_size = params['field_size']\n",
    "        self.embedding_size = params['embedding_size']\n",
    "        self.get_embeddings = get_embeddings\n",
    "        self.fea_name = params['fea_name']\n",
    "        self.max_len = params['max_len'] \n",
    "\n",
    "        feature_embeddings = torch.empty(self.feature_size+2, self.embedding_size, \n",
    "                              dtype=torch.float32, device=self.device, \n",
    "                              requires_grad=True)\n",
    "        nn.init.normal_(feature_embeddings)\n",
    "        self.feature_embeddings = nn.Parameter(feature_embeddings)\n",
    "        \n",
    "    def forward(self, feature_values, feature_idx):\n",
    "        batch_size = feature_values.shape[0]\n",
    "        \n",
    "        # feature index padding and mask\n",
    "        feature_idx_concat = []\n",
    "        for t in self.fea_name:\n",
    "            feature_idx_concat = feature_idx_concat + list(feature_idx[t])\n",
    "        seqLen = torch.tensor(list(map(len, feature_idx_concat)), dtype=torch.float32, device=self.device)\n",
    "        feature_idx_padded = pad_sequence(feature_idx_concat, batch_first=True, padding_value=self.feature_size)[:, 0:self.max_len].to(self.device)\n",
    "        mask = feature_idx_padded != self.feature_size\n",
    "        feature_weight = torch.ones_like(feature_idx_padded, dtype=torch.float32, device=self.device)\n",
    "        feature_weight.masked_fill_(mask == 0, 0)\n",
    "        \n",
    "        # get embeddings and average\n",
    "        embeddings = self.feature_embeddings[feature_idx_padded, :]\n",
    "        embeddings = torch.einsum('ble,bl->ble', embeddings, feature_weight)\n",
    "        embeddings = embeddings.sum(dim=1)\n",
    "        embeddings = embeddings / seqLen.reshape(embeddings.shape[0], 1)\n",
    "        embeddings = embeddings.reshape(self.field_size, batch_size, -1)\n",
    "        embeddings = torch.transpose(embeddings, 0, 1)\n",
    "\n",
    "        # feature values padding and average\n",
    "        feature_values_concat = []\n",
    "        for t in self.fea_name:\n",
    "            feature_values_concat = feature_values_concat + list(feature_values[t])\n",
    "        feature_values_padded = pad_sequence(feature_values_concat, batch_first=True, padding_value=0)[:, 0:self.max_len].to(self.device)\n",
    "        feature_values_padded = feature_values_padded.reshape(self.field_size, batch_size, -1)\n",
    "        feature_values_padded = torch.transpose(feature_values_padded, 0, 1)\n",
    "        feature_values_padded = feature_values_padded.sum(dim=2)\n",
    "        seqLen = torch.transpose(seqLen.reshape(self.field_size, batch_size), 0, 1)\n",
    "        feature_values = feature_values_padded / seqLen\n",
    "\n",
    "        # second order\n",
    "        temp1 = torch.pow(torch.einsum('bf,bfk->bk', (feature_values, embeddings)), 2)\n",
    "        temp2 = torch.einsum('bf,bfk->bk', (torch.pow(feature_values, 2), torch.pow(embeddings, 2)))\n",
    "        second_order = temp1-temp2\n",
    "        if self.get_embeddings:\n",
    "            return second_order, embeddings\n",
    "        else:\n",
    "            return second_order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self, params, use_batchnorm=True, use_dropout=True):\n",
    "        super(MLP, self).__init__()\n",
    "        \n",
    "        self.embedding_size = params['embedding_size']\n",
    "        self.field_size = params['field_size']\n",
    "        self.hidden_dims = params['hidden_dims']\n",
    "        self.device = params['device']\n",
    "        self.p = params['p']\n",
    "        self.use_batchnorm = use_batchnorm\n",
    "        self.use_dropout = use_dropout\n",
    "\n",
    "\n",
    "        \n",
    "        self.input_dim = self.field_size * self.embedding_size\n",
    "        self.num_layers = len(self.hidden_dims)\n",
    "\n",
    "        \n",
    "        ## deep weights\n",
    "        self.deep_layers = nn.Sequential()\n",
    "\n",
    "        net_dims = [self.input_dim]+self.hidden_dims\n",
    "        for i in range(self.num_layers):\n",
    "            self.deep_layers.add_module('fc%d' % (i+1), nn.Linear(net_dims[i], net_dims[i+1]).to(self.device))\n",
    "            if self.use_batchnorm:\n",
    "                self.deep_layers.add_module('bn%d' % (i+1), nn.BatchNorm1d(net_dims[i+1]).to(self.device))\n",
    "            self.deep_layers.add_module('relu%d' % (i+1), nn.ReLU().to(self.device)) \n",
    "            if self.use_dropout:\n",
    "                self.deep_layers.add_module('dropout%d' % (i+1), nn.Dropout(self.p).to(self.device))\n",
    "    \n",
    "    def forward(self, embeddings):\n",
    "        deepInput = embeddings.reshape(embeddings.shape[0], self.input_dim)\n",
    "        deepOut = self.deep_layers(deepInput)\n",
    "        return deepOut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CIN(nn.Module):\n",
    "    '''xDeepFM CIN Module\n",
    "    '''\n",
    "    def __init__(self, params):\n",
    "        super(CIN, self).__init__()\n",
    "        # parse params\n",
    "        self.split_half = params['split_half']\n",
    "        self.field_size = params['field_size']\n",
    "        self.hidden_dims = params['cin_hidden_dims']\n",
    "        self.num_layers = len(self.hidden_dims)\n",
    "        \n",
    "        self.net_dims = [self.field_size]+self.hidden_dims\n",
    "        self.hidden_dims_split_half = [self.field_size]\n",
    "        self.conv1ds = nn.ModuleList()\n",
    "        for i in range(self.num_layers):\n",
    "#             h_weights['h_weight%d' % (i+1)] = torch.empty(net_dims[i], self.field_size)\n",
    "#             nn.init.normal_(h_weights['h_weight%d' % (i+1)])\n",
    "            self.conv1ds.append(nn.Conv1d(self.net_dims[0]*self.hidden_dims_split_half[-1], self.net_dims[i+1], 1))\n",
    "            if self.split_half:\n",
    "                self.hidden_dims_split_half.append(self.net_dims[i+1] // 2)\n",
    "            else:\n",
    "                self.hidden_dims_split_half.append(self.net_dims[i+1])\n",
    "        \n",
    "    def forward(self, inputs):\n",
    "        res = []\n",
    "        h = [inputs]\n",
    "        for i in range(self.num_layers):\n",
    "            temp = torch.einsum('bhd,bmd->bhmd', h[-1], h[0])\n",
    "            temp = temp.reshape(inputs.shape[0], h[-1].shape[1]*inputs.shape[1], inputs.shape[2])\n",
    "            # b * hi * d\n",
    "            temp = self.conv1ds[i](temp)\n",
    "            if self.split_half:\n",
    "                next_hidden, hi = torch.split(temp, 2*[temp.shape[1]//2], 1)\n",
    "            else:\n",
    "                next_hidden, hi = temp, temp\n",
    "            h.append(next_hidden)\n",
    "            res.append(hi)\n",
    "        res = torch.cat(res, dim=1)\n",
    "        # b * (h1 + h2 + ... + hn)\n",
    "        res = torch.sum(res, dim=2)\n",
    "        return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}