1.data-preprocessing-and-feature-engineering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BcNP6wnl8Yfq"
   },
   "source": [
    "# 数据预处理与特征工程"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kgpyTPSZdkxv"
   },
   "source": [
    "## 数据一览"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JEBQLW6V8t9F",
    "outputId": "1c10ecd1-55bc-46cd-8066-3c0d404fd86f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  wechat_algo_data.zip\n",
      "   creating: wechat_algo_data/\n",
      "  inflating: wechat_algo_data/test_a.csv  \n",
      "  inflating: wechat_algo_data/feed_info.csv  \n",
      "  inflating: wechat_algo_data/feed_embeddings.csv  \n",
      "  inflating: wechat_algo_data/README.md  \n",
      "  inflating: wechat_algo_data/user_action.csv  \n",
      "  inflating: wechat_algo_data/submit_demo_初赛a.csv  \n"
     ]
    }
   ],
   "source": [
    "!unzip wechat_algo_data.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "6XXpcyyLdsjL"
   },
   "outputs": [],
   "source": [
    "# 数据处理工具库\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RjCNQCURdvc0",
    "outputId": "c2d4ff16-2bc3-46bb-9462-8b0b43bf6dfc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feed_embeddings.csv  README.md\t\t    test_a.csv\n",
      "feed_info.csv\t     submit_demo_初赛a.csv  user_action.csv\n"
     ]
    }
   ],
   "source": [
    "!ls wechat_algo_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 474
    },
    "id": "55sHVsRxeSnj",
    "outputId": "24681838-7a30-4026-f2a1-f4a7d53c4e0e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feedid</th>\n",
       "      <th>authorid</th>\n",
       "      <th>videoplayseconds</th>\n",
       "      <th>description</th>\n",
       "      <th>ocr</th>\n",
       "      <th>asr</th>\n",
       "      <th>bgm_song_id</th>\n",
       "      <th>bgm_singer_id</th>\n",
       "      <th>manual_keyword_list</th>\n",
       "      <th>machine_keyword_list</th>\n",
       "      <th>manual_tag_list</th>\n",
       "      <th>machine_tag_list</th>\n",
       "      <th>description_char</th>\n",
       "      <th>ocr_char</th>\n",
       "      <th>asr_char</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>43549</td>\n",
       "      <td>6165</td>\n",
       "      <td>38</td>\n",
       "      <td>104741 122649 8109 117252 65632 23463 118668 4...</td>\n",
       "      <td>139499 59421 82007 142955 27736 83577 52394 11...</td>\n",
       "      <td>142955 27736 83577 103956 32010 34170 89740 90...</td>\n",
       "      <td>19356.0</td>\n",
       "      <td>11703.0</td>\n",
       "      <td>15506;7715;17582</td>\n",
       "      <td>26334;219;25209;7715;18541</td>\n",
       "      <td>81;269;159;6</td>\n",
       "      <td>269 0.8525666;81 0.8525666;8 1.1e-07;306 0.0;2...</td>\n",
       "      <td>26439 5247 6426 3827 1882 26018 20744 22204 30...</td>\n",
       "      <td>25926 8491 13394 2203 26439 6243 33054 16435 1...</td>\n",
       "      <td>2203 26439 6243 33054 16435 16307 17070 24908 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>77432</td>\n",
       "      <td>9386</td>\n",
       "      <td>60</td>\n",
       "      <td>35753 27736 146603 73055 11794 101761 11794 81...</td>\n",
       "      <td>35753 146603 73055 11794 101761 67496 16933 52...</td>\n",
       "      <td>146739 14368 79290 79213 47366 8109 33194 1198...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8199;18322;4243</td>\n",
       "      <td>24078;19924</td>\n",
       "      <td>194;267;159;6</td>\n",
       "      <td>267 0.99293476;194 0.99293476</td>\n",
       "      <td>31010 32495 6243 13923 15360 30483 2709 26084 ...</td>\n",
       "      <td>31010 32495 13923 15360 30483 2709 26084 15160...</td>\n",
       "      <td>7259 20851 5061 26207 17573 17531 15117 20072 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12921</td>\n",
       "      <td>2996</td>\n",
       "      <td>15</td>\n",
       "      <td>114413 107973 117252 27736 41035 32715 125374 ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10288.0</td>\n",
       "      <td>11354.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11696;6926</td>\n",
       "      <td>202;23;160;6</td>\n",
       "      <td>23 0.64771646;202 0.64771646</td>\n",
       "      <td>27077 10050 27464 20072 26018 6243 21492 14750...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   feedid  ...                                           asr_char\n",
       "0   43549  ...  2203 26439 6243 33054 16435 16307 17070 24908 ...\n",
       "1   77432  ...  7259 20851 5061 26207 17573 17531 15117 20072 ...\n",
       "2   12921  ...                                                NaN\n",
       "\n",
       "[3 rows x 15 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# feed数据\n",
    "feed_df = pd.read_csv(\"wechat_algo_data/feed_info.csv\")\n",
    "feed_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "id": "4Bl0Wuz0evy8",
    "outputId": "abad5a6b-226f-4694-8c1f-0f57c835201b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userid</th>\n",
       "      <th>feedid</th>\n",
       "      <th>date_</th>\n",
       "      <th>device</th>\n",
       "      <th>read_comment</th>\n",
       "      <th>comment</th>\n",
       "      <th>like</th>\n",
       "      <th>play</th>\n",
       "      <th>stay</th>\n",
       "      <th>click_avatar</th>\n",
       "      <th>forward</th>\n",
       "      <th>follow</th>\n",
       "      <th>favorite</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>71474</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>500</td>\n",
       "      <td>5366</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>73916</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>250</td>\n",
       "      <td>1533</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>50282</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>750</td>\n",
       "      <td>1302</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userid  feedid  date_  device  ...  click_avatar  forward  follow  favorite\n",
       "0       8   71474      1       1  ...             0        0       0         0\n",
       "1       8   73916      1       1  ...             0        0       0         0\n",
       "2       8   50282      1       1  ...             0        0       0         0\n",
       "\n",
       "[3 rows x 13 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# action数据\n",
    "action_df = pd.read_csv('./wechat_algo_data/user_action.csv')\n",
    "action_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "id": "u4ZFXRojfAr4",
    "outputId": "a5593f42-641f-4ed5-f8d6-084ab83f0608"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userid</th>\n",
       "      <th>feedid</th>\n",
       "      <th>device</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14298</td>\n",
       "      <td>67227</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>68356</td>\n",
       "      <td>91864</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>49925</td>\n",
       "      <td>104657</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userid  feedid  device\n",
       "0   14298   67227       1\n",
       "1   68356   91864       2\n",
       "2   49925  104657       2"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 预测数据\n",
    "test = pd.read_csv('./wechat_algo_data/test_a.csv')\n",
    "test.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "s2pFscGhfOEq"
   },
   "source": [
    "## 数据合并与预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "FQkm-YYp9uE8"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import copy\n",
    "from tensorflow.python.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "HP6u0ypb9y-v"
   },
   "outputs": [],
   "source": [
    "# 对list字段进行切分和映射编码\n",
    "def split(column):\n",
    "    if not isinstance(column,str):\n",
    "        return []\n",
    "    keys = column.strip().split(';')\n",
    "    for key in keys:\n",
    "        if key not in key2index:\n",
    "            key2index[key] = len(key2index) + 1\n",
    "    return list(map(lambda x: key2index[x], keys))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "dSsqSI2p-FKb"
   },
   "outputs": [],
   "source": [
    "def preprocess(sample,dense_features):\n",
    "    '''\n",
    "    特征工程：对数值型特征做对数变换; id型特征+1; 缺失值补充0。\n",
    "    '''\n",
    "    sample[dense_features] = sample[dense_features].fillna(0.0)\n",
    "    sample[dense_features] = np.log(sample[dense_features] + 1.0)\n",
    "    \n",
    "    sample[[\"authorid\", \"bgm_song_id\", \"bgm_singer_id\"]] += 1  # 0 用于填未知\n",
    "    sample[[\"authorid\", \"bgm_song_id\", \"bgm_singer_id\", \"videoplayseconds\"]] = sample[[\"authorid\", \"bgm_song_id\", \"bgm_singer_id\", \"videoplayseconds\"]].fillna(0)\n",
    "    sample[\"videoplayseconds\"] = np.log(sample[\"videoplayseconds\"] + 1.0)\n",
    "    sample[[\"authorid\", \"bgm_song_id\", \"bgm_singer_id\"]] = sample[[\"authorid\", \"bgm_song_id\", \"bgm_singer_id\"]].astype(int)\n",
    "    return sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "6_k0sj47-HwE"
   },
   "outputs": [],
   "source": [
    "# 合并数据\n",
    "test['date_'] = 15\n",
    "action_df = pd.concat([action_df,test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "5vfgLz46-kAz"
   },
   "outputs": [],
   "source": [
    "# 标签列\n",
    "target = [\"read_comment\", \"like\", \"click_avatar\", \"forward\"]\n",
    "# 稀疏特征\n",
    "sparse_features = ['userid', 'feedid', 'authorid', 'bgm_song_id', 'bgm_singer_id']\n",
    "# 变长序列特征\n",
    "varlen_features = ['manual_tag_list','manual_keyword_list']\n",
    "# 稠密特征\n",
    "dense_features = ['videoplayseconds']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "aTkpJTAU--I3"
   },
   "outputs": [],
   "source": [
    "# 数据合并\n",
    "feed_df = feed_df[['feedid', 'authorid', 'videoplayseconds', 'bgm_song_id', 'bgm_singer_id','manual_tag_list','manual_keyword_list']]\n",
    "data = action_df.merge(feed_df, how='left',on='feedid') #行为数据拼接，作者id，bgm_song_id \n",
    "data = preprocess(data,dense_features) #特征处理\n",
    "data = data[dense_features+sparse_features+varlen_features+['date_']+target]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1n1xXawW_CtH",
    "outputId": "484f7fef-026f-4005-9d2a-38015d5ef26c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "manual_keyword_list 字段最长的取值序列长度为 18\n",
      "manual_tag_list 字段最长的取值序列长度为 11\n"
     ]
    }
   ],
   "source": [
    "# 变长特征编码\n",
    "encoder = {}\n",
    "global key2index\n",
    "for f in ['manual_keyword_list','manual_tag_list']:\n",
    "    key2index = {}\n",
    "    f_list = list(map(split, data[f].values))\n",
    "    f_length = np.array(list(map(len, f_list)))\n",
    "    max_len = max(f_length)\n",
    "    print(f'{f} 字段最长的取值序列长度为 {max_len}')\n",
    "    # Notice : padding=`post`\n",
    "    data[f] = list(pad_sequences(f_list, maxlen=max_len, padding='post', ))\n",
    "    encoder[f] = copy.copy(key2index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4FL9zD0r_LQ2",
    "outputId": "74890c1b-f668-4cdd-c08c-8510af66eb30"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "编码ID字段：userid\n",
      "编码ID字段：feedid\n",
      "编码ID字段：authorid\n",
      "编码ID字段：bgm_song_id\n",
      "编码ID字段：bgm_singer_id\n"
     ]
    }
   ],
   "source": [
    "# 稀疏特征编码\n",
    "for featid in sparse_features:\n",
    "    print(f\"编码ID字段：{featid}\")\n",
    "    encoder[featid] = {uid:ucode+1 for ucode,uid in enumerate(data[featid].unique())} \n",
    "    data[featid] = data[featid].apply(lambda x: encoder[featid].get(x,0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Vn_7SE6Z_O6u",
    "outputId": "ed6b594c-3b91-4c93-c3f0-2135f374aff8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "数据维度： (7739867, 13)\n",
      "数据字段： ['videoplayseconds', 'userid', 'feedid', 'authorid', 'bgm_song_id', 'bgm_singer_id', 'manual_tag_list', 'manual_keyword_list', 'date_', 'read_comment', 'like', 'click_avatar', 'forward']\n",
      "不同的date_取值:  [ 1  2  3  5  6  7  8 10 11 12 13 14  4  9 15]\n"
     ]
    }
   ],
   "source": [
    "print('数据维度：', data.shape)\n",
    "print('数据字段：', data.columns.tolist())\n",
    "print('不同的date_取值: ', data['date_'].unique())\n",
    "# 如果资源比较少，可以在这里进行数据采样\n",
    "data = data.sample(frac = 1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FS41CUfzxlMe",
    "outputId": "4490cbb4-021c-46ba-94bd-0163d359a7e4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘data_and_feature’: File exists\n"
     ]
    }
   ],
   "source": [
    "# 或者手动创建文件夹data_and_feature\n",
    "!mkdir data_and_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "rEn01L9K_a_N"
   },
   "outputs": [],
   "source": [
    "# 构建训练集，验证集和测试集\n",
    "# 第14天样本作为验证集\n",
    "train = data[data['date_'] < 14].drop(['date_'],axis = 1)\n",
    "val = data[data['date_'] == 14].drop(['date_'],axis = 1)  \n",
    "test = data[data['date_'] == 15].drop(['date_'],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "O0osmrVe5xVU",
    "outputId": "70f7991c-aa57-4be9-cee2-27033d8b15cf"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "import joblib\n",
    "del action_df\n",
    "del feed_df\n",
    "del data\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DyhIAqXE58U3"
   },
   "outputs": [],
   "source": [
    "joblib.dump(train, './data_and_feature/train.txt')\n",
    "joblib.dump(val, './data_and_feature/val.txt')\n",
    "joblib.dump(test, './data_and_feature/test.txt')\n",
    "joblib.dump(encoder, './data_and_feature/encoder.txt')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "1.data_preprocessing_and_feature_engineering.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}