Cantonese_preprocessing_updated.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3c16b077",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pycantonese in c:\\programdata\\anaconda3\\lib\\site-packages (3.4.0)\n",
      "Requirement already satisfied: pylangacq<0.17.0,>=0.16.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from pycantonese) (0.16.2)\n",
      "Requirement already satisfied: wordseg==0.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from pycantonese) (0.0.2)\n",
      "Requirement already satisfied: requests<=3.0.0,>=2.18.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from pylangacq<0.17.0,>=0.16.0->pycantonese) (2.28.1)\n",
      "Requirement already satisfied: python-dateutil<=3.0.0,>=2.0.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from pylangacq<0.17.0,>=0.16.0->pycantonese) (2.8.2)\n",
      "Requirement already satisfied: tabulate[widechars]<=0.9.0,>=0.8.9 in c:\\programdata\\anaconda3\\lib\\site-packages (from pylangacq<0.17.0,>=0.16.0->pycantonese) (0.8.10)\n",
      "Requirement already satisfied: six>=1.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from python-dateutil<=3.0.0,>=2.0.0->pylangacq<0.17.0,>=0.16.0->pycantonese) (1.16.0)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<=3.0.0,>=2.18.0->pylangacq<0.17.0,>=0.16.0->pycantonese) (2.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<=3.0.0,>=2.18.0->pylangacq<0.17.0,>=0.16.0->pycantonese) (2023.11.17)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<=3.0.0,>=2.18.0->pylangacq<0.17.0,>=0.16.0->pycantonese) (1.26.11)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<=3.0.0,>=2.18.0->pylangacq<0.17.0,>=0.16.0->pycantonese) (3.3)\n",
      "Requirement already satisfied: wcwidth in c:\\programdata\\anaconda3\\lib\\site-packages (from tabulate[widechars]<=0.9.0,>=0.8.9->pylangacq<0.17.0,>=0.16.0->pycantonese) (0.2.5)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\programdata\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\programdata\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\programdata\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\programdata\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\programdata\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -ensorflow-intel (c:\\programdata\\anaconda3\\lib\\site-packages)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1949095"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Accessing the Yip-Matthews CantoneseEnglish bilingual corpus (Yip and Matthews, 2007)\n",
    "!pip install pycantonese\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "00a0eb2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1949095"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pycantonese\n",
    "url = (\n",
    "    \"https://childes.talkbank.org/\"\n",
    "    \"data/Biling/YipMatthews.zip\"\n",
    ")\n",
    "corpus = pycantonese.read_chat(url)\n",
    "corpus.n_files()\n",
    "501\n",
    "len(corpus.words())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e021d7fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['佢哋',\n",
       " '唔使',\n",
       " '佢',\n",
       " '可以',\n",
       " '睇',\n",
       " '即',\n",
       " '都',\n",
       " '冇',\n",
       " '返',\n",
       " '哩',\n",
       " '我哋',\n",
       " '梗係',\n",
       " '其他',\n",
       " '然之後',\n",
       " '會',\n",
       " '因為',\n",
       " '同埋',\n",
       " '又',\n",
       " '去',\n",
       " '知',\n",
       " '而',\n",
       " '唔係',\n",
       " '跟住',\n",
       " '同',\n",
       " '譬如',\n",
       " '唔',\n",
       " '好',\n",
       " '最',\n",
       " '喺',\n",
       " '點解',\n",
       " '你哋',\n",
       " '之後',\n",
       " '要',\n",
       " '太',\n",
       " '邊個',\n",
       " '而家',\n",
       " '個',\n",
       " '但係',\n",
       " '一定',\n",
       " '吖',\n",
       " '喎',\n",
       " '成日',\n",
       " '原來',\n",
       " '噉',\n",
       " '咁',\n",
       " '哩樣',\n",
       " '嗰啲',\n",
       " '幾',\n",
       " '嘅',\n",
       " '幾多',\n",
       " '可能',\n",
       " '有啲',\n",
       " '我',\n",
       " '哩啲',\n",
       " '係',\n",
       " '一啲',\n",
       " '咪',\n",
       " '有冇',\n",
       " '諗',\n",
       " '你',\n",
       " '嘢',\n",
       " '點',\n",
       " '過',\n",
       " '嗰個',\n",
       " '話',\n",
       " '有',\n",
       " '不如',\n",
       " '哩個',\n",
       " '乜嘢',\n",
       " '咩',\n",
       " '由',\n",
       " '未',\n",
       " '呀',\n",
       " '啲',\n",
       " '喺度',\n",
       " '得',\n",
       " '已經',\n",
       " '噉樣',\n",
       " '乜',\n",
       " '啊',\n",
       " '咗',\n",
       " '自己',\n",
       " '多',\n",
       " '嗯',\n",
       " '哦',\n",
       " '如果',\n",
       " '人哋',\n",
       " '再',\n",
       " '講',\n",
       " '真係',\n",
       " '想',\n",
       " '不過',\n",
       " '即係',\n",
       " '就',\n",
       " '覺得',\n",
       " '到',\n",
       " '應該',\n",
       " '嗰度',\n",
       " '嗰',\n",
       " '哩度',\n",
       " '呢',\n",
       " '或者',\n",
       " '所以',\n",
       " '點樣',\n",
       " '我',\n",
       " '哋',\n",
       " '應該']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words = pycantonese.stop_words()\n",
    "\n",
    "stop_words= list(stop_words)\n",
    "my_stopwords = ['我','哋','應該']  #手動加入stop words\n",
    "stop_words += my_stopwords\n",
    "stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "318bb7ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "107"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6b419a56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Post_number</th>\n",
       "      <th>Title</th>\n",
       "      <th>Main_content</th>\n",
       "      <th>Youtube_link</th>\n",
       "      <th>Comments</th>\n",
       "      <th>Category</th>\n",
       "      <th>Classification</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3000101</td>\n",
       "      <td>想平平地$200收 2005年曼聯盃筆set限量版</td>\n",
       "      <td>有無巴打幫幫手</td>\n",
       "      <td>None</td>\n",
       "      <td>佢屋企人dum咗, \\n\\n有無巴打幫幫手, 李克勤可能有</td>\n",
       "      <td>6</td>\n",
       "      <td>體育台</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3000104</td>\n",
       "      <td>深水埗某小學：祝賀李家超校友當選行政長官</td>\n",
       "      <td>呢間小學算唔算藍店</td>\n",
       "      <td>None</td>\n",
       "      <td>我無票, 呢間小學算唔算藍店\\n\\n, 咁學校要出下力啦\\n拆左間校舍用黎起樓, 咁順便補祝...</td>\n",
       "      <td>5</td>\n",
       "      <td>時事台</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3000105</td>\n",
       "      <td>天文台同cc台一樣，雷聲大雨點少[sosad]</td>\n",
       "      <td>琴日帶咗成日風話落大暴雨[核彈]\\n又話好大雞獲 要做足準備\\n點知原來得兩滴貨仔</td>\n",
       "      <td>None</td>\n",
       "      <td>, 咪撚燈啦屌 成日流流長, 琴日帶咗成日風話落大暴雨[核彈]\\n又話好大雞獲 要做足準備\\...</td>\n",
       "      <td>6</td>\n",
       "      <td>體育台</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3000107</td>\n",
       "      <td>【多圖】陳芷尤轉戰新加坡榮升女一撇甩「張振朗舊愛」稱號 比堅尼照大曬惹火身材</td>\n",
       "      <td>//陳芷尤（Apple）畀人嘅認識可能係擁有惹火身材嘅張振朗舊愛，喺無綫一直無發圍機會，演藝...</td>\n",
       "      <td>None</td>\n",
       "      <td>\\n\\n\\n\\n原文：, \\n\\n\\n\\n//陳芷尤（Apple）畀人嘅認識可能係擁有惹火身...</td>\n",
       "      <td>7</td>\n",
       "      <td>娛樂台</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>3000109</td>\n",
       "      <td>唔好再講「地鐵」，係蝗語黎</td>\n",
       "      <td>就奇！\\n依一兩年係連登見到越黎越多蝗語特警亂鳩咁黎，\\n將一啲香港沿用多年、又或者係一啲書...</td>\n",
       "      <td>None</td>\n",
       "      <td>奶奶 香港講開係指 老公個阿媽\\n而蝗語奶奶 香港用係係阿嫲 \\n\\n同一原理媳婦 香港用開...</td>\n",
       "      <td>5</td>\n",
       "      <td>時事台</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799</th>\n",
       "      <td>3000999</td>\n",
       "      <td>人生冇希望</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://youtu.be/65nfbW-27ps</td>\n",
       "      <td>, 瞓咗, 成日唔準時點搶, 打個丁 聽朝又係新一天, YouTube, 終於到</td>\n",
       "      <td>30</td>\n",
       "      <td>感情台</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>800</th>\n",
       "      <td>3001000</td>\n",
       "      <td>二萬八蚊可以買架車改裝車中泊</td>\n",
       "      <td>已經有埋密斗俾你慢慢改\\n呢啲車真係邊度搵</td>\n",
       "      <td>None</td>\n",
       "      <td>, 運屍車？, 已經有埋密斗俾你慢慢改\\n呢啲車真係邊度搵\\n\\n, 都似係，重要出唔到牌,...</td>\n",
       "      <td>20</td>\n",
       "      <td>汽車台</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>803</th>\n",
       "      <td>3001003</td>\n",
       "      <td>紅磡維港．星岸錄兩宗撻訂　料買家各損失樓價28%</td>\n",
       "      <td>紅磡維港．星岸錄兩宗撻訂　料買家各損失樓價28%　涉逾2000萬撰文：黃祐樺出版：\\n\\n2...</td>\n",
       "      <td>None</td>\n",
       "      <td>紅磡維港．星岸錄兩宗撻訂　料買家各損失樓價28%　涉逾2000萬撰文：黃祐樺出版：\\n\\n2...</td>\n",
       "      <td>37</td>\n",
       "      <td>房屋台</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>804</th>\n",
       "      <td>3001004</td>\n",
       "      <td>壓力好大</td>\n",
       "      <td>好難頂</td>\n",
       "      <td>None</td>\n",
       "      <td>壓力山大\\n\\n你們可以打我了, 學業?家庭?財政?感情?, 好難頂, 加油, 正垃圾</td>\n",
       "      <td>30</td>\n",
       "      <td>感情台</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>806</th>\n",
       "      <td>3001006</td>\n",
       "      <td>納指11500點同馬奇諾防線一樣堅不可摧</td>\n",
       "      <td>點跌都跌唔穿</td>\n",
       "      <td>None</td>\n",
       "      <td>, Ok bye bye, 條線一路向後推, 好心你訓啦, , 雖然有D幾好笑，但有D ov...</td>\n",
       "      <td>15</td>\n",
       "      <td>財經台</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>432 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Post_number                                   Title  \\\n",
       "0        3000101               想平平地$200收 2005年曼聯盃筆set限量版   \n",
       "3        3000104                    深水埗某小學：祝賀李家超校友當選行政長官   \n",
       "4        3000105                 天文台同cc台一樣，雷聲大雨點少[sosad]   \n",
       "6        3000107  【多圖】陳芷尤轉戰新加坡榮升女一撇甩「張振朗舊愛」稱號 比堅尼照大曬惹火身材   \n",
       "8        3000109                           唔好再講「地鐵」，係蝗語黎   \n",
       "..           ...                                     ...   \n",
       "799      3000999                                   人生冇希望   \n",
       "800      3001000                          二萬八蚊可以買架車改裝車中泊   \n",
       "803      3001003                紅磡維港．星岸錄兩宗撻訂　料買家各損失樓價28%   \n",
       "804      3001004                                    壓力好大   \n",
       "806      3001006                    納指11500點同馬奇諾防線一樣堅不可摧   \n",
       "\n",
       "                                          Main_content  \\\n",
       "0                                              有無巴打幫幫手   \n",
       "3                                            呢間小學算唔算藍店   \n",
       "4            琴日帶咗成日風話落大暴雨[核彈]\\n又話好大雞獲 要做足準備\\n點知原來得兩滴貨仔   \n",
       "6    //陳芷尤（Apple）畀人嘅認識可能係擁有惹火身材嘅張振朗舊愛，喺無綫一直無發圍機會，演藝...   \n",
       "8    就奇！\\n依一兩年係連登見到越黎越多蝗語特警亂鳩咁黎，\\n將一啲香港沿用多年、又或者係一啲書...   \n",
       "..                                                 ...   \n",
       "799                                                NaN   \n",
       "800                              已經有埋密斗俾你慢慢改\\n呢啲車真係邊度搵   \n",
       "803  紅磡維港．星岸錄兩宗撻訂　料買家各損失樓價28%　涉逾2000萬撰文：黃祐樺出版：\\n\\n2...   \n",
       "804                                                好難頂   \n",
       "806                                             點跌都跌唔穿   \n",
       "\n",
       "                     Youtube_link  \\\n",
       "0                            None   \n",
       "3                            None   \n",
       "4                            None   \n",
       "6                            None   \n",
       "8                            None   \n",
       "..                            ...   \n",
       "799  https://youtu.be/65nfbW-27ps   \n",
       "800                          None   \n",
       "803                          None   \n",
       "804                          None   \n",
       "806                          None   \n",
       "\n",
       "                                              Comments Category Classification  \n",
       "0                        佢屋企人dum咗, \\n\\n有無巴打幫幫手, 李克勤可能有        6            體育台  \n",
       "3    我無票, 呢間小學算唔算藍店\\n\\n, 咁學校要出下力啦\\n拆左間校舍用黎起樓, 咁順便補祝...        5            時事台  \n",
       "4    , 咪撚燈啦屌 成日流流長, 琴日帶咗成日風話落大暴雨[核彈]\\n又話好大雞獲 要做足準備\\...        6            體育台  \n",
       "6    \\n\\n\\n\\n原文：, \\n\\n\\n\\n//陳芷尤（Apple）畀人嘅認識可能係擁有惹火身...        7            娛樂台  \n",
       "8    奶奶 香港講開係指 老公個阿媽\\n而蝗語奶奶 香港用係係阿嫲 \\n\\n同一原理媳婦 香港用開...        5            時事台  \n",
       "..                                                 ...      ...            ...  \n",
       "799           , 瞓咗, 成日唔準時點搶, 打個丁 聽朝又係新一天, YouTube, 終於到       30            感情台  \n",
       "800  , 運屍車？, 已經有埋密斗俾你慢慢改\\n呢啲車真係邊度搵\\n\\n, 都似係，重要出唔到牌,...       20            汽車台  \n",
       "803  紅磡維港．星岸錄兩宗撻訂　料買家各損失樓價28%　涉逾2000萬撰文：黃祐樺出版：\\n\\n2...       37            房屋台  \n",
       "804        壓力山大\\n\\n你們可以打我了, 學業?家庭?財政?感情?, 好難頂, 加油, 正垃圾       30            感情台  \n",
       "806  , Ok bye bye, 條線一路向後推, 好心你訓啦, , 雖然有D幾好笑，但有D ov...       15            財經台  \n",
       "\n",
       "[432 rows x 7 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv('merged_updated.csv')\n",
    "\n",
    "\n",
    "df = df[df['Category'] != 'None']\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "221b3c87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Main_content</th>\n",
       "      <th>Comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>想平平地$200收 2005年曼聯盃筆set限量版</td>\n",
       "      <td>有無巴打幫幫手</td>\n",
       "      <td>佢屋企人dum咗, \\n\\n有無巴打幫幫手, 李克勤可能有</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>深水埗某小學：祝賀李家超校友當選行政長官</td>\n",
       "      <td>呢間小學算唔算藍店</td>\n",
       "      <td>我無票, 呢間小學算唔算藍店\\n\\n, 咁學校要出下力啦\\n拆左間校舍用黎起樓, 咁順便補祝...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>天文台同cc台一樣，雷聲大雨點少[sosad]</td>\n",
       "      <td>琴日帶咗成日風話落大暴雨[核彈]\\n又話好大雞獲 要做足準備\\n點知原來得兩滴貨仔</td>\n",
       "      <td>, 咪撚燈啦屌 成日流流長, 琴日帶咗成日風話落大暴雨[核彈]\\n又話好大雞獲 要做足準備\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>【多圖】陳芷尤轉戰新加坡榮升女一撇甩「張振朗舊愛」稱號 比堅尼照大曬惹火身材</td>\n",
       "      <td>//陳芷尤（Apple）畀人嘅認識可能係擁有惹火身材嘅張振朗舊愛，喺無綫一直無發圍機會，演藝...</td>\n",
       "      <td>\\n\\n\\n\\n原文：, \\n\\n\\n\\n//陳芷尤（Apple）畀人嘅認識可能係擁有惹火身...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>唔好再講「地鐵」，係蝗語黎</td>\n",
       "      <td>就奇！\\n依一兩年係連登見到越黎越多蝗語特警亂鳩咁黎，\\n將一啲香港沿用多年、又或者係一啲書...</td>\n",
       "      <td>奶奶 香港講開係指 老公個阿媽\\n而蝗語奶奶 香港用係係阿嫲 \\n\\n同一原理媳婦 香港用開...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>P牌仔衰兩次疑問</td>\n",
       "      <td>佢本身上年p牌個陣衰左一次cut雙白，比人加監左半年，之後佢臨完p個一個月就比人斷正衝燈 前...</td>\n",
       "      <td>佢乜撚？我就我啦, 時辰未到姐 等運輸處信, 我都唔建議佢考, 到佢換正式牌個陣換唔到咪知洗...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>有無人個男朋友好思想好細路 唔成熟？</td>\n",
       "      <td>咁你地點解會同佢一齊？</td>\n",
       "      <td>唔重要啦已經 過左去, 唔算直接係既\\n之前有次冇睇message佢以為我睇左唔覆佢\\n佢嗰...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>[圖] 呢個紋身圍住手臂到底有咩意思？</td>\n",
       "      <td>唔通係⋯⋯</td>\n",
       "      <td>, 仲要黑肉不得了\\n\\n, not , Kevin, 啲人話紋呢個圈人都係鍾意玩主奴, 紋...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>有冇邊間電子錢包支援Debit card支付</td>\n",
       "      <td>好似間間都唔比咁\\n淨係可以加credit card\\n係咪間間夾埋</td>\n",
       "      <td>Citi plus 有呀, 既然同credit card無分別咁點解唔用credit car...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>痴撚線而家啲印尼黑幫惡到</td>\n",
       "      <td>我同我個friend去印尼帶貨\\n咁佢哋請我地去夜總會啦\\n無啦啦佢哋攞兩枝槍指住我個頭\\n...</td>\n",
       "      <td>, 做緊生意就認真啦，唔撚係玩玩下, 有聲既, blocked, 我同我個friend去印尼...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Title  \\\n",
       "0                想平平地$200收 2005年曼聯盃筆set限量版   \n",
       "3                     深水埗某小學：祝賀李家超校友當選行政長官   \n",
       "4                  天文台同cc台一樣，雷聲大雨點少[sosad]   \n",
       "6   【多圖】陳芷尤轉戰新加坡榮升女一撇甩「張振朗舊愛」稱號 比堅尼照大曬惹火身材   \n",
       "8                            唔好再講「地鐵」，係蝗語黎   \n",
       "14                                P牌仔衰兩次疑問   \n",
       "15                      有無人個男朋友好思想好細路 唔成熟？   \n",
       "16                     [圖] 呢個紋身圍住手臂到底有咩意思？   \n",
       "17                  有冇邊間電子錢包支援Debit card支付   \n",
       "18                            痴撚線而家啲印尼黑幫惡到   \n",
       "\n",
       "                                         Main_content  \\\n",
       "0                                             有無巴打幫幫手   \n",
       "3                                           呢間小學算唔算藍店   \n",
       "4           琴日帶咗成日風話落大暴雨[核彈]\\n又話好大雞獲 要做足準備\\n點知原來得兩滴貨仔   \n",
       "6   //陳芷尤（Apple）畀人嘅認識可能係擁有惹火身材嘅張振朗舊愛，喺無綫一直無發圍機會，演藝...   \n",
       "8   就奇！\\n依一兩年係連登見到越黎越多蝗語特警亂鳩咁黎，\\n將一啲香港沿用多年、又或者係一啲書...   \n",
       "14  佢本身上年p牌個陣衰左一次cut雙白，比人加監左半年，之後佢臨完p個一個月就比人斷正衝燈 前...   \n",
       "15                                        咁你地點解會同佢一齊？   \n",
       "16                                              唔通係⋯⋯   \n",
       "17                 好似間間都唔比咁\\n淨係可以加credit card\\n係咪間間夾埋   \n",
       "18  我同我個friend去印尼帶貨\\n咁佢哋請我地去夜總會啦\\n無啦啦佢哋攞兩枝槍指住我個頭\\n...   \n",
       "\n",
       "                                             Comments  \n",
       "0                       佢屋企人dum咗, \\n\\n有無巴打幫幫手, 李克勤可能有  \n",
       "3   我無票, 呢間小學算唔算藍店\\n\\n, 咁學校要出下力啦\\n拆左間校舍用黎起樓, 咁順便補祝...  \n",
       "4   , 咪撚燈啦屌 成日流流長, 琴日帶咗成日風話落大暴雨[核彈]\\n又話好大雞獲 要做足準備\\...  \n",
       "6   \\n\\n\\n\\n原文：, \\n\\n\\n\\n//陳芷尤（Apple）畀人嘅認識可能係擁有惹火身...  \n",
       "8   奶奶 香港講開係指 老公個阿媽\\n而蝗語奶奶 香港用係係阿嫲 \\n\\n同一原理媳婦 香港用開...  \n",
       "14  佢乜撚？我就我啦, 時辰未到姐 等運輸處信, 我都唔建議佢考, 到佢換正式牌個陣換唔到咪知洗...  \n",
       "15  唔重要啦已經 過左去, 唔算直接係既\\n之前有次冇睇message佢以為我睇左唔覆佢\\n佢嗰...  \n",
       "16  , 仲要黑肉不得了\\n\\n, not , Kevin, 啲人話紋呢個圈人都係鍾意玩主奴, 紋...  \n",
       "17  Citi plus 有呀, 既然同credit card無分別咁點解唔用credit car...  \n",
       "18  , 做緊生意就認真啦，唔撚係玩玩下, 有聲既, blocked, 我同我個friend去印尼...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 提取廣東話內容的columns\n",
    "\n",
    "df_content = df[['Title','Main_content','Comments']]\n",
    "df_content.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f04e3ae3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2777159090.py:28: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Title'] = df_content['Title'].apply(preprocess_text)\n",
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2777159090.py:29: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Main_content'] = df_content['Main_content'].apply(preprocess_text)\n",
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2777159090.py:30: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Comments'] = df_content['Comments'].apply(preprocess_text)\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "def preprocess_text(text):\n",
    "    if isinstance(text, str):  # 檢查 text 是否為字符串類型\n",
    "        text = re.sub(r'[^\\w\\s]', '', text)\n",
    "        text = text.replace('\\n', '')\n",
    "        \n",
    "        # Stop Word Removal\n",
    "        stop_words = []  # Stop word list, customized according to needs\n",
    "        words = [word for word in text if word not in stop_words]\n",
    "        \n",
    "        # Word Segmentation\n",
    "        words = list(text)\n",
    "        \n",
    "        # Text Cleaning\n",
    "        words = [re.sub(r'[^\\u4e00-\\u9fff\\s]', '', word) for word in words]\n",
    "        \n",
    "        # Lowercasing\n",
    "        words = [word.lower() for word in words]\n",
    "        \n",
    "        # Return preprocessed text\n",
    "        processed_text = ' '.join(words)\n",
    "        return processed_text\n",
    "    else:\n",
    "        return ''  # 非字符串類型返回空字符串\n",
    "    \n",
    "# apply function\n",
    "df_content['Title'] = df_content['Title'].apply(preprocess_text)\n",
    "df_content['Main_content'] = df_content['Main_content'].apply(preprocess_text)\n",
    "df_content['Comments'] = df_content['Comments'].apply(preprocess_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3f0d71e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Main_content</th>\n",
       "      <th>Comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>想 平 平 地    收       年 曼 聯 盃 筆    限 量 版</td>\n",
       "      <td>有 無 巴 打 幫 幫 手</td>\n",
       "      <td>佢 屋 企 人    咗   有 無 巴 打 幫 幫 手   李 克 勤 可 能 有</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>深 水 埗 某 小 學 祝 賀 李 家 超 校 友 當 選 行 政 長 官</td>\n",
       "      <td>呢 間 小 學 算 唔 算 藍 店</td>\n",
       "      <td>我 無 票   呢 間 小 學 算 唔 算 藍 店   咁 學 校 要 出 下 力 啦 拆 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>天 文 台 同   台 一 樣 雷 聲 大 雨 點 少</td>\n",
       "      <td>琴 日 帶 咗 成 日 風 話 落 大 暴 雨 核 彈 又 話 好 大 雞 獲   要 做 ...</td>\n",
       "      <td>咪 撚 燈 啦 屌   成 日 流 流 長   琴 日 帶 咗 成 日 風 話 落 大 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>多 圖 陳 芷 尤 轉 戰 新 加 坡 榮 升 女 一 撇 甩 張 振 朗 舊 愛 稱 號 ...</td>\n",
       "      <td>陳 芷 尤      畀 人 嘅 認 識 可 能 係 擁 有 惹 火 身 材 嘅 張 振 朗...</td>\n",
       "      <td>原 文   陳 芷 尤      畀 人 嘅 認 識 可 能 係 擁 有 惹 火 身 材 嘅...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>唔 好 再 講 地 鐵 係 蝗 語 黎</td>\n",
       "      <td>就 奇 依 一 兩 年 係 連 登 見 到 越 黎 越 多 蝗 語 特 警 亂 鳩 咁 黎 ...</td>\n",
       "      <td>奶 奶   香 港 講 開 係 指   老 公 個 阿 媽 而 蝗 語 奶 奶   香 港 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799</th>\n",
       "      <td>人 生 冇 希 望</td>\n",
       "      <td></td>\n",
       "      <td>瞓 咗   成 日 唔 準 時 點 搶   打 個 丁   聽 朝 又 係 新 一 天 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>800</th>\n",
       "      <td>二 萬 八 蚊 可 以 買 架 車 改 裝 車 中 泊</td>\n",
       "      <td>已 經 有 埋 密 斗 俾 你 慢 慢 改 呢 啲 車 真 係 邊 度 搵</td>\n",
       "      <td>運 屍 車   已 經 有 埋 密 斗 俾 你 慢 慢 改 呢 啲 車 真 係 邊 度 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>803</th>\n",
       "      <td>紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 　 料 買 家 各 損 失 樓 價</td>\n",
       "      <td>紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 　 料 買 家 各 損 失 樓 價   　 涉 ...</td>\n",
       "      <td>紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 　 料 買 家 各 損 失 樓 價   　 涉 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>804</th>\n",
       "      <td>壓 力 好 大</td>\n",
       "      <td>好 難 頂</td>\n",
       "      <td>壓 力 山 大 你 們 可 以 打 我 了   學 業 家 庭 財 政 感 情   好 難 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>806</th>\n",
       "      <td>納 指      點 同 馬 奇 諾 防 線 一 樣 堅 不 可 摧</td>\n",
       "      <td>點 跌 都 跌 唔 穿</td>\n",
       "      <td>條 線 一 路 向 後 推   好 心 你 訓 啦     ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>432 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Title  \\\n",
       "0                想 平 平 地    收       年 曼 聯 盃 筆    限 量 版   \n",
       "3                深 水 埗 某 小 學 祝 賀 李 家 超 校 友 當 選 行 政 長 官   \n",
       "4                     天 文 台 同   台 一 樣 雷 聲 大 雨 點 少        \n",
       "6    多 圖 陳 芷 尤 轉 戰 新 加 坡 榮 升 女 一 撇 甩 張 振 朗 舊 愛 稱 號 ...   \n",
       "8                                  唔 好 再 講 地 鐵 係 蝗 語 黎   \n",
       "..                                                 ...   \n",
       "799                                          人 生 冇 希 望   \n",
       "800                        二 萬 八 蚊 可 以 買 架 車 改 裝 車 中 泊   \n",
       "803          紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 　 料 買 家 各 損 失 樓 價     \n",
       "804                                            壓 力 好 大   \n",
       "806                 納 指      點 同 馬 奇 諾 防 線 一 樣 堅 不 可 摧   \n",
       "\n",
       "                                          Main_content  \\\n",
       "0                                        有 無 巴 打 幫 幫 手   \n",
       "3                                    呢 間 小 學 算 唔 算 藍 店   \n",
       "4    琴 日 帶 咗 成 日 風 話 落 大 暴 雨 核 彈 又 話 好 大 雞 獲   要 做 ...   \n",
       "6    陳 芷 尤      畀 人 嘅 認 識 可 能 係 擁 有 惹 火 身 材 嘅 張 振 朗...   \n",
       "8    就 奇 依 一 兩 年 係 連 登 見 到 越 黎 越 多 蝗 語 特 警 亂 鳩 咁 黎 ...   \n",
       "..                                                 ...   \n",
       "799                                                      \n",
       "800              已 經 有 埋 密 斗 俾 你 慢 慢 改 呢 啲 車 真 係 邊 度 搵   \n",
       "803  紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 　 料 買 家 各 損 失 樓 價   　 涉 ...   \n",
       "804                                              好 難 頂   \n",
       "806                                        點 跌 都 跌 唔 穿   \n",
       "\n",
       "                                              Comments  \n",
       "0           佢 屋 企 人    咗   有 無 巴 打 幫 幫 手   李 克 勤 可 能 有  \n",
       "3    我 無 票   呢 間 小 學 算 唔 算 藍 店   咁 學 校 要 出 下 力 啦 拆 ...  \n",
       "4      咪 撚 燈 啦 屌   成 日 流 流 長   琴 日 帶 咗 成 日 風 話 落 大 ...  \n",
       "6    原 文   陳 芷 尤      畀 人 嘅 認 識 可 能 係 擁 有 惹 火 身 材 嘅...  \n",
       "8    奶 奶   香 港 講 開 係 指   老 公 個 阿 媽 而 蝗 語 奶 奶   香 港 ...  \n",
       "..                                                 ...  \n",
       "799    瞓 咗   成 日 唔 準 時 點 搶   打 個 丁   聽 朝 又 係 新 一 天 ...  \n",
       "800    運 屍 車   已 經 有 埋 密 斗 俾 你 慢 慢 改 呢 啲 車 真 係 邊 度 ...  \n",
       "803  紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 　 料 買 家 各 損 失 樓 價   　 涉 ...  \n",
       "804  壓 力 山 大 你 們 可 以 打 我 了   學 業 家 庭 財 政 感 情   好 難 ...  \n",
       "806                  條 線 一 路 向 後 推   好 心 你 訓 啦     ...  \n",
       "\n",
       "[432 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "96b87995",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2186064603.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Title'] = df_content['Title'].astype(str)\n",
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2186064603.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Main_content'] = df_content['Main_content'].astype(str)\n",
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2186064603.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Comments'] = df_content['Comments'].astype(str)\n",
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2186064603.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Title'] = df_content['Title'].apply(lambda x: ' '.join([word for word in re.split('\\W+', x) if word.lower() not in stop_words]))\n",
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2186064603.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Main_content'] = df_content['Main_content'].apply(lambda x: ' '.join([word for word in re.split('\\W+', x) if word.lower() not in stop_words]))\n",
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2186064603.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Comments'] = df_content['Comments'].apply(lambda x: ' '.join([word for word in re.split('\\W+', x) if word.lower() not in stop_words]))\n"
     ]
    }
   ],
   "source": [
    "## 去除廣東話常用詞 (stop words)\n",
    "\n",
    "df_content['Title'] = df_content['Title'].astype(str)\n",
    "df_content['Main_content'] = df_content['Main_content'].astype(str)\n",
    "df_content['Comments'] = df_content['Comments'].astype(str)\n",
    "\n",
    "df_content['Title'] = df_content['Title'].apply(lambda x: ' '.join([word for word in re.split('\\W+', x) if word.lower() not in stop_words]))\n",
    "df_content['Main_content'] = df_content['Main_content'].apply(lambda x: ' '.join([word for word in re.split('\\W+', x) if word.lower() not in stop_words]))\n",
    "df_content['Comments'] = df_content['Comments'].apply(lambda x: ' '.join([word for word in re.split('\\W+', x) if word.lower() not in stop_words]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b684d236",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Main_content</th>\n",
       "      <th>Comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>平 平 地 收 年 曼 聯 盃 筆 限 量 版</td>\n",
       "      <td>無 巴 打 幫 幫 手</td>\n",
       "      <td>屋 企 人 無 巴 打 幫 幫 手 李 克 勤 可 能</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>深 水 埗 某 小 學 祝 賀 李 家 超 校 友 當 選 行 政 長 官</td>\n",
       "      <td>間 小 學 算 算 藍 店</td>\n",
       "      <td>無 票 間 小 學 算 算 藍 店 學 校 出 下 力 啦 拆 左 間 校 舍 用 黎 起 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>天 文 台 台 一 樣 雷 聲 大 雨 少</td>\n",
       "      <td>琴 日 帶 成 日 風 落 大 暴 雨 核 彈 大 雞 獲 做 足 準 備 原 來 兩 滴 貨 仔</td>\n",
       "      <td>撚 燈 啦 屌 成 日 流 流 長 琴 日 帶 成 日 風 落 大 暴 雨 核 彈 大 雞...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>圖 陳 芷 尤 轉 戰 新 加 坡 榮 升 女 一 撇 甩 張 振 朗 舊 愛 稱 號 比 ...</td>\n",
       "      <td>陳 芷 尤 畀 人 認 識 可 能 擁 惹 火 身 材 張 振 朗 舊 愛 無 綫 一 直 ...</td>\n",
       "      <td>原 文 陳 芷 尤 畀 人 認 識 可 能 擁 惹 火 身 材 張 振 朗 舊 愛 無 綫 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>地 鐵 蝗 語 黎</td>\n",
       "      <td>奇 依 一 兩 年 連 登 見 越 黎 越 蝗 語 特 警 亂 鳩 黎 將 一 香 港 沿 ...</td>\n",
       "      <td>奶 奶 香 港 開 指 老 公 阿 媽 蝗 語 奶 奶 香 港 用 阿 嫲 一 原 理 媳 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799</th>\n",
       "      <td>人 生 希 望</td>\n",
       "      <td></td>\n",
       "      <td>瞓 成 日 準 時 搶 打 丁 聽 朝 新 一 天 終 於</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>800</th>\n",
       "      <td>二 萬 八 蚊 可 以 買 架 車 改 裝 車 中 泊</td>\n",
       "      <td>已 經 埋 密 斗 俾 慢 慢 改 車 真 邊 度 搵</td>\n",
       "      <td>運 屍 車 已 經 埋 密 斗 俾 慢 慢 改 車 真 邊 度 搵 似 重 出 牌 無 出...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>803</th>\n",
       "      <td>紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 料 買 家 各 損 失 樓 價</td>\n",
       "      <td>紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 料 買 家 各 損 失 樓 價 涉 逾 萬 撰 ...</td>\n",
       "      <td>紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 料 買 家 各 損 失 樓 價 涉 逾 萬 撰 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>804</th>\n",
       "      <td>壓 力 大</td>\n",
       "      <td>難 頂</td>\n",
       "      <td>壓 力 山 大 們 可 以 打 了 學 業 家 庭 財 政 感 情 難 頂 加 油 正 垃 圾</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>806</th>\n",
       "      <td>納 指 馬 奇 諾 防 線 一 樣 堅 不 可 摧</td>\n",
       "      <td>跌 跌 穿</td>\n",
       "      <td>條 線 一 路 向 後 推 心 訓 啦 雖 然 笑 但 左 力 跌 了 馬 奇 諾 防 線...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>432 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Title  \\\n",
       "0                              平 平 地 收 年 曼 聯 盃 筆 限 量 版   \n",
       "3                深 水 埗 某 小 學 祝 賀 李 家 超 校 友 當 選 行 政 長 官   \n",
       "4                               天 文 台 台 一 樣 雷 聲 大 雨 少    \n",
       "6    圖 陳 芷 尤 轉 戰 新 加 坡 榮 升 女 一 撇 甩 張 振 朗 舊 愛 稱 號 比 ...   \n",
       "8                                            地 鐵 蝗 語 黎   \n",
       "..                                                 ...   \n",
       "799                                            人 生 希 望   \n",
       "800                        二 萬 八 蚊 可 以 買 架 車 改 裝 車 中 泊   \n",
       "803             紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 料 買 家 各 損 失 樓 價    \n",
       "804                                              壓 力 大   \n",
       "806                          納 指 馬 奇 諾 防 線 一 樣 堅 不 可 摧   \n",
       "\n",
       "                                          Main_content  \\\n",
       "0                                          無 巴 打 幫 幫 手   \n",
       "3                                        間 小 學 算 算 藍 店   \n",
       "4    琴 日 帶 成 日 風 落 大 暴 雨 核 彈 大 雞 獲 做 足 準 備 原 來 兩 滴 貨 仔   \n",
       "6    陳 芷 尤 畀 人 認 識 可 能 擁 惹 火 身 材 張 振 朗 舊 愛 無 綫 一 直 ...   \n",
       "8    奇 依 一 兩 年 連 登 見 越 黎 越 蝗 語 特 警 亂 鳩 黎 將 一 香 港 沿 ...   \n",
       "..                                                 ...   \n",
       "799                                                      \n",
       "800                        已 經 埋 密 斗 俾 慢 慢 改 車 真 邊 度 搵   \n",
       "803  紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 料 買 家 各 損 失 樓 價 涉 逾 萬 撰 ...   \n",
       "804                                                難 頂   \n",
       "806                                              跌 跌 穿   \n",
       "\n",
       "                                              Comments  \n",
       "0                          屋 企 人 無 巴 打 幫 幫 手 李 克 勤 可 能  \n",
       "3    無 票 間 小 學 算 算 藍 店 學 校 出 下 力 啦 拆 左 間 校 舍 用 黎 起 ...  \n",
       "4     撚 燈 啦 屌 成 日 流 流 長 琴 日 帶 成 日 風 落 大 暴 雨 核 彈 大 雞...  \n",
       "6    原 文 陳 芷 尤 畀 人 認 識 可 能 擁 惹 火 身 材 張 振 朗 舊 愛 無 綫 ...  \n",
       "8    奶 奶 香 港 開 指 老 公 阿 媽 蝗 語 奶 奶 香 港 用 阿 嫲 一 原 理 媳 ...  \n",
       "..                                                 ...  \n",
       "799                      瞓 成 日 準 時 搶 打 丁 聽 朝 新 一 天 終 於  \n",
       "800   運 屍 車 已 經 埋 密 斗 俾 慢 慢 改 車 真 邊 度 搵 似 重 出 牌 無 出...  \n",
       "803  紅 磡 維 港 星 岸 錄 兩 宗 撻 訂 料 買 家 各 損 失 樓 價 涉 逾 萬 撰 ...  \n",
       "804    壓 力 山 大 們 可 以 打 了 學 業 家 庭 財 政 感 情 難 頂 加 油 正 垃 圾  \n",
       "806   條 線 一 路 向 後 推 心 訓 啦 雖 然 笑 但 左 力 跌 了 馬 奇 諾 防 線...  \n",
       "\n",
       "[432 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0ce6a9db",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2008976353.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Main_content'] = df_content['Main_content'].apply(lambda x: characters_to_jyutping(x) if isinstance(x, str) else x)\n",
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2008976353.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Title'] = df_content['Title'].apply(lambda x: characters_to_jyutping(x) if isinstance(x, str) else x)\n",
      "C:\\Users\\yeung\\AppData\\Local\\Temp\\ipykernel_21212\\2008976353.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_content['Comments'] = df_content['Comments'].apply(lambda x: characters_to_jyutping(x) if isinstance(x, str) else x)\n"
     ]
    }
   ],
   "source": [
    "# 同時為句子進行分詞和加入拼音\n",
    "\n",
    "import pandas as pd\n",
    "from pycantonese import characters_to_jyutping\n",
    "\n",
    "\n",
    "# 在指定列上應用characters_to_jyutping函數\n",
    "df_content['Main_content'] = df_content['Main_content'].apply(lambda x: characters_to_jyutping(x) if isinstance(x, str) else x)\n",
    "df_content['Title'] = df_content['Title'].apply(lambda x: characters_to_jyutping(x) if isinstance(x, str) else x)\n",
    "df_content['Comments'] = df_content['Comments'].apply(lambda x: characters_to_jyutping(x) if isinstance(x, str) else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "677b6041",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Main_content</th>\n",
       "      <th>Comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[(平平, ping4ping4), (地, dei6), (收, sau1), (年, n...</td>\n",
       "      <td>[(無, mou5), (巴打, baa1daa2), (幫, bong1), (幫手, b...</td>\n",
       "      <td>[(屋企人, uk1kei2jan4), (無, mou5), (巴打, baa1daa2)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[(深水埗, sam1seoi2bou2), (某, mau5), (小學, siu2hok...</td>\n",
       "      <td>[(間, gaan1), (小學, siu2hok6), (算, syun3), (算, s...</td>\n",
       "      <td>[(無, mou5), (票, piu3), (間, gaan1), (小學, siu2ho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[(天文, tin1man4), (台, toi4), (台, toi4), (一樣, ja...</td>\n",
       "      <td>[(琴日, kam4jat6), (帶, daai3), (成日, seng4jat6), ...</td>\n",
       "      <td>[(撚, nan2), (燈, dang1), (啦, la1), (屌, diu2), (...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[(圖, tou4), (陳, can4), (芷, zi2), (尤, jau4), (轉...</td>\n",
       "      <td>[(陳, can4), (芷, zi2), (尤, jau4), (畀人, bei2jan4...</td>\n",
       "      <td>[(原文, jyun4man4), (陳, can4), (芷, zi2), (尤, jau...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[(地鐵, dei6tit3), (蝗, wong4), (語, jyu5), (黎, la...</td>\n",
       "      <td>[(奇, kei4), (依, ji1), (一, jat1), (兩, loeng5), ...</td>\n",
       "      <td>[(奶奶, naai1naai1), (香港, hoeng1gong2), (開, hoi1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799</th>\n",
       "      <td>[(人生, jan4sang1), (希望, hei1mong6)]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[(瞓, fan3), (成日, seng4jat6), (準時, zeon2si4), (...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>800</th>\n",
       "      <td>[(二萬, ji6maan6), (八, baat3), (蚊, man1), (可以, h...</td>\n",
       "      <td>[(已經, ji5ging1), (埋, maai4), (密斗, mat6dau2), (...</td>\n",
       "      <td>[(運, wan6), (屍, si1), (車, ce1), (已經, ji5ging1)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>803</th>\n",
       "      <td>[(紅磡, hung4ham3), (維港, wai4gong2), (星, sing1),...</td>\n",
       "      <td>[(紅磡, hung4ham3), (維港, wai4gong2), (星, sing1),...</td>\n",
       "      <td>[(紅磡, hung4ham3), (維港, wai4gong2), (星, sing1),...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>804</th>\n",
       "      <td>[(壓力, aat3lik6), (大, daai6)]</td>\n",
       "      <td>[(難頂, naan4ding2)]</td>\n",
       "      <td>[(壓力, aat3lik6), (山, saan1), (大, daai6), (們, m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>806</th>\n",
       "      <td>[(納指, naap6zi2), (馬, maa5), (奇, kei4), (諾, nok...</td>\n",
       "      <td>[(跌, dit3), (跌, dit3), (穿, cyun1)]</td>\n",
       "      <td>[(條線, tiu4sin3), (一路, jat1lou6), (向後, hoeng3ha...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>432 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Title  \\\n",
       "0    [(平平, ping4ping4), (地, dei6), (收, sau1), (年, n...   \n",
       "3    [(深水埗, sam1seoi2bou2), (某, mau5), (小學, siu2hok...   \n",
       "4    [(天文, tin1man4), (台, toi4), (台, toi4), (一樣, ja...   \n",
       "6    [(圖, tou4), (陳, can4), (芷, zi2), (尤, jau4), (轉...   \n",
       "8    [(地鐵, dei6tit3), (蝗, wong4), (語, jyu5), (黎, la...   \n",
       "..                                                 ...   \n",
       "799                 [(人生, jan4sang1), (希望, hei1mong6)]   \n",
       "800  [(二萬, ji6maan6), (八, baat3), (蚊, man1), (可以, h...   \n",
       "803  [(紅磡, hung4ham3), (維港, wai4gong2), (星, sing1),...   \n",
       "804                       [(壓力, aat3lik6), (大, daai6)]   \n",
       "806  [(納指, naap6zi2), (馬, maa5), (奇, kei4), (諾, nok...   \n",
       "\n",
       "                                          Main_content  \\\n",
       "0    [(無, mou5), (巴打, baa1daa2), (幫, bong1), (幫手, b...   \n",
       "3    [(間, gaan1), (小學, siu2hok6), (算, syun3), (算, s...   \n",
       "4    [(琴日, kam4jat6), (帶, daai3), (成日, seng4jat6), ...   \n",
       "6    [(陳, can4), (芷, zi2), (尤, jau4), (畀人, bei2jan4...   \n",
       "8    [(奇, kei4), (依, ji1), (一, jat1), (兩, loeng5), ...   \n",
       "..                                                 ...   \n",
       "799                                                 []   \n",
       "800  [(已經, ji5ging1), (埋, maai4), (密斗, mat6dau2), (...   \n",
       "803  [(紅磡, hung4ham3), (維港, wai4gong2), (星, sing1),...   \n",
       "804                                 [(難頂, naan4ding2)]   \n",
       "806                 [(跌, dit3), (跌, dit3), (穿, cyun1)]   \n",
       "\n",
       "                                              Comments  \n",
       "0    [(屋企人, uk1kei2jan4), (無, mou5), (巴打, baa1daa2)...  \n",
       "3    [(無, mou5), (票, piu3), (間, gaan1), (小學, siu2ho...  \n",
       "4    [(撚, nan2), (燈, dang1), (啦, la1), (屌, diu2), (...  \n",
       "6    [(原文, jyun4man4), (陳, can4), (芷, zi2), (尤, jau...  \n",
       "8    [(奶奶, naai1naai1), (香港, hoeng1gong2), (開, hoi1...  \n",
       "..                                                 ...  \n",
       "799  [(瞓, fan3), (成日, seng4jat6), (準時, zeon2si4), (...  \n",
       "800  [(運, wan6), (屍, si1), (車, ce1), (已經, ji5ging1)...  \n",
       "803  [(紅磡, hung4ham3), (維港, wai4gong2), (星, sing1),...  \n",
       "804  [(壓力, aat3lik6), (山, saan1), (大, daai6), (們, m...  \n",
       "806  [(條線, tiu4sin3), (一路, jat1lou6), (向後, hoeng3ha...  \n",
       "\n",
       "[432 rows x 3 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c436b41e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_content.to_csv('preprocessed_data.csv', index=False, encoding='utf-8-sig')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}