PCFG_learn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#-*- coding: utf-8 -*-\n",
    "\n",
    "import os\n",
    "import codecs\n",
    "import random\n",
    "import pickle\n",
    "from time import time\n",
    "import numpy as np\n",
    "import string\n",
    "import re\n",
    "import nltk\n",
    "from collections import defaultdict\n",
    "from nltk.corpus.reader.bracket_parse import BracketParseCorpusReader\n",
    "\n",
    "PCFG_UNARY_RULES_FREQ_FILE = os.path.join(\"model\", \"PCFG_unary_freq.pkl\")\n",
    "PCFG_BINARY_RULES_FREQ_FILE = os.path.join(\"model\", \"PCFG_binary_freq.pkl\")\n",
    "PCFG_POSTAGS_FREQ_FILE = os.path.join(\"model\", \"PCFG_postags_freq.pkl\")\n",
    "PCFG_UNARY_RULES_DICT_FILE = os.path.join(\"model\", \"PCFG_unary_dict.pkl\")\n",
    "PCFG_BINARY_RULES_DICT_FILE = os.path.join(\"model\", \"PCFG_binary_dict.pkl\")\n",
    "PCFG_POSTAGS_DICT_FILE = os.path.join(\"model\", \"PCFG_postags_dict.pkl\")\n",
    "PCFG_NT_SET_FILE = os.path.join(\"model\", \"PCFG_non_terminals_set.pkl\")\n",
    "PCFG_T_SET_FILE = os.path.join(\"model\", \"PCFG_terminals_set.pkl\")\n",
    "PCFG_POSTAGS_SET_FILE = os.path.join(\"model\", \"PCFG_postags_set.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> Reading corpus treebanks from file...\n",
      ">>> Corpus treebanks read done in 0.002s.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "t0 = time()\n",
    "print (\">>> Reading corpus treebanks from file...\")\n",
    "\n",
    "corpus_root = r\"corpus\"\n",
    "train_file_pattern = r\".*_train.tb\"\n",
    "\n",
    "ptb_train = BracketParseCorpusReader(corpus_root, train_file_pattern)\n",
    "\n",
    "print (\">>> Corpus treebanks read done in %0.3fs.\\n\" % (time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> Parsing collection of rules and words...\n",
      "Size of dictionary: 9094, of which 5017 are rare words\n",
      "Number of word occurrances: 53453\n",
      "Number of POS tags: 29\n",
      "\n",
      "Size of unary rules: 40\n",
      "Number of unary rule occurrances: 8588\n",
      "Number of unary lhs: 10\n",
      "Number of unary rhs: 26\n",
      "\n",
      "Size of binary rules: 2918\n",
      "Number of binary rule occurrances: 50974\n",
      "Number of binary lhs: 741\n",
      "Number of binary rhs: 2101\n",
      "\n",
      "Number of non-terminal symbols: 741\n",
      "Number of terminal symbols: 4078\n",
      "\n",
      ">>> Collection of rules and words parsing done in 4.359s.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "t0 = time()\n",
    "print (\">>> Parsing collection of rules and words...\")\n",
    "\n",
    "# Objects for unary rules (A -> B)\n",
    "unary_rules_freq = defaultdict(float)\n",
    "unary_rules_cnt_by_lhs = defaultdict(int)\n",
    "unary_rules_occur_cnt = 0\n",
    "unary_lhs_set = set()\n",
    "unary_rhs_set = set()\n",
    "\n",
    "# Objects for binary rules (A -> BC)\n",
    "binary_rules_freq = defaultdict(float)\n",
    "binary_rules_cnt_by_lhs = defaultdict(int)\n",
    "binary_rules_occur_cnt = 0\n",
    "binary_lhs_set = set()\n",
    "binary_rhs_set = set()\n",
    "\n",
    "# Objects for terminal rules (POS -> <word>)\n",
    "postags_freq = defaultdict(float)\n",
    "postags_cnt_by_pos = defaultdict(int)\n",
    "postags_occur_cnt = 0\n",
    "words_occur_cnt = defaultdict(int)\n",
    "postags_set = set()\n",
    "words_set = set()\n",
    "\n",
    "trees = ptb_train.parsed_sents()\n",
    "for tree in trees:\n",
    "    t = tree.copy()\n",
    "    t.chomsky_normal_form(horzMarkov=2)\n",
    "    #t.collapse_unary(collapsePOS=True, collapseRoot=False)\n",
    "    prods = t.productions()\n",
    "    for prod in prods:\n",
    "        lhs = prod.lhs().symbol()\n",
    "        rh = prod.rhs()\n",
    "        #rhs = ' '.join([r.symbol() if isinstance(r, nltk.grammar.Nonterminal) else r for r in rh])\n",
    "        if isinstance(rh[0], unicode): # Ternimal production (POS -> <word>)\n",
    "            rhs = rh[0]\n",
    "            postags_freq[(lhs, rhs)] += 1\n",
    "            postags_cnt_by_pos[lhs] += 1\n",
    "            postags_occur_cnt += 1\n",
    "            words_occur_cnt[rhs] += 1\n",
    "            postags_set.add(lhs)\n",
    "            words_set.add(rhs)\n",
    "        else: # Non-terminal production (A -> BC | A -> B)\n",
    "            if len(rh) == 1: # Unary production (A -> B)    \n",
    "                rhs = rh[0].symbol()\n",
    "                unary_rules_freq[(lhs, rhs)] += 1\n",
    "                unary_rules_cnt_by_lhs[lhs] += 1\n",
    "                unary_rules_occur_cnt += 1\n",
    "                unary_lhs_set.add(lhs)\n",
    "                unary_rhs_set.add(rhs)\n",
    "            elif len(rh) == 2:\n",
    "                rhs = tuple([nt.symbol() for nt in rh])\n",
    "                binary_rules_freq[(lhs, rhs)] += 1\n",
    "                binary_rules_cnt_by_lhs[lhs] += 1\n",
    "                binary_rules_occur_cnt += 1\n",
    "                binary_lhs_set.add(lhs)\n",
    "                binary_rhs_set.add(rhs)\n",
    "\n",
    "# Replace rare words in the postags_freq with '<UNK>'\n",
    "rare_words = set([w for w in words_set if words_occur_cnt[w] < 2])\n",
    "T_set = words_set.copy()\n",
    "T_set.difference_update(rare_words)\n",
    "T_set.add(u\"<UNK>\")\n",
    "\n",
    "pw_pairs = list(postags_freq.keys())\n",
    "for (pos, w) in pw_pairs:\n",
    "    if w in rare_words:\n",
    "        postags_freq[(pos, u\"<UNK>\")] += postags_freq[(pos, w)]\n",
    "        postags_freq.pop((pos, w))\n",
    "\n",
    "# Normalization to ensure that the sum of outgoing weights is 1 for each NT node\n",
    "for (pos, w) in postags_freq:\n",
    "    postags_freq[(pos, w)] /= postags_cnt_by_pos[pos]\n",
    "\n",
    "for (lhs, rhs) in unary_rules_freq:\n",
    "    unary_rules_freq[(lhs, rhs)] /= (unary_rules_cnt_by_lhs[lhs] + binary_rules_cnt_by_lhs[lhs])\n",
    "    \n",
    "for (lhs, rhs) in binary_rules_freq:\n",
    "    binary_rules_freq[(lhs, rhs)] /= (binary_rules_cnt_by_lhs[lhs] + unary_rules_cnt_by_lhs[lhs])\n",
    "\n",
    "with codecs.open(PCFG_UNARY_RULES_FREQ_FILE, 'wb') as f:\n",
    "    pickle.dump(unary_rules_freq, f)\n",
    "f.close()\n",
    "\n",
    "with codecs.open(PCFG_BINARY_RULES_FREQ_FILE, 'wb') as f:\n",
    "    pickle.dump(binary_rules_freq, f)\n",
    "f.close()\n",
    "\n",
    "with codecs.open(PCFG_POSTAGS_FREQ_FILE, 'wb') as f:\n",
    "    pickle.dump(postags_freq, f)\n",
    "f.close()\n",
    "\n",
    "# Construct the rhs -> lhs dictionary for quick parent lookup in CYK algorithm\n",
    "unary_rules_dict = {}\n",
    "binary_rules_dict = {}\n",
    "postags_dict = {}\n",
    "\n",
    "for rhs in unary_rhs_set:\n",
    "    unary_rules_dict[rhs] = {}\n",
    "for (lhs, rhs) in unary_rules_freq:\n",
    "    unary_rules_dict[rhs][lhs] = unary_rules_freq[(lhs, rhs)]\n",
    "    \n",
    "for rhs in binary_rhs_set:\n",
    "    binary_rules_dict[rhs] = {}\n",
    "for (lhs, rhs) in binary_rules_freq:\n",
    "    binary_rules_dict[rhs][lhs] = binary_rules_freq[(lhs, rhs)]\n",
    "    \n",
    "for w in T_set:\n",
    "    postags_dict[w] = {}\n",
    "for (pos, w) in postags_freq:\n",
    "    postags_dict[w][pos] = postags_freq[(pos, w)]\n",
    "\n",
    "with codecs.open(PCFG_UNARY_RULES_DICT_FILE, 'wb') as f:\n",
    "    pickle.dump(unary_rules_dict, f)\n",
    "f.close()\n",
    "\n",
    "with codecs.open(PCFG_BINARY_RULES_DICT_FILE, 'wb') as f:\n",
    "    pickle.dump(binary_rules_dict, f)\n",
    "f.close()\n",
    "\n",
    "with codecs.open(PCFG_POSTAGS_DICT_FILE, 'wb') as f:\n",
    "    pickle.dump(postags_dict, f)\n",
    "f.close()\n",
    "\n",
    "# Store the set of non-terminals and terminals\n",
    "NT_set = unary_lhs_set.union(binary_lhs_set)\n",
    "\n",
    "with codecs.open(PCFG_NT_SET_FILE, 'wb') as f:\n",
    "    pickle.dump(NT_set, f)\n",
    "f.close()\n",
    "\n",
    "with codecs.open(PCFG_T_SET_FILE, 'wb') as f:\n",
    "    pickle.dump(T_set, f)\n",
    "f.close()\n",
    "\n",
    "with codecs.open(PCFG_POSTAGS_SET_FILE, 'wb') as f:\n",
    "    pickle.dump(postags_set, f)\n",
    "f.close()\n",
    "        \n",
    "print (\"Size of dictionary: %d, of which %d are rare words\" % (len(words_set), len(rare_words)))\n",
    "print (\"Number of word occurrances: %d\" % postags_occur_cnt)\n",
    "print (\"Number of POS tags: %d\\n\" % len(postags_set))\n",
    "\n",
    "print (\"Size of unary rules: %d\" % len(unary_rules_freq))\n",
    "print (\"Number of unary rule occurrances: %d\" % unary_rules_occur_cnt)\n",
    "print (\"Number of unary lhs: %d\" % len(unary_lhs_set))\n",
    "print (\"Number of unary rhs: %d\\n\" % len(unary_rhs_set))\n",
    "\n",
    "print (\"Size of binary rules: %d\" % len(binary_rules_freq))\n",
    "print (\"Number of binary rule occurrances: %d\" % binary_rules_occur_cnt)\n",
    "print (\"Number of binary lhs: %d\" % len(binary_lhs_set))\n",
    "print (\"Number of binary rhs: %d\\n\" % len(binary_rhs_set))\n",
    "\n",
    "print (\"Number of non-terminal symbols: %d\" % len(NT_set))\n",
    "print (\"Number of terminal symbols: %d\\n\" % len(T_set))\n",
    "\n",
    "print (\">>> Collection of rules and words parsing done in %0.3fs.\\n\" % (time() - t0))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}