Country_dom_intl_filter.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"[jinwoo-hojin]country_filter.ipynb","provenance":[{"file_id":"1gdv8MNsfPcoWfbqZa9mYpT36jL2RgTOQ","timestamp":1605553874046},{"file_id":"14OC9V9xjAHlPx6NPhdA5y2YyoDcLoREv","timestamp":1605470718490}],"collapsed_sections":[],"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"0_EhmDss77Jk"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"snlmJTUT819V","executionInfo":{"status":"ok","timestamp":1605554908638,"user_tz":300,"elapsed":3056,"user":{"displayName":"Jin Jung","photoUrl":"","userId":"05270040223697743336"}},"outputId":"011840ae-7d10-4425-8d8f-db4760eaba2a","colab":{"base_uri":"https://localhost:8080/"}},"source":["import os\n","import pandas as pd\n","import datetime\n","\n","\n","!pip install locationtagger\n","\n","import nltk\n","nltk.download('punkt')\n","nltk.download('averaged_perceptron_tagger')\n","nltk.download('maxent_ne_chunker')\n","nltk.download('words')\n","import locationtagger"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: locationtagger in /usr/local/lib/python3.6/dist-packages (0.0.1)\n","Requirement already satisfied: pycountry in /usr/local/lib/python3.6/dist-packages (from locationtagger) (20.7.3)\n","Requirement already satisfied: newspaper3k in /usr/local/lib/python3.6/dist-packages (from locationtagger) (0.2.8)\n","Requirement already satisfied: spacy in /usr/local/lib/python3.6/dist-packages (from locationtagger) (2.2.4)\n","Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (from locationtagger) (3.2.5)\n","Requirement already satisfied: feedparser>=5.2.1 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (6.0.2)\n","Requirement already satisfied: Pillow>=3.3.0 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (7.0.0)\n","Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (2.8.1)\n","Requirement already satisfied: PyYAML>=3.11 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (3.13)\n","Requirement already satisfied: beautifulsoup4>=4.4.1 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (4.6.3)\n","Requirement already satisfied: cssselect>=0.9.2 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (1.1.0)\n","Requirement already satisfied: feedfinder2>=0.0.4 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (0.0.4)\n","Requirement already satisfied: jieba3k>=0.35.1 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (0.35.1)\n","Requirement already satisfied: requests>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (2.23.0)\n","Requirement already satisfied: tinysegmenter==0.3 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (0.3)\n","Requirement already satisfied: lxml>=3.6.0 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (4.2.6)\n","Requirement already satisfied: tldextract>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from newspaper3k->locationtagger) (3.0.2)\n","Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (3.0.2)\n","Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (0.8.0)\n","Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (4.41.1)\n","Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (1.0.3)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (50.3.2)\n","Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (1.18.5)\n","Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (0.4.1)\n","Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (2.0.4)\n","Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (1.0.2)\n","Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (1.0.0)\n","Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (7.4.0)\n","Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy->locationtagger) (1.1.3)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk->locationtagger) (1.15.0)\n","Requirement already satisfied: sgmllib3k in /usr/local/lib/python3.6/dist-packages (from feedparser>=5.2.1->newspaper3k->locationtagger) (1.0.0)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.10.0->newspaper3k->locationtagger) (2020.6.20)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.10.0->newspaper3k->locationtagger) (2.10)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.10.0->newspaper3k->locationtagger) (1.24.3)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.10.0->newspaper3k->locationtagger) (3.0.4)\n","Requirement already satisfied: requests-file>=1.4 in /usr/local/lib/python3.6/dist-packages (from tldextract>=2.0.1->newspaper3k->locationtagger) (1.5.1)\n","Requirement already satisfied: filelock>=3.0.8 in /usr/local/lib/python3.6/dist-packages (from tldextract>=2.0.1->newspaper3k->locationtagger) (3.0.12)\n","Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy->locationtagger) (2.0.0)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy->locationtagger) (3.4.0)\n","[nltk_data] Downloading package punkt to /root/nltk_data...\n","[nltk_data]   Package punkt is already up-to-date!\n","[nltk_data] Downloading package averaged_perceptron_tagger to\n","[nltk_data]     /root/nltk_data...\n","[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n","[nltk_data]       date!\n","[nltk_data] Downloading package maxent_ne_chunker to\n","[nltk_data]     /root/nltk_data...\n","[nltk_data]   Package maxent_ne_chunker is already up-to-date!\n","[nltk_data] Downloading package words to /root/nltk_data...\n","[nltk_data]   Package words is already up-to-date!\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"H57XJL8B1LK2"},"source":["import pycountry\n","countries = {}\n","for con in pycountry.countries:\n","  countries[con.alpha_2] = con.name"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"gaWPJjPt8B6x"},"source":["old_data_path = '/content/drive/My Drive/capstone_data/data/1_pre-Ngram_processed'\n","# countries = ['AU','BD','CA','GB','GH','HK','IE','IN','JM','KE','LK','MY','NG','NZ','PH','PK','SG','TZ','US','ZA']"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"fOyDDvKf8B9Y","executionInfo":{"status":"ok","timestamp":1605555653987,"user_tz":300,"elapsed":3163,"user":{"displayName":"Jin Jung","photoUrl":"","userId":"05270040223697743336"}},"outputId":"ad744ae1-df56-4d1a-86a9-1ee632fd8811","colab":{"base_uri":"https://localhost:8080/","height":197}},"source":["# 'In' 'NG'\n","country = 'NG'\n","filename = country + '_pre-preprocess.csv'\n","path = os.path.join(old_data_path, filename)\n","df = pd.read_csv(path, index_col=[0])\n","df.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>article_id</th>\n","      <th>article_title</th>\n","      <th>publisher</th>\n","      <th>year</th>\n","      <th>article_text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>51128383</td>\n","      <td>OMSL: SAA Doesn't Constitute Threat to Nationa...</td>\n","      <td>thestreetjournal.org</td>\n","      <td>2019</td>\n","      <td>OMSL SAA Doesn't Constitute Threat to Nationa...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>51083517</td>\n","      <td>EFCC Grills Zamfara INEC Officials over Divers...</td>\n","      <td>thestreetjournal.org</td>\n","      <td>2019</td>\n","      <td>EFCC Grills Zamfara INEC Officials over Diver...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>51247758</td>\n","      <td>Alleged N1.4bn Fraud: How Usoro was Paid N700m...</td>\n","      <td>thestreetjournal.org</td>\n","      <td>2019</td>\n","      <td>Alleged N1.4bn Fraud How Usoro was Paid N700m...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>51123139</td>\n","      <td>Opinion: Adesuwa of Aruwa Capital Management: ...</td>\n","      <td>thestreetjournal.org</td>\n","      <td>2019</td>\n","      <td>Opinion Adesuwa of Aruwa Capital Management A...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>51070131</td>\n","      <td>Kidnappers of NSCDC Officials Demand N6m Ranso...</td>\n","      <td>thestreetjournal.org</td>\n","      <td>2019</td>\n","      <td>Kidnappers of NSCDC Officials Demand N6m Rans...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   article_id  ...                                       article_text\n","0    51128383  ...   OMSL SAA Doesn't Constitute Threat to Nationa...\n","1    51083517  ...   EFCC Grills Zamfara INEC Officials over Diver...\n","2    51247758  ...   Alleged N1.4bn Fraud How Usoro was Paid N700m...\n","3    51123139  ...   Opinion Adesuwa of Aruwa Capital Management A...\n","4    51070131  ...   Kidnappers of NSCDC Officials Demand N6m Rans...\n","\n","[5 rows x 5 columns]"]},"metadata":{"tags":[]},"execution_count":66}]},{"cell_type":"code","metadata":{"id":"BOJa0RjniThE","executionInfo":{"status":"ok","timestamp":1605555653988,"user_tz":300,"elapsed":1588,"user":{"displayName":"Jin Jung","photoUrl":"","userId":"05270040223697743336"}},"outputId":"4dfd00ce-d632-4262-e8ae-2b2289093082","colab":{"base_uri":"https://localhost:8080/"}},"source":["df.shape[0]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["79270"]},"metadata":{"tags":[]},"execution_count":67}]},{"cell_type":"code","metadata":{"id":"Fy43t4ooaLgc"},"source":["\n","counter =0\n","a = datetime.datetime.now()\n","def country_ner(text):\n","  if len(text)>1000000:\n","    return []\n","\n","  entities = locationtagger.find_locations(text = text)\n","  \n","  global counter\n","  counter +=1\n","\n","  if counter % 100 == 0:  # i.e. print in multiples of 500\n","    b = datetime.datetime.now()\n","    print(\"num files: {}, time took so far: {}, % completed: {:0.4f}% \".format(counter, b-a, counter/df.shape[0]*100.0))\n","\n","  return entities.country_mentions\n","\n","\n","\n","def isDomestic(row, country):\n","\n","  country_list = list(dict(row['country_mention']).keys())\n","  if len(country_list) > 0:\n","    if country not in country_list:\n","      return False \n","\n","  return True \n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XEf0XbXae9MP"},"source":["# df"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"VPrETHw594Ri","executionInfo":{"status":"error","timestamp":1605555731457,"user_tz":300,"elapsed":60595,"user":{"displayName":"Jin Jung","photoUrl":"","userId":"05270040223697743336"}},"outputId":"ae47708e-97ca-4a14-ec8b-cb96dcdcb4fd","colab":{"base_uri":"https://localhost:8080/","height":370}},"source":["df['country_mention'] = df.article_text.apply(country_ner)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["num files: 100, time took so far: 0:00:39.315892, % completed: 10.5042% \n"],"name":"stdout"},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-74-00768359c489>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'country_mention'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marticle_text\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcountry_ner\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[1;32m   4210\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4211\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4212\u001b[0;31m                 \u001b[0mmapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4213\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4214\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32mpandas/_libs/lib.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n","\u001b[0;32m<ipython-input-72-89387a398dca>\u001b[0m in \u001b[0;36mcountry_ner\u001b[0;34m(text)\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m   \u001b[0mentities\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlocationtagger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_locations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m   \u001b[0;32mglobal\u001b[0m \u001b[0mcounter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/locationtagger/__init__.py\u001b[0m in \u001b[0;36mfind_locations\u001b[0;34m(url, text)\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mfind_locations\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0me\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNamedEntityExtractor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_named_entities\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mlocs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLocationExtractor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnamed_entities\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/locationtagger/locationextractor.py\u001b[0m in \u001b[0;36mfind_named_entities\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mword_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m         \u001b[0mnes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mne_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpos_tag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/chunk/__init__.py\u001b[0m in \u001b[0;36mne_chunk\u001b[0;34m(tagged_tokens, binary)\u001b[0m\n\u001b[1;32m    175\u001b[0m         \u001b[0mchunker_pickle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_MULTICLASS_NE_CHUNKER\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    176\u001b[0m     \u001b[0mchunker\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunker_pickle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 177\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mchunker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtagged_tokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mne_chunk_sents\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtagged_sentences\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbinary\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/chunk/named_entity.py\u001b[0m in \u001b[0;36mparse\u001b[0;34m(self, tokens)\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0mEach\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0mshould\u001b[0m \u001b[0mbe\u001b[0m \u001b[0ma\u001b[0m \u001b[0mpos\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mtagged\u001b[0m \u001b[0mword\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m         \"\"\"\n\u001b[0;32m--> 122\u001b[0;31m         \u001b[0mtagged\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tagger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    123\u001b[0m         \u001b[0mtree\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tagged_to_parse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtagged\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtree\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/tag/sequential.py\u001b[0m in \u001b[0;36mtag\u001b[0;34m(self, tokens)\u001b[0m\n\u001b[1;32m     61\u001b[0m         \u001b[0mtags\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m             \u001b[0mtags\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag_one\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/tag/sequential.py\u001b[0m in \u001b[0;36mtag_one\u001b[0;34m(self, tokens, index, history)\u001b[0m\n\u001b[1;32m     81\u001b[0m         \u001b[0mtag\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtagger\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_taggers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m             \u001b[0mtag\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtagger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchoose_tag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtag\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/tag/sequential.py\u001b[0m in \u001b[0;36mchoose_tag\u001b[0;34m(self, tokens, index, history)\u001b[0m\n\u001b[1;32m    636\u001b[0m         \u001b[0;31m# higher than that cutoff first; otherwise, return None.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    637\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cutoff_prob\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 638\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_classifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclassify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatureset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    639\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    640\u001b[0m         \u001b[0mpdist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_classifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprob_classify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatureset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/classify/maxent.py\u001b[0m in \u001b[0;36mclassify\u001b[0;34m(self, featureset)\u001b[0m\n\u001b[1;32m    144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    145\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclassify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeatureset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 146\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprob_classify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatureset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mprob_classify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeatureset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/nltk/classify/maxent.py\u001b[0m in \u001b[0;36mprob_classify\u001b[0;34m(self, featureset)\u001b[0m\n\u001b[1;32m    153\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_logarithmic\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m                 \u001b[0mtotal\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m                 \u001b[0;32mfor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mf_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf_val\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfeature_vector\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    156\u001b[0m                     \u001b[0mtotal\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_weights\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mf_id\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mf_val\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m                 \u001b[0mprob_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"cell_type":"code","metadata":{"id":"v71guQIhRtGH"},"source":["country_name = countries.get(country)\n","df['Domestic'] = df.apply(lambda row: isDomestic(row, country_name),axis=1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"zuMJeqKmbzmY","executionInfo":{"status":"ok","timestamp":1605554698583,"user_tz":300,"elapsed":266,"user":{"displayName":"Jin Jung","photoUrl":"","userId":"05270040223697743336"}},"outputId":"f7bc36ff-2f76-46e1-94ad-ed68bf75dc96","colab":{"base_uri":"https://localhost:8080/"}},"source":["# df.Domestic.value_counts(normalize=True) "],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["True     0.706\n","False    0.294\n","Name: Domestic, dtype: float64"]},"metadata":{"tags":[]},"execution_count":23}]},{"cell_type":"code","metadata":{"id":"rM4Bm-wre8oo"},"source":["df_domestic = df[df.Domestic == True]\n","df_international = df[df.Domestic == False]\n","\n","new_data_path = '/content/drive/My Drive/capstone_data/data/2_domestic_filter'\n","\n","domestic_output_filename = country + '_domestic.csv'\n","international_output_filename = country + '_international.csv'\n","\n","df_domestic.to_csv(os.path.join(new_data_path, domestic_output_filename))\n","df_international.to_csv(os.path.join(new_data_path, international_output_filename))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Az_rUqU6gFGF"},"source":[""],"execution_count":null,"outputs":[]}]}