phase-one-energy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd             # data package\n",
    "import matplotlib.pyplot as plt # graphics \n",
    "import datetime as dt\n",
    "import numpy as np\n",
    "from census import Census # This is new...\n",
    "\n",
    "import requests, io             # internet and input tools  \n",
    "import zipfile as zf            # zip file tools \n",
    "import os  \n",
    "\n",
    "#import weightedcalcs as wc\n",
    "#import numpy as np\n",
    "\n",
    "import pyarrow as pa\n",
    "import pyarrow.parquet as pq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfproducts = pd.read_csv(\".\\\\data\"+ \"\\\\annex-6-1.csv\", dtype = {\"hs4\": str, \"hs4_o\": str})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['2701', '2709', '271012250', '271111', '271112', '271113', '271311', '271312', '290511']\n"
     ]
    }
   ],
   "source": [
    "is_energy = dfproducts.high_catagory == \"3. Energy\"\n",
    "\n",
    "energy_products = list(dfproducts.loc[is_energy,\"hs4_o\"])\n",
    "\n",
    "energy_products.remove(\"27111990\")\n",
    "\n",
    "print(energy_products)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "trade_type = \"exports\"\n",
    "\n",
    "my_key = \"&key=34e40301bda77077e24c859c6c6c0b721ad73fc7\"\n",
    "# This is my key. I'm nice and I have it posted. If you will be doing more with this\n",
    "# please get your own key!\n",
    "\n",
    "end_use = \"hs?get=ALL_VAL_MO,UNIT_QY1,QTY_1_MO\"\n",
    "\n",
    "url = \"https://api.census.gov/data/timeseries/intltrade/exports/\" + end_use "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_energy_quantity(commodity, url):\n",
    "\n",
    "    url = url + my_key + \"&time==from+2013-01\" + \"&COMM_LVL=HS10\" + \"&E_COMMODITY=\" + commodity + \"*\"\n",
    "\n",
    "    url = url + \"&CTY_CODE=5700\"\n",
    "\n",
    "    r = requests.get(url) \n",
    "    \n",
    "    print(r)\n",
    "    \n",
    "    df = pd.DataFrame(r.json()[1:]) # This then converts it to a dataframe\n",
    "    # Note that the first entry is the labels\n",
    "\n",
    "    df.columns = r.json()[0]\n",
    "\n",
    "    df.time = pd.to_datetime(df.time, format=\"%Y-%m\")\n",
    "    \n",
    "    df[\"hs4_o\"] = commodity\n",
    "    \n",
    "    #df[\"hs4\"] = df[\"E_COMMODITY\"].str[0:4]\n",
    "    \n",
    "    df[\"hs6\"] = df[\"E_COMMODITY\"].str[0:6]\n",
    "    \n",
    "    print(df[\"UNIT_QY1\"].unique(), len(df[\"UNIT_QY1\"].unique()))\n",
    "    \n",
    "    df[\"china_exports\"] = df[\"ALL_VAL_MO\"].astype(float)\n",
    "    \n",
    "    df[\"china_exports_quant\"] = df[\"QTY_1_MO\"].astype(float)\n",
    "    \n",
    "    df[\"quant_type\"] = df[\"UNIT_QY1\"].astype(str)\n",
    "    \n",
    "    df.drop([\"ALL_VAL_MO\", \"E_COMMODITY\", \"COMM_LVL\", \"QTY_1_MO\", \"UNIT_QY1\"], axis = 1, inplace = True)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n",
      "['BBL'] 1\n"
     ]
    }
   ],
   "source": [
    "df = get_energy_quantity(\"271012250\", url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2701\n",
      "<Response [200]>\n",
      "['T'] 1\n",
      "2709\n",
      "<Response [200]>\n",
      "['BBL'] 1\n",
      "271012250\n",
      "<Response [200]>\n",
      "['BBL'] 1\n",
      "271111\n",
      "<Response [200]>\n",
      "['M3'] 1\n",
      "271112\n",
      "<Response [200]>\n",
      "['BBL' 'M3'] 2\n",
      "271113\n",
      "<Response [200]>\n",
      "['BBL'] 1\n",
      "271311\n",
      "<Response [200]>\n",
      "['T'] 1\n",
      "271312\n",
      "<Response [200]>\n",
      "['T'] 1\n",
      "290511\n",
      "<Response [200]>\n",
      "['L'] 1\n"
     ]
    }
   ],
   "source": [
    "energy = pd.DataFrame([])\n",
    "\n",
    "for xxx in energy_products:\n",
    "    \n",
    "    print(xxx)\n",
    "    energy = energy.append(get_energy_quantity(xxx, url))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>CTY_CODE</th>\n",
       "      <th>hs4_o</th>\n",
       "      <th>hs6</th>\n",
       "      <th>china_exports</th>\n",
       "      <th>china_exports_quant</th>\n",
       "      <th>quant_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>2020-09-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>290511</td>\n",
       "      <td>290511</td>\n",
       "      <td>409390.0</td>\n",
       "      <td>248558.0</td>\n",
       "      <td>L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>2020-10-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>290511</td>\n",
       "      <td>290511</td>\n",
       "      <td>227096.0</td>\n",
       "      <td>142500.0</td>\n",
       "      <td>L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>2020-11-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>290511</td>\n",
       "      <td>290511</td>\n",
       "      <td>224346.0</td>\n",
       "      <td>155559.0</td>\n",
       "      <td>L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>2020-12-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>290511</td>\n",
       "      <td>290511</td>\n",
       "      <td>170909.0</td>\n",
       "      <td>173953.0</td>\n",
       "      <td>L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>2021-01-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>290511</td>\n",
       "      <td>290511</td>\n",
       "      <td>136868.0</td>\n",
       "      <td>29958.0</td>\n",
       "      <td>L</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         time CTY_CODE   hs4_o     hs6  china_exports  china_exports_quant  \\\n",
       "92 2020-09-01     5700  290511  290511       409390.0             248558.0   \n",
       "93 2020-10-01     5700  290511  290511       227096.0             142500.0   \n",
       "94 2020-11-01     5700  290511  290511       224346.0             155559.0   \n",
       "95 2020-12-01     5700  290511  290511       170909.0             173953.0   \n",
       "96 2021-01-01     5700  290511  290511       136868.0              29958.0   \n",
       "\n",
       "   quant_type  \n",
       "92          L  \n",
       "93          L  \n",
       "94          L  \n",
       "95          L  \n",
       "96          L  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "energy.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_energy = energy.merge(dfproducts.loc[is_energy,:], left_on = \"hs4_o\", right_on = \"hs4_o\", how = \"left\", indicator = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnames = ['time', 'CTY_CODE', 'china_exports', 'hs6', 'china_exports_quant', 'quant_type',\n",
    "          'hs4', 'hs4_o', 'description', 'low_catagory', 'high_catagory', '_merge']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>CTY_CODE</th>\n",
       "      <th>hs4_o</th>\n",
       "      <th>hs6</th>\n",
       "      <th>china_exports</th>\n",
       "      <th>china_exports_quant</th>\n",
       "      <th>quant_type</th>\n",
       "      <th>description</th>\n",
       "      <th>low_catagory</th>\n",
       "      <th>high_catagory</th>\n",
       "      <th>hs4</th>\n",
       "      <th>_merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270111</td>\n",
       "      <td>7837.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013-02-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013-03-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2013-04-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2013-05-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270111</td>\n",
       "      <td>7124.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        time CTY_CODE hs4_o     hs6  china_exports  china_exports_quant  \\\n",
       "0 2013-01-01     5700  2701  270111         7837.0                 11.0   \n",
       "1 2013-02-01     5700  2701  270111            0.0                  0.0   \n",
       "2 2013-03-01     5700  2701  270111            0.0                  0.0   \n",
       "3 2013-04-01     5700  2701  270111            0.0                  0.0   \n",
       "4 2013-05-01     5700  2701  270111         7124.0                 10.0   \n",
       "\n",
       "  quant_type                                        description low_catagory  \\\n",
       "0          T  Coal; briquett,es  ovoids and similar solid fu...         Coal   \n",
       "1          T  Coal; briquett,es  ovoids and similar solid fu...         Coal   \n",
       "2          T  Coal; briquett,es  ovoids and similar solid fu...         Coal   \n",
       "3          T  Coal; briquett,es  ovoids and similar solid fu...         Coal   \n",
       "4          T  Coal; briquett,es  ovoids and similar solid fu...         Coal   \n",
       "\n",
       "  high_catagory   hs4 _merge  \n",
       "0     3. Energy  2701   both  \n",
       "1     3. Energy  2701   both  \n",
       "2     3. Energy  2701   both  \n",
       "3     3. Energy  2701   both  \n",
       "4     3. Energy  2701   both  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_energy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>CTY_CODE</th>\n",
       "      <th>hs4_o</th>\n",
       "      <th>hs6</th>\n",
       "      <th>china_exports</th>\n",
       "      <th>china_exports_quant</th>\n",
       "      <th>quant_type</th>\n",
       "      <th>description</th>\n",
       "      <th>low_catagory</th>\n",
       "      <th>high_catagory</th>\n",
       "      <th>hs4</th>\n",
       "      <th>_merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270111</td>\n",
       "      <td>7837.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013-02-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013-03-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2013-04-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270111</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2013-05-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270111</td>\n",
       "      <td>7124.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351</th>\n",
       "      <td>2020-08-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>2020-09-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>2020-10-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>354</th>\n",
       "      <td>2020-11-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>2020-12-01</td>\n",
       "      <td>5700</td>\n",
       "      <td>2701</td>\n",
       "      <td>270120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>T</td>\n",
       "      <td>Coal; briquett,es  ovoids and similar solid fu...</td>\n",
       "      <td>Coal</td>\n",
       "      <td>3. Energy</td>\n",
       "      <td>2701</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>356 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          time CTY_CODE hs4_o     hs6  china_exports  china_exports_quant  \\\n",
       "0   2013-01-01     5700  2701  270111         7837.0                 11.0   \n",
       "1   2013-02-01     5700  2701  270111            0.0                  0.0   \n",
       "2   2013-03-01     5700  2701  270111            0.0                  0.0   \n",
       "3   2013-04-01     5700  2701  270111            0.0                  0.0   \n",
       "4   2013-05-01     5700  2701  270111         7124.0                 10.0   \n",
       "..         ...      ...   ...     ...            ...                  ...   \n",
       "351 2020-08-01     5700  2701  270120            0.0                  0.0   \n",
       "352 2020-09-01     5700  2701  270120            0.0                  0.0   \n",
       "353 2020-10-01     5700  2701  270120            0.0                  0.0   \n",
       "354 2020-11-01     5700  2701  270120            0.0                  0.0   \n",
       "355 2020-12-01     5700  2701  270120            0.0                  0.0   \n",
       "\n",
       "    quant_type                                        description  \\\n",
       "0            T  Coal; briquett,es  ovoids and similar solid fu...   \n",
       "1            T  Coal; briquett,es  ovoids and similar solid fu...   \n",
       "2            T  Coal; briquett,es  ovoids and similar solid fu...   \n",
       "3            T  Coal; briquett,es  ovoids and similar solid fu...   \n",
       "4            T  Coal; briquett,es  ovoids and similar solid fu...   \n",
       "..         ...                                                ...   \n",
       "351          T  Coal; briquett,es  ovoids and similar solid fu...   \n",
       "352          T  Coal; briquett,es  ovoids and similar solid fu...   \n",
       "353          T  Coal; briquett,es  ovoids and similar solid fu...   \n",
       "354          T  Coal; briquett,es  ovoids and similar solid fu...   \n",
       "355          T  Coal; briquett,es  ovoids and similar solid fu...   \n",
       "\n",
       "    low_catagory high_catagory   hs4 _merge  \n",
       "0           Coal     3. Energy  2701   both  \n",
       "1           Coal     3. Energy  2701   both  \n",
       "2           Coal     3. Energy  2701   both  \n",
       "3           Coal     3. Energy  2701   both  \n",
       "4           Coal     3. Energy  2701   both  \n",
       "..           ...           ...   ...    ...  \n",
       "351         Coal     3. Energy  2701   both  \n",
       "352         Coal     3. Energy  2701   both  \n",
       "353         Coal     3. Energy  2701   both  \n",
       "354         Coal     3. Energy  2701   both  \n",
       "355         Coal     3. Energy  2701   both  \n",
       "\n",
       "[356 rows x 12 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_energy[df_energy.low_catagory == \"Coal\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_file = \".\\\\data\"+ \"\\\\phaseone-energy.parquet\"\n",
    "\n",
    "pq.write_table(pa.Table.from_pandas(df_energy), out_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}