DT_RF_classification_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import random\n",
    "import dill\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dill.dump_session('notebook_Q1.db')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dill.load_session('notebook_Q1.db')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveInPickle(data, pickleFile):\n",
    "    file = open(pickleFile,\"wb\")\n",
    "    pickle.dump(data,file)\n",
    "    file.close()\n",
    "def loadFromPickle(pickleFile):\n",
    "    file = open(pickleFile,'rb')\n",
    "    pickleData = pickle.load(file)\n",
    "    file.close()\n",
    "    return pickleData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handleCategoricalFeature(dataSet, feature):\n",
    "        featureValues = list(set(dataSet[feature]))\n",
    "        featureMap = {}\n",
    "        featureMap[feature] = {}\n",
    "        i = 0\n",
    "        for f in featureValues:\n",
    "            featureMap[feature][f] =  i\n",
    "            i+=1\n",
    "        print(featureMap)\n",
    "        dataSet.replace(featureMap, inplace=True)\n",
    "        return dataSet[feature]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTrainTestData(dataSetFile):\n",
    "    dataSet = pd.read_csv(dataSetFile)\n",
    "    naColumnMean = dataSet[\"pm2.5\"].mean(axis = 0, skipna = True)\n",
    "    dataSet[\"pm2.5\"] = dataSet[\"pm2.5\"].fillna(naColumnMean)\n",
    "    dataSet[\"cbwd\"] =    handleCategoricalFeature(dataSet, \"cbwd\")\n",
    "    dataSet = dataSet.drop([\"No\"], axis = 1)\n",
    "    trainSet = dataSet.loc[dataSet['year'].isin([2012,2010])]\n",
    "    testSet = dataSet.loc[dataSet['year'].isin([2013,2011])]\n",
    "    return trainSet, testSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getFeaturesAndLabels(dataSet, targetColumn):\n",
    "    dataLabels = dataSet[targetColumn]\n",
    "    data = dataSet\n",
    "    data = data.drop([targetColumn], axis = 1)\n",
    "    data = data.drop([\"No\"], axis = 1)\n",
    "    return data, dataLabels\n",
    "\n",
    "def getFeaturesListAndValues(data):\n",
    "    featuresMap = {}\n",
    "    features = list(data.columns)\n",
    "    features.pop(1)\n",
    "    for feature in features:\n",
    "        featuresMap[feature] = []\n",
    "        featureValues = data[feature]\n",
    "        featureValues = list(set(featureValues))\n",
    "        featureValues = sorted(featureValues)\n",
    "        featuresMap[feature] = featureValues\n",
    "    return featuresMap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculateGiniImpurity(R1, R2, classes):\n",
    "    if(len(R1)==0 or len(R2)==0):\n",
    "        return 1\n",
    "    features = list(R1.columns)\n",
    "    features.pop(1)\n",
    "    gini1 = gini2 = 0\n",
    "    total = len(R1) + len(R2)\n",
    "    l1 = len(R1)\n",
    "    l2 = len(R2)\n",
    "    for c in classes:\n",
    "        p = len(R1.loc[R1['month'] == c])/l1\n",
    "        gini1 += p*(1-p)\n",
    "    \n",
    "    for c in classes:\n",
    "        p = len(R2.loc[R2['month'] == c])/l2\n",
    "        gini2 += p*(1-p)\n",
    "    \n",
    "    gini = (gini1*l1+gini2*l2)/(l1+l2)\n",
    "    return gini\n",
    "\n",
    "def getGiniValue(nLeft, nRight):\n",
    "    l1 = np.sum(nLeft)\n",
    "    l2 = np.sum(nRight)\n",
    "    l = l1+l2\n",
    "    g1 = nLeft/l1\n",
    "    g2 = nRight/l2\n",
    "    g1 = g1*(1-g1)\n",
    "    g2 = g2*(1-g2)\n",
    "    g1 = np.sum(g1)\n",
    "    g2 = np.sum(g2)\n",
    "    g = (g1*l1+g2*l2)/l\n",
    "    return g\n",
    "\n",
    "def getParentGini(data):\n",
    "    df = data.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "    g1 = df/len(data)\n",
    "    g1 = g1*(1-g1)\n",
    "    g1 = g1[\"Count\"].sum()\n",
    "    return g1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'cbwd': {'NW': 0, 'NE': 1, 'cv': 2, 'SE': 3}}\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generateDecisionTree(data, maxDepth):\n",
    "    dTree = []\n",
    "    minGini = getParentGini(data)\n",
    "    parentGini = minGini\n",
    "    l = len(data)\n",
    "#     minGini = 1\n",
    "    classes = sorted(list(set(data[\"month\"])))\n",
    "    df1 = data.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "    for feature in features:\n",
    "        dataSorted = data.sort_values([feature])\n",
    "        nRight = np.array(df1[\"Count\"])\n",
    "        nLeft = np.array([0]*len(classes))\n",
    "        for i in range(0,l-1):\n",
    "            row = dataSorted.iloc[i]\n",
    "            c = row[\"month\"]\n",
    "            indx = classes.index(c)\n",
    "            nRight[indx]-=1\n",
    "            nLeft[indx]+=1\n",
    "            if(row[feature] == dataSorted.iloc[i+1][feature]):\n",
    "                continue\n",
    "            gini = getGiniValue(nLeft, nRight)\n",
    "            if(minGini> gini):\n",
    "                minGini = gini\n",
    "                minFeature = feature\n",
    "                threshold = (row[feature] + dataSorted.iloc[i+1][feature])/2\n",
    "#                 if(minGini==0):\n",
    "#                     break\n",
    "#         if(minGini==0):\n",
    "#             break\n",
    "    \n",
    "#     print(\"split at\", maxDepth)\n",
    "   \n",
    "#     print(\"minGini\", minGini)\n",
    "\n",
    "    if(parentGini==minGini):\n",
    "        df = data.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "        df = df.sort_values(by ='Count' , ascending=False)\n",
    "        return df.loc[0][\"month\"]\n",
    "    \n",
    "    R1 = data.loc[data[minFeature] < threshold]\n",
    "    R2 = data.loc[data[minFeature] >= threshold]\n",
    "    \n",
    "#     if(len(R1)==0 or len(R2)==0):\n",
    "#         df = data.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "#         df = df.sort_values(by ='Count' , ascending=False)\n",
    "#         return df.loc[0][\"month\"]\n",
    "    \n",
    "    if(minGini == 0 or maxDepth == 0):    \n",
    "        df = R1.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "        df = df.sort_values(by ='Count' , ascending=False)\n",
    "        DT1 = df.loc[0][\"month\"]\n",
    "        df = R2.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "        df = df.sort_values(by ='Count' , ascending=False)\n",
    "        DT2 = df.loc[0][\"month\"]\n",
    "    else:    \n",
    "        DT1 = generateDecisionTree(R1,maxDepth-1)\n",
    "        DT2 = generateDecisionTree(R2,maxDepth-1)\n",
    "    \n",
    "    dTree = [minFeature, threshold, DT1, DT2]\n",
    "    \n",
    "    return dTree\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generateDTreeWithParams(data, maxDepth, maxWidth = math.pow(2, 500)):\n",
    "    dTree = []\n",
    "    minGini = getParentGini(data)\n",
    "    parentGini = minGini\n",
    "    l = len(data)\n",
    "    classes = sorted(list(set(data[\"month\"])))\n",
    "    df1 = data.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "    for feature in features:\n",
    "        dataSorted = data.sort_values([feature])\n",
    "        nRight = np.array(df1[\"Count\"])\n",
    "        nLeft = np.array([0]*len(classes))\n",
    "        for i in range(0,l-1):\n",
    "            row = dataSorted.iloc[i]\n",
    "            c = row[\"month\"]\n",
    "            indx = classes.index(c)\n",
    "            nRight[indx]-=1\n",
    "            nLeft[indx]+=1\n",
    "            if(row[feature] == dataSorted.iloc[i+1][feature]):\n",
    "                continue\n",
    "            gini = getGiniValue(nLeft, nRight)\n",
    "            if(minGini> gini):\n",
    "                minGini = gini\n",
    "                minFeature = feature\n",
    "                threshold = (row[feature] + dataSorted.iloc[i+1][feature])/2\n",
    "\n",
    "    if(parentGini==minGini or maxWidth<=0):\n",
    "        df = data.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "        df = df.sort_values(by ='Count' , ascending=False)\n",
    "        return df.loc[0][\"month\"], maxWidth-1\n",
    "    \n",
    "    R1 = data.loc[data[minFeature] < threshold]\n",
    "    R2 = data.loc[data[minFeature] >= threshold]\n",
    "    \n",
    "    if(minGini == 0 or maxDepth == 0):    \n",
    "        df = R1.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "        df = df.sort_values(by ='Count' , ascending=False)\n",
    "        DT1 = df.loc[0][\"month\"]\n",
    "        df = R2.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "        df = df.sort_values(by ='Count' , ascending=False)\n",
    "        DT2 = df.loc[0][\"month\"]\n",
    "        width = maxWidth - 2\n",
    "    else:    \n",
    "        DT1, width = generateDTreeWithParams(R1,maxDepth-1, maxWidth)\n",
    "        DT2, width = generateDTreeWithParams(R2,maxDepth-1, width)\n",
    "    \n",
    "    dTree = [minFeature, threshold, DT1, DT2]\n",
    "    \n",
    "    return dTree, width\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainDecisionTree(trainSet, maxDepth, maxWidth = 0):\n",
    "    target = \"month\"\n",
    "    dTree = generateDecisionTree(trainSet, maxDepth)\n",
    "    return dTree\n",
    "    \n",
    "def testDecisionTree(test, dTree):\n",
    "    if(isinstance(dTree, list)):\n",
    "        minFeature = dTree[0]\n",
    "        threshold = dTree[1]\n",
    "        if(test[minFeature] < threshold):\n",
    "            return testDecisionTree(test, dTree[2])\n",
    "        else:\n",
    "            return testDecisionTree(test, dTree[3])\n",
    "    else:\n",
    "        return dTree\n",
    "    \n",
    "def getAccuracy(results):\n",
    "    count = 0\n",
    "    for i in range(len(results)):\n",
    "        if(results[i][0]==results[i][1]):\n",
    "            count+=1\n",
    "    return count\n",
    "\n",
    "def getTestResults(testSet, dTree):\n",
    "    results = []\n",
    "    for i in range(len(testSet)):\n",
    "        testData = testSet.iloc[i]\n",
    "        result = testDecisionTree(testData, dTree)\n",
    "        results.append((result, testData[\"month\"]))\n",
    "    count = getAccuracy(results)\n",
    "    print(\"Accuracy\", 100*count/len(testSet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "######################-----classification using Bagged Decision Tree ----########################################\n",
    "\n",
    "def generateBaggedDataSet(trainSet, bagSize, n):\n",
    "    bagTrainSet = []\n",
    "    l = len(trainSet)\n",
    "    size = int(l*bagSize/100)\n",
    "    for i in range(n):\n",
    "        bagTrainSet.append(trainSet.sample(n = size))\n",
    "    \n",
    "    return bagTrainSet\n",
    "\n",
    "def trainBagDecisionTree(bagTrainSet, maxDepth):\n",
    "    bagDtrees = []\n",
    "    for i in range(len(bagTrainSet)):\n",
    "        dTree = generateDecisionTree(bagTrainSet[i], maxDepth)\n",
    "        print(\"done\")\n",
    "        bagDtrees.append(dTree)\n",
    "    return bagDtrees\n",
    "\n",
    "def testBagDtree(bagDtrees, testSet):\n",
    "    bagResults = []\n",
    "    for i in range(len(testSet)):\n",
    "        testData = testSet.iloc[i]\n",
    "        result = []\n",
    "        for dTree in bagDtrees:\n",
    "            y = testDecisionTree(testData, dTree)\n",
    "            result.append(y)\n",
    "        bagResults.append((result, testData[\"month\"]))\n",
    "#     print(bagResults)\n",
    "    count = getBagAccuracy(bagResults)\n",
    "    print(\"Accuracy\", 100*count/len(testSet))\n",
    "    \n",
    "def testDecisionTree(test, dTree):\n",
    "    if(isinstance(dTree, list)):\n",
    "        minFeature = dTree[0]\n",
    "        threshold = dTree[1]\n",
    "        if(test[minFeature] < threshold):\n",
    "            return testDecisionTree(test, dTree[2])\n",
    "        else:\n",
    "            return testDecisionTree(test, dTree[3])\n",
    "    else:\n",
    "        return dTree\n",
    "    \n",
    "def getBagAccuracy(bagResults):\n",
    "    accuracy = 0\n",
    "    count = []\n",
    "    classes = []\n",
    "    for results in bagResults:\n",
    "        count = []\n",
    "        classes = []\n",
    "        result = results[0]\n",
    "        truth = results[1]\n",
    "        for y in result:\n",
    "            if(y not in classes):\n",
    "                count.append(0)\n",
    "                classes.append(y)\n",
    "            indx =  classes.index(y)\n",
    "            count[indx]+=1\n",
    "        \n",
    "        indx = count.index(max(count))\n",
    "        if(classes[indx] == truth):\n",
    "            accuracy+=1\n",
    "    return accuracy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generateRandomForestTree(data, maxDepth):\n",
    "    dTree = []\n",
    "    minGini = getParentGini(data)\n",
    "    parentGini = minGini\n",
    "    l = len(data)\n",
    "    classes = sorted(list(set(data[\"month\"])))\n",
    "    df1 = data.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "    randomFeatures = random.sample(features, math.ceil(math.sqrt(len(features))))\n",
    "    print(randomFeatures)\n",
    "    for feature in randomFeatures:\n",
    "        dataSorted = data.sort_values([feature])\n",
    "        nRight = np.array(df1[\"Count\"])\n",
    "        nLeft = np.array([0]*len(classes))\n",
    "        for i in range(0,l-1):\n",
    "            row = dataSorted.iloc[i]\n",
    "            c = row[\"month\"]\n",
    "            indx = classes.index(c)\n",
    "            nRight[indx]-=1\n",
    "            nLeft[indx]+=1\n",
    "            if(row[feature] == dataSorted.iloc[i+1][feature]):\n",
    "                continue\n",
    "            gini = getGiniValue(nLeft, nRight)\n",
    "            if(minGini> gini):\n",
    "                minGini = gini\n",
    "                minFeature = feature\n",
    "                threshold = (row[feature] + dataSorted.iloc[i+1][feature])/2\n",
    "\n",
    "    if(parentGini==minGini):\n",
    "        df = data.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "        df = df.sort_values(by ='Count' , ascending=False)\n",
    "        return df.loc[0][\"month\"]\n",
    "    \n",
    "    R1 = data.loc[data[minFeature] < threshold]\n",
    "    R2 = data.loc[data[minFeature] >= threshold]\n",
    "\n",
    "    if(minGini == 0 or maxDepth == 0):    \n",
    "        df = R1.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "        df = df.sort_values(by ='Count' , ascending=False)\n",
    "        DT1 = df.loc[0][\"month\"]\n",
    "        df = R2.groupby(['month'], sort=True).size().reset_index(name='Count')\n",
    "        df = df.sort_values(by ='Count' , ascending=False)\n",
    "        DT2 = df.loc[0][\"month\"]\n",
    "    else:    \n",
    "        DT1 = generateDecisionTree(R1,maxDepth-1)\n",
    "        DT2 = generateDecisionTree(R2,maxDepth-1)\n",
    "    \n",
    "    dTree = [minFeature, threshold, DT1, DT2]\n",
    "    \n",
    "    return dTree\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### random forest classification########################################\n",
    "\n",
    "def generateRandomDataSet(trainSet, bagSize, n):\n",
    "    bagTrainSet = []\n",
    "    l = len(trainSet)\n",
    "    size = int(l*bagSize/100)\n",
    "    for i in range(n):\n",
    "        bagTrainSet.append(trainSet.sample(n = size))\n",
    "    \n",
    "    return bagTrainSet\n",
    "\n",
    "def trainRandomForest(bagTrainSet, maxDepth):\n",
    "    bagDtrees = []\n",
    "    for i in range(len(bagTrainSet)):\n",
    "        dTree = generateRandomForestTree(bagTrainSet[i], maxDepth)\n",
    "        print(\"done\")\n",
    "        bagDtrees.append(dTree)\n",
    "    return bagDtrees\n",
    "\n",
    "def testRandomForest(bagDtrees, testSet):\n",
    "    bagResults = []\n",
    "    for i in range(len(testSet)):\n",
    "        testData = testSet.iloc[i]\n",
    "        result = []\n",
    "        for dTree in bagDtrees:\n",
    "            y = testDecisionTree(testData, dTree)\n",
    "            result.append(y)\n",
    "        bagResults.append((result, testData[\"month\"]))\n",
    "    print(bagResults)\n",
    "    count = getBagAccuracy(bagResults)\n",
    "    print(\"Accuracy\", count/len(testSet))\n",
    "    \n",
    "    \n",
    "def getBagAccuracy(bagResults):\n",
    "    accuracy = 0\n",
    "    count = []\n",
    "    classes = []\n",
    "    for results in bagResults:\n",
    "        count = []\n",
    "        classes = []\n",
    "        result = results[0]\n",
    "        truth = results[1]\n",
    "        for y in result:\n",
    "            if(y not in classes):\n",
    "                count.append(0)\n",
    "                classes.append(y)\n",
    "            indx =  classes.index(y)\n",
    "            count[indx]+=1\n",
    "        \n",
    "        indx = count.index(max(count))\n",
    "        if(classes[indx] == truth):\n",
    "            accuracy+=1\n",
    "    return accuracy\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################################### REGRESSION ############################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "##########################################      REGRESSION   ######################################################\n",
    "\n",
    "def getRegressionFeatures(features, target):\n",
    "    features.remove(target)\n",
    "    return features\n",
    "\n",
    "def getRegionError(nLeft, nRight):\n",
    "    meanLeft = np.mean(nLeft)\n",
    "    meanRight = np.mean(nRight)\n",
    "    nLeft = nLeft-meanLeft\n",
    "    nRight = nRight - meanRight\n",
    "    nLeft = np.square(nLeft)\n",
    "    nRight = np.square(nRight)\n",
    "    error = np.sum(nLeft) + np.sum(nRight)\n",
    "    return error\n",
    "\n",
    "def getParentError(regionValues):\n",
    "    meanRegion = np.mean(regionValues)\n",
    "    regionValues = regionValues-meanRegion\n",
    "    regionValues = np.square(regionValues)\n",
    "    error = np.sum(regionValues)\n",
    "    return error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainRegressionTree(trainSet, maxDepth, target,  maxWidth = 0):\n",
    "    dTree = generateRegressionTree(trainSet, maxDepth, target)\n",
    "    return dTree\n",
    "    \n",
    "def testRegressionTree(test, dTree):\n",
    "    if(isinstance(dTree, list)):\n",
    "        minFeature = dTree[0]\n",
    "        threshold = dTree[1]\n",
    "        if(test[minFeature] < threshold):\n",
    "            return testRegressionTree(test, dTree[2])\n",
    "        else:\n",
    "            return testRegressionTree(test, dTree[3])\n",
    "    else:\n",
    "        return dTree\n",
    "    \n",
    "def getMSE(results):\n",
    "    mse = 0\n",
    "    error = 0\n",
    "    sd = 0\n",
    "    elements = []\n",
    "    for i in range(len(results)):\n",
    "        elements.append(results[i][0]-results[i][1])\n",
    "        error+= results[i][0]-results[i][1]\n",
    "        mse += np.square(results[i][0]-results[i][1])\n",
    "    sd = np.std(elements)\n",
    "    error = error/len(elements)\n",
    "    return mse, [error, sd]\n",
    "\n",
    "def getRegressionResults(testSet, dTree, target):\n",
    "    results = []\n",
    "    for i in range(len(testSet)):\n",
    "        testData = testSet.iloc[i]\n",
    "        result = testRegressionTree(testData, dTree)\n",
    "#         print(result, testData[target])\n",
    "        results.append((result, testData[target]))\n",
    "    count, error = getMSE(results)\n",
    "#     print(count)\n",
    "    print(\"error mean\", error[0])\n",
    "    print(\"standard deviation error\", error[1])\n",
    "    print(\"MSE\", count/len(testSet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### ---------- Regression Decision Tree -----------\n",
    "\n",
    "def generateRegressionTree(data, maxDepth, target):\n",
    "    dTree = []\n",
    "    targetValues = sorted(list(data[target]))\n",
    "#     print(targetValues)\n",
    "#     minError = getParentError(targetValues)\n",
    "#     parentError = minError\n",
    "    minError = math.inf\n",
    "    l = len(data)\n",
    "#     print(regressionFeatures)\n",
    "    for feature in regressionFeatures:\n",
    "        dataSorted = data.sort_values([feature])\n",
    "        nRight = []\n",
    "        nRight[:] = targetValues\n",
    "        nLeft = []\n",
    "        for i in range(0,l-1):\n",
    "            row = dataSorted.iloc[i]\n",
    "            y = row[target]\n",
    "#             print(y)\n",
    "            nRight.remove(y)\n",
    "            nLeft.append(y)\n",
    "            if(row[feature] == dataSorted.iloc[i+1][feature]):\n",
    "                continue\n",
    "            error = getRegionError(nLeft, nRight)\n",
    "            if(minError> error):\n",
    "                minError = error\n",
    "                minFeature = feature\n",
    "                threshold = (row[feature] + dataSorted.iloc[i+1][feature])/2\n",
    "\n",
    "    \n",
    "#     print(\"split at\", maxDepth)\n",
    "\n",
    "    if(minError==math.inf):\n",
    "        targetMean = data[target].mean()\n",
    "        return targetMean\n",
    "    \n",
    "    R1 = data.loc[data[minFeature] < threshold]\n",
    "    R2 = data.loc[data[minFeature] >= threshold]\n",
    "    \n",
    "    if(maxDepth == 0):    \n",
    "        DT1 = R1[target].mean()\n",
    "        DT2 = R2[target].mean()\n",
    "    else:    \n",
    "        DT1 = generateRegressionTree(R1,maxDepth-1, target)\n",
    "        DT2 = generateRegressionTree(R2,maxDepth-1, target)\n",
    "    \n",
    "    dTree = [minFeature, threshold, DT1, DT2]\n",
    "    \n",
    "    return dTree\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### ---------- Regression Decision Tree -----------\n",
    "\n",
    "def generateRTreeWithWidth(data, maxDepth, target, maxWidth = math.pow(2, 500)):\n",
    "    dTree = []\n",
    "    targetValues = sorted(list(data[target]))\n",
    "    minError = math.inf\n",
    "    l = len(data)\n",
    "#     print(regressionFeatures)\n",
    "    for feature in regressionFeatures:\n",
    "        dataSorted = data.sort_values([feature])\n",
    "        nRight = []\n",
    "        nRight[:] = targetValues\n",
    "        nLeft = []\n",
    "        for i in range(0,l-1):\n",
    "            row = dataSorted.iloc[i]\n",
    "            y = row[target]\n",
    "#             print(y)\n",
    "            nRight.remove(y)\n",
    "            nLeft.append(y)\n",
    "            if(row[feature] == dataSorted.iloc[i+1][feature]):\n",
    "                continue\n",
    "            error = getRegionError(nLeft, nRight)\n",
    "            if(minError> error):\n",
    "                minError = error\n",
    "                minFeature = feature\n",
    "                threshold = (row[feature] + dataSorted.iloc[i+1][feature])/2\n",
    "\n",
    "    \n",
    "#     print(\"split at\", maxDepth)\n",
    "\n",
    "    if(minError==math.inf or maxWidth <= 0):\n",
    "        targetMean = data[target].mean()\n",
    "        return targetMean, maxWidth-1\n",
    "    \n",
    "    R1 = data.loc[data[minFeature] < threshold]\n",
    "    R2 = data.loc[data[minFeature] >= threshold]\n",
    "    \n",
    "    if(maxDepth == 0):    \n",
    "        DT1 = R1[target].mean()\n",
    "        DT2 = R2[target].mean()\n",
    "        width = maxWidth - 2\n",
    "    else:    \n",
    "        DT1, width = generateRTreeWithWidth(R1,maxDepth-1, target, maxWidth)\n",
    "        DT2, width = generateRTreeWithWidth(R2,maxDepth-1, target, width)\n",
    "    \n",
    "    dTree = [minFeature, threshold, DT1, DT2]\n",
    "    \n",
    "    return dTree, width\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################### -----  Regression using Bagged Decision Tree  ----- ########################################\n",
    "\n",
    "def generateBaggedDataSet(trainSet, bagSize, n):\n",
    "    bagTrainSet = []\n",
    "    l = len(trainSet)\n",
    "    size = int(l*bagSize/100)\n",
    "    for i in range(n):\n",
    "        bagTrainSet.append(trainSet.sample(n = size))\n",
    " \n",
    "    return bagTrainSet\n",
    "\n",
    "def trainBagRegressionTree(bagTrainSet, maxDepth, target):\n",
    "    bagDtrees = []\n",
    "    for i in range(len(bagTrainSet)):\n",
    "        dTree = generateRegressionTree(bagTrainSet[i], maxDepth, target)\n",
    "        print(\"done\")\n",
    "        bagDtrees.append(dTree)\n",
    "    return bagDtrees\n",
    "\n",
    "def testBagRtree(bagDtrees, testSet, target):\n",
    "    bagResults = []\n",
    "    for i in range(len(testSet)):\n",
    "        testData = testSet.iloc[i]\n",
    "        result = []\n",
    "        for dTree in bagDtrees:\n",
    "            y = testRegressionTree(testData, dTree)\n",
    "            result.append(y)\n",
    "        result = np.array(result)\n",
    "        resultMean = np.mean(result)\n",
    "        bagResults.append((resultMean, testData[target]))\n",
    "    count, error = getBagMSE(bagResults)\n",
    "    print(\"error mean\", error[0])\n",
    "    print(\"standard deviation error\", error[1])\n",
    "    print(\"MSE\", count/len(testSet))\n",
    "    \n",
    "\n",
    "def getBagMSE(bagResults):\n",
    "    elements = []\n",
    "    error = 0\n",
    "    bagMse = 0\n",
    "    for i in range(len(bagResults)):\n",
    "        error += bagResults[i][0]-bagResults[i][1]\n",
    "        elements.append(bagResults[i][0]-bagResults[i][1])\n",
    "        bagMse += np.square(bagResults[i][0]-bagResults[i][1])\n",
    "    sd = np.std(elements)\n",
    "    error = error/len(elements)\n",
    "    return bagMse, [error, sd]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "################################  Regression Random Forest ###################################\n",
    "\n",
    "def generateRandomForestRegTree(data, maxDepth, target):\n",
    "    dTree = []\n",
    "    targetValues = sorted(list(data[target]))\n",
    "    minError = math.inf\n",
    "    l = len(data)\n",
    "    randomFeatures = random.sample(regressionFeatures, math.ceil(math.sqrt(len(regressionFeatures))))\n",
    "    for feature in randomFeatures:\n",
    "        dataSorted = data.sort_values([feature])\n",
    "        nRight = []\n",
    "        nRight[:] = targetValues\n",
    "        nLeft = []\n",
    "        for i in range(0,l-1):\n",
    "            row = dataSorted.iloc[i]\n",
    "            y = row[target]\n",
    "#             print(y)\n",
    "            nRight.remove(y)\n",
    "            nLeft.append(y)\n",
    "            if(row[feature] == dataSorted.iloc[i+1][feature]):\n",
    "                continue\n",
    "            error = getRegionError(nLeft, nRight)\n",
    "            if(minError> error):\n",
    "                minError = error\n",
    "                minFeature = feature\n",
    "                threshold = (row[feature] + dataSorted.iloc[i+1][feature])/2\n",
    "\n",
    "    \n",
    "#     print(\"split at\", maxDepth)\n",
    "\n",
    "    if(minError==math.inf):\n",
    "        targetMean = data[target].mean()\n",
    "        return targetMean\n",
    "    \n",
    "    R1 = data.loc[data[minFeature] < threshold]\n",
    "    R2 = data.loc[data[minFeature] >= threshold]\n",
    "    \n",
    "    if(maxDepth == 0):    \n",
    "        DT1 = R1[target].mean()\n",
    "        DT2 = R2[target].mean()\n",
    "    else:    \n",
    "        DT1 = generateRandomForestRegTree(R1,maxDepth-1, target)\n",
    "        DT2 = generateRandomForestRegTree(R2,maxDepth-1, target)\n",
    "    \n",
    "    dTree = [minFeature, threshold, DT1, DT2]\n",
    "    \n",
    "    return dTree\n",
    "\n",
    "\n",
    "def trainRegRandomForest(bagTrainSet, maxDepth, target):\n",
    "    bagDtrees = []\n",
    "    for i in range(len(bagTrainSet)):\n",
    "        dTree = generateRandomForestRegTree(bagTrainSet[i], maxDepth, target)\n",
    "        print(\"done\")\n",
    "        bagDtrees.append(dTree)\n",
    "    return bagDtrees\n",
    "\n",
    "def testRegRandomForest(bagDtrees, testSet):\n",
    "    bagResults = []\n",
    "    for i in range(len(testSet)):\n",
    "        testData = testSet.iloc[i]\n",
    "        result = []\n",
    "        for dTree in bagDtrees:\n",
    "            y = testRegressionTree(testData, dTree)\n",
    "            result.append(y)\n",
    "        result = np.array(result)\n",
    "        resultMean = np.mean(result)\n",
    "        bagResults.append((resultMean, testData[target]))\n",
    "    count, error = getBagMSE(bagResults)\n",
    "    print(\"error mean\", error[0])\n",
    "    print(\"standard deviation error\", error[1])\n",
    "    print(\"MSE\", count/len(testSet))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##################################################################################################################\n",
    "### command line argument \n",
    "argumentList = sys.argv \n",
    "dataSetFile = sys.argv[1] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### code start from here\n",
    "trainSet, testSet = getTrainTestData(dataSetFile)\n",
    "featuresMap = getFeaturesListAndValues(trainSet)\n",
    "featuresMap[\"DEWP\"] = sorted(featuresMap[\"DEWP\"])\n",
    "features = list(featuresMap.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "depth = 3\n",
      "Accuracy 10.468036529680365\n",
      "depth = 5\n",
      "Accuracy 15.256849315068493\n",
      "depth = 7\n",
      "Accuracy 20.422374429223744\n",
      "depth = 10\n",
      "Accuracy 32.19178082191781\n",
      "depth = 20\n",
      "Accuracy 38.89840182648402\n",
      "depth = 30\n",
      "Accuracy 38.89840182648402\n",
      "depth = 50\n",
      "Accuracy 38.89840182648402\n",
      "depth = 100\n",
      "Accuracy 38.89840182648402\n"
     ]
    }
   ],
   "source": [
    "####using precomputed decision trees\n",
    "##### Classification using Decision Tree with maxDepth param\n",
    "print(\"Classification using Decision Tree with maxDepth param\")\n",
    "path = \"decision_trees_pickle\"\n",
    "depthList = [3,5,7,10,20,30,50,100]\n",
    "\n",
    "for depth in depthList:\n",
    "    file = path+\"/dTree_d\"+str(depth)\n",
    "    dTree = loadFromPickle(file)\n",
    "    print(\"depth =\",depth,)\n",
    "    getTestResults(testSet, dTree)\n",
    "    \n",
    "### run this code to mnaually test a decision tree\n",
    "# dTree = trainDecisionTree(trainSet, 25)\n",
    "# print(\"Training Accuracy :\")\n",
    "# getTestResults(trainSet, dTree)\n",
    "# print(\"Testing Accuracy :\")\n",
    "# getTestResults(testSet, dTree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "depth = 5\n",
      "width = 5\n",
      "Accuracy 8.43607305936073\n",
      "width = 10\n",
      "Accuracy 8.43607305936073\n",
      "width = 20\n",
      "Accuracy 8.05365296803653\n"
     ]
    }
   ],
   "source": [
    "##### Classification using Decision Tree with maxDepth param and maxWidth param\n",
    "\n",
    "print(\"\\nClassification using Decision Tree with maxDepth param and maxWidth param\")\n",
    "\n",
    "depth = 5\n",
    "widthList = [5, 10, 20]\n",
    "print(\"depth =\",depth,)\n",
    "for width in widthList:\n",
    "    file = path+\"/dTree_d\"+str(depth)+\"_w\"+str(width)\n",
    "    dTree = loadFromPickle(file)\n",
    "    print(\"width =\",width)\n",
    "    getTestResults(testSet, dTree)\n",
    "\n",
    "########## run this code to manually test \n",
    "# maxDepth = 5\n",
    "# maxWidth = 20 \n",
    "# dTree, width = generateDTreeWithParams(trainSet, maxDepth, maxWidth)\n",
    "# print(\"Training Accuracy :\")\n",
    "# getTestResults(trainSet, dTree)\n",
    "# print(\"Testing Accuracy :\")\n",
    "# getTestResults(testSet, dTree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "depth = 3 n = 5\n",
      "Accuracy 10.673515981735159\n",
      "depth = 3 n = 10\n",
      "Accuracy 12.174657534246576\n",
      "depth = 3 n = 15\n",
      "Accuracy 14.126712328767123\n",
      "depth = 5 n = 5\n",
      "Accuracy 16.48972602739726\n",
      "depth = 5 n = 10\n",
      "Accuracy 19.337899543378995\n",
      "depth = 5 n = 15\n",
      "Accuracy 19.726027397260275\n"
     ]
    }
   ],
   "source": [
    "### bagged decision tree classification\n",
    "\n",
    "print(\"\\nbagged decision tree classification\")\n",
    "\n",
    "depthList = [3, 5]\n",
    "nList = [5, 10, 15]\n",
    "for depth in depthList:\n",
    "    for n in nList:\n",
    "        file = path+\"/baggedDTree_d\"+str(depth)+\"_n\"+str(n)\n",
    "        dTree = loadFromPickle(file)\n",
    "        print(\"depth =\",depth, \"n =\", n)\n",
    "        testBagDtree(dTree, testSet)\n",
    "\n",
    "\n",
    "### run this code to manually test\n",
    "# bagSize = 10 ###### bagSize denotes the perentage of training data to be taken for training\n",
    "# n = 15  #### number of decision trees taking part in bagged decision tree training\n",
    "# bagTrainSet = generateBaggedDataSet(trainSet, bagSize, n)\n",
    "# bagDtrees = trainBagDecisionTree(bagTrainSet, maxDepth = 5)\n",
    "# saveInPickle(bagDtrees, \"baggedDTree_d5_n15\")\n",
    "# testBagDtree(bagDtrees, testSet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "depth = 5 n = 5\n",
      "Accuracy 20.8675799086758\n",
      "depth = 5 n = 10\n",
      "Accuracy 18.46461187214612\n",
      "depth = 5 n = 15\n",
      "Accuracy 19.09246575342466\n"
     ]
    }
   ],
   "source": [
    "##### random forest classification  ########################################\n",
    "print(\"\\nrandom forest classification\")\n",
    "\n",
    "depthList = [5]\n",
    "nList = [5, 10, 15]\n",
    "for depth in depthList:\n",
    "    for n in nList:\n",
    "        file = path+\"/randomForest_d\"+str(depth)+\"_n\"+str(n)\n",
    "        dTree = loadFromPickle(file)\n",
    "        print(\"depth =\",depth, \"n =\", n)\n",
    "        testBagDtree(dTree, testSet)\n",
    "\n",
    "### run this code to manually test        \n",
    "# bagSize = 10 ###### bagSize denotes the perentage of training data to be taken for training\n",
    "# n = 10  #### number of decision trees taking part in bagged decision tree training\n",
    "# bagTrainSet = generateRandomDataSet(trainSet, bagSize, n)\n",
    "# randomForest = trainRandomForest(bagTrainSet, maxDepth = 5)\n",
    "# saveInPickle(randomForest, \"randomForest_d5_n10\")\n",
    "# testBagDtree(randomForest, testSet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "regression using decision tree\n",
      "depth = 3\n",
      "error mean -98.00118172295589\n",
      "standard deviation error 93.41026603879334\n",
      "MSE 18329.70942053402\n",
      "depth = 5\n",
      "error mean -6.492356414352403\n",
      "standard deviation error 79.49131000272936\n",
      "MSE 6361.019057760899\n",
      "depth = 10\n",
      "error mean -14.107162193792345\n",
      "standard deviation error 85.16963327443298\n",
      "MSE 7452.878457263348\n",
      "depth = 20\n",
      "error mean -13.658723239657148\n",
      "standard deviation error 92.11119361907664\n",
      "MSE 8671.03271046856\n"
     ]
    }
   ],
   "source": [
    "#### Regression using decision tree with maxDepth\n",
    "\n",
    "print(\"\\nregression using decision tree\")\n",
    "\n",
    "depthList = [3,5,10,20]\n",
    "target=\"pm2.5\"\n",
    "for depth in depthList:\n",
    "    file = path+\"/RTree_d\"+str(depth)\n",
    "    dTree = loadFromPickle(file)\n",
    "    print(\"depth =\",depth,)\n",
    "    getRegressionResults(testSet, dTree, target)\n",
    "    \n",
    "# regressionFeatures = list(trainSet.columns)\n",
    "# regressionFeatures.remove(target)\n",
    "# print(regressionFeatures)\n",
    "# target = \"pm2.5\"\n",
    "# maxDepth = 20\n",
    "\n",
    "# dTree = trainRegressionTree(trainSet, maxDepth, target)\n",
    "# print(\"Training Accuracy :\")\n",
    "# getRegressionResults(trainSet, dTree, target)\n",
    "# print(\"Testing Accuracy :\")\n",
    "# getRegressionResults(testSet, dTree, target)\n",
    "# saveInPickle(dTree, \"RTree_d\"+str(maxDepth))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "regression using bagged decision tree\n",
      "depth = 10 n = 5\n",
      "error mean -8.734479877863011\n",
      "standard deviation error 75.8623815644502\n",
      "MSE 5831.392075366973\n",
      "depth = 10 n = 10\n",
      "error mean -8.618251012317113\n",
      "standard deviation error 72.4729240413074\n",
      "MSE 5326.598969608431\n",
      "depth = 10 n = 15\n",
      "error mean -8.656993967499071\n",
      "standard deviation error 72.1550563083018\n",
      "MSE 5281.295695407537\n"
     ]
    }
   ],
   "source": [
    "#### regression using bagged decision tree\n",
    "print(\"\\nregression using bagged decision tree\")\n",
    "depthList = [10]\n",
    "nList = [5, 10, 15]\n",
    "for depth in depthList:\n",
    "    for n in nList:\n",
    "        file = path+\"/baggedRegTree_d\"+str(depth)+\"_n\"+str(n)\n",
    "        dTree = loadFromPickle(file)\n",
    "        print(\"depth =\",depth, \"n =\", n)\n",
    "        testBagRtree(dTree, testSet, target)\n",
    "\n",
    "        \n",
    "# bagSize = 10 ###### bagSize denotes the perentage of training data to be taken for training\n",
    "# n = 10  #### number of decision trees taking part in bagged decision tree training\n",
    "# target = \"pm2.5\"\n",
    "# maxDepth = 10\n",
    "# bagTrainSet = generateBaggedDataSet(trainSet, bagSize, n)\n",
    "# bagDtrees = trainBagRegressionTree(bagTrainSet,maxDepth, target)\n",
    "# testBagRtree(bagDtrees, testSet, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "regression using Random forest\n",
      "depth = 10 n = 5\n",
      "error mean -7.653774585011568\n",
      "standard deviation error 77.10757638595479\n",
      "MSE 6004.15860151401\n",
      "depth = 10 n = 10\n",
      "error mean -6.010047673735416\n",
      "standard deviation error 75.01752098500705\n",
      "MSE 5663.749127776518\n",
      "depth = 10 n = 15\n",
      "error mean -7.026487414689301\n",
      "standard deviation error 74.23671597993476\n",
      "MSE 5560.461524874287\n"
     ]
    }
   ],
   "source": [
    "#### regression using Random forest\n",
    "print(\"\\nregression using Random forest\")\n",
    "depthList = [10]\n",
    "nList = [5, 10, 15]\n",
    "for depth in depthList:\n",
    "    for n in nList:\n",
    "        file = path+\"/Reg_RandomForest_d\"+str(depth)+\"_n\"+str(n)\n",
    "        dTree = loadFromPickle(file)\n",
    "        print(\"depth =\",depth, \"n =\", n)\n",
    "        testRegRandomForest(dTree, testSet)\n",
    "        \n",
    "        \n",
    "# bagSize = 10 ###### bagSize denotes the perentage of training data to be taken for training\n",
    "# n = 10  #### number of decision trees taking part in bagged decision tree training\n",
    "# maxDepth = 10\n",
    "# bagTrainSet = generateRandomDataSet(trainSet, bagSize, n)\n",
    "# randomForest = trainRegRandomForest(bagTrainSet, maxDepth, target)\n",
    "# testRegRandomForest(randomForest, testSet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Classification using Decision Tree with maxDepth param\n",
    "\n",
    "# depthList = [3,5,7,10,20,30,50,100]\n",
    "\n",
    "# for depth in depthList:\n",
    "#     file = path+\"/dTree_d\"+str(depth)\n",
    "#     dTree = loadFromPickle(file)\n",
    "#     print(\"depth =\",depth,)\n",
    "#     getTestResults(testSet, dTree)\n",
    "\n",
    "    \n",
    "# dTree = trainDecisionTree(trainSet, 25)\n",
    "# print(\"Training Accuracy :\")\n",
    "# getTestResults(trainSet, dTree)\n",
    "# print(\"Testing Accuracy :\")\n",
    "# getTestResults(testSet, dTree)\n",
    "# saveInPickle(dTree, \"DTree_d25\")\n",
    "\n",
    "# ##### Classification using Decision Tree with maxDepth param and maxWidth param\n",
    "# maxDepth = 5\n",
    "# maxWidth = \n",
    "# dTree, width = generateDTreeWithParams(trainSet, maxDepth, maxWidth)\n",
    "# print(\"Training Accuracy :\")\n",
    "# getTestResults(trainSet, dTree)\n",
    "# print(\"Testing Accuracy :\")\n",
    "# getTestResults(testSet, dTree)\n",
    "# saveInPickle(dTree, \"DTree_d\"+str(maxDepth)+\"_w\"+str(maxWidth))\n",
    "\n",
    "# ### bagged decision tree classification\n",
    "# bagSize = 10 ###### bagSize denotes the perentage of training data to be taken for training\n",
    "# n = 15  #### number of decision trees taking part in bagged decision tree training\n",
    "# bagTrainSet = generateBaggedDataSet(trainSet, bagSize, n)\n",
    "# bagDtrees = trainBagDecisionTree(bagTrainSet, maxDepth = 5)\n",
    "# saveInPickle(bagDtrees, \"baggedDTree_d5_n15\")\n",
    "# testBagDtree(bagDtrees, testSet)\n",
    "\n",
    "# ##### random forest classification########################################\n",
    "# bagSize = 10 ###### bagSize denotes the perentage of training data to be taken for training\n",
    "# n = 10  #### number of decision trees taking part in bagged decision tree training\n",
    "# bagTrainSet = generateRandomDataSet(trainSet, bagSize, n)\n",
    "# randomForest = trainRandomForest(bagTrainSet, maxDepth = 5)\n",
    "# saveInPickle(randomForest, \"randomForest_d5_n10\")\n",
    "# testBagDtree(randomForest, testSet)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}