Mushroom Data/Mushroom - Decision Tree.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ddedae15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: SPARK_LOCAL_HOSTNAME=localhost\n"
     ]
    }
   ],
   "source": [
    "# đặt biến môi trường \n",
    "%env SPARK_LOCAL_HOSTNAME=localhost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "34803a70",
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "afe8bfe9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Welcome to\n",
      "      ____              __\n",
      "     / __/__  ___ _____/ /__\n",
      "    _\\ \\/ _ \\/ _ `/ __/  '_/\n",
      "   /__ / .__/\\_,_/_/ /_/\\_\\   version 3.2.0\n",
      "      /_/\n",
      "\n",
      "Using Python version 3.9.7 (default, Sep 16 2021 16:59:28)\n",
      "Spark context Web UI available at http://localhost:4040\n",
      "Spark context available as 'sc' (master = local[*], app id = local-1642964313358).\n",
      "SparkSession available as 'spark'.\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.classification import DecisionTreeClassifier\n",
    "from pyspark.ml.feature import StringIndexer, VectorIndexer, VectorAssembler\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "import scipy\n",
    "from pyspark.python.pyspark.shell import spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bed71438",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[class: string, cap-shape: string, cap-surface: string, cap-color: string, bruises: string, odor: string, gill-attachment: string, gill-spacing: string, gill-size: string, gill-color: string, stalk-shape: string, stalk-root: string, stalk-surface-above-ring: string, stalk-surface-below-ring: string, stalk-color-above-ring: string, stalk-color-below-ring: string, veil-type: string, veil-color: string, ring-number: string, ring-type: string, spore-print-color: string, population: string, habitat: string]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = spark.read.load(\"C:/Users/Admin/Documents/mushrooms.csv\", format=\"csv\", header=True, delimiter=\",\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6a32b2f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+---------+-----------+---------+-------+----+---------------+------------+---------+----------+-----------+----------+------------------------+------------------------+----------------------+----------------------+---------+----------+-----------+---------+-----------------+----------+-------+\n",
      "|class|cap-shape|cap-surface|cap-color|bruises|odor|gill-attachment|gill-spacing|gill-size|gill-color|stalk-shape|stalk-root|stalk-surface-above-ring|stalk-surface-below-ring|stalk-color-above-ring|stalk-color-below-ring|veil-type|veil-color|ring-number|ring-type|spore-print-color|population|habitat|\n",
      "+-----+---------+-----------+---------+-------+----+---------------+------------+---------+----------+-----------+----------+------------------------+------------------------+----------------------+----------------------+---------+----------+-----------+---------+-----------------+----------+-------+\n",
      "|    p|        x|          s|        n|      t|   p|              f|           c|        n|         k|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         s|      u|\n",
      "|    e|        x|          s|        y|      t|   a|              f|           c|        b|         k|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         n|      g|\n",
      "|    e|        b|          s|        w|      t|   l|              f|           c|        b|         n|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         n|      m|\n",
      "|    p|        x|          y|        w|      t|   p|              f|           c|        n|         n|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         s|      u|\n",
      "|    e|        x|          s|        g|      f|   n|              f|           w|        b|         k|          t|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        e|                n|         a|      g|\n",
      "|    e|        x|          y|        y|      t|   a|              f|           c|        b|         n|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         n|      g|\n",
      "|    e|        b|          s|        w|      t|   a|              f|           c|        b|         g|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         n|      m|\n",
      "|    e|        b|          y|        w|      t|   l|              f|           c|        b|         n|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         s|      m|\n",
      "|    p|        x|          y|        w|      t|   p|              f|           c|        n|         p|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         v|      g|\n",
      "|    e|        b|          s|        y|      t|   a|              f|           c|        b|         g|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         s|      m|\n",
      "|    e|        x|          y|        y|      t|   l|              f|           c|        b|         g|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         n|      g|\n",
      "|    e|        x|          y|        y|      t|   a|              f|           c|        b|         n|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         s|      m|\n",
      "|    e|        b|          s|        y|      t|   a|              f|           c|        b|         w|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         s|      g|\n",
      "|    p|        x|          y|        w|      t|   p|              f|           c|        n|         k|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         v|      u|\n",
      "|    e|        x|          f|        n|      f|   n|              f|           w|        b|         n|          t|         e|                       s|                       f|                     w|                     w|        p|         w|          o|        e|                k|         a|      g|\n",
      "|    e|        s|          f|        g|      f|   n|              f|           c|        n|         k|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         y|      u|\n",
      "|    e|        f|          f|        w|      f|   n|              f|           w|        b|         k|          t|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        e|                n|         a|      g|\n",
      "|    p|        x|          s|        n|      t|   p|              f|           c|        n|         n|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         s|      g|\n",
      "|    p|        x|          y|        w|      t|   p|              f|           c|        n|         n|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         s|      u|\n",
      "|    p|        x|          s|        n|      t|   p|              f|           c|        n|         k|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         s|      u|\n",
      "+-----+---------+-----------+---------+-------+----+---------------+------------+---------+----------+-----------+----------+------------------------+------------------------+----------------------+----------------------+---------+----------+-----------+---------+-----------------+----------+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d7ff0377",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+---------------+-----------------+---------------+-------------+----------+---------------------+------------------+---------------+----------------+-----------------+----------------+------------------------------+------------------------------+----------------------------+----------------------------+---------------+----------------+-----------------+---------------+-----------------------+----------------+-------------+\n",
      "|class_index|cap-shape_index|cap-surface_index|cap-color_index|bruises_index|odor_index|gill-attachment_index|gill-spacing_index|gill-size_index|gill-color_index|stalk-shape_index|stalk-root_index|stalk-surface-above-ring_index|stalk-surface-below-ring_index|stalk-color-above-ring_index|stalk-color-below-ring_index|veil-type_index|veil-color_index|ring-number_index|ring-type_index|spore-print-color_index|population_index|habitat_index|\n",
      "+-----------+---------------+-----------------+---------------+-------------+----------+---------------------+------------------+---------------+----------------+-----------------+----------------+------------------------------+------------------------------+----------------------------+----------------------------+---------------+----------------+-----------------+---------------+-----------------------+----------------+-------------+\n",
      "|        1.0|            0.0|              1.0|            0.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             7.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             2.0|          4.0|\n",
      "|        0.0|            0.0|              1.0|            3.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             7.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             3.0|          1.0|\n",
      "|        0.0|            3.0|              1.0|            4.0|          1.0|       5.0|                  0.0|               0.0|            0.0|             3.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             3.0|          5.0|\n",
      "|        1.0|            0.0|              0.0|            4.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             3.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             2.0|          4.0|\n",
      "|        0.0|            0.0|              1.0|            1.0|          0.0|       0.0|                  0.0|               1.0|            0.0|             7.0|              0.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            1.0|                    1.0|             4.0|          1.0|\n",
      "|        0.0|            0.0|              0.0|            3.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             3.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             3.0|          1.0|\n",
      "|        0.0|            3.0|              1.0|            4.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             4.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             3.0|          5.0|\n",
      "|        0.0|            3.0|              0.0|            4.0|          1.0|       5.0|                  0.0|               0.0|            0.0|             3.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             2.0|          5.0|\n",
      "|        1.0|            0.0|              0.0|            4.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             1.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             0.0|          1.0|\n",
      "|        0.0|            3.0|              1.0|            3.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             4.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             2.0|          5.0|\n",
      "|        0.0|            0.0|              0.0|            3.0|          1.0|       5.0|                  0.0|               0.0|            0.0|             4.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             3.0|          1.0|\n",
      "|        0.0|            0.0|              0.0|            3.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             3.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             2.0|          5.0|\n",
      "|        0.0|            3.0|              1.0|            3.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             2.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             2.0|          1.0|\n",
      "|        1.0|            0.0|              0.0|            4.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             7.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             0.0|          4.0|\n",
      "|        0.0|            0.0|              2.0|            0.0|          0.0|       0.0|                  0.0|               1.0|            0.0|             3.0|              0.0|             2.0|                           0.0|                           2.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            1.0|                    2.0|             4.0|          1.0|\n",
      "|        0.0|            4.0|              2.0|            1.0|          0.0|       0.0|                  0.0|               0.0|            1.0|             7.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             1.0|          4.0|\n",
      "|        0.0|            1.0|              2.0|            4.0|          0.0|       0.0|                  0.0|               1.0|            0.0|             7.0|              0.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            1.0|                    1.0|             4.0|          1.0|\n",
      "|        1.0|            0.0|              1.0|            0.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             3.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             2.0|          1.0|\n",
      "|        1.0|            0.0|              0.0|            4.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             3.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             2.0|          4.0|\n",
      "|        1.0|            0.0|              1.0|            0.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             7.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             2.0|          4.0|\n",
      "+-----------+---------------+-----------------+---------------+-------------+----------+---------------------+------------------+---------------+----------------+-----------------+----------------+------------------------------+------------------------------+----------------------------+----------------------------+---------------+----------------+-----------------+---------------+-----------------------+----------------+-------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "\n",
    "indexers = [StringIndexer(inputCol=column, outputCol=column+\"_index\").fit(df) for column in df.columns ]\n",
    "\n",
    "stages = indexers\n",
    "pipeline = Pipeline().setStages(stages)\n",
    "df_r = pipeline.fit(df).transform(df)\n",
    "\n",
    "clolumnsToDrop = ['class','cap-shape','cap-surface','cap-color','bruises','odor','gill-attachment','gill-spacing','gill-size','gill-color','stalk-shape','stalk-root','stalk-surface-above-ring','stalk-surface-below-ring','stalk-color-above-ring','stalk-color-below-ring','veil-type','veil-color','ring-number','ring-type','spore-print-color','population','habitat']\n",
    "df_r = df_r.drop(*clolumnsToDrop)\n",
    "df_r.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8bb8d4ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "assem = VectorAssembler(inputCols=[\"class_index\", \"odor_index\"], outputCol='features')\n",
    "df_r = assem.transform(df_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "deaa5c7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "labelIndexer = StringIndexer(inputCol=\"class_index\", outputCol=\"indexedLabel\").fit(df_r)\n",
    "\n",
    "featureIndexer =\\\n",
    "                VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(df_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7fc80563",
   "metadata": {},
   "outputs": [],
   "source": [
    "(trainingData, testData) = df_r.randomSplit([0.8, 0.2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0e614d97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+------------+---------+\n",
      "|prediction|indexedLabel| features|\n",
      "+----------+------------+---------+\n",
      "|       0.0|         0.0|(2,[],[])|\n",
      "|       0.0|         0.0|(2,[],[])|\n",
      "|       0.0|         0.0|(2,[],[])|\n",
      "|       0.0|         0.0|(2,[],[])|\n",
      "|       0.0|         0.0|(2,[],[])|\n",
      "+----------+------------+---------+\n",
      "only showing top 5 rows\n",
      "\n",
      "DecisionTreeClassificationModel: uid=DecisionTreeClassifier_e37a3b2257eb, depth=1, numNodes=3, numClasses=2, numFeatures=2\n",
      "Decision Tree - Test Accuracy = 1\n",
      "Decision Tree - Test Error = 0\n",
      "The Confusion Matrix for Decision Tree Model is :\n",
      "[[864   0 864   0   0   0   0   0   0   0   0   0]\n",
      " [284 532 144 264 212  32  16   0   4   4   0   0]\n",
      " [302 232 174 132 212  64  40  38   4   4   0   0]\n",
      " [348 236 144 112 192   0  16   0   0   0   0   0]\n",
      " [ 12 292   0 280 136   8  24   0   0   0   0   0]\n",
      " [ 64 312   0 216 100  32   0   0   4   4   0   0]\n",
      " [144 160 144   0  20   0  16   0   4   4   0   0]\n",
      " [108  76   0  64 160   0   0   0   0   0   0   0]\n",
      " [ 24   0  24   0   0  24  24   0   0   0   0   0]\n",
      " [ 70   0   6   4   0   0   0   6   0   0   0   0]\n",
      " [ 64   0   0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   8   8   8   0   0   0   0   0]]\n",
      "The precision score for Decision Tree Model is: 0.22968980797636632\n",
      "The recall score for Decision Tree Model is: 0.22968980797636632\n"
     ]
    }
   ],
   "source": [
    "dt = DecisionTreeClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\")\n",
    "\n",
    "\n",
    "pipeline = Pipeline(stages=[labelIndexer, featureIndexer, dt])\n",
    "\n",
    "\n",
    "model = pipeline.fit(trainingData)\n",
    "\n",
    "predictions = model.transform(testData)\n",
    "\n",
    "predictions.select(\"prediction\", \"indexedLabel\", \"features\").show(5)\n",
    "\n",
    "evaluator = MulticlassClassificationEvaluator(\n",
    "    labelCol=\"indexedLabel\", predictionCol=\"prediction\", metricName=\"accuracy\")\n",
    "\n",
    "accuracy = evaluator.evaluate(predictions)\n",
    "\n",
    "y_true = df_r.select(\"gill-color_index\").rdd.flatMap(lambda x: x).collect()\n",
    "y_pred = df_r.select(\"cap-color_index\").rdd.flatMap(lambda x: x).collect()\n",
    "\n",
    "confusionmatrix = confusion_matrix(y_true, y_pred)\n",
    "\n",
    "precision = precision_score(y_true, y_pred, average='micro')\n",
    "\n",
    "recall = recall_score(y_true, y_pred, average='micro')\n",
    "\n",
    "treeModel = model.stages[2]\n",
    "\n",
    "print(treeModel)\n",
    "print(\"Decision Tree - Test Accuracy = %g\" % (accuracy))\n",
    "print(\"Decision Tree - Test Error = %g\" % (1.0 - accuracy))\n",
    "\n",
    "print(\"The Confusion Matrix for Decision Tree Model is :\\n\" + str(confusionmatrix))\n",
    "\n",
    "print(\"The precision score for Decision Tree Model is: \" + str(precision))\n",
    "\n",
    "print(\"The recall score for Decision Tree Model is: \" + str(recall))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23a22c6d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}