Mushroom Data/Mushroom - Random Forest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "17a43691",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: SPARK_LOCAL_HOSTNAME=localhost\n"
     ]
    }
   ],
   "source": [
    "# đặt biến môi trường \n",
    "%env SPARK_LOCAL_HOSTNAME=localhost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "485dd163",
   "metadata": {},
   "outputs": [],
   "source": [
    "#khởi động spark\n",
    "import findspark\n",
    "findspark.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "05c99cc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Welcome to\n",
      "      ____              __\n",
      "     / __/__  ___ _____/ /__\n",
      "    _\\ \\/ _ \\/ _ `/ __/  '_/\n",
      "   /__ / .__/\\_,_/_/ /_/\\_\\   version 3.2.0\n",
      "      /_/\n",
      "\n",
      "Using Python version 3.9.7 (default, Sep 16 2021 16:59:28)\n",
      "Spark context Web UI available at http://localhost:4041\n",
      "Spark context available as 'sc' (master = local[*], app id = local-1642773769261).\n",
      "SparkSession available as 'spark'.\n"
     ]
    }
   ],
   "source": [
    "# thêm các thư viện spark để thực hiện random \n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml.feature import IndexToString, StringIndexer, VectorIndexer, VectorAssembler\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from pyspark.python.pyspark.shell import spark\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3f8b3cc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[class: string, cap-shape: string, cap-surface: string, cap-color: string, bruises: string, odor: string, gill-attachment: string, gill-spacing: string, gill-size: string, gill-color: string, stalk-shape: string, stalk-root: string, stalk-surface-above-ring: string, stalk-surface-below-ring: string, stalk-color-above-ring: string, stalk-color-below-ring: string, veil-type: string, veil-color: string, ring-number: string, ring-type: string, spore-print-color: string, population: string, habitat: string]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = spark.read.load(\"C:/Users/Admin/Documents/mushrooms.csv\", format=\"csv\", header=True, delimiter=\",\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dd885664",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+---------+-----------+---------+-------+----+---------------+------------+---------+----------+-----------+----------+------------------------+------------------------+----------------------+----------------------+---------+----------+-----------+---------+-----------------+----------+-------+\n",
      "|class|cap-shape|cap-surface|cap-color|bruises|odor|gill-attachment|gill-spacing|gill-size|gill-color|stalk-shape|stalk-root|stalk-surface-above-ring|stalk-surface-below-ring|stalk-color-above-ring|stalk-color-below-ring|veil-type|veil-color|ring-number|ring-type|spore-print-color|population|habitat|\n",
      "+-----+---------+-----------+---------+-------+----+---------------+------------+---------+----------+-----------+----------+------------------------+------------------------+----------------------+----------------------+---------+----------+-----------+---------+-----------------+----------+-------+\n",
      "|    p|        x|          s|        n|      t|   p|              f|           c|        n|         k|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         s|      u|\n",
      "|    e|        x|          s|        y|      t|   a|              f|           c|        b|         k|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         n|      g|\n",
      "|    e|        b|          s|        w|      t|   l|              f|           c|        b|         n|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         n|      m|\n",
      "|    p|        x|          y|        w|      t|   p|              f|           c|        n|         n|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         s|      u|\n",
      "|    e|        x|          s|        g|      f|   n|              f|           w|        b|         k|          t|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        e|                n|         a|      g|\n",
      "|    e|        x|          y|        y|      t|   a|              f|           c|        b|         n|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         n|      g|\n",
      "|    e|        b|          s|        w|      t|   a|              f|           c|        b|         g|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         n|      m|\n",
      "|    e|        b|          y|        w|      t|   l|              f|           c|        b|         n|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         s|      m|\n",
      "|    p|        x|          y|        w|      t|   p|              f|           c|        n|         p|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         v|      g|\n",
      "|    e|        b|          s|        y|      t|   a|              f|           c|        b|         g|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         s|      m|\n",
      "|    e|        x|          y|        y|      t|   l|              f|           c|        b|         g|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         n|      g|\n",
      "|    e|        x|          y|        y|      t|   a|              f|           c|        b|         n|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         s|      m|\n",
      "|    e|        b|          s|        y|      t|   a|              f|           c|        b|         w|          e|         c|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         s|      g|\n",
      "|    p|        x|          y|        w|      t|   p|              f|           c|        n|         k|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         v|      u|\n",
      "|    e|        x|          f|        n|      f|   n|              f|           w|        b|         n|          t|         e|                       s|                       f|                     w|                     w|        p|         w|          o|        e|                k|         a|      g|\n",
      "|    e|        s|          f|        g|      f|   n|              f|           c|        n|         k|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         y|      u|\n",
      "|    e|        f|          f|        w|      f|   n|              f|           w|        b|         k|          t|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        e|                n|         a|      g|\n",
      "|    p|        x|          s|        n|      t|   p|              f|           c|        n|         n|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                k|         s|      g|\n",
      "|    p|        x|          y|        w|      t|   p|              f|           c|        n|         n|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         s|      u|\n",
      "|    p|        x|          s|        n|      t|   p|              f|           c|        n|         k|          e|         e|                       s|                       s|                     w|                     w|        p|         w|          o|        p|                n|         s|      u|\n",
      "+-----+---------+-----------+---------+-------+----+---------------+------------+---------+----------+-----------+----------+------------------------+------------------------+----------------------+----------------------+---------+----------+-----------+---------+-----------------+----------+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9fed76cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+---------------+-----------------+---------------+-------------+----------+---------------------+------------------+---------------+----------------+-----------------+----------------+------------------------------+------------------------------+----------------------------+----------------------------+---------------+----------------+-----------------+---------------+-----------------------+----------------+-------------+\n",
      "|class_index|cap-shape_index|cap-surface_index|cap-color_index|bruises_index|odor_index|gill-attachment_index|gill-spacing_index|gill-size_index|gill-color_index|stalk-shape_index|stalk-root_index|stalk-surface-above-ring_index|stalk-surface-below-ring_index|stalk-color-above-ring_index|stalk-color-below-ring_index|veil-type_index|veil-color_index|ring-number_index|ring-type_index|spore-print-color_index|population_index|habitat_index|\n",
      "+-----------+---------------+-----------------+---------------+-------------+----------+---------------------+------------------+---------------+----------------+-----------------+----------------+------------------------------+------------------------------+----------------------------+----------------------------+---------------+----------------+-----------------+---------------+-----------------------+----------------+-------------+\n",
      "|        1.0|            0.0|              1.0|            0.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             7.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             2.0|          4.0|\n",
      "|        0.0|            0.0|              1.0|            3.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             7.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             3.0|          1.0|\n",
      "|        0.0|            3.0|              1.0|            4.0|          1.0|       5.0|                  0.0|               0.0|            0.0|             3.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             3.0|          5.0|\n",
      "|        1.0|            0.0|              0.0|            4.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             3.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             2.0|          4.0|\n",
      "|        0.0|            0.0|              1.0|            1.0|          0.0|       0.0|                  0.0|               1.0|            0.0|             7.0|              0.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            1.0|                    1.0|             4.0|          1.0|\n",
      "|        0.0|            0.0|              0.0|            3.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             3.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             3.0|          1.0|\n",
      "|        0.0|            3.0|              1.0|            4.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             4.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             3.0|          5.0|\n",
      "|        0.0|            3.0|              0.0|            4.0|          1.0|       5.0|                  0.0|               0.0|            0.0|             3.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             2.0|          5.0|\n",
      "|        1.0|            0.0|              0.0|            4.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             1.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             0.0|          1.0|\n",
      "|        0.0|            3.0|              1.0|            3.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             4.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             2.0|          5.0|\n",
      "|        0.0|            0.0|              0.0|            3.0|          1.0|       5.0|                  0.0|               0.0|            0.0|             4.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             3.0|          1.0|\n",
      "|        0.0|            0.0|              0.0|            3.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             3.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             2.0|          5.0|\n",
      "|        0.0|            3.0|              1.0|            3.0|          1.0|       4.0|                  0.0|               0.0|            0.0|             2.0|              1.0|             3.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             2.0|          1.0|\n",
      "|        1.0|            0.0|              0.0|            4.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             7.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             0.0|          4.0|\n",
      "|        0.0|            0.0|              2.0|            0.0|          0.0|       0.0|                  0.0|               1.0|            0.0|             3.0|              0.0|             2.0|                           0.0|                           2.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            1.0|                    2.0|             4.0|          1.0|\n",
      "|        0.0|            4.0|              2.0|            1.0|          0.0|       0.0|                  0.0|               0.0|            1.0|             7.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             1.0|          4.0|\n",
      "|        0.0|            1.0|              2.0|            4.0|          0.0|       0.0|                  0.0|               1.0|            0.0|             7.0|              0.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            1.0|                    1.0|             4.0|          1.0|\n",
      "|        1.0|            0.0|              1.0|            0.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             3.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    2.0|             2.0|          1.0|\n",
      "|        1.0|            0.0|              0.0|            4.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             3.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             2.0|          4.0|\n",
      "|        1.0|            0.0|              1.0|            0.0|          1.0|       6.0|                  0.0|               0.0|            1.0|             7.0|              1.0|             2.0|                           0.0|                           0.0|                         0.0|                         0.0|            0.0|             0.0|              0.0|            0.0|                    1.0|             2.0|          4.0|\n",
      "+-----------+---------------+-----------------+---------------+-------------+----------+---------------------+------------------+---------------+----------------+-----------------+----------------+------------------------------+------------------------------+----------------------------+----------------------------+---------------+----------------+-----------------+---------------+-----------------------+----------------+-------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "\n",
    "indexers = [StringIndexer(inputCol=column, outputCol=column+\"_index\").fit(df) for column in df.columns ]\n",
    "\n",
    "stages = indexers\n",
    "pipeline = Pipeline().setStages(stages)\n",
    "df_r = pipeline.fit(df).transform(df)\n",
    "\n",
    "clolumnsToDrop = ['class','cap-shape','cap-surface','cap-color','bruises','odor','gill-attachment','gill-spacing','gill-size','gill-color','stalk-shape','stalk-root','stalk-surface-above-ring','stalk-surface-below-ring','stalk-color-above-ring','stalk-color-below-ring','veil-type','veil-color','ring-number','ring-type','spore-print-color','population','habitat']\n",
    "df_r = df_r.drop(*clolumnsToDrop)\n",
    "df_r.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c1654b89",
   "metadata": {},
   "outputs": [],
   "source": [
    "assem = VectorAssembler(inputCols=[\"class_index\", \"odor_index\"], outputCol='features')\n",
    "df_r = assem.transform(df_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f4f4093b",
   "metadata": {},
   "outputs": [],
   "source": [
    "labelIndexer = StringIndexer(inputCol=\"class_index\", outputCol=\"indexedLabel\").fit(df_r)\n",
    "\n",
    "featureIndexer =\\\n",
    "                VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(df_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f678ceec",
   "metadata": {},
   "outputs": [],
   "source": [
    "(trainingData, testData) = df_r.randomSplit([0.8, 0.2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2a682c9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+-----------+--------+\n",
      "|predictedLabel|class_index|features|\n",
      "+--------------+-----------+--------+\n",
      "|           0.0|        0.0|   [0.0]|\n",
      "|           0.0|        0.0|   [0.0]|\n",
      "|           0.0|        0.0|   [0.0]|\n",
      "|           0.0|        0.0|   [0.0]|\n",
      "|           0.0|        0.0|   [0.0]|\n",
      "+--------------+-----------+--------+\n",
      "only showing top 5 rows\n",
      "\n",
      "RandomForestClassificationModel: uid=RandomForestClassifier_bb6fc3ff7b22, numTrees=10, numClasses=2, numFeatures=1\n",
      "Random Forest - Test Accuracy = 1\n",
      "Random Forest - Test Error = 0\n",
      "The Confusion Matrix for Random Forest Model is :\n",
      "[[1184  288  288  288   48   48  128    0   12    0]\n",
      " [1032  744    0    0    0    0    0   64    0    0]\n",
      " [ 624  288  288  288    0    0    0    0   12    0]\n",
      " [  24  648    0    0  200  200    0    0    0    0]\n",
      " [ 448   96    0    0  152  152  128   64    0    0]\n",
      " [  72   96    0    0    0    0    0    0    0    0]\n",
      " [  80    0    0    0    0    0    0   64    0    0]\n",
      " [  32    0    0    0    0    0    0    0   12    0]\n",
      " [  16    0    0    0    0    0    0    0    0    0]\n",
      " [  16    0    0    0    0    0    0    0    0    0]]\n",
      "The precision score for Random Forest Model is: 0.2914820285573609\n",
      "The recall score for Random Forest Model is: 0.2914820285573609\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\", numTrees=10)\n",
    "\n",
    "# Chuyển label\n",
    "labelConverter = IndexToString(inputCol=\"prediction\", outputCol=\"predictedLabel\",\n",
    "                               labels=labelIndexer.labels)\n",
    "\n",
    "y_true = df_r.select(\"cap-color_index\").rdd.flatMap(lambda x: x).collect()\n",
    "y_pred = df_r.select(\"odor_index\").rdd.flatMap(lambda x: x).collect()\n",
    "\n",
    "pipeline = Pipeline(stages=[labelIndexer, featureIndexer, rf, labelConverter])\n",
    "\n",
    "# Train model.\n",
    "model = pipeline.fit(trainingData)\n",
    "\n",
    "# dự đoán\n",
    "predictions = model.transform(testData)\n",
    "\n",
    "\n",
    "predictions.select(\"predictedLabel\", \"class_index\", \"features\").show(5)\n",
    "\n",
    "evaluator = MulticlassClassificationEvaluator(\n",
    "    labelCol=\"indexedLabel\", predictionCol=\"prediction\", metricName=\"accuracy\")\n",
    "\n",
    "accuracy = evaluator.evaluate(predictions)\n",
    "\n",
    "confusionmatrix = confusion_matrix(y_true, y_pred)\n",
    "\n",
    "precision = precision_score(y_true, y_pred, average='micro')\n",
    "\n",
    "recall = recall_score(y_true, y_pred, average='micro')\n",
    "\n",
    "rfModel = model.stages[2]\n",
    "print(rfModel)\n",
    "print(\"Random Forest - Test Accuracy = %g\" % (accuracy))\n",
    "print(\"Random Forest - Test Error = %g\" % (1.0 - accuracy))\n",
    "\n",
    "print(\"The Confusion Matrix for Random Forest Model is :\\n\" + str(confusionmatrix))\n",
    "\n",
    "print(\"The precision score for Random Forest Model is: \" + str(precision))\n",
    "\n",
    "print(\"The recall score for Random Forest Model is: \" + str(recall))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dab954c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}