main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ac7a8235",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fd984fc0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\SPARK\\\\spark-3.2.1-bin-hadoop3.2'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import findspark\n",
    "findspark.init()\n",
    "findspark.find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b26b10d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "# os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
    "# os.environ[\"SPARK_HOME\"] = \"spark-3.2.1-bin-hadoop3.2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "597b098f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1184313f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark import SparkContext, SparkConf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "925f383d",
   "metadata": {},
   "outputs": [],
   "source": [
    "conf = pyspark.SparkConf().setAppName('Netflix').setMaster('local')\n",
    "sc = pyspark.SparkContext(conf=conf)\n",
    "spark = SparkSession(sc)\n",
    "\n",
    "# sc = pyspark.SparkContext(appName=\"Netflix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "12b6acf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# spark = SparkSession.builder.getOrCreate()"
   ]
  },
  {
   "cell_type": "raw",
   "id": "e326550a",
   "metadata": {},
   "source": [
    "combineddata*.txt is a file that contains: movie_id: user_id, rating, date\n",
    "\n",
    "c_combineddata.csv is a file that transforms combineddata.txt into movie_id, user_id, rating, date# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4d079654",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.types import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09fcbc23",
   "metadata": {},
   "source": [
    "# Combined Data 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bcabad5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_schema = StructType([\n",
    "    StructField('ID Film',IntegerType(), False),\n",
    "    StructField('ID User',IntegerType(), False),\n",
    "    StructField('Rating',IntegerType(), False),\n",
    "    StructField('Tanggal',DateType(), False)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "238738a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = spark.read.csv(\n",
    "    'Cleaned Input/dataset/c_combined_data_1.csv', \n",
    "    header=True, \n",
    "    schema=data_schema\n",
    ").cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "da1de615",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------+------+----------+\n",
      "|ID Film|ID User|Rating|   Tanggal|\n",
      "+-------+-------+------+----------+\n",
      "|      1| 822109|     5|2005-05-13|\n",
      "|      1| 885013|     4|2005-10-19|\n",
      "|      1|  30878|     4|2005-12-26|\n",
      "|      1| 823519|     3|2004-05-03|\n",
      "|      1| 893988|     3|2005-11-17|\n",
      "|      1| 124105|     4|2004-08-05|\n",
      "|      1|1248029|     3|2004-04-22|\n",
      "|      1|1842128|     4|2004-05-09|\n",
      "|      1|2238063|     3|2005-05-11|\n",
      "|      1|1503895|     4|2005-05-19|\n",
      "|      1|2207774|     5|2005-06-06|\n",
      "|      1|2590061|     3|2004-08-12|\n",
      "|      1|   2442|     3|2004-04-14|\n",
      "|      1| 543865|     4|2004-05-28|\n",
      "|      1|1209119|     4|2004-03-23|\n",
      "|      1| 804919|     4|2004-06-10|\n",
      "|      1|1086807|     3|2004-12-28|\n",
      "|      1|1711859|     4|2005-05-08|\n",
      "|      1| 372233|     5|2005-11-23|\n",
      "|      1|1080361|     3|2005-03-28|\n",
      "+-------+-------+------+----------+\n",
      "only showing top 20 rows\n",
      "\n",
      "Number of Rows 24053523\n"
     ]
    }
   ],
   "source": [
    "df_1.show()\n",
    "row = df_1.count()\n",
    "print(f'Number of Rows {row}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "935bc0d3",
   "metadata": {},
   "source": [
    "# Combined Data 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "14fa2c46",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------+------+----------+\n",
      "|ID Film|ID User|Rating|   Tanggal|\n",
      "+-------+-------+------+----------+\n",
      "|   4500| 573364|     3|2005-06-20|\n",
      "|   4500|1696725|     3|2004-02-27|\n",
      "|   4500|1253431|     3|2004-03-31|\n",
      "|   4500|1265574|     2|2003-09-01|\n",
      "|   4500|1049643|     1|2003-11-15|\n",
      "|   4500|1601348|     4|2005-04-05|\n",
      "|   4500|1495289|     5|2005-07-09|\n",
      "|   4500|1254903|     3|2003-09-02|\n",
      "|   4500|2604070|     3|2005-05-15|\n",
      "|   4500|1006473|     5|2005-05-23|\n",
      "|   4500|1989892|     3|2004-04-06|\n",
      "|   4500|1517471|     4|2003-12-24|\n",
      "|   4500|1478381|     4|2005-05-21|\n",
      "|   4500| 923084|     2|2004-11-15|\n",
      "|   4500|2446292|     4|2005-10-06|\n",
      "|   4500|2554745|     3|2003-05-07|\n",
      "|   4500|1133125|     5|2004-08-10|\n",
      "|   4500| 349528|     4|2003-08-11|\n",
      "|   4500|1614895|     5|2004-08-29|\n",
      "|   4500| 424958|     4|2005-08-02|\n",
      "+-------+-------+------+----------+\n",
      "only showing top 20 rows\n",
      "\n",
      "Number of Rows 26977590\n"
     ]
    }
   ],
   "source": [
    "df_2 = spark.read.csv(\n",
    "    'Cleaned Input/dataset/c_combined_data_2.csv', \n",
    "    header=True, \n",
    "    schema=data_schema\n",
    ").cache()\n",
    "\n",
    "df_2.show()\n",
    "row = df_2.count()\n",
    "print(f'Number of Rows {row}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "223e68d8",
   "metadata": {},
   "source": [
    "# Combined Data 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "37fcba05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------+------+----------+\n",
      "|ID Film|ID User|Rating|   Tanggal|\n",
      "+-------+-------+------+----------+\n",
      "|   9211|2435457|     2|2005-06-01|\n",
      "|   9211|2338545|     3|2001-02-17|\n",
      "|   9211|2218269|     1|2002-12-27|\n",
      "|   9211| 441153|     4|2002-10-11|\n",
      "|   9211|1921624|     2|2005-08-31|\n",
      "|   9211|2096652|     3|2004-05-31|\n",
      "|   9211| 818736|     2|2004-02-17|\n",
      "|   9211| 284560|     3|2003-07-27|\n",
      "|   9211|1211224|     5|2004-05-08|\n",
      "|   9211|1984086|     1|2004-09-16|\n",
      "|   9211|1389539|     3|2005-06-07|\n",
      "|   9211| 454575|     2|2004-10-23|\n",
      "|   9211| 149028|     3|2003-06-19|\n",
      "|   9211|1105843|     2|2005-04-07|\n",
      "|   9211| 353813|     2|2004-08-05|\n",
      "|   9211| 903779|     3|2001-05-21|\n",
      "|   9211| 639194|     2|2003-10-19|\n",
      "|   9211| 308031|     4|2000-09-07|\n",
      "|   9211|1794933|     1|2003-02-14|\n",
      "|   9211| 625085|     3|2004-05-21|\n",
      "+-------+-------+------+----------+\n",
      "only showing top 20 rows\n",
      "\n",
      "Number of Rows 22601628\n"
     ]
    }
   ],
   "source": [
    "df_3 = spark.read.csv(\n",
    "    'Cleaned Input/dataset/c_combined_data_3.csv', \n",
    "    header=True, \n",
    "    schema=data_schema\n",
    ").cache()\n",
    "\n",
    "df_3.show()\n",
    "row = df_3.count()\n",
    "print(f'Number of Rows {row}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdab4e90",
   "metadata": {},
   "source": [
    "# Combined Data 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "45dffaf0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------+------+----------+\n",
      "|ID Film|ID User|Rating|   Tanggal|\n",
      "+-------+-------+------+----------+\n",
      "|  13368| 659432|     3|2005-03-16|\n",
      "|  13368| 751812|     2|2002-12-16|\n",
      "|  13368|2625420|     2|2004-05-25|\n",
      "|  13368|1650301|     1|2005-08-30|\n",
      "|  13368|2269227|     4|2005-10-27|\n",
      "|  13368|2220672|     4|2002-08-19|\n",
      "|  13368|2500511|     4|2003-08-11|\n",
      "|  13368|1452058|     2|2005-01-29|\n",
      "|  13368|1624891|     3|2002-07-27|\n",
      "|  13368| 970031|     3|2004-04-14|\n",
      "|  13368| 345673|     4|2005-04-07|\n",
      "|  13368|1426869|     5|2005-04-21|\n",
      "|  13368|1037088|     2|2005-09-14|\n",
      "|  13368|2079559|     5|2005-10-08|\n",
      "|  13368|2175560|     5|2005-01-12|\n",
      "|  13368| 946805|     4|2005-02-09|\n",
      "|  13368| 767843|     1|2005-05-04|\n",
      "|  13368| 464031|     3|2001-02-08|\n",
      "|  13368|2473764|     4|2005-09-26|\n",
      "|  13368|2339119|     3|2005-11-27|\n",
      "+-------+-------+------+----------+\n",
      "only showing top 20 rows\n",
      "\n",
      "Number of Rows 26847421\n"
     ]
    }
   ],
   "source": [
    "df_4 = spark.read.csv(\n",
    "    'Cleaned Input/dataset/c_combined_data_4.csv', \n",
    "    header=True, \n",
    "    schema=data_schema\n",
    ").cache()\n",
    "\n",
    "df_4.show()\n",
    "row = df_4.count()\n",
    "print(f'Number of Rows {row}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "558013f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------+------+----------+\n",
      "|ID Film|ID User|Rating|   Tanggal|\n",
      "+-------+-------+------+----------+\n",
      "|      1| 822109|     5|2005-05-13|\n",
      "|      1| 885013|     4|2005-10-19|\n",
      "|      1|  30878|     4|2005-12-26|\n",
      "|      1| 823519|     3|2004-05-03|\n",
      "|      1| 893988|     3|2005-11-17|\n",
      "|      1| 124105|     4|2004-08-05|\n",
      "|      1|1248029|     3|2004-04-22|\n",
      "|      1|1842128|     4|2004-05-09|\n",
      "|      1|2238063|     3|2005-05-11|\n",
      "|      1|1503895|     4|2005-05-19|\n",
      "|      1|2207774|     5|2005-06-06|\n",
      "|      1|2590061|     3|2004-08-12|\n",
      "|      1|   2442|     3|2004-04-14|\n",
      "|      1| 543865|     4|2004-05-28|\n",
      "|      1|1209119|     4|2004-03-23|\n",
      "|      1| 804919|     4|2004-06-10|\n",
      "|      1|1086807|     3|2004-12-28|\n",
      "|      1|1711859|     4|2005-05-08|\n",
      "|      1| 372233|     5|2005-11-23|\n",
      "|      1|1080361|     3|2005-03-28|\n",
      "+-------+-------+------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "all_samples = df_1.union(df_2.union(df_3.union(df_4)))\n",
    "all_samples.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "91cd0757",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100480162"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_samples.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bf9f2c9",
   "metadata": {},
   "source": [
    "# Cek data apakah ada yang kosong"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "fa7665f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------+------+----------+\n",
      "|ID Film|ID User|Rating|   Tanggal|\n",
      "+-------+-------+------+----------+\n",
      "|      1| 822109|     5|2005-05-13|\n",
      "|      1| 885013|     4|2005-10-19|\n",
      "|      1|  30878|     4|2005-12-26|\n",
      "|      1| 823519|     3|2004-05-03|\n",
      "|      1| 893988|     3|2005-11-17|\n",
      "|      1| 124105|     4|2004-08-05|\n",
      "|      1|1248029|     3|2004-04-22|\n",
      "|      1|1842128|     4|2004-05-09|\n",
      "|      1|2238063|     3|2005-05-11|\n",
      "|      1|1503895|     4|2005-05-19|\n",
      "|      1|2207774|     5|2005-06-06|\n",
      "|      1|2590061|     3|2004-08-12|\n",
      "|      1|   2442|     3|2004-04-14|\n",
      "|      1| 543865|     4|2004-05-28|\n",
      "|      1|1209119|     4|2004-03-23|\n",
      "|      1| 804919|     4|2004-06-10|\n",
      "|      1|1086807|     3|2004-12-28|\n",
      "|      1|1711859|     4|2005-05-08|\n",
      "|      1| 372233|     5|2005-11-23|\n",
      "|      1|1080361|     3|2005-03-28|\n",
      "+-------+-------+------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "all_samples.show()\n",
    "#all_samples[['ID Film', 'ID User', 'Rating', 'Tanggal']].summary().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab1e9d56",
   "metadata": {},
   "source": [
    "Tidak ada data yang kosong\n",
    "Jumlah data ada 100.480.162\n",
    "Data diambil dari 1999-11-11 hingga 2005-12-31"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e0ed4366",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import year\n",
    "added_column = all_samples.withColumn('year', year(all_samples['Tanggal']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3cb72ed6",
   "metadata": {},
   "outputs": [],
   "source": [
    "year_list = range(1999,2006)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2a23e7ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_count = []\n",
    "for y in year_list:\n",
    "    user_count.append(added_column.select('ID User').where(added_column.year == y).count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "448cb2df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2178, 924443, 1769031, 4342871, 9985324, 30206362, 53249953]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_count"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f514d355",
   "metadata": {},
   "source": [
    "# Split Data Training dan Testing\n",
    "## Tahun 1999 - 2004 Training, 2005 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "031aef48",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_dataset_1 = added_column.select('ID User','ID Film','Rating').where(added_column.year <= 2004)\n",
    "\n",
    "dataset_2005 = added_column.select('ID User','ID Film','Rating').where(added_column.year == 2005)\n",
    "(training_dataset_2, testing_dataset) = dataset_2005.randomSplit([0.5, 0.5])\n",
    "\n",
    "training_dataset = training_dataset_1.union(training_dataset_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "47422bcf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------+------+\n",
      "|ID User|ID Film|Rating|\n",
      "+-------+-------+------+\n",
      "| 823519|      1|     3|\n",
      "| 124105|      1|     4|\n",
      "|1248029|      1|     3|\n",
      "|1842128|      1|     4|\n",
      "|2590061|      1|     3|\n",
      "|   2442|      1|     3|\n",
      "| 543865|      1|     4|\n",
      "|1209119|      1|     4|\n",
      "| 804919|      1|     4|\n",
      "|1086807|      1|     3|\n",
      "| 558634|      1|     4|\n",
      "|2165002|      1|     4|\n",
      "|1181550|      1|     3|\n",
      "|1227322|      1|     4|\n",
      "| 427928|      1|     4|\n",
      "| 786312|      1|     3|\n",
      "|1133214|      1|     4|\n",
      "|1537427|      1|     4|\n",
      "| 525356|      1|     2|\n",
      "|1910569|      1|     4|\n",
      "+-------+-------+------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "training_dataset.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1740344",
   "metadata": {},
   "source": [
    "# Split Data Training dan Testing\n",
    "## Tahun 1999 - 2004 Training, 2005 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "341052cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_dataset = added_column.select('ID User','ID Film','Rating').where(added_column.year <= 2004)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4f256f60",
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_dataset = added_column.select('ID User','ID Film','Rating').where(added_column.year == 2005)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "687df94f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73860294"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_dataset.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9602e94e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26619868"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_dataset.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85ff55b7",
   "metadata": {},
   "source": [
    "# testing_dataset.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "8f126add",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.recommendation import ALS, ALSModel\n",
    "from pyspark.mllib.evaluation import RegressionMetrics, RankingMetrics\n",
    "from pyspark.ml.evaluation import RegressionEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5c05fce0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Exception while sending command.\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\INTEL\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3444, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"C:\\Users\\INTEL\\AppData\\Local\\Temp/ipykernel_1616/1087180402.py\", line 5, in <module>\n",
      "    model = als.fit(training_dataset)\n",
      "  File \"C:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\pyspark\\ml\\base.py\", line 161, in fit\n",
      "    return self._fit(dataset)\n",
      "  File \"C:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\pyspark\\ml\\wrapper.py\", line 335, in _fit\n",
      "    java_model = self._fit_java(dataset)\n",
      "  File \"C:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\pyspark\\ml\\wrapper.py\", line 332, in _fit_java\n",
      "    return self._java_obj.fit(dataset._jdf)\n",
      "  File \"C:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\java_gateway.py\", line 1321, in __call__\n",
      "    return_value = get_return_value(\n",
      "  File \"C:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\pyspark\\sql\\utils.py\", line 111, in deco\n",
      "    return f(*a, **kw)\n",
      "  File \"C:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\protocol.py\", line 326, in get_return_value\n",
      "    raise Py4JJavaError(\n",
      "py4j.protocol.Py4JJavaError: <unprintable Py4JJavaError object>\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\INTEL\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2064, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'Py4JJavaError' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\clientserver.py\", line 475, in send_command\n",
      "    answer = smart_decode(self.stream.readline()[:-1])\n",
      "  File \"C:\\Users\\INTEL\\anaconda3\\lib\\socket.py\", line 704, in readinto\n",
      "    return self._sock.recv_into(b)\n",
      "ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\java_gateway.py\", line 1038, in send_command\n",
      "    response = connection.send_command(command)\n",
      "  File \"C:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\clientserver.py\", line 503, in send_command\n",
      "    raise Py4JNetworkError(\n",
      "py4j.protocol.Py4JNetworkError: Error while sending or receiving\n"
     ]
    },
    {
     "ename": "ConnectionRefusedError",
     "evalue": "[WinError 10061] No connection could be made because the target machine actively refused it",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
      "    \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_1616/1087180402.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m           implicitPrefs=False)\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mals\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtraining_dataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\pyspark\\ml\\base.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, dataset, params)\u001b[0m\n\u001b[0;32m    160\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 161\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    162\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\pyspark\\ml\\wrapper.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, dataset)\u001b[0m\n\u001b[0;32m    334\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 335\u001b[1;33m         \u001b[0mjava_model\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit_java\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    336\u001b[0m         \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_create_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjava_model\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\pyspark\\ml\\wrapper.py\u001b[0m in \u001b[0;36m_fit_java\u001b[1;34m(self, dataset)\u001b[0m\n\u001b[0;32m    331\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_transfer_params_to_java\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 332\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_java_obj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jdf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    333\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m   1320\u001b[0m         \u001b[0manswer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1321\u001b[1;33m         return_value = get_return_value(\n\u001b[0m\u001b[0;32m   1322\u001b[0m             answer, self.gateway_client, self.target_id, self.name)\n",
      "\u001b[1;32mC:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\pyspark\\sql\\utils.py\u001b[0m in \u001b[0;36mdeco\u001b[1;34m(*a, **kw)\u001b[0m\n\u001b[0;32m    110\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 111\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    112\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[1;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[0;32m    325\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0manswer\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mREFERENCE_TYPE\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 326\u001b[1;33m                 raise Py4JJavaError(\n\u001b[0m\u001b[0;32m    327\u001b[0m                     \u001b[1;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31m<class 'str'>\u001b[0m: (<class 'ConnectionRefusedError'>, ConnectionRefusedError(10061, 'No connection could be made because the target machine actively refused it', None, 10061, None))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mConnectionRefusedError\u001b[0m                    Traceback (most recent call last)",
      "    \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\u001b[0m in \u001b[0;36mshowtraceback\u001b[1;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[0;32m   2067\u001b[0m                                             value, tb, tb_offset=tb_offset)\n\u001b[0;32m   2068\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2069\u001b[1;33m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_showtraceback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0metype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2070\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcall_pdb\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2071\u001b[0m                         \u001b[1;31m# drop into debugger\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\ipykernel\\zmqshell.py\u001b[0m in \u001b[0;36m_showtraceback\u001b[1;34m(self, etype, evalue, stb)\u001b[0m\n\u001b[0;32m    541\u001b[0m             \u001b[1;34m'traceback'\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mstb\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    542\u001b[0m             \u001b[1;34m'ename'\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0metype\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 543\u001b[1;33m             \u001b[1;34m'evalue'\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    544\u001b[0m         }\n\u001b[0;32m    545\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\protocol.py\u001b[0m in \u001b[0;36m__str__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    469\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__str__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    470\u001b[0m         \u001b[0mgateway_client\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjava_exception\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_gateway_client\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 471\u001b[1;33m         \u001b[0manswer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgateway_client\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexception_cmd\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    472\u001b[0m         \u001b[0mreturn_value\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_return_value\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0manswer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgateway_client\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    473\u001b[0m         \u001b[1;31m# Note: technically this should return a bytestring 'str' rather than\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\java_gateway.py\u001b[0m in \u001b[0;36msend_command\u001b[1;34m(self, command, retry, binary)\u001b[0m\n\u001b[0;32m   1034\u001b[0m          \u001b[1;32mif\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mbinary\u001b[0m\u001b[0;31m`\u001b[0m \u001b[1;32mis\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1035\u001b[0m         \"\"\"\n\u001b[1;32m-> 1036\u001b[1;33m         \u001b[0mconnection\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_connection\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1037\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1038\u001b[0m             \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconnection\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\clientserver.py\u001b[0m in \u001b[0;36m_get_connection\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    279\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    280\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mconnection\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mconnection\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msocket\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 281\u001b[1;33m             \u001b[0mconnection\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_create_new_connection\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    282\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mconnection\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    283\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\clientserver.py\u001b[0m in \u001b[0;36m_create_new_connection\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    286\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjava_parameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython_parameters\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    287\u001b[0m             self.gateway_property, self)\n\u001b[1;32m--> 288\u001b[1;33m         \u001b[0mconnection\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconnect_to_java_server\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    289\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_thread_connection\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconnection\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    290\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mconnection\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\SPARK\\spark-3.2.1-bin-hadoop3.2\\python\\lib\\py4j-0.10.9.3-src.zip\\py4j\\clientserver.py\u001b[0m in \u001b[0;36mconnect_to_java_server\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    400\u001b[0m                 self.socket = self.ssl_context.wrap_socket(\n\u001b[0;32m    401\u001b[0m                     self.socket, server_hostname=self.java_address)\n\u001b[1;32m--> 402\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msocket\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjava_address\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjava_port\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    403\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstream\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msocket\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmakefile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"rb\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    404\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_connected\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mConnectionRefusedError\u001b[0m: [WinError 10061] No connection could be made because the target machine actively refused it"
     ]
    }
   ],
   "source": [
    "als = ALS(maxIter=5, regParam=0.01, \n",
    "          userCol=\"ID User\", itemCol=\"ID Film\", ratingCol=\"Rating\",\n",
    "          coldStartStrategy=\"drop\",\n",
    "          implicitPrefs=False)\n",
    "model = als.fit(training_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae8bb0f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate the model by computing the RMSE on the test data\n",
    "predictions = model.transform(testing_dataset)\n",
    "evaluator = RegressionEvaluator(metricName=\"rmse\", labelCol=\"Rating\",\n",
    "                                predictionCol=\"prediction\")\n",
    "\n",
    "rmse = evaluator.evaluate(predictions)\n",
    "print(\"Root-mean-square error = \" + str(rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c36c6223",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}