NUS_DATATHON_CHAMPION_DataMinds.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### The cell below is for you to keep track of the libraries used and install those libraries quickly\n",
    "##### Ensure that the proper library names are used and the syntax of `%pip install PACKAGE_NAME` is followed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%pip install pandas \n",
    "#%pip install matplotlib\n",
    "#%pip3 install pandas\n",
    "#%pip3 install matplotlib\n",
    "#%pip3 install numpy\n",
    "#%pip3 install scikit-learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **DO NOT CHANGE** the filepath variable\n",
    "##### Instead, create a folder named 'data' in your current working directory and \n",
    "##### have the .csv file inside that. A relative path *must* be used when loading data into pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import modules\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import math\n",
    "from matplotlib.pyplot import subplots\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "filepath = \"./data/catA_train.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **ALL** Code for machine learning and dataset analysis should be entered below. \n",
    "##### Ensure that your code is clear and readable.\n",
    "##### Comments and Markdown notes are advised to direct attention to pieces of code you deem useful."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import necessary libraries\n",
    "## We use sklearn library for RandomForestRegressor implementation\n",
    "# Import necessary libraries\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "import joblib\n",
    "\n",
    "# Save the base model to an HDF5 file\n",
    "\n",
    "# Load the dataset\n",
    "filepath = 'data/catA_train.csv'\n",
    "dataset = pd.read_csv(filepath)\n",
    "\n",
    "# Modification of Dataset & EDA\n",
    "columns_to_drop = [\"AccountID\",\"Company\",\"Industry\",\"8-Digit SIC Code\",\"8-Digit SIC Description\",\"Entity Type\",\"Parent Company\",\"Parent Country\",\"Ownership Type\",\"Company Description\",\"Sales (Global Ultimate Total USD)\",\"Fiscal Year End\",\"Global Ultimate Company\",\"Global Ultimate Country\",\"Domestic Ultimate Company\"]\n",
    "\n",
    "dataset = dataset.drop(columns=[col for col in columns_to_drop if col in dataset.columns], errors='ignore')\n",
    "dataset = dataset[dataset[\"Company Status (Active/Inactive)\"] == \"Active\"]\n",
    "dataset[\"Import/Export Status\"] = dataset[\"Import/Export Status\"].replace({'': '0','Imports':1, 'Exports':2,'Both Imports & Exports': 3})\n",
    "dataset = dataset.drop([\"Company Status (Active/Inactive)\"], axis=1)\n",
    "\n",
    "\n",
    "# Extract features and target variable\n",
    "X = dataset.drop(['Sales (Domestic Ultimate Total USD)'], axis=1)  # Features\n",
    "y = dataset['Sales (Domestic Ultimate Total USD)']  # Target variable\n",
    "\n",
    "\n",
    "### ML: Random Forest Regressor\n",
    "# Split the dataset into training and testing sets\n",
    "# Using 20% data for testing and 80% data for training\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Create a pipeline with preprocessing (standardization) and the Random Forest Regressor\n",
    "MLmodel = Pipeline([\n",
    "    ('scaler', StandardScaler()),  # Standardize features\n",
    "    ('regressor', RandomForestRegressor())  # Random Forest Regressor\n",
    "])\n",
    "\n",
    "# Train the model\n",
    "MLmodel.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = MLmodel.predict(X_test)\n",
    "result = list(y_pred)\n",
    "\n",
    "# Evaluate the model\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "\n",
    "print(\"predicted domestic sales figures: \",result)\n",
    "print(f'Mean Squared Error: {mse}')\n",
    "print(f'R-squared Score: {r2}')\n",
    "\n",
    "joblib.dump(MLmodel, 'mlmodel.h5')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The cell below is **NOT** to be removed\n",
    "##### The function is to be amended so that it accepts the given input (dataframe) and returns the required output (list). \n",
    "##### It is recommended to test the function out prior to submission\n",
    "-------------------------------------------------------------------------------------------------------------------------------\n",
    "##### The hidden_data parsed into the function below will have the same layout columns wise as the dataset *SENT* to you\n",
    "##### Thus, ensure that steps taken to modify the initial dataset to fit into the model are also carried out in the function below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################\n",
    "#Test Hidden_data with loaded model#\n",
    "####################################\n",
    "\n",
    "### After running Load_model.py, a model is trained and tested. Load the mlmodel.h5 created in the directory\n",
    "\n",
    "def testing_hidden_data(hidden_data: pd.DataFrame) -> list:\n",
    "    ### Modify dataset to fit mlmodel\n",
    "    dataset = hidden_data\n",
    "    columns_to_drop = [\"AccountID\",\"Company\",\"Industry\",\"8-Digit SIC Code\",\"8-Digit SIC Description\",\"Entity Type\",\"Parent Company\",\"Parent Country\",\"Ownership Type\",\"Company Description\",\"Sales (Global Ultimate Total USD)\",\"Fiscal Year End\",\"Global Ultimate Company\",\"Global Ultimate Country\",\"Domestic Ultimate Company\"]\n",
    "    dataset = dataset.drop(columns=[col for col in columns_to_drop if col in dataset.columns], errors='ignore')\n",
    "    dataset = dataset[dataset[\"Company Status (Active/Inactive)\"] == \"Active\"]\n",
    "    dataset[\"Import/Export Status\"] = dataset[\"Import/Export Status\"].replace({'': '0','Imports':1, 'Exports':2,'Both Imports & Exports': 3})\n",
    "    dataset = dataset.drop([\"Company Status (Active/Inactive)\"], axis=1)\n",
    "    \n",
    "    \n",
    "    loaded_model = joblib.load('./mlmodel.h5')\n",
    "    result = list(loaded_model.predict(dataset))\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Cell to check testing_hidden_data function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This cell should output a list of predictions.\n",
    "filepath = '' # Fill in your test_hidden_data path\n",
    "test_df = pd.read_csv(filepath)\n",
    "test_df = test_df.drop(columns=['Sales (Domestic Ultimate Total USD)'])\n",
    "print(testing_hidden_data(test_df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Please have the filename renamed and ensure that it can be run with the requirements above being met. All the best!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}