Part4_Normalization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5581c1f4",
   "metadata": {},
   "source": [
    "# Normalization on Vector Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed7ae8ad",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks (FPCKNN Imputation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8b4438f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_2_FinalData_NewFeatures_KnnImputation/\"\n",
    "\n",
    "newf_imputed_data = []\n",
    "labels = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_NewFeatures_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        newf_imputed_data.append(pickle.load(f))\n",
    "        \n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(newf_imputed_data[i]).any() or np.isinf(newf_imputed_data[i]).any()))  \n",
    "\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_Labels_NewFeatures_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        labels.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "371ab35f",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks (Mean Imputation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0b3cd703",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_0_FinalData_BaseLineImputation/\"\n",
    "\n",
    "newf_imputed_data = []\n",
    "labels = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_NewFeatures_MeanImputation\" +\".pkl\", 'rb') as f:\n",
    "        newf_imputed_data.append(pickle.load(f))\n",
    "        \n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(newf_imputed_data[i]).any() or np.isinf(newf_imputed_data[i]).any()))  \n",
    "\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_Labels_NewFeatures_MeanImputation\" +\".pkl\", 'rb') as f:\n",
    "        labels.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d68fd48",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks (Nextvalue Imputation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3ae2bdcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_0_FinalData_BaseLineImputation/\"\n",
    "\n",
    "newf_imputed_data = []\n",
    "labels = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_NewFeatures_NextvalueImputation\" +\".pkl\", 'rb') as f:\n",
    "        newf_imputed_data.append(pickle.load(f))\n",
    "        \n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(newf_imputed_data[i]).any() or np.isinf(newf_imputed_data[i]).any()))  \n",
    "\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_Labels_NewFeatures_NextvalueImputation\" +\".pkl\", 'rb') as f:\n",
    "        labels.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ca025e9",
   "metadata": {},
   "source": [
    "### Zscore Normalization Technique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ffae663c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import stats\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "from tqdm import tqdm\n",
    "\n",
    "def zscore(start_partition, end_partition, data, labels, data_dir, name):\n",
    "    \n",
    "    num_new_features = 9\n",
    "    num_attributes = 24\n",
    "    num_all_features = num_attributes * num_new_features\n",
    "\n",
    "    for i in range(start_partition-1,end_partition):\n",
    "        new_partition = np.zeros((np.array(data[i]).shape[0], num_all_features))\n",
    "        new_partition = np.array(data[i])\n",
    "\n",
    "        with tqdm(num_all_features) as pbar:\n",
    "            for j in range(0,num_all_features):\n",
    "                new_column = np.zeros(new_partition.shape[0])  \n",
    "                new_column = new_partition[:,j]\n",
    "\n",
    "                \n",
    "                if np.std(new_column)== 0.0:\n",
    "                    zscore = np.ones(new_partition.shape[0])\n",
    "                else:\n",
    "                    zscore = stats.zscore(new_column)\n",
    "                if (np.isnan(zscore).any() or np.isinf(zscore).any()):\n",
    "                    print('nan-zscore')\n",
    "\n",
    "                new_column = zscore\n",
    "\n",
    "\n",
    "                new_partition[:,j] = new_column\n",
    "                pbar.update(1)\n",
    "\n",
    "            with open(data_dir + \"Partition\" + str(i+1) + \"_\" + name +\".pkl\", 'wb') as f:\n",
    "                pickle.dump(new_partition, f)\n",
    "            with open(data_dir + \"Partition\" + str(i+1) + \"_Labels_\" + name +\".pkl\", 'wb') as f:\n",
    "                pickle.dump(labels[i], f)\n",
    "\n",
    "        \n",
    "                   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd09d732",
   "metadata": {},
   "source": [
    "## MinMax Normalization Technique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a96d3f8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import stats\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "from tqdm import tqdm\n",
    "\n",
    "def minmax(start_partition, end_partition, data, labels, data_dir, name):\n",
    "    \n",
    "    num_new_features = 9\n",
    "    num_attributes = 24\n",
    "    num_all_features = num_attributes * num_new_features\n",
    "\n",
    "    for i in range(start_partition-1,end_partition):\n",
    "        new_partition = np.zeros((np.array(data[i]).shape[0], num_all_features))\n",
    "        new_partition = np.array(data[i])\n",
    "\n",
    "        with tqdm(num_all_features) as pbar:\n",
    "            for j in range(0,num_all_features):\n",
    "                new_column = np.zeros(new_partition.shape[0])  \n",
    "                new_column = new_partition[:,j]\n",
    "\n",
    "                \n",
    "                if np.std(new_column)== 0.0:\n",
    "                    minmax = np.ones(new_partition.shape[0])\n",
    "                else:\n",
    "                    data_2d = new_column.reshape(-1, 1)\n",
    "                    minmax = scaler.fit_transform(data_2d).flatten()\n",
    "                if (np.isnan(minmax).any() or np.isinf(minmax).any()):\n",
    "                    print('nan-zscore')\n",
    "\n",
    "                new_column = minmax\n",
    "\n",
    "\n",
    "                new_partition[:,j] = new_column\n",
    "                pbar.update(1)\n",
    "\n",
    "            with open(data_dir + \"Partition\" + str(i+1) + \"_\" + name +\".pkl\", 'wb') as f:\n",
    "                pickle.dump(new_partition, f)\n",
    "            with open(data_dir + \"Partition\" + str(i+1) + \"_Labels_\" + name +\".pkl\", 'wb') as f:\n",
    "                pickle.dump(labels[i], f)\n",
    "\n",
    "        \n",
    "                   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2fc139eb",
   "metadata": {},
   "source": [
    "### Saving Z-score Normalization and FPCKNN Imputation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e9912acd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:00, 949.52it/s] \n",
      "216it [00:00, 848.12it/s] \n",
      "216it [00:00, 1781.60it/s]\n",
      "216it [00:00, 1520.89it/s]\n",
      "216it [00:00, 939.91it/s] \n"
     ]
    }
   ],
   "source": [
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_1_FinalData_NewFeatures_ZM_KnnImputation/\"\n",
    "name = 'NewFeatures_ZNorm_KnnImputation'\n",
    "\n",
    "zscore(1,5,newf_imputed_data,labels, data_dir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bbc459e",
   "metadata": {},
   "source": [
    "### Saving MinMax Normalization and FPCKNN Imputation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "be752f8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:00, 1034.65it/s]\n",
      "216it [00:00, 878.62it/s] \n",
      "216it [00:00, 1814.47it/s]\n",
      "216it [00:00, 1534.84it/s]\n",
      "216it [00:00, 1048.31it/s]\n"
     ]
    }
   ],
   "source": [
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_1_FinalData_NewFeatures_ZM_KnnImputation/\"\n",
    "name = 'NewFeatures_MinMaxNorm_KnnImputation'\n",
    "\n",
    "minmax(1,5,newf_imputed_data,labels, data_dir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbfd4a59",
   "metadata": {},
   "source": [
    "### Saving MinMax Normalization and Mean Imputation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "02fd9f8b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:00, 990.77it/s] \n",
      "216it [00:00, 798.41it/s] \n",
      "216it [00:00, 1700.95it/s]\n",
      "216it [00:00, 1451.52it/s]\n",
      "216it [00:00, 881.63it/s] \n"
     ]
    }
   ],
   "source": [
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_0_FinalData_ZM_BaseLineImputation/\"\n",
    "name = 'NewFeatures_MinMaxNorm_MeanImputation'\n",
    "\n",
    "minmax(1,5,newf_imputed_data,labels, data_dir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aef23b50",
   "metadata": {},
   "source": [
    "### Saving Z-score Normalization and Mean Imputation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "45ddb504",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:00, 937.13it/s] \n",
      "216it [00:00, 716.38it/s]\n",
      "216it [00:00, 1676.03it/s]\n",
      "216it [00:00, 1411.09it/s]\n",
      "216it [00:00, 911.91it/s] \n"
     ]
    }
   ],
   "source": [
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_0_FinalData_ZM_BaseLineImputation/\"\n",
    "name = 'NewFeatures_ZNorm_MeanImputation'\n",
    "\n",
    "zscore(1,5,newf_imputed_data,labels, data_dir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5623511",
   "metadata": {},
   "source": [
    "### Saving MinMax Normalization and Nextvalue Imputation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dddc8b2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:00, 717.84it/s]\n",
      "216it [00:00, 611.33it/s]\n",
      "216it [00:00, 1258.78it/s]\n",
      "216it [00:00, 867.79it/s] \n",
      "216it [00:00, 681.77it/s]\n"
     ]
    }
   ],
   "source": [
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_0_FinalData_ZM_BaseLineImputation/\"\n",
    "name = 'NewFeatures_MinMaxNorm_NextvalueImputation'\n",
    "\n",
    "minmax(1,5,newf_imputed_data,labels, data_dir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51750192",
   "metadata": {},
   "source": [
    "### Saving Z-score Normalization and Nextvalue Imputation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6124b772",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:00, 679.21it/s]\n",
      "216it [00:00, 556.04it/s]\n",
      "216it [00:00, 1157.89it/s]\n",
      "216it [00:00, 998.19it/s] \n",
      "216it [00:00, 704.72it/s]\n"
     ]
    }
   ],
   "source": [
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_0_FinalData_ZM_BaseLineImputation/\"\n",
    "name = 'NewFeatures_ZNorm_NextvalueImputation'\n",
    "\n",
    "zscore(1,5,newf_imputed_data,labels, data_dir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40cf08dd",
   "metadata": {},
   "source": [
    "# LSBZM Normalization Technique for Vector Data (our Novelty)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8ef69cd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "from scipy import stats\n",
    "from tqdm import tqdm\n",
    "\n",
    "def lsbzm(start_partition, end_partition, data, labels, data_dir, name):\n",
    "\n",
    "    num_new_features = 9\n",
    "    num_attributes = 24\n",
    "    num_all_features = num_attributes * num_new_features\n",
    "    \n",
    "\n",
    "    for i in range(start_partition-1,end_partition):\n",
    "        new_partition = np.zeros((np.array(data[i]).shape[0], num_all_features))\n",
    "        new_partition = np.array(data[i])\n",
    "\n",
    "        with tqdm(num_all_features) as pbar:\n",
    "            for j in range(0,num_all_features):\n",
    "                new_column = np.zeros(new_partition.shape[0])  \n",
    "                new_column = new_partition[:,j]\n",
    "\n",
    "                minmax = np.zeros(new_partition.shape[0])\n",
    "                all_positive = np.zeros(new_partition.shape[0])\n",
    "\n",
    "                the_min = np.min(new_column)\n",
    "                the_max = np.max(new_column)\n",
    "                skewness = stats.skew(new_column)\n",
    "                \n",
    "                        \n",
    "                if (the_max - the_min > 100000):\n",
    "                    if (skewness > 1):\n",
    "                        if (the_min < 0):\n",
    "                            all_positive = new_column + 2*abs(the_min)\n",
    "                        else:\n",
    "                            all_positive = new_column + 0.1\n",
    "                            \n",
    "                        if np.std(all_positive)== 0.0:\n",
    "                            minmax = np.ones(new_partition.shape[0])\n",
    "                        else:\n",
    "                            log = np.log(all_positive)\n",
    "                            data_2d = log.reshape(-1, 1)\n",
    "                            minmax = scaler.fit_transform(data_2d).flatten()\n",
    "                        if (np.isnan(minmax).any() or np.isinf(minmax).any()):\n",
    "                            print('nan-log')\n",
    "                            \n",
    "\n",
    "                    elif (skewness < -1):\n",
    "                        if (the_min < 0):\n",
    "                            all_positive = new_column + abs(the_min)\n",
    "                        else:\n",
    "                            all_positive = new_column + 0.1\n",
    "\n",
    "                        if np.std(all_positive)== 0.0:\n",
    "                            minmax = np.ones(new_partition.shape[0])\n",
    "                        else:\n",
    "                            sqrt = np.sqrt(all_positive)\n",
    "                            data_2d = sqrt.reshape(-1, 1)\n",
    "                            minmax = scaler.fit_transform(data_2d).flatten()\n",
    "                        if (np.isnan(minmax).any() or np.isinf(minmax).any()):\n",
    "                            print('nan-sqrt')\n",
    "\n",
    "                    else:\n",
    "                        if np.std(new_column)== 0.0:\n",
    "                            minmax = np.ones(new_partition.shape[0])\n",
    "                        else:\n",
    "                            zscore = stats.zscore(new_column)\n",
    "                            data_2d = zscore.reshape(-1, 1)\n",
    "                            minmax = scaler.fit_transform(data_2d).flatten()\n",
    "                        if (np.isnan(minmax).any() or np.isinf(minmax).any()):\n",
    "                            print('nan-zscore')\n",
    "                \n",
    "                        \n",
    "                elif (the_max < 1 and the_min > -1):\n",
    "                    \n",
    "                    if (skewness > 1):\n",
    "                        if (the_min < 0):\n",
    "                            all_positive = new_column + 2*abs(the_min)\n",
    "                        else:\n",
    "                            all_positive = new_column + 0.1\n",
    "                            \n",
    "                        if np.std(all_positive)== 0.0:\n",
    "                            minmax = np.ones(new_partition.shape[0])\n",
    "                        else:\n",
    "                            log = np.log(all_positive)\n",
    "                            data_2d = log.reshape(-1, 1)\n",
    "                            minmax = scaler.fit_transform(data_2d).flatten()\n",
    "                        if (np.isnan(minmax).any() or np.isinf(minmax).any()):\n",
    "                            print('nan-log') \n",
    "\n",
    "                    elif (skewness < -1):\n",
    "                        if (the_min < 0):\n",
    "                            all_positive = new_column + abs(the_min)\n",
    "                        else:\n",
    "                            all_positive = new_column + 0.1\n",
    "\n",
    "                        if np.std(all_positive)== 0.0:\n",
    "                            minmax = np.ones(new_partition.shape[0])\n",
    "                        else:\n",
    "                            sqrt = np.sqrt(all_positive)\n",
    "                            data_2d = sqrt.reshape(-1, 1)\n",
    "                            minmax = scaler.fit_transform(data_2d).flatten()\n",
    "                        if (np.isnan(minmax).any() or np.isinf(minmax).any()):\n",
    "                            print('nan-sqrt') \n",
    "\n",
    "                    else:\n",
    "                        if np.std(new_column)== 0.0:\n",
    "                            minmax = np.ones(new_partition.shape[0])\n",
    "                        else:\n",
    "                            zscore = stats.zscore(new_column)\n",
    "                            data_2d = zscore.reshape(-1, 1)\n",
    "                            minmax = scaler.fit_transform(data_2d).flatten()\n",
    "                        if (np.isnan(minmax).any() or np.isinf(minmax).any()):\n",
    "                            print('nan-zscore')\n",
    "                            \n",
    "                        \n",
    "                else:\n",
    "                    if (skewness > 1 or skewness < -1):\n",
    "                        if (the_min < 0):\n",
    "                            all_positive = new_column + 2*abs(the_min)\n",
    "                        else:\n",
    "                            all_positive = new_column + 0.1\n",
    "\n",
    "                        if np.std(all_positive)== 0.0:\n",
    "                            minmax = np.ones(new_partition.shape[0])\n",
    "                        else:\n",
    "                            boxcox, values = stats.boxcox(all_positive)\n",
    "                            data_2d = boxcox.reshape(-1, 1)\n",
    "                            minmax = scaler.fit_transform(data_2d).flatten()\n",
    "                        if (np.isnan(minmax).any() or np.isinf(minmax).any()):\n",
    "                            print('nan-boxcox')\n",
    "\n",
    "                            \n",
    "                    else:\n",
    "                        if np.std(new_column)== 0.0:\n",
    "                            minmax = np.ones(new_partition.shape[0])\n",
    "                        else:\n",
    "                            zscore = stats.zscore(new_column)\n",
    "                            data_2d = zscore.reshape(-1, 1)\n",
    "                            minmax = scaler.fit_transform(data_2d).flatten()\n",
    "                        if (np.isnan(minmax).any() or np.isinf(minmax).any()):\n",
    "                            print('nan-zscore')\n",
    "\n",
    "                new_column = minmax\n",
    "\n",
    "                new_partition[:,j] = new_column\n",
    "\n",
    "                pbar.update(1)\n",
    "\n",
    "            with open(data_dir + \"Partition\" + str(i+1) + \"_\" + name + \".pkl\", 'wb') as f:\n",
    "                pickle.dump(new_partition, f)\n",
    "            with open(data_dir + \"Partition\" + str(i+1) + \"_Labels_\" + name +\".pkl\", 'wb') as f:\n",
    "                pickle.dump(labels[i], f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4932b3cd",
   "metadata": {},
   "source": [
    "### Saving LSBZM Normalization and FPCKNN Imputation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "5d24e12b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:02, 102.16it/s]\n",
      "216it [00:02, 85.43it/s] \n",
      "216it [00:01, 206.00it/s]\n",
      "216it [00:01, 150.81it/s]\n",
      "216it [00:02, 103.78it/s]\n"
     ]
    }
   ],
   "source": [
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_2_FinalData_NewFeatures_LSBZM_KnnImputation/\"\n",
    "name = 'NewFeatures_LSBZM_KnnImputation'\n",
    "\n",
    "lsbzm(1,5,newf_imputed_data,labels, data_dir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a700f4ae",
   "metadata": {},
   "source": [
    "### Saving LSBZM Normalization and Mean Imputation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cddeede5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:03, 66.15it/s] \n",
      "216it [00:04, 47.05it/s]\n",
      "216it [00:01, 115.50it/s]\n",
      "216it [00:02, 79.83it/s] \n",
      "216it [00:03, 63.81it/s]\n"
     ]
    }
   ],
   "source": [
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_0_FinalData_ZM_BaseLineImputation/\"\n",
    "name = 'NewFeatures_LSBZM_MeanImputation'\n",
    "\n",
    "lsbzm(1,5,newf_imputed_data,labels, data_dir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "095b8422",
   "metadata": {},
   "source": [
    "### Saving LSBZM Normalization and Nextvalue Imputation Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "033e557a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:02, 77.40it/s] \n",
      "216it [00:04, 47.34it/s]\n",
      "216it [00:01, 112.66it/s]\n",
      "216it [00:02, 79.96it/s] \n",
      "216it [00:03, 63.52it/s]\n"
     ]
    }
   ],
   "source": [
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_0_FinalData_ZM_BaseLineImputation/\"\n",
    "name = 'NewFeatures_LSBZM_NextvalueImputation'\n",
    "\n",
    "lsbzm(1,5,newf_imputed_data,labels, data_dir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a69414d",
   "metadata": {},
   "source": [
    "## Data Without B and C Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "46fd7b81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_6_FinalData_WithoutB&C_NewFeatures_KnnImputation/\"\n",
    "\n",
    "imputed_wBandC_data = []\n",
    "wBandC_labels = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_WithoutB&C_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        imputed_wBandC_data.append(pickle.load(f))\n",
    "        \n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(imputed_wBandC_data[i]).any() or np.isinf(imputed_wBandC_data[i]).any()))  \n",
    "\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_Labels_WithoutB&C_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        wBandC_labels.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e582709a",
   "metadata": {},
   "source": [
    "### Saving FPCKNN Imputation Data (Without B and C Classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3fa9518e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:01, 135.53it/s]\n",
      "216it [00:01, 113.87it/s]\n",
      "216it [00:00, 234.22it/s]\n",
      "216it [00:01, 180.31it/s]\n",
      "216it [00:01, 133.26it/s]\n"
     ]
    }
   ],
   "source": [
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_5_FinalData_NewFeatures_WithoutB&C_LSBZM_KnnImputation/\"\n",
    "name = 'NewFeatures_WithoutB&C_LSBZM_KnnImputation'\n",
    "\n",
    "lsbzm(1,5,imputed_wBandC_data,wBandC_labels, data_dir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8100f0cb",
   "metadata": {},
   "source": [
    "## Data Without C Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4557034e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_7_FinalData_WithoutC_NewFeatures_KnnImputation/\"\n",
    "\n",
    "imputed_wC_data = []\n",
    "wC_labels = []\n",
    "\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_WithoutC_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        imputed_wC_data.append(pickle.load(f))\n",
    "        \n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(imputed_wC_data[i]).any() or np.isinf(imputed_wC_data[i]).any()))  \n",
    "\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_Labels_WithoutC_KnnImputation\" +\".pkl\", 'rb') as f:\n",
    "        wC_labels.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc9c970e",
   "metadata": {},
   "source": [
    "### Saving FPCKNN Imputation Data (Without C Class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d28477e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:01, 121.13it/s]\n",
      "216it [00:02, 107.66it/s]\n",
      "216it [00:00, 236.22it/s]\n",
      "216it [00:01, 182.00it/s]\n",
      "216it [00:01, 127.33it/s]\n"
     ]
    }
   ],
   "source": [
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_6_FinalData_NewFeatures_WithoutC_LSBZM_KnnImputation/\"\n",
    "name = 'NewFeatures_WithoutC_LSBZM_KnnImputation'\n",
    "\n",
    "lsbzm(1,5,imputed_wC_data, wC_labels, data_dir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67a7527c",
   "metadata": {},
   "source": [
    "### Missing value Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "cbca877f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "# Loading the Raw Data\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_2_FinalData_NewFeatures_LSBZM_KnnImputation/\"\n",
    "normalized_data = []\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_NewFeatures_LSBZM_KnnImputation\" + \".pkl\", 'rb') as f:\n",
    "        normalized_data.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(normalized_data[i]).any() or np.isinf(normalized_data[i]).any()))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "342130f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n",
      "P2 Nan-Value: False\n",
      "P3 Nan-Value: False\n",
      "P4 Nan-Value: False\n",
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "# Loading the Raw Data\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/5_1_FinalData_NewFeatures_ZM_KnnImputation/\"\n",
    "normalized_data = []\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \"_NewFeatures_ZNorm_KnnImputation\" + \".pkl\", 'rb') as f:\n",
    "        normalized_data.append(pickle.load(f))\n",
    "    print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(normalized_data[i]).any() or np.isinf(normalized_data[i]).any()))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ecb5257",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}