Example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import random_split\n",
    "from src.Datasets import CIFAR100Dataset\n",
    "from utils_DGMMC import DGMMClassifier, train_from_features_PCA, test_from_features_PCA, get_means_bandwidth_from_features, CrossEntropy\n",
    "from utils import get_trained_PCA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyper Paremeters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Code running on : cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
    "print('Code running on :', device)\n",
    "\n",
    "P = 80 # percentage of cumulative variance ration to keep\n",
    "G = 1 # Number of Gaussian component in each GMMs\n",
    "embedding = 'IMAGEBIND'\n",
    "\n",
    "classes = 100\n",
    "batch_size = 64\n",
    "nb_epochs = 30"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "EXPERIMENT_PATH = os.path.join('experiments_tutorial_on_CIFAR100')\n",
    "if os.path.isdir(EXPERIMENT_PATH) is False:\n",
    "        os.mkdir(EXPERIMENT_PATH)\n",
    "        \n",
    "FEATURES_ABOSLUTE_PATH = os.path.join('/home/jeremy/Documents/Datasets/CIFAR100', 'Features') # Path to directory containing all the features provided by CLIP or ImageBind\n",
    "\n",
    "embeding_folder = os.path.join(EXPERIMENT_PATH, 'IMAGEBIND')\n",
    "if os.path.isdir(embeding_folder) is False:\n",
    "    os.mkdir(embeding_folder)\n",
    "\n",
    "SDGM_folder_path = os.path.join(embeding_folder, 'DGMMC')\n",
    "if os.path.isdir(SDGM_folder_path) is False:\n",
    "    os.mkdir(SDGM_folder_path)\n",
    "\n",
    "results_path = os.path.join(SDGM_folder_path, 'results')\n",
    "if os.path.isdir(results_path) is False:\n",
    "    os.mkdir(results_path)\n",
    "\n",
    "models_path = os.path.join(SDGM_folder_path, 'models')\n",
    "if os.path.isdir(models_path) is False:\n",
    "    os.mkdir(models_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Datasets and Dataloaders\n",
    "\n",
    "Custom datasets have been created for each datasets in order to load directly the features provided by the pretrained CNN (CLIP or ImageBind)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainset = CIFAR100Dataset(os.path.join(FEATURES_ABOSLUTE_PATH, embedding, 'train'), train=True)\n",
    "train_ds, val_ds = random_split(trainset, [math.floor(0.90*len(trainset)), len(trainset) - math.floor(0.90*len(trainset))])\n",
    "\n",
    "trainloader = torch.utils.data.DataLoader(train_ds, batch_size=batch_size, shuffle=True,num_workers = 8, pin_memory = False)\n",
    "valloader = torch.utils.data.DataLoader(val_ds, batch_size=batch_size, shuffle=False,num_workers = 8, pin_memory = False)\n",
    "\n",
    "testset = CIFAR100Dataset(os.path.join(FEATURES_ABOSLUTE_PATH, embedding, 'test'), train=False)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False,num_workers = 8, pin_memory = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PCA decomposition\n",
    "\n",
    "Here we use the function \"get_trained_PCA\" to train the PCA using the training data o the dataset. Then, we compute the cumulative variate ratio and only keep the $d$ first eigenvectors that contains $P$ of the variance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:01<00:00, 591.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164\n"
     ]
    }
   ],
   "source": [
    "\n",
    "pca = get_trained_PCA(trainloader, 1024)\n",
    "\n",
    "cumsum = np.cumsum(pca.explained_variance_ratio_)\n",
    "\n",
    "d = np.argmax(cumsum >= P/100) + 1\n",
    "\n",
    "print(d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialisation of the model\n",
    "Using our classifier DGMMC, we can initilize the means of each Gaussian around the means o each class computed using the training data of the dataset. This allows to converge much quicker and reduce training time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing : 100%|██████████| 704/704 [00:03<00:00, 180.13batchs/s]\n"
     ]
    }
   ],
   "source": [
    "init_means, init_stds = get_means_bandwidth_from_features(classes, trainloader, pca, d)\n",
    "\n",
    "model = DGMMClassifier(d,classes, G, init_means)\n",
    "model.to(device)\n",
    "\n",
    "criterion = CrossEntropy()\n",
    "\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9, nesterov=True)\n",
    "\n",
    "scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=nb_epochs, eta_min=1e-4)\n",
    "best_loss = math.inf\n",
    "\n",
    "model_path = os.path.join(models_path, 'model.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training procedure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 134.09it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 138.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 0/30] tr_loss: 18.9126 -- tr_acc: 0.858 -- val_loss: 15.5153 -- val_acc: 0.878\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 126.79it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 149.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 1/30] tr_loss: 16.3650 -- tr_acc: 0.888 -- val_loss: 15.0756 -- val_acc: 0.883\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 118.44it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 127.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 2/30] tr_loss: 15.8110 -- tr_acc: 0.894 -- val_loss: 14.5125 -- val_acc: 0.882\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:04<00:00, 143.65it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 139.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 3/30] tr_loss: 15.0426 -- tr_acc: 0.898 -- val_loss: 13.6574 -- val_acc: 0.881\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 125.88it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 178.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 4/30] tr_loss: 13.7519 -- tr_acc: 0.899 -- val_loss: 11.8545 -- val_acc: 0.881\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:04<00:00, 141.04it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 141.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 5/30] tr_loss: 7.7186 -- tr_acc: 0.901 -- val_loss: 5.1879 -- val_acc: 0.890\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:03<00:00, 194.46it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 122.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 6/30] tr_loss: 5.5312 -- tr_acc: 0.917 -- val_loss: 5.0733 -- val_acc: 0.884\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 134.12it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 158.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 7/30] tr_loss: 5.3871 -- tr_acc: 0.919 -- val_loss: 4.9670 -- val_acc: 0.891\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 123.85it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 92.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 8/30] tr_loss: 5.2418 -- tr_acc: 0.917 -- val_loss: 4.8679 -- val_acc: 0.889\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 131.84it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 129.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 9/30] tr_loss: 5.0863 -- tr_acc: 0.918 -- val_loss: 4.7733 -- val_acc: 0.881\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 128.12it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 137.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 10/30] tr_loss: 4.9247 -- tr_acc: 0.918 -- val_loss: 4.6424 -- val_acc: 0.884\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:06<00:00, 110.40it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 177.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 11/30] tr_loss: 4.7499 -- tr_acc: 0.921 -- val_loss: 4.4975 -- val_acc: 0.885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:06<00:00, 117.02it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 166.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 12/30] tr_loss: 4.5753 -- tr_acc: 0.921 -- val_loss: 4.4046 -- val_acc: 0.883\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:04<00:00, 149.87it/s]\n",
      "100%|██████████| 79/79 [00:01<00:00, 74.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 13/30] tr_loss: 4.3911 -- tr_acc: 0.921 -- val_loss: 4.2252 -- val_acc: 0.884\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:06<00:00, 108.24it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 134.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 14/30] tr_loss: 4.2087 -- tr_acc: 0.923 -- val_loss: 4.0792 -- val_acc: 0.887\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:04<00:00, 153.11it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 155.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 15/30] tr_loss: 4.0240 -- tr_acc: 0.923 -- val_loss: 3.9732 -- val_acc: 0.880\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 138.75it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 149.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 16/30] tr_loss: 3.8349 -- tr_acc: 0.924 -- val_loss: 3.8112 -- val_acc: 0.884\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:04<00:00, 151.30it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 161.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 17/30] tr_loss: 3.6518 -- tr_acc: 0.925 -- val_loss: 3.6639 -- val_acc: 0.881\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:04<00:00, 168.60it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 99.05it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 18/30] tr_loss: 3.4791 -- tr_acc: 0.925 -- val_loss: 3.4789 -- val_acc: 0.889\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:06<00:00, 111.14it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 158.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 19/30] tr_loss: 3.3058 -- tr_acc: 0.928 -- val_loss: 3.3848 -- val_acc: 0.885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 123.81it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 134.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 20/30] tr_loss: 3.1464 -- tr_acc: 0.929 -- val_loss: 3.2597 -- val_acc: 0.884\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:04<00:00, 159.38it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 120.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 21/30] tr_loss: 3.0006 -- tr_acc: 0.931 -- val_loss: 3.1441 -- val_acc: 0.885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 129.01it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 148.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 22/30] tr_loss: 2.8660 -- tr_acc: 0.931 -- val_loss: 3.0387 -- val_acc: 0.891\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 120.13it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 131.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 23/30] tr_loss: 2.7438 -- tr_acc: 0.933 -- val_loss: 2.9552 -- val_acc: 0.884\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 127.20it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 106.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 24/30] tr_loss: 2.6337 -- tr_acc: 0.934 -- val_loss: 2.8754 -- val_acc: 0.891\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 119.07it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 133.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 25/30] tr_loss: 2.5381 -- tr_acc: 0.935 -- val_loss: 2.7978 -- val_acc: 0.888\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 119.38it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 132.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 26/30] tr_loss: 2.4521 -- tr_acc: 0.935 -- val_loss: 2.7246 -- val_acc: 0.886\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:04<00:00, 148.15it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 112.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 27/30] tr_loss: 2.3785 -- tr_acc: 0.935 -- val_loss: 2.6827 -- val_acc: 0.888\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:05<00:00, 129.93it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 100.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 28/30] tr_loss: 2.3098 -- tr_acc: 0.936 -- val_loss: 2.6204 -- val_acc: 0.888\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 704/704 [00:06<00:00, 105.51it/s]\n",
      "100%|██████████| 79/79 [00:00<00:00, 140.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 29/30] tr_loss: 2.2462 -- tr_acc: 0.937 -- val_loss: 2.5808 -- val_acc: 0.890\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 157/157 [00:01<00:00, 106.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test: test_loss: 2.74961 -- test_acc: 0.887\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "tr = []\n",
    "val = []\n",
    "for epoch in range(nb_epochs):\n",
    "\n",
    "    model, train_loss, train_acc = train_from_features_PCA(classes, device, model, trainloader, criterion, optimizer, pca, d)\n",
    "    tr.append(np.hstack((train_loss, train_acc)))\n",
    "\n",
    "    val_loss, val_acc = test_from_features_PCA(classes, device, model, valloader, criterion, pca, d)\n",
    "    val.append(np.hstack((val_loss, val_acc)))\n",
    "\n",
    "    print(\"[Epoch {}/{}] tr_loss: {:.4f} -- tr_acc: {:.3f} -- val_loss: {:.4f} -- val_acc: {:.3f}\".format(epoch, nb_epochs, train_loss, train_acc, val_loss, val_acc))\n",
    "\n",
    "    if val_loss < best_loss:\n",
    "        torch.save(model, model_path)\n",
    "        best_loss = val_loss\n",
    "\n",
    "    scheduler.step()\n",
    "\n",
    "best_model = torch.load(model_path)\n",
    "best_model.eval()\n",
    "best_model.to(device)\n",
    "\n",
    "test_loss, test_acc = test_from_features_PCA(classes, device, best_model, testloader, criterion, pca, d)\n",
    "print(\"Test: test_loss: {:.5f} -- test_acc: {:.3f}\".format(test_loss, test_acc))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Savind the results and the training details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr = np.stack(tr, axis=0)\n",
    "df_tr = pd.DataFrame(tr, columns=['loss', 'acc'])\n",
    "fpath = os.path.join(results_path, 'train.csv')\n",
    "\n",
    "val = np.stack(val, axis=0)\n",
    "df_val = pd.DataFrame(val, columns=['loss', 'acc'])\n",
    "fpath = os.path.join(results_path, 'val.csv')\n",
    "df_val.to_csv(fpath, sep=';')\n",
    "\n",
    "te = np.vstack((test_loss, test_acc)).transpose()\n",
    "df_test = pd.DataFrame(te, columns=['loss', 'acc'])\n",
    "fpath = os.path.join(results_path, 'test.csv')\n",
    "df_test.to_csv(fpath, sep=';')\n",
    "\n",
    "feat_infos = np.vstack((P/100, d)).transpose()\n",
    "df_feat_info = pd.DataFrame(feat_infos, columns=['P', 'Features_kept'])\n",
    "fpath = os.path.join(results_path, 'Features.csv')\n",
    "df_feat_info.to_csv(fpath, sep=';')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "GMMC",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}