yor_CTC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "id": "jvbdbirgHmnb",
    "outputId": "02b9700e-9186-4a7f-e2aa-1bc68f29a3d9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "#mriviere@fb.com\n",
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 507
    },
    "colab_type": "code",
    "id": "U222FEV6H3Kd",
    "outputId": "9a41e9ca-4963-45e9-a287-fc9d56daecd3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting torchaudio\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e9/0a/40e53c686c2af65b2a4e818d11d9b76fa79178440caf99f3ceb2a32c3b04/torchaudio-0.5.1-cp36-cp36m-manylinux1_x86_64.whl (3.2MB)\n",
      "\u001b[K     |████████████████████████████████| 3.2MB 8.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: torch==1.5.1 in /usr/local/lib/python3.6/dist-packages (from torchaudio) (1.5.1+cu101)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch==1.5.1->torchaudio) (1.18.5)\n",
      "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch==1.5.1->torchaudio) (0.16.0)\n",
      "Installing collected packages: torchaudio\n",
      "Successfully installed torchaudio-0.5.1\n",
      "Requirement already satisfied: PyDrive in /usr/local/lib/python3.6/dist-packages (1.3.1)\n",
      "Requirement already satisfied: google-api-python-client>=1.2 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (1.7.12)\n",
      "Requirement already satisfied: PyYAML>=3.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (3.13)\n",
      "Requirement already satisfied: oauth2client>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (4.1.3)\n",
      "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (0.0.3)\n",
      "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (3.0.1)\n",
      "Requirement already satisfied: httplib2<1dev,>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (0.17.4)\n",
      "Requirement already satisfied: six<2dev,>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (1.12.0)\n",
      "Requirement already satisfied: google-auth>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (1.17.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.2.8)\n",
      "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.4.8)\n",
      "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (4.6)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.4.1->google-api-python-client>=1.2->PyDrive) (4.1.0)\n",
      "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.4.1->google-api-python-client>=1.2->PyDrive) (47.3.1)\n",
      "Collecting soundfile\n",
      "  Downloading https://files.pythonhosted.org/packages/eb/f2/3cbbbf3b96fb9fa91582c438b574cff3f45b29c772f94c400e2c99ef5db9/SoundFile-0.10.3.post1-py2.py3-none-any.whl\n",
      "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.6/dist-packages (from soundfile) (1.14.0)\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi>=1.0->soundfile) (2.20)\n",
      "Installing collected packages: soundfile\n",
      "Successfully installed soundfile-0.10.3.post1\n"
     ]
    }
   ],
   "source": [
    "!pip install torchaudio\n",
    "!pip install PyDrive\n",
    "!pip install soundfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MSIi6VyMIBgb"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from random import randint, uniform\n",
    "import re\n",
    "import numpy as np\n",
    "import wave\n",
    "import contextlib\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "DGRmDPlvIOMF",
    "outputId": "e0a58da9-2c07-4065-801e-6342aefd0c4d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/gdrive/My Drive/SR/speech\n"
     ]
    }
   ],
   "source": [
    "cd gdrive/My Drive/SR/speech"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "70JBGumqIi0Z"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OEVo_Y9NJAwH"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YERPAqtbablr"
   },
   "outputs": [],
   "source": [
    "# !unzip text_files.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ulAjI9GUbaJz"
   },
   "outputs": [],
   "source": [
    "#!unzip text_files.zip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-Y1S2JquIDE_"
   },
   "source": [
    "## Data parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5BvJ6SHfIMqf"
   },
   "outputs": [],
   "source": [
    "result = []\n",
    "n=10000\n",
    "\n",
    "with open(\"alltext.txt\",\"r\") as f:\n",
    "    for line in f.readlines():\n",
    "        line = re.sub(\"[A-Za-z0-9,.\\(\\)\\\"\\\":;]\",\"\",line)\n",
    "        line=line.split()\n",
    "        i=0\n",
    "        if n<=0:\n",
    "            break\n",
    "        p=uniform(0,1)\n",
    "        if p>0.1:\n",
    "            continue\n",
    "        while i<len(line) and n>0:\n",
    "            window = randint(10,20)\n",
    "            sentence = line[i:i+window]\n",
    "            i+=window\n",
    "            if len(sentence)<5:\n",
    "                continue\n",
    "            result.append(\" \".join(sentence))\n",
    "            n-=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iFk9h2xYKccC"
   },
   "outputs": [],
   "source": [
    "with open(\"alltext.txt\",\"w\") as out:\n",
    "    out.write(\" ## si \\n\".join(result)+\" ## si\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "D8Ew0XiLKrg8"
   },
   "source": [
    "## Calculating recordings lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "GH59G9zIKtTR",
    "outputId": "893858a1-7b19-458d-f207-a0d19c314fb7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total is  7862.625374999993  s ,  2.1840626041666646  h\n"
     ]
    }
   ],
   "source": [
    "\n",
    "total = 0\n",
    "corrupted=0\n",
    "files=0\n",
    "for directory in os.listdir(\"./sections/\"):\n",
    "    if os.path.isdir(\"./sections/\"+directory):\n",
    "        for file in os.listdir(\"./sections/\"+directory):\n",
    "            if file.endswith(\".wav\"):\n",
    "                files+=1\n",
    "                fname = \"./sections/\"+directory+\"/\"+file\n",
    "                try:\n",
    "                    with contextlib.closing(wave.open(fname,'r')) as f:\n",
    "                        frames = f.getnframes()\n",
    "                        rate = f.getframerate()\n",
    "                        duration = frames / float(rate)\n",
    "                        total+=duration\n",
    "                except Exception as e:\n",
    "                    corrupted+=1\n",
    "\n",
    "print(\"total is \",total,\" s\",\", \",total/(60*60),\" h\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "wUtlb_J_Vf9k",
    "outputId": "d6d9a47a-8b43-4a15-de92-f2cc404fce5a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1001\n",
      "45\n"
     ]
    }
   ],
   "source": [
    "print(files)\n",
    "print(corrupted)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nOetPDbyVjOC"
   },
   "source": [
    "## Organizing data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "72kskVChVpXQ"
   },
   "source": [
    "### spliting records to train, val and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BiIqTXdOWFuF"
   },
   "outputs": [],
   "source": [
    "!mkdir data\n",
    "!mkdir data/records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ApIfPwx2Vr6n"
   },
   "outputs": [],
   "source": [
    "# Extracting all the non-corrupted files\n",
    "wav_files = []\n",
    "all_linkers = dict()\n",
    "for directory in os.listdir(\"./recordings/\"): # parent directory of the recordings, it should contain folders that contain wav,json and txt files\n",
    "    if os.path.isdir(\"./recordings/\"+directory):\n",
    "        linker = []\n",
    "        to_remove = []\n",
    "        linker_file = [file for file in os.listdir(\"./recordings/\"+directory) if file.endswith(\".txt\")][0]\n",
    "        linker_data = open(\"./recordings/\"+directory+\"/\"+linker_file).readlines()\n",
    "        for file in os.listdir(\"./recordings/\"+directory):                \n",
    "            if file.endswith(\".wav\"):\n",
    "                fname = \"./recordings/\"+directory+\"/\"+file\n",
    "                try:\n",
    "                    with contextlib.closing(wave.open(fname,'r')) as f:\n",
    "                        frames = f.getnframes()\n",
    "                        rate = f.getframerate()\n",
    "                        duration = frames / float(rate)\n",
    "                        wav_files.append(fname)\n",
    "                except Exception as e:\n",
    "                    to_remove.append(file)\n",
    "        for file in to_remove:\n",
    "            i=0\n",
    "            while i<len(linker_data):\n",
    "                if file in linker_data[i]:\n",
    "                    linker_data.pop(i)\n",
    "                i+=1\n",
    "        \n",
    "        linker.extend(linker_data)\n",
    "        all_linkers[directory] = linker\n",
    "        \n",
    "\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "00Jxx7shdnAc",
    "outputId": "23135a81-0542-4f9a-b509-6f6bc8f395bd"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['200625-230031_yor_874_elicit', '200630-123859_yor_874_elicit', '200630-112655_yor_874_elicit', '200629-222626_yor_874_elicit', '200629-234938_yor_874_elicit', '200630-092358_yor_874_elicit', '200630-110532_yor_874_elicit', '200625-215447_yor_874_elicit', '200630-101743_yor_874_elicit', '200629-230040_yor_874_elicit', '200625-221631_yor_874_elicit', '200630-003859_yor_874_elicit', '200626-001537_yor_874_elicit', '200625-223759_yor_874_elicit', '200630-095103_yor_874_elicit', '200629-232814_yor_874_elicit', '200625-232021_yor_874_elicit', '200630-115031_yor_874_elicit', '200629-205402_yor_874_elicit', '200625-235253_yor_874_elicit', '200630-121116_yor_874_elicit', '200630-001255_yor_874_elicit'])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_linkers.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lrEmZrZ1WPBJ"
   },
   "outputs": [],
   "source": [
    "to_copy = \" \".join(wav_files)\n",
    "!cp -t data/records/ {to_copy}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "LgqOPi2jA1g6",
    "outputId": "36ef6e4d-93df-444b-d459-43975c3e0872"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LpPjmFmVfnTE"
   },
   "outputs": [],
   "source": [
    "# for f in to_copy:\n",
    "\n",
    "#   !cp -t data/records/ {f}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xu1F04oZ01vK"
   },
   "outputs": [],
   "source": [
    "new_linkers = dict()\n",
    "for section,linker in all_linkers.items():\n",
    "    for i,link in enumerate(linker):\n",
    "        text_file_name = link.split(\" \")[0].split(\"/\")[-1]\n",
    "        #print(link)\n",
    "#         break\n",
    "        line = link.split(\";\")[0].split(\"(\")[1].split(\")\")[0].strip()\n",
    "        wav = link.split(\";\")[1].strip().split(\"/\")[-1]\n",
    "        #print(wav,line)\n",
    "\n",
    "        linker[i] = wav+\":\"+line\n",
    "        new_linkers[text_file_name] = linker\n",
    "#     break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y5RS8bcc03eF"
   },
   "outputs": [],
   "source": [
    "# out_linker = open(\"./data/linker.txt\",\"w\")\n",
    "# out_linker.write(\"\\n\".join(linker))\n",
    "# out_linker.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2q5Qef8g0-g_"
   },
   "outputs": [],
   "source": [
    "!mkdir ./data/records/train\n",
    "!mkdir ./data/records/test\n",
    "!mkdir ./data/records/val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_train_val_test(wav_files, num_splits, val_split):\n",
    "    np.random.seed(0)\n",
    "    all_indices = np.random.permutation(len(wav_files))\n",
    "\n",
    "    # calculate length of splits\n",
    "    if num_splits > 2:\n",
    "        index_start = len(all_indices)//num_splits\n",
    "    else:\n",
    "        index_start = 0\n",
    "        \n",
    "    indices = all_indices[index_start:]\n",
    "\n",
    "    if num_splits > 2:\n",
    "        extra_idx = all_indices[:index_start]\n",
    "    test_idx = indices[:len(indices)//2]\n",
    "    validation_portion = int(len(indices)//2 * val_split)\n",
    "    train_idx = indices[len(indices)//2:-validation_portion]\n",
    "    valid_idx = indices[-validation_portion:]\n",
    "\n",
    "    # prepare paths for copy\n",
    "    extra_files = []\n",
    "    if num_splits > 2:\n",
    "        extra_files = [wav_files[i].split(\"/\")[-1] for i in extra_idx]\n",
    "    train_set_files = [wav_files[i].split(\"/\")[-1] for i in train_idx]\n",
    "    valid_set_files = [wav_files[i].split(\"/\")[-1] for i in valid_idx]\n",
    "    test_set_files = [wav_files[i].split(\"/\")[-1] for i in test_idx]\n",
    "\n",
    "    # copy to folders\n",
    "    if len(extra_files) >= 1:\n",
    "        to_copy_extra = \"./data/records/\" + \" ./data/records/\".join(extra_files)\n",
    "    else:\n",
    "        to_copy_extra = []\n",
    "    to_copy_train = \"./data/records/\" + \" ./data/records/\".join(train_set_files)\n",
    "    to_copy_valid = \"./data/records/\" + \" ./data/records/\".join(valid_set_files)\n",
    "    to_copy_test = \"./data/records/\" + \" ./data/records/\".join(test_set_files)\n",
    "\n",
    "    return to_copy_train, to_copy_valid, to_copy_test, to_copy_extra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_copy_train, to_copy_valid, to_copy_test, to_copy_extra = split_train_val_test(wav_files, 3, 0.2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QNV2DO_w1Cid"
   },
   "outputs": [],
   "source": [
    "# import numpy as np\n",
    "# np.random.seed(0)\n",
    "# indices = np.random.permutation(len(wav_files))\n",
    "\n",
    "# test_idx = indices[:len(indices)//2]\n",
    "# validation_portion = int(len(indices)//2 * 0.2)\n",
    "# train_idx = indices[len(indices)//2:-validation_portion]\n",
    "# valid_idx = indices[-validation_portion:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kFJQ4ul21J1i"
   },
   "outputs": [],
   "source": [
    "# train_set_files = [wav_files[i].split(\"/\")[-1] for i in train_idx]\n",
    "# valid_set_files = [wav_files[i].split(\"/\")[-1] for i in valid_idx]\n",
    "# test_set_files = [wav_files[i].split(\"/\")[-1] for i in test_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9bU6407k1Qj_"
   },
   "outputs": [],
   "source": [
    "# to_copy_train = \"./data/records/\" + \" ./data/records/\".join(train_set_files)\n",
    "# to_copy_valid = \"./data/records/\" + \" ./data/records/\".join(valid_set_files)\n",
    "# to_copy_test = \"./data/records/\" + \" ./data/records/\".join(test_set_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LWq10rHY1Wu-"
   },
   "outputs": [],
   "source": [
    "!mv -t data/records/train/ {to_copy_train}\n",
    "!mv -t data/records/val/ {to_copy_valid}\n",
    "!mv -t data/records/test/ {to_copy_test}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_5JOKvFC1hvc"
   },
   "source": [
    "### making of chars.txt file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "line = text_file[idx]\n",
    "#line = line.split(\"##\")[0].strip()\n",
    "line = re.sub(exclude, \"\", line)\n",
    "#line = re.sub(\"[\\[\\]|٪%«»_ـ]\",\"\",line) # Clean unnecessary characters from the data, this is for arabic\n",
    "#line = re.sub(\"[,،;؛μ\\\\\\/≈٠١٢٣٤٥٦٧٨٩ταίηο–ēιäɛθκ-]\",\"\",line) # Clean unnecessary characters from the data, this is for arabic\n",
    "text_data.append((line,file.split(\".\")[0]))\n",
    "char_set = set(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LQC2plL31jXt"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "chars = {\" \":1,\"ε\":0}\n",
    "text_data = []\n",
    "char_idx = 2\n",
    "for section_id,linker in new_linkers.items():\n",
    "    text_file = open(\"./split_text/\"+section_id,\"r\").readlines() # Modify this to the directory of your txt file that you recorded with\n",
    "    \n",
    "    for link in linker:\n",
    "        #print(link)\n",
    "        file,idx = link.split(\":\")[0],int(link.split(\":\")[1].split(\" \")[1])-1\n",
    "        line = text_file[idx]\n",
    "        line = line.split(\"##\")[0].strip()\n",
    "        \n",
    "        line = re.sub(\"[\\n\\.,''-''̀''́'''!-]\",\"\",line) # Clean unnecessary characters from the data, this is for arabic\n",
    "        #line = re.sub(\"[—\\u200b]\",\"\",line) # Clean unnecessary characters from the data, this is for arabic\n",
    "        text_data.append((line,file.split(\".\")[0]))\n",
    "        char_set = set(line)\n",
    "        for c in char_set:\n",
    "            if c not in chars:\n",
    "                chars[c]=char_idx\n",
    "                char_idx+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hsTXKTEXEZEH"
   },
   "outputs": [],
   "source": [
    "raw_text = \"\\n\".join([wav+\":\"+line for line,wav in text_data])\n",
    "with open(\"data/raw_text_file.txt\",\"w\") as f:\n",
    "    f.write(raw_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DLDzdlsFEc86"
   },
   "outputs": [],
   "source": [
    "indices_text = []\n",
    "for line,wav in text_data:\n",
    "    line = list(line)\n",
    "    indicies = []\n",
    "    for c in line:\n",
    "        indicies.append(str(chars[c]))\n",
    "    indices_text.append(wav+\" \"+\" \".join(indicies))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "5wTMESizEebI",
    "outputId": "188a0d5c-cddf-42aa-e46f-85afa389c31f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "Id7zllFNEiMg",
    "outputId": "fcd3fbe1-ec2b-478a-aac1-b48e074e9843"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{' ': 1,\n",
       " 'ε': 0,\n",
       " 's': 2,\n",
       " 'g': 3,\n",
       " 'p': 4,\n",
       " 'ò': 5,\n",
       " 'v': 6,\n",
       " 'é': 7,\n",
       " 'k': 8,\n",
       " 'e': 9,\n",
       " 'ọ': 10,\n",
       " 'ṣ': 11,\n",
       " 'd': 12,\n",
       " 'è': 13,\n",
       " 'i': 14,\n",
       " 'í': 15,\n",
       " 'ẹ': 16,\n",
       " 'ú': 17,\n",
       " 'ù': 18,\n",
       " 'r': 19,\n",
       " 'l': 20,\n",
       " 'j': 21,\n",
       " 'ì': 22,\n",
       " 'á': 23,\n",
       " 'y': 24,\n",
       " 't': 25,\n",
       " 'n': 26,\n",
       " 'a': 27,\n",
       " 'à': 28,\n",
       " 'b': 29,\n",
       " 'o': 30,\n",
       " 'h': 31,\n",
       " 'w': 32,\n",
       " 'ó': 33,\n",
       " 'f': 34,\n",
       " 'u': 35,\n",
       " 'm': 36,\n",
       " 'c': 37,\n",
       " '–': 38,\n",
       " 'ń': 39,\n",
       " 'ǹ': 40,\n",
       " 'z': 41,\n",
       " 'ḿ': 42,\n",
       " 'ί': 43,\n",
       " 'ὸ': 44}"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "X2ZyPsMfEnLy"
   },
   "outputs": [],
   "source": [
    "indicies_text = \"\\n\".join(indices_text)\n",
    "with open(\"data/chars.txt\",\"w\") as f:\n",
    "    f.write(indicies_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_zEGGI6dEqzW"
   },
   "outputs": [],
   "source": [
    "with open(\"data/charset.txt\",\"w\") as js:\n",
    "    js.write(str(chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "q5SEkbFaEtj1"
   },
   "outputs": [],
   "source": [
    "with open(\"data/charset.txt\") as js:\n",
    "    charset = eval(js.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "knb56UiMEyg9",
    "outputId": "2e3c688c-6d54-4e60-c63c-cb31f9561dd5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{' ': 1,\n",
       " '\"': 85,\n",
       " '%': 84,\n",
       " \"'\": 13,\n",
       " ',': 36,\n",
       " '-': 56,\n",
       " '.': 79,\n",
       " '/': 67,\n",
       " '0': 23,\n",
       " '1': 51,\n",
       " '2': 24,\n",
       " '3': 63,\n",
       " '4': 66,\n",
       " '5': 53,\n",
       " '6': 25,\n",
       " '7': 72,\n",
       " '8': 54,\n",
       " '9': 64,\n",
       " ':': 73,\n",
       " ';': 69,\n",
       " '?': 80,\n",
       " 'A': 43,\n",
       " 'B': 20,\n",
       " 'C': 37,\n",
       " 'D': 40,\n",
       " 'E': 31,\n",
       " 'F': 59,\n",
       " 'G': 62,\n",
       " 'H': 44,\n",
       " 'I': 42,\n",
       " 'J': 49,\n",
       " 'K': 41,\n",
       " 'L': 55,\n",
       " 'M': 29,\n",
       " 'N': 50,\n",
       " 'O': 48,\n",
       " 'P': 35,\n",
       " 'Q': 57,\n",
       " 'R': 34,\n",
       " 'S': 8,\n",
       " 'T': 61,\n",
       " 'U': 83,\n",
       " 'V': 60,\n",
       " 'W': 70,\n",
       " 'X': 93,\n",
       " 'Y': 88,\n",
       " 'Z': 94,\n",
       " ']': 87,\n",
       " 'a': 10,\n",
       " 'b': 38,\n",
       " 'c': 16,\n",
       " 'd': 7,\n",
       " 'e': 6,\n",
       " 'f': 30,\n",
       " 'g': 19,\n",
       " 'h': 33,\n",
       " 'i': 11,\n",
       " 'j': 46,\n",
       " 'k': 22,\n",
       " 'l': 9,\n",
       " 'm': 4,\n",
       " 'n': 2,\n",
       " 'o': 5,\n",
       " 'p': 18,\n",
       " 'q': 26,\n",
       " 'r': 14,\n",
       " 's': 12,\n",
       " 't': 17,\n",
       " 'u': 15,\n",
       " 'v': 28,\n",
       " 'w': 74,\n",
       " 'x': 47,\n",
       " 'y': 39,\n",
       " 'z': 52,\n",
       " '«': 82,\n",
       " '»': 81,\n",
       " 'À': 92,\n",
       " 'Â': 89,\n",
       " 'Æ': 103,\n",
       " 'È': 104,\n",
       " 'É': 45,\n",
       " 'Î': 99,\n",
       " 'à': 21,\n",
       " 'á': 100,\n",
       " 'â': 71,\n",
       " 'ä': 91,\n",
       " 'æ': 107,\n",
       " 'ç': 68,\n",
       " 'è': 32,\n",
       " 'é': 3,\n",
       " 'ê': 27,\n",
       " 'ë': 98,\n",
       " 'í': 102,\n",
       " 'î': 65,\n",
       " 'ï': 90,\n",
       " 'ó': 101,\n",
       " 'ô': 58,\n",
       " 'ö': 95,\n",
       " 'ù': 76,\n",
       " 'û': 77,\n",
       " 'ü': 97,\n",
       " 'Œ': 96,\n",
       " 'œ': 78,\n",
       " 'ε': 0,\n",
       " '‑': 106,\n",
       " '–': 86,\n",
       " '’': 75,\n",
       " '…': 105}"
      ]
     },
     "execution_count": 30,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "charset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jzzuGAMna4NG"
   },
   "outputs": [],
   "source": [
    "from pydrive.auth import GoogleAuth\n",
    "from pydrive.drive import GoogleDrive\n",
    "from google.colab import auth\n",
    "from oauth2client.client import GoogleCredentials\n",
    "auth.authenticate_user()\n",
    "gauth = GoogleAuth()\n",
    "gauth.credentials = GoogleCredentials.get_application_default()\n",
    "drive = GoogleDrive(gauth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ppRP3ZqP8vZP"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchaudio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "-m9QGAOmGFrH",
    "outputId": "30fc0120-09b2-402e-bbdc-ba8335f5ecb9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'CPC_audio' already exists and is not an empty directory.\n"
     ]
    }
   ],
   "source": [
    "! git clone https://github.com/facebookresearch/CPC_audio.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 622
    },
    "colab_type": "code",
    "id": "diDLgjhBFzZI",
    "outputId": "ec4694d3-a0a1-4d66-c6eb-6d0ab538f466"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘checkpoint_data’: File exists\n",
      "--2020-07-06 11:32:04--  https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_30.pt\n",
      "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 172.67.9.4, 104.22.75.142, 104.22.74.142, ...\n",
      "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|172.67.9.4|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 113599715 (108M) [application/octet-stream]\n",
      "Saving to: ‘checkpoint_data/checkpoint_30.pt.1’\n",
      "\n",
      "checkpoint_30.pt.1  100%[===================>] 108.34M  12.0MB/s    in 10s     \n",
      "\n",
      "2020-07-06 11:32:15 (10.6 MB/s) - ‘checkpoint_data/checkpoint_30.pt.1’ saved [113599715/113599715]\n",
      "\n",
      "--2020-07-06 11:32:17--  https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_logs.json\n",
      "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 172.67.9.4, 104.22.75.142, 104.22.74.142, ...\n",
      "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|172.67.9.4|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 20786 (20K) [text/plain]\n",
      "Saving to: ‘checkpoint_data/checkpoint_logs.json.1’\n",
      "\n",
      "checkpoint_logs.jso 100%[===================>]  20.30K   127KB/s    in 0.2s    \n",
      "\n",
      "2020-07-06 11:32:18 (127 KB/s) - ‘checkpoint_data/checkpoint_logs.json.1’ saved [20786/20786]\n",
      "\n",
      "--2020-07-06 11:32:19--  https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_args.json\n",
      "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 172.67.9.4, 104.22.75.142, 104.22.74.142, ...\n",
      "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|172.67.9.4|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 2063 (2.0K) [text/plain]\n",
      "Saving to: ‘checkpoint_data/checkpoint_args.json.1’\n",
      "\n",
      "checkpoint_args.jso 100%[===================>]   2.01K  --.-KB/s    in 0.001s  \n",
      "\n",
      "2020-07-06 11:32:19 (1.80 MB/s) - ‘checkpoint_data/checkpoint_args.json.1’ saved [2063/2063]\n",
      "\n",
      "checkpoint_30.pt    checkpoint_args.json    checkpoint_logs.json\n",
      "checkpoint_30.pt.1  checkpoint_args.json.1  checkpoint_logs.json.1\n"
     ]
    }
   ],
   "source": [
    "\n",
    "!mkdir checkpoint_data\n",
    "!wget https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_30.pt -P checkpoint_data\n",
    "!wget https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_logs.json -P checkpoint_data\n",
    "!wget https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_args.json -P checkpoint_data\n",
    "!ls checkpoint_data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YZ1Wywx3BNGS"
   },
   "source": [
    "# Part 1 : contrastive predictive coding\n",
    "\n",
    "Contrastive Predictive Coding (CPC) is a method of unsupervised training for speech models. The idea behind it is pretty simple:\n",
    "\n",
    "\n",
    "1.   The raw audio wave is passed through a convolutional network: the ```encoder```\n",
    "2.   Then, the encoder's output is given to a recurrent network the ```context```\n",
    "3. A third party network, the ```prediction_network``` will try to predict the  future embeddings of the encoder using the output of the context network.\n",
    "\n",
    "In order to avoid a collapse to trivial solutions, the prediction_network doesn't try to reconstruct the future features. Instead, using the context output $c_t$ at time $t$ it is trained to discriminate the real  encoder representatioin $g_{t+k}$ at time $t+k$ from several other features $(g_n)_n$ taken elsewhere in the batch. Thus the loss becomes:\n",
    "\n",
    "\\\\[ \\mathcal{L}_c = - \\frac{1}{K} \\sum_{k=1}^K \\text{Cross_entropy}(\\phi_k(c_t), g_{t+k}) \\\\]\n",
    "\n",
    "Or:\n",
    "\n",
    "\\\\[ \\mathcal{L}_c = - \\frac{1}{K} \\sum_{k=1}^K \\log \\frac{ \\exp\\left(\\phi_k(c_t)^\\top g_{t+k}\\right) }{  \\sum_{\\mathbf{n}\\in\\mathcal{N}_t} \\exp\\left(\\phi_k(c_t)^\\top g_n\\right)} \\\\]\n",
    "\n",
    "Where:\n",
    "\n",
    "\n",
    "*   $\\phi_k$ is the prediction network for the kth timestep\n",
    "*   $\\mathcal{N}_t$ is the set of all negative examples sampled for timestep $t$\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "frPFYXuPfNPs"
   },
   "source": [
    "## Exercice 1 : Building the model\n",
    "\n",
    "In this exercise, we will build and train a small CPC model using the repository CPC_audio.\n",
    "\n",
    "The code below loads a context and an encoder newtorks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "8g-xPSdLRdui",
    "outputId": "fa41e437-47a4-4c7e-b719-c02a71d8b6c7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Errno 2] No such file or directory: './CPC_audio'\n",
      "/content/gdrive/My Drive/SR/speech/CPC_audio\n"
     ]
    }
   ],
   "source": [
    "%cd ./CPC_audio\n",
    "from cpc.model import CPCEncoder, CPCAR\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "DIM_ENCODER=256\n",
    "DIM_CONTEXT=256\n",
    "KEEP_HIDDEN_VECTOR=False\n",
    "N_LEVELS_CONTEXT=1\n",
    "CONTEXT_RNN=\"LSTM\"\n",
    "N_PREDICTIONS=12\n",
    "LEARNING_RATE=2e-4\n",
    "N_NEGATIVE_SAMPLE =128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7Wx8WkrQk9bQ"
   },
   "outputs": [],
   "source": [
    "encoder = CPCEncoder(DIM_ENCODER).to(device)\n",
    "context = CPCAR(DIM_ENCODER, DIM_CONTEXT, KEEP_HIDDEN_VECTOR, 1, mode=CONTEXT_RNN).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "f9BrweAIla4J"
   },
   "outputs": [],
   "source": [
    "# Several functions that will be necessary to load the data later\n",
    "from cpc.dataset import findAllSeqs, AudioBatchData, parseSeqLabels\n",
    "SIZE_WINDOW = 20480\n",
    "BATCH_SIZE=8\n",
    "def load_dataset(path_dataset, file_extension='.wav', phone_label_dict=None):\n",
    "  data_list, speakers = findAllSeqs(path_dataset, extension=file_extension)\n",
    "  dataset = AudioBatchData(path_dataset, SIZE_WINDOW, data_list, phone_label_dict, len(speakers))\n",
    "  return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "58pQ7ysXk9ZO"
   },
   "source": [
    "Now build a new class, ```CPCModel``` which will"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rR5IYRTpRF8T"
   },
   "outputs": [],
   "source": [
    "class CPCModel(torch.nn.Module):\n",
    "\n",
    "\n",
    "    def __init__(self,\n",
    "                 encoder,\n",
    "                 AR):\n",
    "\n",
    "        super(CPCModel, self).__init__()\n",
    "        self.gEncoder = encoder\n",
    "        self.gAR = AR\n",
    "\n",
    "    def forward(self, batch_data):\n",
    "        \n",
    "\n",
    "        encoder_output = self.gEncoder(batch_data)\n",
    "        #print(encoder_output.shape)\n",
    "        # The output of the encoder data does not have the good format \n",
    "        # indeed it is Batch_size x Hidden_size x temp size\n",
    "        # while the context requires Batch_size  x temp size x Hidden_size\n",
    "        # thus you need to permute\n",
    "        context_input = encoder_output.permute(0, 2, 1)\n",
    "\n",
    "        context_output = self.gAR(context_input)\n",
    "        #print(context_output.shape)\n",
    "        return context_output, encoder_output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XUJgm6Rl4vS4"
   },
   "source": [
    "Let's test your code !\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PR4PoduBo7Bb"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D1x1n4mv4y03"
   },
   "outputs": [],
   "source": [
    "audio = torchaudio.load(\"../data/records/train/200624-200537_fra_ac6_elicit_23.wav\")[0]\n",
    "audio = audio.view(1, 1, -1)\n",
    "cpc_model = CPCModel(encoder, context).to(device)\n",
    "context_output, encoder_output = cpc_model(audio.to(device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "H12H8zK37xeL",
    "outputId": "9abe2e34-a152-41b2-c2d5-35a1066ed060"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.0000,  0.0000,  0.0000,  ..., -0.0013, -0.0013, -0.0016]]])"
      ]
     },
     "execution_count": 86,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "audio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "X27ce8Hy3C2p"
   },
   "source": [
    "## Exercise 2 : CPC loss\n",
    "\n",
    "We will define a class ```CPCCriterion``` which will hold the prediction networks $\\phi_k$ defined above and perform the classification loss $\\mathcal{L}_c$.\n",
    "\n",
    "a) In this exercise, the $\\phi_k$ will be a linear transform, ie:\n",
    "\n",
    "\\\\[ \\phi_k(c_t) = \\mathbf{A}_k c_t\\\\]\n",
    "\n",
    "Using the class [torch.nn.Linear](https://pytorch.org/docs/stable/nn.html#torch.nn.Linear), define the transformations $\\phi_k$ in the code below and complete the function ```get_prediction_k``` which computes $\\phi_k(c_t)$ for a given batch of vectors $c_t$.\n",
    "\n",
    "b) Using both ```get_prediction_k```  and ```sample_negatives``` defined below, write the forward function which will take as input two batches of features $c_t$ and $g_t$ and outputs the classification loss $\\mathcal{L}_c$ and the average acuracy for all predictions. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UlAs-z1fBc3W"
   },
   "outputs": [],
   "source": [
    "# Exercice 2: write the CPC loss\n",
    "# a) Write the negative sampling (with some help)\n",
    "# ERRATUM: it's really hard, the sampling will be provided\n",
    "\n",
    "class CPCCriterion(torch.nn.Module):\n",
    "\n",
    "  def __init__(self,\n",
    "               K,\n",
    "               dim_context,\n",
    "               dim_encoder,\n",
    "               n_negative):\n",
    "    super(CPCCriterion, self).__init__()\n",
    "    self.K_ = K\n",
    "    self.dim_context = dim_context\n",
    "    self.dim_encoder = dim_encoder\n",
    "    self.n_negative = n_negative\n",
    "\n",
    "    self.predictors = torch.nn.ModuleList() \n",
    "    for k in range(self.K_):\n",
    "      # TO COMPLETE !\n",
    "      \n",
    "      # A affine transformation in pytorch is equivalent to a nn.Linear layer\n",
    "      # To get a linear transformation you must set bias=False\n",
    "      # input dimension of the layer = dimension of the encoder\n",
    "      # output dimension of the layer = dimension of the context\n",
    "      self.predictors.append(torch.nn.Linear(dim_context, dim_encoder, bias=False))\n",
    "\n",
    "  def get_prediction_k(self, context_data):\n",
    "\n",
    "    #TO COMPLETE !\n",
    "    output = [] \n",
    "    # For each time step k\n",
    "    for k in range(self.K_):\n",
    "\n",
    "      # We need to compute phi_k = A_k * c_t\n",
    "      phi_k = self.predictors[k](context_data)\n",
    "      output.append(phi_k)\n",
    "\n",
    "    return output\n",
    "\n",
    "\n",
    "  def sample_negatives(self, encoded_data):\n",
    "    r\"\"\"\n",
    "    Sample some negative examples in the given encoded data.\n",
    "    Input:\n",
    "    - encoded_data size: B x T x H\n",
    "    Returns\n",
    "    - outputs of size B x (n_negative + 1) x (T - K_) x H\n",
    "      outputs[:, 0, :, :] contains the positive example\n",
    "      outputs[:, 1:, :, :] contains negative example sampled in the batch\n",
    "    - labels, long tensor of size B x (T - K_)\n",
    "      Since the positive example is always at coordinates 0 for all sequences \n",
    "      in the batch and all timestep in the sequence, labels is just a tensor\n",
    "      full of zeros !\n",
    "    \"\"\"\n",
    "    batch_size, time_size, dim_encoded = encoded_data.size()\n",
    "    window_size = time_size - self.K_\n",
    "    outputs = []\n",
    "\n",
    "    neg_ext = encoded_data.contiguous().view(-1, dim_encoded)\n",
    "    n_elem_sampled = self.n_negative * window_size * batch_size\n",
    "    # Draw nNegativeExt * batchSize negative samples anywhere in the batch\n",
    "    batch_idx = torch.randint(low=0, high=batch_size,\n",
    "                              size=(n_elem_sampled, ),\n",
    "                              device=encoded_data.device)\n",
    "\n",
    "    seq_idx = torch.randint(low=1, high=time_size,\n",
    "                            size=(n_elem_sampled, ),\n",
    "                            device=encoded_data.device)\n",
    "\n",
    "    base_idx = torch.arange(0, window_size, device=encoded_data.device)\n",
    "    base_idx = base_idx.view(1, 1, window_size)\n",
    "    base_idx = base_idx.expand(1, self.n_negative, window_size)\n",
    "    base_idx = base_idx.expand(batch_size, self.n_negative, window_size)\n",
    "    seq_idx += base_idx.contiguous().view(-1)\n",
    "    seq_idx = torch.remainder(seq_idx, time_size)\n",
    "\n",
    "    ext_idx = seq_idx + batch_idx * time_size\n",
    "    neg_ext = neg_ext[ext_idx].view(batch_size, self.n_negative,\n",
    "                                    window_size, dim_encoded)\n",
    "    label_loss = torch.zeros((batch_size, window_size),\n",
    "                              dtype=torch.long,\n",
    "                              device=encoded_data.device)\n",
    "\n",
    "    for k in range(1, self.K_ + 1):\n",
    "\n",
    "      # Positive samples\n",
    "      if k < self.K_:\n",
    "          pos_seq = encoded_data[:, k:-(self.K_-k)]\n",
    "      else:\n",
    "          pos_seq = encoded_data[:, k:]\n",
    "\n",
    "      pos_seq = pos_seq.view(batch_size, 1, pos_seq.size(1), dim_encoded)\n",
    "      full_seq = torch.cat((pos_seq, neg_ext), dim=1)\n",
    "      outputs.append(full_seq)\n",
    "\n",
    "    return outputs, label_loss\n",
    "\n",
    "  def forward(self, encoded_data, context_data):\n",
    "\n",
    "    # TO COMPLETE:\n",
    "    # Perform the full cpc criterion\n",
    "    # Returns 2 values:\n",
    "    # - the average classification loss avg_loss\n",
    "    # - the average classification acuracy avg_acc\n",
    "\n",
    "    # Reminder : The permuation !\n",
    "    encoded_data = encoded_data.permute(0, 2, 1)\n",
    "\n",
    "    # First we need to sample the negative examples\n",
    "    negative_samples, labels = self.sample_negatives(encoded_data)\n",
    "\n",
    "    # Then we must compute phi_k\n",
    "    phi_k = self.get_prediction_k(context_data)\n",
    "\n",
    "    # Finally we must get the dot product between phi_k and negative_samples \n",
    "    # for each k\n",
    "\n",
    "    #The total loss is the average of all losses\n",
    "    avg_loss = 0\n",
    "\n",
    "    # Average acuracy\n",
    "    avg_acc = 0\n",
    "\n",
    "    for k in range(self.K_):\n",
    "      B, N_sampled, S_small, H = negative_samples[k].size() \n",
    "      B, S, H = phi_k[k].size()\n",
    "\n",
    "      # As told before S = S_small + K. For segments too far in the sequence\n",
    "      # there are no positive exmples anyway, so we must shorten phi_k\n",
    "      phi = phi_k[k][:, :S_small]\n",
    "\n",
    "      # Now the dot product\n",
    "      # You have several ways to do that, let's do the simple but non optimal \n",
    "      # one\n",
    "      # pytorch has a matrix product function https://pytorch.org/docs/stable/torch.html#torch.bmm\n",
    "      # But it takes only 3D tensors of the same batch size !\n",
    "      # To begin negative_samples is a 4D tensor ! \n",
    "      # We want to compute the dot product for each features, of each sequence\n",
    "      # of the batch. Thus we are trying to compute a dot product for all\n",
    "      # B* N_sampled * S_small 1D vector of negative_samples[k]\n",
    "      # Or, a 1D tensor of size H is also a matrix of size 1 x H\n",
    "      # Then, we must view it as a 3D tensor of size (B* N_sampled * S_small, 1, H)\n",
    "      negative_sample_k  =  negative_samples[k].view(B* N_sampled* S_small, 1, H)\n",
    "\n",
    "      # But now phi and negative_sample_k no longer have the same batch size !\n",
    "      # No worries, we can expand phi so that each sequence of the batch\n",
    "      # is repeated N_sampled times\n",
    "      phi = phi.view(B, 1,S_small, H).expand(B, N_sampled, S_small, H)\n",
    "\n",
    "      # And now we can view it as a 3D tensor \n",
    "      phi  = phi.contiguous().view(B * N_sampled * S_small, H, 1)\n",
    "\n",
    "      # We can finally get the dot product !\n",
    "      scores = torch.bmm(negative_sample_k, phi)\n",
    "\n",
    "      # Dot_product has a size (B * N_sampled * S_small , 1, 1)\n",
    "      # Let's reorder it a bit\n",
    "      scores = scores.reshape(B, N_sampled, S_small)\n",
    "\n",
    "      # For each elements of the sequence, and each elements sampled, it gives \n",
    "      # a floating score stating the likelihood of this element being the \n",
    "      # true one.\n",
    "      # Now the classification loss, we need to use the Cross Entropy loss\n",
    "      # https://pytorch.org/docs/master/generated/torch.nn.CrossEntropyLoss.html\n",
    "\n",
    "      # For each time-step of each sequence of the batch \n",
    "      # we have N_sampled possible predictions. \n",
    "      # Looking at the documentation of torch.nn.CrossEntropyLoss\n",
    "      # we can see that this loss expect a tensor of size M x C where \n",
    "      # - M is the number of elements with a classification score\n",
    "      # - C is the number of possible classes\n",
    "      # There are N_sampled candidates for each predictions so\n",
    "      # C = N_sampled \n",
    "      # Each timestep of each sequence of the batch has a prediction so\n",
    "      # M = B * S_small\n",
    "      # Thus we need an input vector of size B * S_small, N_sampled\n",
    "      # To begin, we need to permute the axis\n",
    "      scores = scores.permute(0, 2, 1) # Now it has size B , S_small, N_sampled\n",
    "\n",
    "      # Then we can cast it into a 2D tensor\n",
    "      scores = scores.reshape(B * S_small, N_sampled)\n",
    "\n",
    "      # Same thing for the labels \n",
    "      labels = labels.reshape(B * S_small)\n",
    "\n",
    "      # Finally we can get the classification loss\n",
    "      loss_criterion = torch.nn.CrossEntropyLoss()\n",
    "      loss_k = loss_criterion(scores, labels)\n",
    "      avg_loss+= loss_k\n",
    "\n",
    "      # And for the acuracy\n",
    "      # The prediction for each elements is the sample with the highest score\n",
    "      # Thus the tensors of all predictions is the tensors of the index of the \n",
    "      # maximal score for each time-step of each sequence of the batch\n",
    "      predictions = torch.argmax(scores, 1)\n",
    "      acc_k  = (labels == predictions).sum() / (B * S_small)\n",
    "      avg_acc += acc_k\n",
    "\n",
    "    # Normalization\n",
    "    avg_loss = avg_loss / self.K_\n",
    "    avg_acc = avg_acc / self.K_\n",
    "      \n",
    "    return avg_loss , avg_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0cqGXhLf-_O1"
   },
   "source": [
    "Don't forget to test !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sYJSMh5I_QCf"
   },
   "outputs": [],
   "source": [
    "audio = torchaudio.load(\"../data/records/train/200624-200537_fra_ac6_elicit_23.wav\")[0]\n",
    "audio = audio.view(1, 1, -1)\n",
    "cpc_criterion = CPCCriterion(N_PREDICTIONS, DIM_CONTEXT, \n",
    "                             DIM_ENCODER, N_NEGATIVE_SAMPLE).to(device)\n",
    "context_output, encoder_output = cpc_model(audio.to(device))\n",
    "loss, avg = cpc_criterion(encoder_output,context_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zLv7GoE0_4C3"
   },
   "source": [
    "## Exercise 3: Full training loop !\n",
    "\n",
    "You have the model, you have the criterion. All you need now are a data loader and an optimizer to run your training loop.\n",
    "\n",
    "We will use an Adam optimizer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zcg29tqPAIR1"
   },
   "outputs": [],
   "source": [
    "parameters = list(cpc_criterion.parameters()) + list(cpc_model.parameters())\n",
    "optimizer = torch.optim.Adam(parameters, lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hETDwSfTAuF4"
   },
   "source": [
    "And as far as the data loader is concerned, we will rely on the data loader provided by the CPC_audio library."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BjhwgYH_JSi7"
   },
   "outputs": [],
   "source": [
    "from cpc.dataset import parseSeqLabels\n",
    "from cpc.feature_loader import loadModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 555
    },
    "colab_type": "code",
    "id": "y1tmlOp6I6Jv",
    "outputId": "eae7daa5-327b-4b21-b455-7a508f6a98ff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading checkpoint ../checkpoint_data/checkpoint_30.pt\n",
      "Loading the state dict at ../checkpoint_data/checkpoint_30.pt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:00, 81.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved cache file at ../data/records/train/_seqs_cache.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking length...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "612it [00:00, 609139.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done, elapsed: 0.753 seconds\n",
      "Scanned 612 sequences in 0.75 seconds\n",
      "1 chunks computed\n",
      "Joining pool\n",
      "Joined process, elapsed=5.268 secs\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-91-335f4d2b3251>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mcpc_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcpc_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mlabel_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mN_PHONES\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparseSeqLabels\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../data/chars.txt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdataset_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../data/records/train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_extension\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.wav'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphone_label_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabel_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mdataset_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../data/records/val'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_extension\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.wav'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphone_label_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabel_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mdata_loader_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdataset_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"speaker\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-83-e313e61b0ebf>\u001b[0m in \u001b[0;36mload_dataset\u001b[0;34m(path_dataset, file_extension, phone_label_dict)\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mload_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_extension\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.wav'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphone_label_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m   \u001b[0mdata_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspeakers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfindAllSeqs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mextension\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfile_extension\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m   \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAudioBatchData\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSIZE_WINDOW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphone_label_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspeakers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/gdrive/My Drive/SR/speech/CPC_audio/cpc/dataset.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, path, sizeWindow, seqNames, phoneLabelsDict, nSpeakers, nProcessLoader, MAX_SIZE_LOADED)\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneLabelsDict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdeepcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mphoneLabelsDict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadNextPack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadNextPack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdoubleLabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/gdrive/My Drive/SR/speech/CPC_audio/cpc/dataset.py\u001b[0m in \u001b[0;36mloadNextPack\u001b[0;34m(self, first)\u001b[0m\n\u001b[1;32m    128\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Joined process, elapsed={time.time()-start_time:.3f} secs'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnextData\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparseNextDataBlock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    131\u001b[0m             \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnextData\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnextPack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrentPack\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpackageIndex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/gdrive/My Drive/SR/speech/CPC_audio/cpc/dataset.py\u001b[0m in \u001b[0;36mparseNextDataBlock\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneLabelsDict\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneLabels\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneLabelsDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mseqName\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m                 \u001b[0mnewSize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneLabelsDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mseqName\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneSize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m                 \u001b[0mseq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mseq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mnewSize\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: '200625-133626_fra_ac6_elicit_5'"
     ]
    }
   ],
   "source": [
    "checkpoint_path = '../checkpoint_data/checkpoint_30.pt'\n",
    "cpc_model, HIDDEN_CONTEXT_MODEL, HIDDEN_ENCODER_MODEL = loadModel([checkpoint_path])\n",
    "cpc_model = cpc_model.cuda()\n",
    "label_dict, N_PHONES = parseSeqLabels('../data/chars.txt')\n",
    "dataset_train = load_dataset('../data/records/train', file_extension='.wav', phone_label_dict=label_dict)\n",
    "dataset_val = load_dataset('../data/records/val', file_extension='.wav', phone_label_dict=label_dict)\n",
    "data_loader_train = dataset_train.getDataLoader(BATCH_SIZE, \"speaker\", True)\n",
    "data_loader_val = dataset_val.getDataLoader(BATCH_SIZE, \"sequence\", False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 319
    },
    "colab_type": "code",
    "id": "O6KES4RXA0tU",
    "outputId": "1fbf5098-d1f3-4fba-eb48-69f7a3eb88c7"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:00, 52.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved cache file at ../data/records/train/_seqs_cache.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking length...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "614it [00:00, 136252.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done, elapsed: 0.703 seconds\n",
      "Scanned 614 sequences in 0.70 seconds\n",
      "1 chunks computed\n",
      "Joining pool\n",
      "Joined process, elapsed=2.445 secs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:00, 18.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved cache file at ../data/records/val/_seqs_cache.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking length...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "182it [00:00, 259294.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done, elapsed: 4.545 seconds\n",
      "Scanned 182 sequences in 4.55 seconds\n",
      "1 chunks computed\n",
      "Joining pool\n",
      "Joined process, elapsed=1.968 secs\n"
     ]
    }
   ],
   "source": [
    "dataset_train = load_dataset('../data/records/train')\n",
    "dataset_val = load_dataset('../data/records/val')\n",
    "data_loader_train = dataset_train.getDataLoader(BATCH_SIZE, \"speaker\", True)\n",
    "data_loader_val = dataset_train.getDataLoader(BATCH_SIZE, \"sequence\", False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uCoMCPL0A8VI"
   },
   "source": [
    "Now that everything is ready, complete and test the ```train_step``` function below which trains the model for one epoch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U-hIH3p8BsZr"
   },
   "outputs": [],
   "source": [
    "def train_step(data_loader,\n",
    "               cpc_model,\n",
    "               cpc_criterion,\n",
    "               optimizer):\n",
    "  \n",
    "  avg_loss = 0\n",
    "  avg_acc = 0\n",
    "  n_items = 0\n",
    "\n",
    "  for step, data in enumerate(data_loader):\n",
    "    x,y = data\n",
    "    bs = len(x)\n",
    "    optimizer.zero_grad()\n",
    "    context_output, encoder_output = cpc_model(x.to(device))\n",
    "    loss , acc = cpc_criterion(encoder_output, context_output)\n",
    "    loss.backward()\n",
    "    n_items+=bs\n",
    "    avg_loss+=loss.item()*bs\n",
    "    avg_acc +=acc.item()*bs\n",
    "  \n",
    "  avg_loss/=n_items\n",
    "  avg_acc/=n_items\n",
    "  return avg_loss, avg_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rIj06giJE_bZ"
   },
   "source": [
    "## Exercise 4 : Validation loop\n",
    "\n",
    "Now complete the validation loop."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Q7Qwi6jDByyt"
   },
   "outputs": [],
   "source": [
    "def validation_step(data_loader,\n",
    "                    cpc_model,\n",
    "                    cpc_criterion):\n",
    "  \n",
    "  avg_loss = 0\n",
    "  avg_acc = 0\n",
    "  n_items = 0\n",
    "\n",
    "  for step, data in enumerate(data_loader):\n",
    "    x,y = data\n",
    "    bs = len(x)\n",
    "    context_output, encoder_output = cpc_model(x.to(device))\n",
    "    loss , acc = cpc_criterion(encoder_output, context_output)\n",
    "    n_items+=bs\n",
    "    avg_loss+=loss.item()*bs\n",
    "    avg_acc+=acc.item()*bs\n",
    "  \n",
    "  avg_loss/=n_items\n",
    "  avg_acc/=n_items\n",
    "  return avg_loss, avg_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "OBVUPKKs2_0U"
   },
   "source": [
    "## Exercise 5: Run everything"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZbXsZIRiB1tm"
   },
   "outputs": [],
   "source": [
    "def run(train_loader,\n",
    "        val_loader,\n",
    "        cpc_model,\n",
    "        cpc_criterion,\n",
    "        optimizer,\n",
    "        n_epochs):\n",
    "  \n",
    "  for epoch in range(n_epochs):\n",
    "\n",
    "    \n",
    "    print(f\"Running epoch {epoch+1} / {n_epochs}\")\n",
    "    avg_loss_train, avg_acc_train = train_step(train_loader, cpc_model, cpc_criterion, optimizer)\n",
    "    print(\"----------------------\")\n",
    "    print(f\"Training dataset\")\n",
    "    print(f\"- average loss : {avg_loss_train}\")\n",
    "    print(f\"- average acuracy : {avg_acc_train}\")\n",
    "    print(\"----------------------\")\n",
    "    with torch.no_grad():\n",
    "      cpc_model.eval()\n",
    "      cpc_criterion.eval()\n",
    "      avg_loss_val, avg_acc_val = validation_step(val_loader, cpc_model, cpc_criterion)\n",
    "      print(f\"Validation dataset\")\n",
    "      print(f\"- average loss : {avg_loss_val}\")\n",
    "      print(f\"- average acuracy : {avg_acc_val}\")\n",
    "      print(\"----------------------\")\n",
    "      print()\n",
    "      cpc_model.train()\n",
    "      cpc_criterion.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 320
    },
    "colab_type": "code",
    "id": "5xx8vN2wpECC",
    "outputId": "1d3f1c38-71cd-4f69-b513-17496c5e1afd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running epoch 1 / 1\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-67-f61c111410c2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_loader_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_loader_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcpc_model\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcpc_criterion\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-66-61f00c60fd28>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(train_loader, val_loader, cpc_model, cpc_criterion, optimizer, n_epochs)\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Running epoch {epoch+1} / {n_epochs}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m     \u001b[0mavg_loss_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mavg_acc_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcpc_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcpc_criterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"----------------------\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Training dataset\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-64-b08fcd668c4e>\u001b[0m in \u001b[0;36mtrain_step\u001b[0;34m(data_loader, cpc_model, cpc_criterion, optimizer)\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mbs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m     \u001b[0mcontext_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoder_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcpc_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m     \u001b[0mloss\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcpc_criterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mencoder_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    548\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: forward() missing 1 required positional argument: 'label'"
     ]
    }
   ],
   "source": [
    "run(data_loader_train, data_loader_val, cpc_model,cpc_criterion,optimizer,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5hoT3_3W6HYY"
   },
   "source": [
    "Once everything is donw, clear the memory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fU5mDOY46KSG"
   },
   "outputs": [],
   "source": [
    "del dataset_train\n",
    "del dataset_val\n",
    "del cpc_model\n",
    "del context\n",
    "del encoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "srPM5r_LB9v-"
   },
   "source": [
    "# Part 2 : Fine tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0Nb_-0IQiJk9"
   },
   "source": [
    "## Exercice 1 : Phone separability with aligned phonemes.\n",
    "\n",
    "One option to evaluate the quality of the features trained with CPC can be to check if they can be used to recognize phonemes. \n",
    "To do so, we can fine-tune a pre-trained model using a limited amount of labelled speech data.\n",
    "We are going to start with a simple evaluation setting where we have the phone labels for each timestep corresponding to a CPC feature.\n",
    "\n",
    "We will work with a model already pre-trained on English data. As far as the fine-tuning dataset is concerned, we will use a 1h subset of [librispeech-100](http://www.openslr.org/12/). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 588
    },
    "colab_type": "code",
    "id": "N-scDMAasXxc",
    "outputId": "c383ebc7-a52c-4711-bf4d-c94d22246e5f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-07-06 10:37:18--  https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_30.pt\n",
      "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.75.142, 172.67.9.4, 104.22.74.142, ...\n",
      "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.75.142|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 113599715 (108M) [application/octet-stream]\n",
      "Saving to: ‘checkpoint_data/checkpoint_30.pt’\n",
      "\n",
      "checkpoint_30.pt    100%[===================>] 108.34M  12.0MB/s    in 10s     \n",
      "\n",
      "2020-07-06 10:37:29 (10.3 MB/s) - ‘checkpoint_data/checkpoint_30.pt’ saved [113599715/113599715]\n",
      "\n",
      "--2020-07-06 10:37:31--  https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_logs.json\n",
      "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.74.142, 172.67.9.4, 104.22.75.142, ...\n",
      "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.74.142|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 20786 (20K) [text/plain]\n",
      "Saving to: ‘checkpoint_data/checkpoint_logs.json’\n",
      "\n",
      "checkpoint_logs.jso 100%[===================>]  20.30K   121KB/s    in 0.2s    \n",
      "\n",
      "2020-07-06 10:37:32 (121 KB/s) - ‘checkpoint_data/checkpoint_logs.json’ saved [20786/20786]\n",
      "\n",
      "--2020-07-06 10:37:33--  https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_args.json\n",
      "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 172.67.9.4, 104.22.74.142, 104.22.75.142, ...\n",
      "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|172.67.9.4|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 2063 (2.0K) [text/plain]\n",
      "Saving to: ‘checkpoint_data/checkpoint_args.json’\n",
      "\n",
      "checkpoint_args.jso 100%[===================>]   2.01K  --.-KB/s    in 0.001s  \n",
      "\n",
      "2020-07-06 10:37:34 (2.10 MB/s) - ‘checkpoint_data/checkpoint_args.json’ saved [2063/2063]\n",
      "\n",
      "checkpoint_30.pt  checkpoint_args.json\tcheckpoint_logs.json\n"
     ]
    }
   ],
   "source": [
    "!mkdir checkpoint_data\n",
    "!wget https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_30.pt -P checkpoint_data\n",
    "!wget https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_logs.json -P checkpoint_data\n",
    "!wget https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/not_hub/2levels_6k_top_ctc/checkpoint_args.json -P checkpoint_data\n",
    "!ls checkpoint_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 589
    },
    "colab_type": "code",
    "id": "SSSaiYo82_oY",
    "outputId": "f978b31a-ac90-4154-ee00-59453d4a6302"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Errno 2] No such file or directory: '/content/CPC_audio'\n",
      "/content/gdrive/My Drive/SR/speech/CPC_audio\n",
      "Loading checkpoint checkpoint_data/checkpoint_30.pt\n",
      "Loading the state dict at checkpoint_data/checkpoint_30.pt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:00, 114.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved cache file at ../data/records/train/_seqs_cache.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking length...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "383it [00:00, 91227.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done, elapsed: 0.432 seconds\n",
      "Scanned 383 sequences in 0.43 seconds\n",
      "1 chunks computed\n",
      "Joining pool\n",
      "Joined process, elapsed=1.458 secs\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-47-1a054df427cb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mcpc_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcpc_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mlabel_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mN_PHONES\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparseSeqLabels\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../data/chars.txt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mdataset_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../data/records/train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_extension\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.wav'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphone_label_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabel_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0mdataset_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../data/records/val'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_extension\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.wav'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphone_label_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabel_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mdata_loader_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdataset_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"speaker\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-35-a65acdc6487d>\u001b[0m in \u001b[0;36mload_dataset\u001b[0;34m(path_dataset, file_extension, phone_label_dict)\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mload_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_extension\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'.flac'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphone_label_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m   \u001b[0mdata_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspeakers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfindAllSeqs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mextension\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfile_extension\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m   \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAudioBatchData\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSIZE_WINDOW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphone_label_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspeakers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/gdrive/My Drive/SR/speech/CPC_audio/cpc/dataset.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, path, sizeWindow, seqNames, phoneLabelsDict, nSpeakers, nProcessLoader, MAX_SIZE_LOADED)\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneLabelsDict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdeepcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mphoneLabelsDict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadNextPack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadNextPack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdoubleLabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/gdrive/My Drive/SR/speech/CPC_audio/cpc/dataset.py\u001b[0m in \u001b[0;36mloadNextPack\u001b[0;34m(self, first)\u001b[0m\n\u001b[1;32m    128\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Joined process, elapsed={time.time()-start_time:.3f} secs'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnextData\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparseNextDataBlock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    131\u001b[0m             \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnextData\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnextPack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrentPack\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpackageIndex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/gdrive/My Drive/SR/speech/CPC_audio/cpc/dataset.py\u001b[0m in \u001b[0;36mparseNextDataBlock\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneLabelsDict\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneLabels\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneLabelsDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mseqName\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m                 \u001b[0mnewSize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneLabelsDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mseqName\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphoneSize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m                 \u001b[0mseq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mseq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mnewSize\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: '200625-133626_fra_ac6_elicit_0'"
     ]
    }
   ],
   "source": [
    "%cd /content/CPC_audio\n",
    "from cpc.dataset import parseSeqLabels\n",
    "from cpc.feature_loader import loadModel\n",
    "\n",
    "checkpoint_path = 'checkpoint_data/checkpoint_30.pt'\n",
    "cpc_model, HIDDEN_CONTEXT_MODEL, HIDDEN_ENCODER_MODEL = loadModel([checkpoint_path])\n",
    "cpc_model = cpc_model.cuda()\n",
    "label_dict, N_PHONES = parseSeqLabels('../data/chars.txt')\n",
    "dataset_train = load_dataset('../data/records/train', file_extension='.wav', phone_label_dict=label_dict)\n",
    "dataset_val = load_dataset('../data/records/val', file_extension='.wav', phone_label_dict=label_dict)\n",
    "data_loader_train = dataset_train.getDataLoader(BATCH_SIZE, \"speaker\", True)\n",
    "data_loader_val = dataset_val.getDataLoader(BATCH_SIZE, \"sequence\", False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3ulgYV3nHcoa"
   },
   "outputs": [],
   "source": [
    "??cpc_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xkKi-qfosng2"
   },
   "source": [
    "Then we will use a simple linear classifier to recognize the phonemes from the features produced by ```cpc_model```. \n",
    "\n",
    "### a) Build the phone classifier \n",
    "\n",
    "Design a class of linear classifiers, ```PhoneClassifier``` that will take as input a batch of sequences of CPC features and output a score vector for each phoneme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4RpAbz-0CXJJ"
   },
   "outputs": [],
   "source": [
    "class PhoneClassifier(torch.nn.Module):\n",
    "\n",
    "  def __init__(self,\n",
    "               input_dim : int,\n",
    "               n_phones : int):\n",
    "    super(PhoneClassifier, self).__init__()\n",
    "    self.linear = torch.nn.Linear(input_dim, n_phones)\n",
    "    \n",
    "\n",
    "  def forward(self, x):\n",
    "    return self.linear(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Zt5oa_nqtH-d"
   },
   "source": [
    "Our phone classifier will then be:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NRBf_83IuLv5"
   },
   "outputs": [],
   "source": [
    "phone_classifier = PhoneClassifier(HIDDEN_CONTEXT_MODEL, N_PHONES).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "z_Vf5AbUhqm4"
   },
   "source": [
    "### b - What would be the correct loss criterion for this task ?\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uhyPM-cgjrtw"
   },
   "outputs": [],
   "source": [
    "loss_criterion = torch.nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Nv4cSxbaplrz"
   },
   "source": [
    "To perform the fine-tuning, we will also need an optimization function.\n",
    "\n",
    "We will use an [Adam optimizer ](https://pytorch.org/docs/stable/optim.html#torch.optim.Adam)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "W5CgYyAlqKxu"
   },
   "outputs": [],
   "source": [
    "parameters = list(phone_classifier.parameters()) + list(cpc_model.parameters())\n",
    "LEARNING_RATE = 2e-4\n",
    "optimizer = torch.optim.Adam(parameters, lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qQB9HS9PvAXc"
   },
   "source": [
    "You might also want to perform this training while freezing the weights of the ```cpc_model```. Indeed, if the pre-training was good enough, then ```cpc_model``` phonemes representation should be linearly separable. In this case the optimizer should be defined like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nRy0gn6awGUQ"
   },
   "outputs": [],
   "source": [
    "optimizer_frozen = torch.optim.Adam(list(phone_classifier.parameters()), lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cO93ngIfj4JW"
   },
   "source": [
    "### c- Now let's build a training loop. \n",
    "Complete the function ```train_one_epoch``` below.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fabqj3wvLwgU"
   },
   "outputs": [],
   "source": [
    "def train_one_epoch(cpc_model, \n",
    "                    phone_classifier, \n",
    "                    loss_criterion, \n",
    "                    data_loader, \n",
    "                    optimizer):\n",
    "\n",
    "  cpc_model.train()\n",
    "  loss_criterion.train()\n",
    "\n",
    "  avg_loss = 0\n",
    "  avg_accuracy = 0\n",
    "  n_items = 0\n",
    "  for step, full_data in enumerate(data_loader):\n",
    "    # Each batch is represented by a Tuple of vectors:\n",
    "    # sequence of size : N x 1 x T\n",
    "    # label of size : N x T\n",
    "    # \n",
    "    # With :\n",
    "    # - N number of sequence in the batch\n",
    "    # - T size of each sequence\n",
    "    sequence, label = full_data\n",
    "    \n",
    "    \n",
    "\n",
    "    bs = len(sequence)\n",
    "    seq_len = label.size(1)\n",
    "    optimizer.zero_grad()\n",
    "    context_out, enc_out, _ = cpc_model(sequence.to(device),label.to(device))\n",
    "\n",
    "    scores = phone_classifier(context_out)\n",
    "\n",
    "    scores = scores.permute(0,2,1)\n",
    "    loss = loss_criterion(scores,label.to(device))\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    avg_loss+=loss.item()*bs\n",
    "    n_items+=bs\n",
    "    correct_labels = scores.argmax(1)\n",
    "    avg_accuracy += ((label==correct_labels.cpu()).float()).mean(1).sum().item()\n",
    "  avg_loss/=n_items\n",
    "  avg_accuracy/=n_items\n",
    "  return avg_loss, avg_accuracy\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "quYtjx_TxIPK"
   },
   "source": [
    "Don't forget to test it !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "50MwxbKhxMKp"
   },
   "outputs": [],
   "source": [
    "avg_loss, avg_accuracy = train_one_epoch(cpc_model, phone_classifier, loss_criterion, data_loader_train, optimizer_frozen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "_o6yk8XKWnYe",
    "outputId": "19b006e6-03ac-4c18-c415-cdeb293b54d3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.3621995804512956, 0.6497440501715266)"
      ]
     },
     "execution_count": 60,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_loss, avg_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EmUkuJ2bwu4Z"
   },
   "source": [
    "### d- Build the validation loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kZJMxj6cwzd3"
   },
   "outputs": [],
   "source": [
    "def validation_step(cpc_model, \n",
    "                    phone_classifier, \n",
    "                    loss_criterion, \n",
    "                    data_loader):\n",
    "  \n",
    "  cpc_model.eval()\n",
    "  phone_classifier.eval()\n",
    "\n",
    "  avg_loss = 0\n",
    "  avg_accuracy = 0\n",
    "  n_items = 0\n",
    "  with torch.no_grad():\n",
    "    for step, full_data in enumerate(data_loader):\n",
    "      # Each batch is represented by a Tuple of vectors:\n",
    "      # sequence of size : N x 1 x T\n",
    "      # label of size : N x T\n",
    "      # \n",
    "      # With :\n",
    "      # - N number of sequence in the batch\n",
    "      # - T size of each sequence\n",
    "      sequence, label = full_data\n",
    "      bs = len(sequence)\n",
    "      seq_len = label.size(1)\n",
    "      context_out, enc_out, _ = cpc_model(sequence.to(device),label.to(device))\n",
    "      scores = phone_classifier(context_out)\n",
    "      scores = scores.permute(0,2,1)\n",
    "      loss = loss_criterion(scores,label.to(device))\n",
    "      avg_loss+=loss.item()*bs\n",
    "      n_items+=bs\n",
    "      correct_labels = scores.argmax(1)\n",
    "      avg_accuracy += ((label==correct_labels.cpu()).float()).mean(1).sum().item()\n",
    "  avg_loss/=n_items\n",
    "  avg_accuracy/=n_items\n",
    "  return avg_loss, avg_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vownVCt7xbVh"
   },
   "source": [
    "### e- Run everything\n",
    "\n",
    "Test this functiion with both ```optimizer``` and ```optimizer_frozen```."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xvO_4nKUxfQx"
   },
   "outputs": [],
   "source": [
    "def run(cpc_model, \n",
    "        phone_classifier, \n",
    "        loss_criterion, \n",
    "        data_loader_train, \n",
    "        data_loader_val, \n",
    "        optimizer,\n",
    "        n_epoch):\n",
    "\n",
    "  for epoch in range(n_epoch):\n",
    "\n",
    "    print(f\"Running epoch {epoch + 1} / {n_epoch}\")\n",
    "    loss_train, acc_train = train_one_epoch(cpc_model, phone_classifier, loss_criterion, data_loader_train, optimizer)\n",
    "    print(\"-------------------\")\n",
    "    print(f\"Training dataset :\")\n",
    "    print(f\"Average loss : {loss_train}. Average accuracy {acc_train}\")\n",
    "\n",
    "    print(\"-------------------\")\n",
    "    print(\"Validation dataset\")\n",
    "    loss_val, acc_val = validation_step(cpc_model, phone_classifier, loss_criterion, data_loader_val)\n",
    "    print(f\"Average loss : {loss_val}. Average accuracy {acc_val}\")\n",
    "    print(\"-------------------\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "ceCEO2h2bxAn",
    "outputId": "2842125e-7f20-434d-9189-33699670f620"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running epoch 1 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 1.0219527069604895. Average accuracy 0.7057498257933105\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 1.0688188383544701. Average accuracy 0.6966259057971015\n",
      "-------------------\n",
      "\n",
      "Running epoch 2 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 0.9813667901793229. Average accuracy 0.7150101397226987\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 1.0425163616304811. Average accuracy 0.7018257472826087\n",
      "-------------------\n",
      "\n",
      "Running epoch 3 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 0.9586697046657233. Average accuracy 0.7197957984919954\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 1.029444672577623. Average accuracy 0.7042572463768116\n",
      "-------------------\n",
      "\n",
      "Running epoch 4 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 0.9506696053679975. Average accuracy 0.7210777810534591\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 0.9972989958265553. Average accuracy 0.7106855751811594\n",
      "-------------------\n",
      "\n",
      "Running epoch 5 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 0.9448814161032387. Average accuracy 0.7218349101272156\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 0.9780038367147031. Average accuracy 0.7143144248188406\n",
      "-------------------\n",
      "\n",
      "Running epoch 6 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 0.929420669805397. Average accuracy 0.7249711888936535\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 0.9746506056923797. Average accuracy 0.715234375\n",
      "-------------------\n",
      "\n",
      "Running epoch 7 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 0.9157625487901743. Average accuracy 0.7281409689465409\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 0.9955525258313055. Average accuracy 0.710739356884058\n",
      "-------------------\n",
      "\n",
      "Running epoch 8 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 0.9135043791459587. Average accuracy 0.7293961504788451\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 0.9721649721048881. Average accuracy 0.7155910326086956\n",
      "-------------------\n",
      "\n",
      "Running epoch 9 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 0.9004762198940013. Average accuracy 0.7319076302530018\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 0.980453244154004. Average accuracy 0.713963428442029\n",
      "-------------------\n",
      "\n",
      "Running epoch 10 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 0.9172716077018834. Average accuracy 0.7282263972269869\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 0.9502466599146525. Average accuracy 0.7199048913043479\n",
      "-------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "run(cpc_model,phone_classifier,loss_criterion,data_loader_train,data_loader_val,optimizer_frozen,n_epoch=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TdfWDiFnylMT"
   },
   "source": [
    "## Exercise 2 : Phone separability without alignment (PER)\n",
    "\n",
    "Aligned data are very practical, but un real life they are rarely available. That's why in this excercise we will consider a fine-tuning with non-aligned phonemes.\n",
    "\n",
    "The model, the optimizer and the phone classifier will stay the same. However, we will replace our phone criterion with a [CTC loss](https://pytorch.org/docs/master/generated/torch.nn.CTCLoss.html). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_9BpM_Lpzgx8"
   },
   "outputs": [],
   "source": [
    "loss_ctc = torch.nn.CTCLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AQpYgTyfzsrq"
   },
   "source": [
    "Besides, we will use a siglthy different dataset class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "id": "9HRxoatlz3ZZ",
    "outputId": "210b7f5f-b722-42dd-97df-56cb2e614a1a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "67it [00:00, 16115.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/CPC_audio\n",
      "Saved cache file at /content/per_data/pack_master/1h/_seqs_cache.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "7it [00:00, 1867.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 287 sequences in 4.05 seconds\n",
      "maxSizeSeq : 273359\n",
      "maxSizePhone : 207\n",
      "minSizePhone : 17\n",
      "Total size dataset 1.0406152430555555 hours\n",
      "Saved cache file at /content/per_data/pack_master/10min/_seqs_cache.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 212 sequences in 2.49 seconds\n",
      "maxSizeSeq : 273760\n",
      "maxSizePhone : 188\n",
      "minSizePhone : 17\n",
      "Total size dataset 0.73 hours\n"
     ]
    }
   ],
   "source": [
    "%cd /content/CPC_audio\n",
    "from cpc.eval.common_voices_eval import SingleSequenceDataset, parseSeqLabels, findAllSeqs\n",
    "path_train_data_per = '/content/per_data/pack_master/1h'\n",
    "path_val_data_per = '/content/per_data/pack_master/10min'\n",
    "path_phone_data_per = '/content/per_data/pack_master/10h_phones.txt'\n",
    "BATCH_SIZE=8\n",
    "\n",
    "phone_labels, N_PHONES = parseSeqLabels(path_phone_data_per)\n",
    "data_train_per, _ = findAllSeqs(path_train_data_per, extension='.flac')\n",
    "dataset_train_non_aligned = SingleSequenceDataset(path_train_data_per, data_train_per, phone_labels)\n",
    "data_loader_train = torch.utils.data.DataLoader(dataset_train_non_aligned, batch_size=BATCH_SIZE,\n",
    "                                                shuffle=True)\n",
    "\n",
    "data_val_per, _ = findAllSeqs(path_val_data_per, extension='.flac')\n",
    "dataset_val_non_aligned = SingleSequenceDataset(path_val_data_per, data_val_per, phone_labels)\n",
    "data_loader_val = torch.utils.data.DataLoader(dataset_val_non_aligned, batch_size=BATCH_SIZE,\n",
    "                                              shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GwAckY62z7s9"
   },
   "source": [
    "### a- Training\n",
    "\n",
    "Since the phonemes are not aligned, there is no simple direct way to get the classification acuracy of a model. Write and test the three functions ```train_one_epoch_ctc```, ```validation_step_ctc``` and ```run_ctc``` as before but without considering the average acuracy of the model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "oYg5YzW8EHl4",
    "outputId": "be8aec47-f0a8-4cb6-b4ff-df232118a6d3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading checkpoint checkpoint_data/checkpoint_30.pt\n",
      "Loading the state dict at checkpoint_data/checkpoint_30.pt\n"
     ]
    }
   ],
   "source": [
    "from cpc.feature_loader import loadModel\n",
    "\n",
    "checkpoint_path = 'checkpoint_data/checkpoint_30.pt'\n",
    "cpc_model, HIDDEN_CONTEXT_MODEL, HIDDEN_ENCODER_MODEL = loadModel([checkpoint_path])\n",
    "cpc_model = cpc_model.cuda()\n",
    "phone_classifier = PhoneClassifier(HIDDEN_CONTEXT_MODEL, N_PHONES).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CFQ2g3PjErdZ"
   },
   "outputs": [],
   "source": [
    "parameters = list(phone_classifier.parameters()) + list(cpc_model.parameters())\n",
    "LEARNING_RATE = 2e-4\n",
    "optimizer = torch.optim.Adam(parameters, lr=LEARNING_RATE)\n",
    "\n",
    "optimizer_frozen = torch.optim.Adam(list(phone_classifier.parameters()), lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zsgjv3cD0oqD"
   },
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "\n",
    "def train_one_epoch_ctc(cpc_model, \n",
    "                        phone_classifier, \n",
    "                        loss_criterion, \n",
    "                        data_loader, \n",
    "                        optimizer):\n",
    "  \n",
    "  cpc_model.train()\n",
    "  loss_criterion.train()\n",
    "\n",
    "  avg_loss = 0\n",
    "  avg_accuracy = 0\n",
    "  n_items = 0\n",
    "  for step, full_data in enumerate(data_loader):\n",
    "\n",
    "    x, x_len, y, y_len = full_data\n",
    "\n",
    "    x_batch_len = x.shape[-1]\n",
    "    x, y = x.to(device), y.to(device)\n",
    "\n",
    "    bs=x.size(0)\n",
    "    optimizer.zero_grad()\n",
    "    context_out, enc_out, _ = cpc_model(x.to(device),y.to(device))\n",
    "  \n",
    "    scores = phone_classifier(context_out)\n",
    "    scores = scores.permute(1,0,2)\n",
    "    scores = F.log_softmax(scores,2)\n",
    "    yhat_len = torch.tensor([int(scores.shape[0]*x_len[i]/x_batch_len) for i in range(scores.shape[1])]) # this is an approximation, should be good enough\n",
    "\n",
    "    loss = loss_criterion(scores,y.to(device),yhat_len,y_len)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    avg_loss+=loss.item()*bs\n",
    "    n_items+=bs\n",
    "  avg_loss/=n_items\n",
    "  return avg_loss\n",
    "\n",
    "def validation_step(cpc_model, \n",
    "                    phone_classifier, \n",
    "                    loss_criterion, \n",
    "                    data_loader):\n",
    "\n",
    "  cpc_model.eval()\n",
    "  phone_classifier.eval()\n",
    "  avg_loss = 0\n",
    "  avg_accuracy = 0\n",
    "  n_items = 0\n",
    "  with torch.no_grad():\n",
    "    for step, full_data in enumerate(data_loader):\n",
    "\n",
    "      x, x_len, y, y_len = full_data\n",
    "\n",
    "      x_batch_len = x.shape[-1]\n",
    "      x, y = x.to(device), y.to(device)\n",
    "\n",
    "      bs=x.size(0)\n",
    "      context_out, enc_out, _ = cpc_model(x.to(device),y.to(device))\n",
    "    \n",
    "      scores = phone_classifier(context_out)\n",
    "      scores = scores.permute(1,0,2)\n",
    "      scores = F.log_softmax(scores,2)\n",
    "      yhat_len = torch.tensor([int(scores.shape[0]*x_len[i]/x_batch_len) for i in range(scores.shape[1])]) # this is an approximation, should be good enough\n",
    "\n",
    "      loss = loss_criterion(scores,y.to(device),yhat_len,y_len)\n",
    "      avg_loss+=loss.item()*bs\n",
    "      n_items+=bs\n",
    "  avg_loss/=n_items\n",
    "\n",
    "  return avg_loss\n",
    "\n",
    "def run_ctc(cpc_model, \n",
    "            phone_classifier, \n",
    "            loss_criterion, \n",
    "            data_loader_train, \n",
    "            data_loader_val, \n",
    "            optimizer,\n",
    "            n_epoch):\n",
    "  for epoch in range(n_epoch):\n",
    "\n",
    "    print(f\"Running epoch {epoch + 1} / {n_epoch}\")\n",
    "    loss_train = train_one_epoch_ctc(cpc_model, phone_classifier, loss_criterion, data_loader_train, optimizer)\n",
    "    print(\"-------------------\")\n",
    "    print(f\"Training dataset :\")\n",
    "    print(f\"Average loss : {loss_train}.\")\n",
    "\n",
    "    print(\"-------------------\")\n",
    "    print(\"Validation dataset\")\n",
    "    loss_val = validation_step(cpc_model, phone_classifier, loss_criterion, data_loader_val)\n",
    "    print(f\"Average loss : {loss_val}\")\n",
    "    print(\"-------------------\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "GSr7tcUdD72c",
    "outputId": "ea9f1f53-6b42-436d-e760-68bfdae72bf4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running epoch 1 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 32.44543953208657.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 32.01081585093132\n",
      "-------------------\n",
      "\n",
      "Running epoch 2 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 30.99022026328774.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 30.300324444522225\n",
      "-------------------\n",
      "\n",
      "Running epoch 3 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 29.319565432888645.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 28.464903420181635\n",
      "-------------------\n",
      "\n",
      "Running epoch 4 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 27.567655403297262.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 26.642856191119876\n",
      "-------------------\n",
      "\n",
      "Running epoch 5 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 25.832834390493538.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 24.82515669546986\n",
      "-------------------\n",
      "\n",
      "Running epoch 6 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 24.127855487636754.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 23.089440377402646\n",
      "-------------------\n",
      "\n",
      "Running epoch 7 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 22.487650171026484.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 21.420966125777547\n",
      "-------------------\n",
      "\n",
      "Running epoch 8 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 20.911723703771205.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 19.82448653813222\n",
      "-------------------\n",
      "\n",
      "Running epoch 9 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 19.407203647640202.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 18.315616182806366\n",
      "-------------------\n",
      "\n",
      "Running epoch 10 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 17.984150039566146.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 16.896408695745244\n",
      "-------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "run_ctc(cpc_model,phone_classifier,loss_ctc,data_loader_train,data_loader_val,optimizer_frozen,n_epoch=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TKrYW4gK1BBF"
   },
   "source": [
    "### b- Evaluation: the Phone Error Rate (PER)\n",
    "\n",
    "In order to compute the similarity between two sequences, we can use the [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance). This distance estimates the minimum number of insertion, deletion and addition to move from one sequence to another. If we normalize this distance by the number of characters in the reference sequence we get the Phone Error Rate (PER).\n",
    "\n",
    "This value can be interpreted as :\n",
    "\\\\[  PER = \\frac{S + D + I}{N} \\\\]\n",
    "\n",
    "Where:\n",
    "\n",
    "\n",
    "*   N is the number of characters in the reference\n",
    "*   S is the number of substitutiion\n",
    "*   I in the number of insertion\n",
    "*   D in the number of deletion\n",
    "\n",
    "For the best possible alignment of the two sequences.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RoBhsx7GNqI_"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def get_PER_sequence(ref_seq, target_seq):\n",
    "\n",
    "  # re = g.split()\n",
    "  # h = h.split()\n",
    "  n = len(ref_seq)\n",
    "  m = len(target_seq)\n",
    "\n",
    "  D = np.zeros((n+1,m+1))\n",
    "  for i in range(1,n+1):\n",
    "    D[i,0] = D[i-1,0]+1\n",
    "  for j in range(1,m+1):\n",
    "    D[0,j] = D[0,j-1]+1\n",
    "  \n",
    "  ### TODO compute the alignment\n",
    "\n",
    "  for i in range(1,n+1):\n",
    "    for j in range(1,m+1):\n",
    "      D[i,j] = min(\n",
    "          D[i-1,j]+1,\n",
    "          D[i-1,j-1]+1,\n",
    "          D[i,j-1]+1,\n",
    "          D[i-1,j-1]+ 0 if ref_seq[i-1]==target_seq[j-1] else float(\"inf\")\n",
    "      )\n",
    "  return D[n,m]/len(ref_seq)\n",
    "  \n",
    "\n",
    "  #return PER"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "r-hr0KK0mgcR"
   },
   "source": [
    "You can test your function below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "AfTb3yOQmvey",
    "outputId": "5db807ca-4c58-4413-ab19-6581d7d07263"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "ref_seq = [0, 1, 1, 2, 0, 2, 2]\n",
    "pred_seq = [1, 1, 2, 2, 0, 0]\n",
    "\n",
    "expected_PER = 4. / 7.\n",
    "print(get_PER_sequence(ref_seq, pred_seq) == expected_PER)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nHiyChl-m_k7"
   },
   "source": [
    "## c- Evaluating the PER of your model on the test dataset\n",
    "\n",
    "Evaluate the PER on the validation dataset. Please notice that you should usually use a separate dataset, called the dev dataset, to perform this operation. However for the sake of simplicity we will work with validation data in this exercise."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DMkX0PoFnclg"
   },
   "outputs": [],
   "source": [
    "import progressbar\n",
    "from multiprocessing import Pool\n",
    "\n",
    "def cut_data(seq, sizeSeq):\n",
    "    maxSeq = sizeSeq.max()\n",
    "    return seq[:, :maxSeq]\n",
    "\n",
    "\n",
    "def prepare_data(data):\n",
    "    seq, sizeSeq, phone, sizePhone = data\n",
    "    seq = seq.cuda()\n",
    "    phone = phone.cuda()\n",
    "    sizeSeq = sizeSeq.cuda().view(-1)\n",
    "    sizePhone = sizePhone.cuda().view(-1)\n",
    "\n",
    "    seq = cut_data(seq.permute(0, 2, 1), sizeSeq).permute(0, 2, 1)\n",
    "    return seq, sizeSeq, phone, sizePhone\n",
    "\n",
    "\n",
    "def get_per(test_dataloader,\n",
    "            cpc_model,\n",
    "            phone_classifier):\n",
    "\n",
    "  downsampling_factor = 160\n",
    "  cpc_model.eval()\n",
    "  phone_classifier.eval()\n",
    "\n",
    "  avgPER = 0\n",
    "  nItems = 0 \n",
    "\n",
    "  print(\"Starting the PER computation through beam search\")\n",
    "  bar = progressbar.ProgressBar(maxval=len(test_dataloader))\n",
    "  bar.start()\n",
    "\n",
    "  for index, data in enumerate(test_dataloader):\n",
    "\n",
    "    bar.update(index)\n",
    "\n",
    "    with torch.no_grad():\n",
    "      \n",
    "        seq, sizeSeq, phone, sizePhone = prepare_data(data)\n",
    "        c_feature, _, _ = cpc_model(seq.to(device),phone.to(device))\n",
    "        sizeSeq = sizeSeq / downsampling_factor\n",
    "        predictions = torch.nn.functional.softmax(\n",
    "        phone_classifier(c_feature), dim=2).cpu()\n",
    "        phone = phone.cpu()\n",
    "        sizeSeq = sizeSeq.cpu()\n",
    "        sizePhone = sizePhone.cpu()\n",
    "\n",
    "        bs = c_feature.size(0)\n",
    "        data_per = [(predictions[b].argmax(1),  phone[b]) for b in range(bs)]\n",
    "        # data_per = [(predictions[b], sizeSeq[b], phone[b], sizePhone[b],\n",
    "        #               \"criterion.module.BLANK_LABEL\") for b in range(bs)]\n",
    "\n",
    "        with Pool(bs) as p:\n",
    "            poolData = p.starmap(get_PER_sequence, data_per)\n",
    "        avgPER += sum([x for x in poolData])\n",
    "        nItems += len(poolData)\n",
    "\n",
    "  bar.finish()\n",
    "\n",
    "  avgPER /= nItems\n",
    "\n",
    "  print(f\"Average PER {avgPER}\")\n",
    "  return avgPER\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "2hvnudh4Osb4",
    "outputId": "acdd8ab7-c766-4a2b-9b6c-b744266d6240"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "N/A% (0 of 27) |                         | Elapsed Time: 0:00:00 ETA:  --:--:--"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting the PER computation through beam search\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (27 of 27) |########################| Elapsed Time: 0:10:44 Time:  0:10:44\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average PER 0.9509821691500522\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9509821691500522"
      ]
     },
     "execution_count": 91,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_per(data_loader_val,cpc_model,phone_classifier)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p8e9D7g8159k"
   },
   "source": [
    "## Exercice 3 : Character error rate (CER) \n",
    "\n",
    "The Character Error Rate (CER) is an evaluation metric similar to the PER but with characters insterad of phonemes. Using the following data, run the functions you defined previously to estimate the CER of your model after fine-tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "id": "cXONmKQOuFSn",
    "outputId": "5a2791ec-ff0e-4749-ce93-92922ffbee01"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "67it [00:00, 10784.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/CPC_audio\n",
      "Saved cache file at /content/per_data/pack_master/1h/_seqs_cache.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "7it [00:00, 1741.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 287 sequences in 2.68 seconds\n",
      "maxSizeSeq : 273359\n",
      "maxSizePhone : 300\n",
      "minSizePhone : 18\n",
      "Total size dataset 1.0406152430555555 hours\n",
      "Saved cache file at /content/per_data/pack_master/10min/_seqs_cache.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 212 sequences in 2.29 seconds\n",
      "maxSizeSeq : 273760\n",
      "maxSizePhone : 273\n",
      "minSizePhone : 29\n",
      "Total size dataset 0.73 hours\n"
     ]
    }
   ],
   "source": [
    "# Load a dataset labelled with the letters of each sequence.\n",
    "%cd /content/CPC_audio\n",
    "from cpc.eval.common_voices_eval import SingleSequenceDataset, parseSeqLabels, findAllSeqs\n",
    "path_train_data_cer = '/content/per_data/pack_master/1h'\n",
    "path_val_data_cer = '/content/per_data/pack_master/10min'\n",
    "path_letter_data_cer = '/content/per_data/pack_master/chars.txt'\n",
    "BATCH_SIZE=8\n",
    "\n",
    "letters_labels, N_LETTERS = parseSeqLabels(path_letter_data_cer)\n",
    "data_train_cer, _ = findAllSeqs(path_train_data_cer, extension='.flac')\n",
    "dataset_train_non_aligned = SingleSequenceDataset(path_train_data_cer, data_train_cer, letters_labels)\n",
    "\n",
    "\n",
    "data_val_cer, _ = findAllSeqs(path_val_data_cer, extension='.flac')\n",
    "dataset_val_non_aligned = SingleSequenceDataset(path_val_data_cer, data_val_cer, letters_labels)\n",
    "\n",
    "\n",
    "# The data loader will generate a tuple of tensors data, labels for each batch\n",
    "# data : size N x T1 x 1 : the audio sequence\n",
    "# label : size N x T2 the sequence of letters corresponding to the audio data\n",
    "# IMPORTANT NOTE: just like the PER the CER is computed with non-aligned phone data.\n",
    "data_loader_train_letters = torch.utils.data.DataLoader(dataset_train_non_aligned, batch_size=BATCH_SIZE,\n",
    "                                                shuffle=True)\n",
    "data_loader_val_letters = torch.utils.data.DataLoader(dataset_val_non_aligned, batch_size=BATCH_SIZE,\n",
    "                                              shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "9h07zI2LjzAU",
    "outputId": "70542ed5-a32e-448f-83ea-6355e601511e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading checkpoint checkpoint_data/checkpoint_30.pt\n",
      "Loading the state dict at checkpoint_data/checkpoint_30.pt\n"
     ]
    }
   ],
   "source": [
    "from cpc.feature_loader import loadModel\n",
    "\n",
    "checkpoint_path = 'checkpoint_data/checkpoint_30.pt'\n",
    "cpc_model, HIDDEN_CONTEXT_MODEL, HIDDEN_ENCODER_MODEL = loadModel([checkpoint_path])\n",
    "cpc_model = cpc_model.cuda()\n",
    "character_classifier = PhoneClassifier(HIDDEN_CONTEXT_MODEL, N_LETTERS).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rHCNg1E7lW1L"
   },
   "outputs": [],
   "source": [
    "parameters = list(character_classifier.parameters()) + list(cpc_model.parameters())\n",
    "LEARNING_RATE = 2e-4\n",
    "optimizer = torch.optim.Adam(parameters, lr=LEARNING_RATE)\n",
    "\n",
    "optimizer_frozen = torch.optim.Adam(list(character_classifier.parameters()), lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "engpkljbk9hj"
   },
   "outputs": [],
   "source": [
    "loss_ctc = torch.nn.CTCLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "9NBHd2s2kxld",
    "outputId": "24067818-b4b2-4f90-a433-a63f851b7587"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running epoch 1 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 17.15224294729166.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 16.621312869103598\n",
      "-------------------\n",
      "\n",
      "Running epoch 2 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 15.531890602378578.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 14.840831449246519\n",
      "-------------------\n",
      "\n",
      "Running epoch 3 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 13.80899079863008.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 12.998893340052021\n",
      "-------------------\n",
      "\n",
      "Running epoch 4 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 12.093861906678526.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 11.24841435599666\n",
      "-------------------\n",
      "\n",
      "Running epoch 5 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 10.522326436076131.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 9.72009142654202\n",
      "-------------------\n",
      "\n",
      "Running epoch 6 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 9.169493901979672.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 8.452587624861732\n",
      "-------------------\n",
      "\n",
      "Running epoch 7 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 8.05292275568822.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 7.4334790514543725\n",
      "-------------------\n",
      "\n",
      "Running epoch 8 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 7.152782953702486.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 6.621052656128509\n",
      "-------------------\n",
      "\n",
      "Running epoch 9 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 6.4348565915247775.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 5.983951862389443\n",
      "-------------------\n",
      "\n",
      "Running epoch 10 / 10\n",
      "-------------------\n",
      "Training dataset :\n",
      "Average loss : 5.864399843282633.\n",
      "-------------------\n",
      "Validation dataset\n",
      "Average loss : 5.487570416870841\n",
      "-------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "run_ctc(cpc_model,character_classifier,loss_ctc,data_loader_train_letters,data_loader_val_letters,optimizer_frozen,n_epoch=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "A8oxFr1jm17P",
    "outputId": "947ea6da-7fd9-4106-e3f1-32cf6d77c4f4"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "N/A% (0 of 27) |                         | Elapsed Time: 0:00:00 ETA:  --:--:--"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting the PER computation through beam search\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (27 of 27) |########################| Elapsed Time: 0:17:48 Time:  0:17:48\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average PER 0.9113886992183796\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9113886992183796"
      ]
     },
     "execution_count": 36,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_per(data_loader_val_letters,cpc_model,character_classifier)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Moteu_CTC.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}