Multimodal_attempt(best_one_so_far).ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "492db99a",
        "outputId": "ed8c119e-8403-4d0c-891a-d4c65716856f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "import json\n",
        "import os\n",
        "\n",
        "# drive à importer : https://drive.google.com/drive/folders/1vn_RM47LA_HdpQwZqdwqNRPm7CXyeeV3?usp=sharing\n",
        "# \n",
        "#1 chargement des datas\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "id": "492db99a"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BKP-lUu4Gmzd"
      },
      "source": [
        "### Preprocessing des transcriptions"
      ],
      "id": "BKP-lUu4Gmzd"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "47bcf75a"
      },
      "outputs": [],
      "source": [
        "#whisperX with segmented 30sec audios length and whistespaces representation \n",
        "folder_path_transcriptions_ad = \"/content/drive/MyDrive/Alzheimer_transcripts/transcriptions/whisperX_seg/white_spaces_ad\"\n",
        "folder_path_transcriptions_cn = \"/content/drive/MyDrive/Alzheimer_transcripts/transcriptions/whisperX_seg/white_spaces_cn\""
      ],
      "id": "47bcf75a"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6d6d201e"
      },
      "outputs": [],
      "source": [
        "def from_json_to_dict(folder_ad, folder_cn):\n",
        "    data_ad = []\n",
        "    data_cn = []\n",
        "    for dir_path in [folder_ad, folder_cn]:\n",
        "        for file_name in sorted(os.listdir(dir_path)):\n",
        "            if file_name.endswith('.json'):\n",
        "                file_path = os.path.join(dir_path, file_name)\n",
        "                with open(file_path, 'r') as f:\n",
        "                    data = json.load(f)\n",
        "                    if dir_path == folder_ad:\n",
        "                        data_ad.append(data)\n",
        "                    elif dir_path == folder_cn:\n",
        "                        data_cn.append(data)\n",
        "    return data_ad, data_cn             \n",
        "                \n",
        "transcripts_data_ad, transcripts_data_cn = from_json_to_dict(folder_path_transcriptions_ad, folder_path_transcriptions_cn)"
      ],
      "id": "6d6d201e"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gakJZ4yaItVp",
        "outputId": "c01b1abb-e4b2-4503-df56-4e7c721e9cfa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "exemple d'une data ad avec whisperX :\n",
            "  {'text': 'Good.', 'label': 1}\n",
            "\n",
            "exemple d'une data cn avec whisperX :\n",
            "  {'text': 'Those are the only action things that I can', 'label': 0}\n"
          ]
        }
      ],
      "source": [
        "from random import choice\n",
        "\n",
        "print(\"\\nexemple d'une data ad avec whisperX :\\n \", choice(transcripts_data_ad))\n",
        "print(\"\\nexemple d'une data cn avec whisperX :\\n \", choice(transcripts_data_cn))\n"
      ],
      "id": "gakJZ4yaItVp"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dd4c6b38"
      },
      "outputs": [],
      "source": [
        "#delete des keys autre que 'text' et 'label'\n",
        "keys_to_keep = ['text', 'label']\n",
        "\n",
        "def delete_keys(d, keys_to_keep):\n",
        "    for key in list(d.keys()):\n",
        "        if key not in keys_to_keep:\n",
        "            del d[key]\n",
        "\n",
        "for data in (transcripts_data_ad + transcripts_data_cn):\n",
        "    delete_keys(data, keys_to_keep)\n",
        "    "
      ],
      "id": "dd4c6b38"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6eaa6fac"
      },
      "outputs": [],
      "source": [
        "transcripts = transcripts_data_ad + transcripts_data_cn\n",
        "\n",
        "texts, labels = zip(*[(d['text'], d['label']) for d in transcripts])"
      ],
      "id": "6eaa6fac"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6oQJZI5uGXbE",
        "outputId": "c2c26b12-161e-45b1-d8aa-f99a4a463385"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(\"Here's a cookie jar and the lid is off the cookie jar. long silence The boy is about to come down on the\",\n",
              " \"and... .. girls . I don't know much about girls, but anyway .. the .. housewife .. who's in the kitchen .\")"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "texts[:2]"
      ],
      "id": "6oQJZI5uGXbE"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pFX9OpEtJ-88"
      },
      "source": [
        "### Audio preprocessing (eGeMAPS features)"
      ],
      "id": "pFX9OpEtJ-88"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ro1m78etKF2I"
      },
      "outputs": [],
      "source": [
        "# %%capture\n",
        "# !pip install opensmile"
      ],
      "id": "Ro1m78etKF2I"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "phz2Wgowsypa"
      },
      "outputs": [],
      "source": [
        "folder_path_seg_audio_ad = \"/content/drive/MyDrive/Alzheimer_transcripts/audio/train/Segmented_audio/ad_segmented\"\n",
        "folder_path_seg_audio_cn = \"/content/drive/MyDrive/Alzheimer_transcripts/audio/train/Segmented_audio/cn_segmented\""
      ],
      "id": "phz2Wgowsypa"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "82Z08efvK1mm"
      },
      "outputs": [],
      "source": [
        "# import audiofile\n",
        "# import opensmile\n",
        "\n",
        "# def get_features(path):\n",
        "#   features_list = []\n",
        "#   for file_name in sorted(os.listdir(path)):\n",
        "#     signal, sampling_rate = audiofile.read(\n",
        "#       path+'/'+file_name,\n",
        "#     )\n",
        "#     smile = opensmile.Smile(\n",
        "#     feature_set=opensmile.FeatureSet.eGeMAPSv02,\n",
        "#     feature_level=opensmile.FeatureLevel.Functionals,\n",
        "#     )\n",
        "\n",
        "#     features_list.append(smile.to_numpy(smile.process_signal(signal, sampling_rate)))\n",
        "#   return features_list\n",
        "\n",
        "# features_ad = get_features(folder_path_seg_audio_ad)\n",
        "# features_cn = get_features(folder_path_seg_audio_cn)\n",
        "\n",
        "# features = features_ad + features_cn"
      ],
      "id": "82Z08efvK1mm"
    },
    {
      "cell_type": "code",
      "source": [
        "# #save features to load again\n",
        "# import pickle\n",
        "# with open('acoustics.pkl', 'wb') as f:\n",
        "#     pickle.dump(features, f)"
      ],
      "metadata": {
        "id": "4FAhc85SfQn1"
      },
      "id": "4FAhc85SfQn1",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "with open('/content/drive/MyDrive/Alzheimer_transcripts/acoustics features/acoustics.pkl', 'rb') as f:\n",
        "    features = pickle.load(f)"
      ],
      "metadata": {
        "id": "bBGH3v4foCsU"
      },
      "id": "bBGH3v4foCsU",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "np.shape(features[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yzj0arIMkqup",
        "outputId": "51ed816b-ca58-4631-cda8-debde439c810"
      },
      "id": "Yzj0arIMkqup",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1, 88, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a0c7074a"
      },
      "outputs": [],
      "source": [
        "# from sklearn.model_selection import train_test_split\n",
        "# import tensorflow as tf\n",
        "# import numpy as np\n",
        "# data = list(zip(texts,features))\n",
        "# new_data = []\n",
        "# for d in data:\n",
        "#   new_data.append([d[0],d[1].reshape(-1)])\n",
        "\n",
        "# data = new_data\n",
        "# np.shape(data[0][1])"
      ],
      "id": "a0c7074a"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-tGU7fdkKzJr"
      },
      "source": [
        "### Modèle"
      ],
      "id": "-tGU7fdkKzJr"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5_mo76zSz9vu"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "\n",
        "def fusion_list(l1, l2):\n",
        "  return list(map(list,list(zip(l1,l2))))\n",
        "\n",
        "\n",
        "\n",
        "features = [f.reshape(-1) for f in features]\n",
        "\n",
        "data = fusion_list(texts, features)\n",
        "x_train, x_test, y_train, y_test = train_test_split(data, labels, test_size = 0.20, stratify = labels)\n",
        "x_train_text, x_train_audio = zip(*x_train)\n",
        "x_test_text, x_test_audio = zip(*x_test)\n",
        "\n",
        "x_train_text = np.array(x_train_text)\n",
        "x_train_audio = np.array(x_train_audio)\n",
        "x_test_text = np.array(x_test_text)\n",
        "x_test_audio = np.array(x_test_audio)\n",
        "y_train = np.array(y_train)\n",
        "y_test = np.array(y_test)\n",
        "\n",
        "# x_train_T = [list(x) for x in zip(*x_train)]\n",
        "# x_test_T = [list(x) for x in zip(*x_test)]\n",
        "# x_train_text = x_train_T[0]\n",
        "# x_train_audio = x_train_T[1]\n",
        "# x_test_text = x_test_T[0]\n",
        "# x_test_audio = x_test_T[1]"
      ],
      "id": "5_mo76zSz9vu"
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "UPLDMqjpRmyb"
      },
      "id": "UPLDMqjpRmyb",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "np.shape(x_train_audio)"
      ],
      "metadata": {
        "id": "8haTaIpKr2Tv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0ff3ff69-f5f9-4a7c-dea1-94f8e4135933"
      },
      "id": "8haTaIpKr2Tv",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(410, 88)"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.shape(x_train_audio[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NZ-NcE_AUe84",
        "outputId": "e766186a-3908-477f-df70-4842456fc98e"
      },
      "id": "NZ-NcE_AUe84",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(88,)"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(x_train_audio) == len(x_train_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FKPD_J8Y_tEk",
        "outputId": "4bd352db-7389-4a3a-ad92-bd46ba0e6a90"
      },
      "id": "FKPD_J8Y_tEk",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(x_train_text)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PkY6sqhK_2BT",
        "outputId": "570afd2d-007a-45cc-825a-13967a44fd32"
      },
      "id": "PkY6sqhK_2BT",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "410"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bb10ff99"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "#!pip install transformers\n",
        "!pip install tensorflow_hub\n",
        "!pip install tensorflow_text\n",
        "#from transformers import BertTokenizer, TFBertModel\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow_text as text"
      ],
      "id": "bb10ff99"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a5bdd5a2"
      },
      "outputs": [],
      "source": [
        "# def build_classifier_model():\n",
        "\n",
        "#   #text\n",
        "#   bert_preprocess = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\", name='preprocessing')\n",
        "#   bert_encoder = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/4\", trainable=True, name='BERT_encoder')\n",
        "\n",
        "#   text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
        "#   #audio_features_input = tf.keras.layers.Input(shape=(88,), dtype=\"float32\", name='feature')\n",
        "\n",
        "#   # Encoder le texte avec BERT\n",
        "#   preprocessing_layer = bert_preprocess\n",
        "#   encoder_inputs = preprocessing_layer(text_input)\n",
        "#   encoder = bert_encoder\n",
        "#   outputs = encoder(encoder_inputs)\n",
        "#   net = outputs['pooled_output']\n",
        "#   # Concaténer l'encodeur de texte de BERT avec les features additionnelles\n",
        "#   #concatenated = tf.keras.layers.Concatenate()([bert_output, audio_features_input])\n",
        "\n",
        "#   # Ajouter une couche dense pour la classification binaire\n",
        "#   #dense1 = tf.keras.layers.Dense(64, activation='relu')(concatenated)\n",
        "#   net = tf.keras.layers.Dropout(0.1)(net)\n",
        "#   net = tf.keras.layers.Dense(1, activation='sigmoid', name='classifier')(net)\n",
        "#   return tf.keras.Model(text_input, net)"
      ],
      "id": "a5bdd5a2"
    },
    {
      "cell_type": "code",
      "source": [
        "def build_classifier_model_multi_modal():\n",
        "\n",
        "  #text\n",
        "  bert_preprocess = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\", name='preprocessing')\n",
        "  bert_encoder = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/4\", trainable=True, name='BERT_encoder')\n",
        "  text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
        "  \n",
        "  # couche : Encoder le texte avec BERT\n",
        "  preprocessing_layer = bert_preprocess\n",
        "  encoder_inputs = preprocessing_layer(text_input)\n",
        "  encoder = bert_encoder\n",
        "  outputs = encoder(encoder_inputs)\n",
        "  bert_net = outputs['pooled_output']\n",
        "  #bert_out = tf.keras.layers.Dense(1, activation='sigmoid')(bert_net)\n",
        "\n",
        "  #couche : audio features\n",
        "  audio_features_input = tf.keras.layers.Input(shape=(88,), dtype='float32', name='audio_features')\n",
        "  audio_out = tf.keras.layers.Dense(1, activation='sigmoid')(audio_features_input)\n",
        "  #audio_out = tf.keras.layers.Dropout(0.1)(audio_features_input)\n",
        "\n",
        "  # Concaténer l'encodeur de texte de BERT avec les features additionnelles\n",
        "\n",
        "  fusion = tf.keras.layers.concatenate([bert_net, audio_out])\n",
        "  #net = tf.keras.layers.Dense(64, activation='relu')(fusion)\n",
        "  net = tf.keras.layers.Dropout(0.1)(fusion)\n",
        " \n",
        "  net = tf.keras.layers.Dense(1, activation=\"sigmoid\", name='classifier')(fusion)\n",
        "\n",
        "  return tf.keras.Model([text_input, audio_features_input], net)"
      ],
      "metadata": {
        "id": "FlbSL4GaFcH1"
      },
      "id": "FlbSL4GaFcH1",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_multi_modal = build_classifier_model_multi_modal()\n",
        "model_multi_modal.summary()\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "08winBlYMCUG",
        "outputId": "a6c067ae-d40c-4d8f-eda7-f563c5a62101"
      },
      "id": "08winBlYMCUG",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Please fix your imports. Module tensorflow.python.training.tracking.data_structures has been moved to tensorflow.python.trackable.data_structures. The old module will be deleted in version 2.11.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.9/dist-packages/tensorflow/python/autograph/pyct/static_analysis/liveness.py:83: Analyzer.lamba_check (from tensorflow.python.autograph.pyct.static_analysis.liveness) is deprecated and will be removed after 2023-09-23.\n",
            "Instructions for updating:\n",
            "Lambda fuctions will be no more assumed to be used in the statement where they are used, or at least in the same block. https://github.com/tensorflow/tensorflow/issues/56089\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " text (InputLayer)              [(None,)]            0           []                               \n",
            "                                                                                                  \n",
            " preprocessing (KerasLayer)     {'input_mask': (Non  0           ['text[0][0]']                   \n",
            "                                e, 128),                                                          \n",
            "                                 'input_word_ids':                                                \n",
            "                                (None, 128),                                                      \n",
            "                                 'input_type_ids':                                                \n",
            "                                (None, 128)}                                                      \n",
            "                                                                                                  \n",
            " audio_features (InputLayer)    [(None, 88)]         0           []                               \n",
            "                                                                                                  \n",
            " BERT_encoder (KerasLayer)      {'default': (None,   109482241   ['preprocessing[0][0]',          \n",
            "                                768),                             'preprocessing[0][1]',          \n",
            "                                 'encoder_outputs':               'preprocessing[0][2]']          \n",
            "                                 [(None, 128, 768),                                               \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768),                                                \n",
            "                                 (None, 128, 768)],                                               \n",
            "                                 'pooled_output': (                                               \n",
            "                                None, 768),                                                       \n",
            "                                 'sequence_output':                                               \n",
            "                                 (None, 128, 768)}                                                \n",
            "                                                                                                  \n",
            " dense (Dense)                  (None, 1)            89          ['audio_features[0][0]']         \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 769)          0           ['BERT_encoder[0][13]',          \n",
            "                                                                  'dense[0][0]']                  \n",
            "                                                                                                  \n",
            " classifier (Dense)             (None, 1)            770         ['concatenate[0][0]']            \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 109,483,100\n",
            "Trainable params: 109,483,099\n",
            "Non-trainable params: 1\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wmli2cMtNS-D"
      },
      "outputs": [],
      "source": [
        "!pip install -q tf-models-official\n",
        "\n",
        "from official.nlp import optimization\n",
        "from keras.optimizers import SGD\n",
        "\n",
        "metrics = tf.metrics.BinaryAccuracy()\n",
        "\n",
        "epochs = 20\n",
        "\n",
        "\n",
        "# steps_per_epoch = 1000\n",
        "# num_train_steps = steps_per_epoch * epochs\n",
        "# num_warmup_steps = int(0.1*num_train_steps)\n",
        "\n",
        "# optimizer = optimization.create_optimizer(init_lr=init_lr,\n",
        "#                                           num_train_steps=num_train_steps,\n",
        "#                                           num_warmup_steps=num_warmup_steps,\n",
        "#                                           optimizer_type='adamw')\n",
        "\n"
      ],
      "id": "wmli2cMtNS-D"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AA56eOu1Nb_K"
      },
      "outputs": [],
      "source": [
        "# #training model\n",
        "# model.compile(optimizer=optimizer,\n",
        "#               loss='binary_crossentropy',\n",
        "#               metrics=metrics)\n",
        "# model.fit(x_train_text, y_train, epochs=epochs)\n",
        "\n",
        "from keras.optimizers import SGD, Adam\n",
        "opt = Adam(learning_rate=0.0001)"
      ],
      "id": "AA56eOu1Nb_K"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T4oIPlSaTnWC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "26070a76-3a70-4378-a9d8-3a59ac51e390"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "13/13 [==============================] - 17s 131ms/step - loss: 0.6599 - binary_accuracy: 0.6089\n",
            "Epoch 2/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.6339 - binary_accuracy: 0.6707\n",
            "Epoch 3/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.5619 - binary_accuracy: 0.7341\n",
            "Epoch 4/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.4306 - binary_accuracy: 0.8098\n",
            "Epoch 5/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.2637 - binary_accuracy: 0.9049\n",
            "Epoch 6/50\n",
            "13/13 [==============================] - 2s 132ms/step - loss: 0.1579 - binary_accuracy: 0.9463\n",
            "Epoch 7/50\n",
            "13/13 [==============================] - 2s 132ms/step - loss: 0.1177 - binary_accuracy: 0.9610\n",
            "Epoch 8/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0740 - binary_accuracy: 0.9780\n",
            "Epoch 9/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0887 - binary_accuracy: 0.9732\n",
            "Epoch 10/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.1435 - binary_accuracy: 0.9439\n",
            "Epoch 11/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0326 - binary_accuracy: 0.9902\n",
            "Epoch 12/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0260 - binary_accuracy: 0.9878\n",
            "Epoch 13/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0658 - binary_accuracy: 0.9805\n",
            "Epoch 14/50\n",
            "13/13 [==============================] - 2s 131ms/step - loss: 0.0430 - binary_accuracy: 0.9878\n",
            "Epoch 15/50\n",
            "13/13 [==============================] - 2s 131ms/step - loss: 0.0260 - binary_accuracy: 0.9902\n",
            "Epoch 16/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0345 - binary_accuracy: 0.9902\n",
            "Epoch 17/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0983 - binary_accuracy: 0.9610\n",
            "Epoch 18/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0869 - binary_accuracy: 0.9659\n",
            "Epoch 19/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0273 - binary_accuracy: 0.9902\n",
            "Epoch 20/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0530 - binary_accuracy: 0.9780\n",
            "Epoch 21/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0717 - binary_accuracy: 0.9756\n",
            "Epoch 22/50\n",
            "13/13 [==============================] - 2s 131ms/step - loss: 0.0654 - binary_accuracy: 0.9780\n",
            "Epoch 23/50\n",
            "13/13 [==============================] - 2s 131ms/step - loss: 0.0477 - binary_accuracy: 0.9829\n",
            "Epoch 24/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0267 - binary_accuracy: 0.9902\n",
            "Epoch 25/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0204 - binary_accuracy: 0.9951\n",
            "Epoch 26/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0171 - binary_accuracy: 0.9927\n",
            "Epoch 27/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0194 - binary_accuracy: 0.9927\n",
            "Epoch 28/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0192 - binary_accuracy: 0.9902\n",
            "Epoch 29/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0173 - binary_accuracy: 0.9927\n",
            "Epoch 30/50\n",
            "13/13 [==============================] - 2s 131ms/step - loss: 0.0178 - binary_accuracy: 0.9927\n",
            "Epoch 31/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0167 - binary_accuracy: 0.9927\n",
            "Epoch 32/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0165 - binary_accuracy: 0.9927\n",
            "Epoch 33/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0168 - binary_accuracy: 0.9927\n",
            "Epoch 34/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0173 - binary_accuracy: 0.9927\n",
            "Epoch 35/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0174 - binary_accuracy: 0.9927\n",
            "Epoch 36/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0168 - binary_accuracy: 0.9927\n",
            "Epoch 37/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0173 - binary_accuracy: 0.9927\n",
            "Epoch 38/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0171 - binary_accuracy: 0.9927\n",
            "Epoch 39/50\n",
            "13/13 [==============================] - 2s 131ms/step - loss: 0.0162 - binary_accuracy: 0.9927\n",
            "Epoch 40/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0173 - binary_accuracy: 0.9927\n",
            "Epoch 41/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0168 - binary_accuracy: 0.9927\n",
            "Epoch 42/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0173 - binary_accuracy: 0.9927\n",
            "Epoch 43/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0180 - binary_accuracy: 0.9927\n",
            "Epoch 44/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0173 - binary_accuracy: 0.9927\n",
            "Epoch 45/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0166 - binary_accuracy: 0.9927\n",
            "Epoch 46/50\n",
            "13/13 [==============================] - 2s 131ms/step - loss: 0.0173 - binary_accuracy: 0.9927\n",
            "Epoch 47/50\n",
            "13/13 [==============================] - 2s 131ms/step - loss: 0.0166 - binary_accuracy: 0.9927\n",
            "Epoch 48/50\n",
            "13/13 [==============================] - 2s 130ms/step - loss: 0.0176 - binary_accuracy: 0.9927\n",
            "Epoch 49/50\n",
            "13/13 [==============================] - 2s 129ms/step - loss: 0.0177 - binary_accuracy: 0.9927\n",
            "Epoch 50/50\n",
            "13/13 [==============================] - 2s 131ms/step - loss: 0.0269 - binary_accuracy: 0.9902\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fa7ff1af760>"
            ]
          },
          "metadata": {},
          "execution_count": 103
        }
      ],
      "source": [
        "from keras.metrics.metrics import accuracy\n",
        "del model_multi_modal\n",
        "model_multi_modal = build_classifier_model_multi_modal()\n",
        "model_multi_modal.compile(optimizer=opt,\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=metrics)\n",
        "\n",
        "model_multi_modal.fit([x_train_text, x_train_audio], y_train, epochs=50)"
      ],
      "id": "T4oIPlSaTnWC"
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "bOPAv1UFWWXC"
      },
      "id": "bOPAv1UFWWXC",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_wGVLr85NoQl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3f21bcb8-3e1b-4003-cc71-4c660f73e0ea"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4/4 [==============================] - 1s 57ms/step - loss: 1.5739 - binary_accuracy: 0.7476\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1.5739384889602661, 0.7475728392601013]"
            ]
          },
          "metadata": {},
          "execution_count": 104
        }
      ],
      "source": [
        "from matplotlib import pyplot as plt\n",
        "import seaborn as sn\n",
        "from sklearn.metrics import confusion_matrix, classification_report, ConfusionMatrixDisplay\n",
        "import numpy as np\n",
        "\n",
        "model_multi_modal.evaluate([x_test_text, x_test_audio], y_test)"
      ],
      "id": "_wGVLr85NoQl"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aw1V23xaN6_i",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "outputId": "5762514a-bb49-495f-f9cd-26bbdd50439e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4/4 [==============================] - 1s 56ms/step\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAATIAAAEGCAYAAADmLRl+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAXoklEQVR4nO3de5weVX3H8c93N3dyIZCYphAEIYFGLIHGgORVCkFrwBsoVREt1th4AS9VW5T2JYitYqultuIlAhIrFxFEFLmIAYpUiSSYpFxsCRFJSDAJIZAQSLLP/vrHzMqTsNmZSZ7LzO73/XqdV565PGd+mw0/zjlz5owiAjOzKutodwBmZnvKiczMKs+JzMwqz4nMzCrPiczMKm9QuwOoN3zs0Bgzca92h2EFbHmo3RFYEc/zLNtiq/akjteesFc8uaGW69zFy7beGhGz9+R6eZQqkY2ZuBdnXPnqdodhBSw7ytN3qmRhLNjjOtZvqLHw1v1znTt44iPj9viCOZQqkZlZFQS16G53EDtwIjOzQgLoplwtcScyMyusG7fIzKzCgmC7u5ZmVmUB1Ny1NLOq8xiZmVVaALWSrZrjRGZmhZVrhMyPKJlZQUFQy1nykNQp6VeSbky3D5K0UNJySd+VNCSrDicyMyskArbnLDl9BKh/2O0LwEURcQjwFDAnqwInMjMrSNRylsyapP2B1wGXpNsCZgHXpqfMB07JqsdjZGZWSADdjRvr/zfg74BR6fa+wMaI6Eq3VwH7ZVXiFpmZFVagRTZO0qK6MrenDkmvB9ZGxOI9jcctMjMrJJkQm3sloPURMX0Xx2YCb5R0MjAMGA18Gdhb0qC0VbY/8HjWRdwiM7NCAtgeHblKn/VEfCoi9o+IA4G3A7dHxBnAHcBp6WlnAjdkxeREZmaFBKJGR66ym84BPiZpOcmY2aVZX3DX0swK6449WmT2RSLiTuDO9PMKYEaR7zuRmVkhBcfIWsKJzMwKErWM8a9WcyIzs0KSFWKdyMyswiLEtuhsdxg7cCIzs8K6PUZmZlWWDPa7a2lmlebBfjOrOA/2m1m/UGvwhNg95URmZoUEYnuUK3WUKxozKz0P9ptZ5QVy19LMqs+D/WZWaRF4+oWZVVsy2O9HlMys4jzYb2aVFqjhCyvuKScyMyvMLTIzq7TkvZZOZGZWafneIt5KTmRmVkjyOjjftTSzCouQu5ZmVn2eEGtmlZasR+YxMjOrtMasECtpGHAXMJQkF10bEedJuhz4M+Dp9NR3R8SSvupyIjOzQpLpFw1pkW0FZkXEZkmDgbsl3Zwe+9uIuDZvRU5kZlZIo561jIgANqebg9MSu1NXuUbszKwSuunIVYBxkhbVlbn19UjqlLQEWAvcFhEL00P/JGmZpIskDc2Kxy0yMyskWcYnd9dyfURM33VdUQOmSdobuF7S4cCngCeAIcA84Bzggr4u4haZmRXWHcpV8oqIjcAdwOyIWBOJrcC3gBlZ33ciM7NCktUvOnKVvkgan7bEkDQceA3wa0kT030CTgHuz4rJXUszKyR5RKkhbaCJwHxJnSSNqmsi4kZJt0saDwhYArw/qyInsgbq3ho88l6IbRA1GHMi/MEHxMrzgs2LoXNkct6kz8DwQ8s1odASp8xZx0lnbEAKbr5iX66/ZHy7QyqhxjyiFBHLgCN72T+raF1NTWSSZgNfBjqBSyLiwmZer900BF72DegcIWJ7sHwOjJqZ3E2e+FHY+9VOXmX20kOf46QzNvDh101m+zbxuStXsPCno1n9aOZNswGnbDP7mzZGljYXLwZOAqYCp0ua2qzrlYEkOkckv+DoSorK9fu2PhwweSu//tUItj7XQXdNLPvFSGae/HT2FweYnruWeUqrNHOwfwawPCJWRMQ24GrgTU28XilELfi/twcPvhpGHQ0jXpH8Mp+4GP7vrcHqLwbd23Zrzp812aO/HsbhMzYzamwXQ4d388pZzzD+D7e1O6xSasRgfyM1s2u5H7CybnsVcPTOJ6UT5OYCjJo4oonhtIY6xZSrobYpePTj8Pzy4A/OhkHjILbD4/8I6y6HCXMzq7IWW7l8GNd89SV8/qoVPL+lgxUPDKe75ib1zsq4Zn/bp19ExLyImB4R00fs3X/GIjpHiZHTYdPPYfB4IYmOIWLsG2FL5s1ka5dbr9qXs2dP4RNvPoTNT3eyakX/+TfZKAF0RUeu0irNvNLjwKS67f3Tff1W11NBbVPSbex+Pth0Dww9ELavS/ZFBM/cAcMOaWOQ1qcx+24HYPx+25h58tPccf3YNkdUTgOpa3kvMFnSQSQJ7O3AO5p4vbbbvg5WngfUggjY+zUw+jjxyNygtjHZN3wK7Pf37Y7UduXTl/yWUWO7qG0XXzl3P559plxLOpdCwVn7rdC0RBYRXZLOBm4lmX5xWUQ80KzrlcHwKWLKVS/ef/C8cv3Sbdc+fqqby1kG3MKKEXETcFMzr2FmrTdgWmRm1j81cGHFhnEiM7NCAtHV3fYJDztwIjOzwgbUGJmZ9UPhrqWZVZzHyMysX3AiM7NKC0TNg/1mVnUe7DezSgsP9ptZfxBOZGZWbQPooXEz67/cIjOzSouAWrcTmZlVnO9amlmlBeXrWpZrVpuZVUAy2J+n9FmLNEzSLyUtlfSApM+k+w+StFDScknflTQkKyInMjMrLCJfybAVmBURRwDTgNmSjgG+AFwUEYcATwFzsipyIjOzwiKUq/RdR0REbE43B6clgFnAten++cApWfF4jMzMCknuWuZuA42TtKhue15EzOvZkNQJLAYOAS4GHgE2RkRXesoqknfk9smJzMwKy9Ft7LE+Iqbvup6oAdMk7Q1cDxy2O/E4kZlZYY2+axkRGyXdAbwK2FvSoLRVlut9uB4jM7NCgnzjY1nJTtL4tCWGpOHAa4CHgDuA09LTzgRuyIrJLTIzKyx/z7JPE4H56ThZB3BNRNwo6UHgakn/CPwKuDSrIicyMysmIBrwiFJELAOO7GX/CmBGkbqcyMyssLLN7HciM7PCCty1bIldJjJJ/0EfXeGI+HBTIjKzUivjs5Z9tcgW9XHMzAaqAKqSyCJifv22pBERsaX5IZlZ2ZWta5k5j0zSq9Lbob9Ot4+Q9NWmR2ZmJSWiO19plTwTYv8NeC3wJEBELAWOa2JMZlZ2kbO0SK67lhGxUtohu9aaE46ZlV5Ua7C/x0pJxwIhaTDwEZLHCMxsoKraGBnwfuAskqU0VpMsgHZWE2Mys9JTztIamS2yiFgPnNGCWMysKrrbHcCO8ty1fJmkH0laJ2mtpBskvawVwZlZCfXMI8tTWiRP1/JK4BqSJ9X/EPgecFUzgzKzcmvQmv0NkyeRjYiI/4yIrrR8BxjW7MDMrMSqMv1C0j7px5slfRK4miS0twE3tSA2MyurCk2/WEySuHoifl/dsQA+1aygzKzcVLLpF309a3lQKwMxs4oIQQsfP8oj18x+SYcDU6kbG4uIbzcrKDMruaq0yHpIOg84niSR3QScBNwNOJGZDVQlS2R57lqeBpwIPBERfwUcAYxpalRmVm5VuWtZ57mI6JbUJWk0sBaY1OS4zKysqrSwYp1F6bvnvklyJ3Mz8ItmBmVm5VaZu5Y9IuKD6cevS7oFGJ2+xsnMBqqqJDJJR/V1LCLua05IZlZ2VWqRfamPYwHManAsbF4zgoWfeWWjq7Umumv1vHaHYAXMeG2DXrvRgDEySZNIZj9MIMkp8yLiy5LOB/4aWJeeem5E9Pk0UV8TYk/Y40jNrP9p3B3JLuDjEXGfpFHAYkm3pccuiogv5q3IL+g1s+IakMgiYg2wJv28SdJDJAu4FpZnHpmZ2Q7Una8A4yQtqitze61POhA4EliY7jpb0jJJl0kamxWPE5mZFZd/Quz6iJheV140qCppJHAd8NGIeAb4GnAwybL6a+h7vB7It0KsJL1T0qfT7QMkzcj+Sc2sP1LkL5l1JS80ug64IiK+DxARv4uIWkR0k8xfzcw3eVpkXwVeBZyebm8CLs7xPTPrrxqw1LWSd0xeCjwUEf9at39i3WmnAvdnhZNnsP/oiDhK0q8AIuIpSUNyfM/M+qvG3LWcCbwL+B9JS9J95wKnS5qWXuVRdlwLsVd5Etl2SZ1ppUgaT+neoWJmrdSICbERcTe9vzOu8ArUeRLZvwPXAy+R9E8kq2H8Q9ELmVk/Eb+/I1kaeZ61vELSYpKlfAScEhF+07jZQFahR5SA5C4lsAX4Uf2+iHismYGZWYlVLZEBP+aFl5AMAw4C/hd4eRPjMrMSq9JD4wBExCvqt9NVMT64i9PNzFqu8LOW6QOeRzcjGDOriKq1yCR9rG6zAzgKWN20iMys3Kp41xIYVfe5i2TM7LrmhGNmlVClFlk6EXZURHyiRfGYWcmJCg32SxoUEV2SZrYyIDOrgKokMuCXJONhSyT9EPge8GzPwZ4n1c1sgMm5skUr5RkjGwY8SbJGf898sgCcyMwGqgoN9r8kvWN5Py8ksB4ly8dm1kpVapF1AiPp/en0kv0YZtZSJcsAfSWyNRFxQcsiMbNqaNxblBqmr0S25y+uM7N+qUpdyxNbFoWZVUtVEllEbGhlIGZWHVV8RMnM7AUVGyMzM3sRUb4BdCcyMyvOLTIzq7oq3bU0M+udE5mZVVoJF1bsaHcAZlZBkbP0QdIkSXdIelDSA5I+ku7fR9Jtkh5O/xybFY4TmZkVpshXMnQBH4+IqcAxwFmSpgKfBBZExGRgQbrdJycyMyuuAS2yiFgTEfelnzcBDwH7AW8C5qenzQdOyQrHY2RmVliBu5bjJC2q254XEfNeVJ90IHAksBCYEBFr0kNPABOyLuJEZmbFBEUWVlwfEdP7OkHSSJIXGn00Ip6RXphuGxEhZadNdy3NrJCel480YIwMSYNJktgVdcvn/07SxPT4RGBtVj1OZGZWXGPuWgq4FHgoIv617tAPgTPTz2cCN2SF466lmRWmaMiM2JnAu4D/kbQk3XcucCFwjaQ5wG+Bt2ZV5ERmZsU0aPWLiLibXT9/Xmg9RCcyMyvMz1qaWeWV7RElJzIzK84tMjOrtIq+adzMbEdOZGZWZT0TYsvEiczMClN3uTKZE5mZFeO3KPV/57zrvzj2FY/x1KbhvPuzpwFw/pwFTJqwEYCRI7axecsQ5nzuLW2M0nZWq8GHZk9h34nb+ey3f8OFZx3Aw0tH0Dk4OHTaFj7yzysZNLjdUZbHgJl+Ieky4PXA2og4vFnXKZtbfjGF6+98Oee++87f7zv/0hcmKZ/1lnvY/NyQNkRmffnBJeOZNHkrWzYnjx/PevNTnPOVxwC48IMv5eYr9+UNZz7ZzhDLpWQtsmY+NH45MLuJ9ZfS0uUTeebZobs4Gpxw1AoW3HtwS2Oyvq1bPZhfLhjNSe94IVHNOHETEkhw6JFbWL/GzbF6jVr9olGalsgi4i5gQ7Pqr6IjDnmCDZuGs2rdmHaHYnW+ft5+vPcfVqNe/mvo2g4Lrh3L9BM2tT6wsgogIl9pkbYv4yNprqRFkhZ1bX223eE01YmvfMStsZK557bR7D2ui8l//Fyvx//jU5M4/JhnecXR/fvfZlHqzldape2JLCLmRcT0iJg+aOhe7Q6naTo7ujlu2qPcvvhl7Q7F6jx4717c85PR/OWMqXz+Ay9l6d2j+MLZBwDwnS9N4OknB/G+8x9vc5Tl0siFFRvFdy1b5E8Oe5zHnhjDuo0j2x2K1XnPuWt4z7nJ8vBLfz6Sa78+nnO+8hg3X7EPi+4czReuWU5H2/93XzIt7jbm4UTWYJ9+z+0cOWU1Y0Y+z7Wfu5Jv3XgUP/75YZw4/RF+usjdyqr4909OYsL+2/joG6YAMPPkjbzzY79rc1TlMWBm9ku6Cjie5C0qq4DzIuLSZl2vLC64bFav+z//7eNbG4gVdsSxmzni2M0A3LxyaZujKbmBksgi4vRm1W1m7TVgWmRm1k8FUCtXJnMiM7PC3CIzs+rzXUszqzq3yMys2ryMj5lVnQCVbLDfc5bNrDBF5CqZ9UiXSVor6f66fedLelzSkrScnFWPE5mZFRMFSrbL6X25r4siYlpabsqqxF1LMyuocc9aRsRdkg7c03rcIjOzwgqsfjGuZ5mutMzNeYmzJS1Lu55js052IjOz4vIvrLi+Z5mutMzLUfvXgIOBacAa4EtZX3DX0syKiebetYyI3y8zIumbwI1Z33GLzMyKa9xg/4tImli3eSpw/67O7eEWmZkVlmdqRa56elnuCzhe0jSSVPgo8L6sepzIzKy4xt217G25r8LrFjqRmVkxAQyUF/SaWf8k8s3abyUnMjMrrrtcTTInMjMrxl1LM+sP3LU0s+pzIjOzavMLes2s6vwWJTPrDzxGZmbV50RmZpUWQLcTmZlVmgf7zaw/cCIzs0oLoFauqf1OZGZWUEA4kZlZ1blraWaV5ruWZtYvuEVmZpXnRGZmlRYBtVq7o9iBE5mZFecWmZlVnhOZmVVb+K6lmVVcQJRsQmxHuwMwswqqdecrGSRdJmmtpPvr9u0j6TZJD6d/js2qx4nMzIqJSF4Hl6dkuxyYvdO+TwILImIysCDd7pMTmZkVF5GvZFYTdwEbdtr9JmB++nk+cEpWPR4jM7PCIv8LesdJWlS3PS8i5mV8Z0JErEk/PwFMyLqIE5mZFVRoYcX1ETF9t68UEZIyL+aupZkV0/PQeJ6ye34naSJA+ufarC84kZlZIQFErZar7KYfAmemn88Ebsj6gruWZlZMNG5hRUlXAceTjKWtAs4DLgSukTQH+C3w1qx6nMjMrLBo0Mz+iDh9F4dOLFKPE5mZFVeymf2KEj38KWkdSVOyvxkHrG93EFZIf/2dvTQixu9JBZJuIfn7yWN9ROw84bXhSpXI+itJi/bkFrS1nn9n1eK7lmZWeU5kZlZ5TmStkfVIhpWPf2cV4jEyM6s8t8jMrPKcyMys8pzImkjSbEn/K2m5pMzF4az9elux1MrPiaxJJHUCFwMnAVOB0yVNbW9UlsPlvHjFUis5J7LmmQEsj4gVEbENuJpk5UsrsV2sWGol50TWPPsBK+u2V6X7zKzBnMjMrPKcyJrncWBS3fb+6T4zazAnsua5F5gs6SBJQ4C3k6x8aWYN5kTWJBHRBZwN3Ao8BFwTEQ+0NyrLkq5Y+gvgUEmr0lVKreT8iJKZVZ5bZGZWeU5kZlZ5TmRmVnlOZGZWeU5kZlZ5TmQVIqkmaYmk+yV9T9KIPajrckmnpZ8v6euBdknHSzp2N67xqKQXvW1nV/t3OmdzwWudL+kTRWO0/sGJrFqei4hpEXE4sA14f/1BSbv1ntKIeG9EPNjHKccDhROZWas4kVXXz4BD0tbSzyT9EHhQUqekf5F0r6Rlkt4HoMRX0vXRfgq8pKciSXdKmp5+ni3pPklLJS2QdCBJwvybtDX4p5LGS7ouvca9kmam391X0k8kPSDpEkBZP4SkH0hanH5n7k7HLkr3L5A0Pt13sKRb0u/8TNJhDfnbtErzm8YrKG15nQTcku46Cjg8In6TJoOnI+KVkoYC/y3pJ8CRwKEka6NNAB4ELtup3vHAN4Hj0rr2iYgNkr4ObI6IL6bnXQlcFBF3SzqA5OmFPwLOA+6OiAskvQ7IMyv+Pek1hgP3SrouIp4E9gIWRcTfSPp0WvfZJC8FeX9EPCzpaOCrwKzd+Gu0fsSJrFqGS1qSfv4ZcClJl++XEfGbdP+fA3/cM/4FjAEmA8cBV0VEDVgt6fZe6j8GuKunrojY1bpcrwamSr9vcI2WNDK9xpvT7/5Y0lM5fqYPSzo1/TwpjfVJoBv4brr/O8D302scC3yv7tpDc1zD+jknsmp5LiKm1e9I/4N+tn4X8KGIuHWn805uYBwdwDER8XwvseQm6XiSpPiqiNgi6U5g2C5Oj/S6G3f+OzDzGFn/cyvwAUmDASRNkbQXcBfwtnQMbSJwQi/fvQc4TtJB6Xf3SfdvAkbVnfcT4EM9G5KmpR/vAt6R7jsJGJsR6xjgqTSJHUbSIuzRAfS0Kt9B0mV9BviNpL9IryFJR2RcwwYAJ7L+5xKS8a/70hdofIOk5X098HB67NskKzzsICLWAXNJunFLeaFr9yPg1J7BfuDDwPT0ZsKDvHD39DMkifABki7mYxmx3gIMkvQQcCFJIu3xLDAj/RlmARek+88A5qTxPYCXDze8+oWZ9QNukZlZ5TmRmVnlOZGZWeU5kZlZ5TmRmVnlOZGZWeU5kZlZ5f0/N4nMGfJMiIsAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "y_predicted = model_multi_modal.predict([x_test_text, x_test_audio])\n",
        "y_predicted = np.round(y_predicted).flatten()\n",
        "\n",
        "cm = confusion_matrix(y_test, y_predicted)\n",
        "disp = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
        "disp.plot()\n",
        "plt.show()"
      ],
      "id": "aw1V23xaN6_i"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Xh4hui1aOCop"
      },
      "outputs": [],
      "source": [
        "print(classification_report(y_test, y_predicted))"
      ],
      "id": "Xh4hui1aOCop"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Itpxrs4QOWop"
      },
      "outputs": [],
      "source": [
        "y_train"
      ],
      "id": "Itpxrs4QOWop"
    },
    {
      "cell_type": "code",
      "source": [
        "y_test"
      ],
      "metadata": {
        "id": "eLVx-TIji615"
      },
      "id": "eLVx-TIji615",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "_VeM4koIi77u"
      },
      "id": "_VeM4koIi77u",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "machine_shape": "hm"
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.16"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}