LLM_Chatbot_Featureform_workshop.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4d71663b",
   "metadata": {},
   "source": [
    "# Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "61dcf437",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: python-dotenv in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (1.0.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "!pip install python-dotenv\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()  # take environment variables from .env.\n",
    "weaviate_cluster = os.getenv(\"WEAVIATE_URL\")\n",
    "weaviate_apikey = os.getenv(\"WEAVIATE_KEY\")\n",
    "openai_org = os.getenv(\"OPENAI_ORG\")\n",
    "openai_key = os.getenv(\"OPENAI_KEY\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd4e323e",
   "metadata": {},
   "source": [
    "# Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "85b6c488-ca42-4e10-b5a4-9066c595cfe7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting featureform\n",
      "  Using cached featureform-1.10.5-py3-none-any.whl (7.9 MB)\n",
      "Requirement already satisfied: sentence-transformers in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (2.2.2)\n",
      "Requirement already satisfied: weaviate-client in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (3.24.2)\n",
      "Requirement already satisfied: pandas==1.3.5 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (1.3.5)\n",
      "Requirement already satisfied: sqlalchemy<2.0.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (1.4.49)\n",
      "Requirement already satisfied: numpy>=1.21.6 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (1.26.1)\n",
      "Requirement already satisfied: typing-extensions>=4.1.1 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (4.8.0)\n",
      "Requirement already satisfied: dataclasses==0.6 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (0.6)\n",
      "Requirement already satisfied: Flask-Cors==3.0.10 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (3.0.10)\n",
      "Requirement already satisfied: requests in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (2.31.0)\n",
      "Requirement already satisfied: pinecone-client in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (2.2.4)\n",
      "Requirement already satisfied: rich in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (13.6.0)\n",
      "Requirement already satisfied: pandasql==0.7.3 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (0.7.3)\n",
      "Collecting flask==2.2.1\n",
      "  Using cached Flask-2.2.1-py3-none-any.whl (101 kB)\n",
      "Requirement already satisfied: dill==0.3.7 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (0.3.7)\n",
      "Requirement already satisfied: typeguard<3.0.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (2.13.3)\n",
      "Requirement already satisfied: click>=7.1.2 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (8.1.7)\n",
      "Requirement already satisfied: validators>=0.20.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (0.22.0)\n",
      "Requirement already satisfied: protobuf>=3.20.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (4.24.4)\n",
      "Requirement already satisfied: grpcio>=1.47.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (1.54.2)\n",
      "Requirement already satisfied: fastparquet in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (0.8.3)\n",
      "Requirement already satisfied: pyarrow in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from featureform) (13.0.0)\n",
      "Requirement already satisfied: Jinja2>=3.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from flask==2.2.1->featureform) (3.1.2)\n",
      "Requirement already satisfied: itsdangerous>=2.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from flask==2.2.1->featureform) (2.1.2)\n",
      "Requirement already satisfied: Werkzeug>=2.2.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from flask==2.2.1->featureform) (3.0.0)\n",
      "Requirement already satisfied: Six in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from Flask-Cors==3.0.10->featureform) (1.16.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from pandas==1.3.5->featureform) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from pandas==1.3.5->featureform) (2023.3.post1)\n",
      "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from sentence-transformers) (4.34.0)\n",
      "Requirement already satisfied: tqdm in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from sentence-transformers) (4.66.1)\n",
      "Requirement already satisfied: torch>=1.6.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from sentence-transformers) (2.1.0)\n",
      "Requirement already satisfied: torchvision in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from sentence-transformers) (0.16.0)\n",
      "Requirement already satisfied: scikit-learn in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from sentence-transformers) (1.3.1)\n",
      "Requirement already satisfied: scipy in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from sentence-transformers) (1.11.3)\n",
      "Requirement already satisfied: nltk in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from sentence-transformers) (3.8.1)\n",
      "Requirement already satisfied: sentencepiece in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from sentence-transformers) (0.1.99)\n",
      "Requirement already satisfied: huggingface-hub>=0.4.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from sentence-transformers) (0.17.3)\n",
      "Requirement already satisfied: authlib<2.0.0,>=1.2.1 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from weaviate-client) (1.2.1)\n",
      "Requirement already satisfied: cryptography>=3.2 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from authlib<2.0.0,>=1.2.1->weaviate-client) (41.0.4)\n",
      "Requirement already satisfied: packaging>=20.9 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (6.0.1)\n",
      "Requirement already satisfied: fsspec in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (2023.9.2)\n",
      "Requirement already satisfied: filelock in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (3.12.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from requests->featureform) (2.0.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from requests->featureform) (2023.7.22)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from requests->featureform) (3.4)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from requests->featureform) (3.3.0)\n",
      "Requirement already satisfied: sympy in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from torch>=1.6.0->sentence-transformers) (1.12)\n",
      "Requirement already satisfied: networkx in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from torch>=1.6.0->sentence-transformers) (3.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (2023.10.3)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (0.4.0)\n",
      "Requirement already satisfied: tokenizers<0.15,>=0.14 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (0.14.1)\n",
      "Requirement already satisfied: cramjam>=2.3.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from fastparquet->featureform) (2.7.0)\n",
      "Requirement already satisfied: joblib in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from nltk->sentence-transformers) (1.3.2)\n",
      "Requirement already satisfied: dnspython>=2.0.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from pinecone-client->featureform) (2.4.2)\n",
      "Requirement already satisfied: loguru>=0.5.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from pinecone-client->featureform) (0.7.2)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from rich->featureform) (2.16.1)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from rich->featureform) (3.0.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from scikit-learn->sentence-transformers) (3.2.0)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from torchvision->sentence-transformers) (10.1.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: cffi>=1.12 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from cryptography>=3.2->authlib<2.0.0,>=1.2.1->weaviate-client) (1.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from Jinja2>=3.0->flask==2.2.1->featureform) (2.1.3)\n",
      "Requirement already satisfied: mdurl~=0.1 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich->featureform) (0.1.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from sympy->torch>=1.6.0->sentence-transformers) (1.3.0)\n",
      "Requirement already satisfied: pycparser in /Users/simba/.pyenv/versions/3.10.12/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=3.2->authlib<2.0.0,>=1.2.1->weaviate-client) (2.21)\n",
      "Installing collected packages: flask, featureform\n",
      "  Attempting uninstall: flask\n",
      "    Found existing installation: Flask 3.0.0\n",
      "    Uninstalling Flask-3.0.0:\n",
      "      Successfully uninstalled Flask-3.0.0\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "featureform-enterprise 0.10.12 requires flask>=3.0.0, but you have flask 2.2.1 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed featureform-1.10.5 flask-2.2.1\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install featureform sentence-transformers weaviate-client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e7c4fe52-0864-47cb-9998-2e5510ebc2f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n"
     ]
    }
   ],
   "source": [
    "import featureform as ff\n",
    "from featureform import local\n",
    "\n",
    "client = ff.Client(local=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1e1f2535-87c0-4f34-9a4c-3cdec39edccc",
   "metadata": {},
   "outputs": [],
   "source": [
    "ff.set_run(\"webinar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fd002883-a033-4995-b58e-99cc4601f68f",
   "metadata": {},
   "outputs": [],
   "source": [
    "episodes = local.register_directory(\n",
    "    name=\"mlops-episodes\",\n",
    "    path=\"data/files\",\n",
    "    description=\"Transcripts from recent MLOps episodes\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2ddd2953-d21a-41d2-b59b-8b68669f73fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying Run: webinar\n",
      "Creating provider local-mode \n",
      "Creating source mlops-episodes  webinar\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MLOps Weekly - 04-03-2023.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MLOps Weekly - Stefan.mp3 (2).csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MLOps Weekly - Atindriyo Sanyal (2).mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MLOps Weekly - 07-27-2022 V2.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MLOps Weekly - David Stein.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>MLOps Weekly - Piero.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>MLOps Weekly - Stefan.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>MLOps Weekly - Liran Hason.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>MLOps Weekly - 02-02-2022.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>MLOps Weekly - Doris.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>MLOps Weekly - 07-22-2022.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>MLOps Weekly - Bob.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>MLOps Weekly - Josh Wills.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>MLOps Weekly - Sam.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>MLOps Weekly - 02-13-2023.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>MLOps Weekly - Chris White.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>MLOps Weekly - Jacopo.mp3.csv</td>\n",
       "      <td>﻿Number;Speaker;Start time;End time;Duration;T...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       filename  \\\n",
       "0             MLOps Weekly - 04-03-2023.mp3.csv   \n",
       "1             MLOps Weekly - Stefan.mp3 (2).csv   \n",
       "2   MLOps Weekly - Atindriyo Sanyal (2).mp3.csv   \n",
       "3          MLOps Weekly - 07-27-2022 V2.mp3.csv   \n",
       "4            MLOps Weekly - David Stein.mp3.csv   \n",
       "5                  MLOps Weekly - Piero.mp3.csv   \n",
       "6                 MLOps Weekly - Stefan.mp3.csv   \n",
       "7            MLOps Weekly - Liran Hason.mp3.csv   \n",
       "8             MLOps Weekly - 02-02-2022.mp3.csv   \n",
       "9                  MLOps Weekly - Doris.mp3.csv   \n",
       "10            MLOps Weekly - 07-22-2022.mp3.csv   \n",
       "11                   MLOps Weekly - Bob.mp3.csv   \n",
       "12            MLOps Weekly - Josh Wills.mp3.csv   \n",
       "13                   MLOps Weekly - Sam.mp3.csv   \n",
       "14            MLOps Weekly - 02-13-2023.mp3.csv   \n",
       "15           MLOps Weekly - Chris White.mp3.csv   \n",
       "16                MLOps Weekly - Jacopo.mp3.csv   \n",
       "\n",
       "                                                 body  \n",
       "0   ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "1   ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "2   ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "3   ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "4   ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "5   ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "6   ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "7   ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "8   ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "9   ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "10  ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "11  ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "12  ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "13  ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "14  ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "15  ﻿Number;Speaker;Start time;End time;Duration;T...  \n",
       "16  ﻿Number;Speaker;Start time;End time;Duration;T...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.dataframe(episodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c94dbb2-5340-4511-8f34-b3c389ca2242",
   "metadata": {},
   "outputs": [],
   "source": [
    "!featureform dash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cfc9309e-2fe7-4630-8b08-eb0450f08b79",
   "metadata": {},
   "outputs": [],
   "source": [
    "@local.df_transformation(inputs=[episodes])\n",
    "def process_episode_files(dir_df):\n",
    "    from io import StringIO\n",
    "    import pandas as pd\n",
    "\n",
    "    episode_dfs = []\n",
    "    for i, row in dir_df.iterrows():\n",
    "        csv_str = StringIO(row[1])\n",
    "        r_df = pd.read_csv(csv_str, sep=\";\")\n",
    "        r_df[\"filename\"] = row[0]\n",
    "        episode_dfs.append(r_df)\n",
    "\n",
    "    return pd.concat(episode_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a41bf161-3784-41fe-945b-aaca1978053f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying Run: webinar\n",
      "Creating provider local-mode \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Number</th>\n",
       "      <th>Speaker</th>\n",
       "      <th>Start time</th>\n",
       "      <th>End time</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Text</th>\n",
       "      <th>filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Simba Khadder</td>\n",
       "      <td>00:00:06.170</td>\n",
       "      <td>00:00:22.730</td>\n",
       "      <td>00:00:16.560</td>\n",
       "      <td>Hey everyone. Simba Khadder here and you are l...</td>\n",
       "      <td>MLOps Weekly - 04-03-2023.mp3.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Mikiko Bazeley</td>\n",
       "      <td>00:00:22.930</td>\n",
       "      <td>00:00:51.720</td>\n",
       "      <td>00:00:28.790</td>\n",
       "      <td>Hey, everyone, my name is Mikiko Bazeley and I...</td>\n",
       "      <td>MLOps Weekly - 04-03-2023.mp3.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Mikiko Bazeley</td>\n",
       "      <td>00:00:51.780</td>\n",
       "      <td>00:01:16.460</td>\n",
       "      <td>00:00:24.680</td>\n",
       "      <td>I've also worked in a ton of different industr...</td>\n",
       "      <td>MLOps Weekly - 04-03-2023.mp3.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Simba Khadder</td>\n",
       "      <td>00:01:16.610</td>\n",
       "      <td>00:01:43.330</td>\n",
       "      <td>00:00:26.720</td>\n",
       "      <td>I know we've been talking about doing this for...</td>\n",
       "      <td>MLOps Weekly - 04-03-2023.mp3.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Mikiko Bazeley</td>\n",
       "      <td>00:01:43.430</td>\n",
       "      <td>00:01:58.780</td>\n",
       "      <td>00:00:15.350</td>\n",
       "      <td>Yeah, it's funny because when I've talked to o...</td>\n",
       "      <td>MLOps Weekly - 04-03-2023.mp3.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>78</td>\n",
       "      <td>Jacopo Tagliabue</td>\n",
       "      <td>00:28:29.840</td>\n",
       "      <td>00:28:54.900</td>\n",
       "      <td>00:00:25.060</td>\n",
       "      <td>My suggestion is always the same is, A, it's a...</td>\n",
       "      <td>MLOps Weekly - Jacopo.mp3.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>79</td>\n",
       "      <td>Jacopo Tagliabue</td>\n",
       "      <td>00:28:54.900</td>\n",
       "      <td>00:29:23.550</td>\n",
       "      <td>00:00:28.650</td>\n",
       "      <td>The second point that I would say the second t...</td>\n",
       "      <td>MLOps Weekly - Jacopo.mp3.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>80</td>\n",
       "      <td>Jacopo Tagliabue</td>\n",
       "      <td>00:29:24.630</td>\n",
       "      <td>00:30:00.400</td>\n",
       "      <td>00:00:35.770</td>\n",
       "      <td>Metaflow is always the same. I see a lot of pe...</td>\n",
       "      <td>MLOps Weekly - Jacopo.mp3.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>81</td>\n",
       "      <td>Simba Khadder</td>\n",
       "      <td>00:30:00.400</td>\n",
       "      <td>00:30:23.670</td>\n",
       "      <td>00:00:23.270</td>\n",
       "      <td>To expand on that, I bet you if you think, wou...</td>\n",
       "      <td>MLOps Weekly - Jacopo.mp3.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>82</td>\n",
       "      <td>Jacopo Tagliabue</td>\n",
       "      <td>00:30:23.670</td>\n",
       "      <td>00:30:27.950</td>\n",
       "      <td>00:00:04.280</td>\n",
       "      <td>Oh, thanks so much for having me and thanks fo...</td>\n",
       "      <td>MLOps Weekly - Jacopo.mp3.csv</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1482 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Number           Speaker    Start time      End time      Duration  \\\n",
       "0        0     Simba Khadder  00:00:06.170  00:00:22.730  00:00:16.560   \n",
       "1        1    Mikiko Bazeley  00:00:22.930  00:00:51.720  00:00:28.790   \n",
       "2        2    Mikiko Bazeley  00:00:51.780  00:01:16.460  00:00:24.680   \n",
       "3        3     Simba Khadder  00:01:16.610  00:01:43.330  00:00:26.720   \n",
       "4        4    Mikiko Bazeley  00:01:43.430  00:01:58.780  00:00:15.350   \n",
       "..     ...               ...           ...           ...           ...   \n",
       "78      78  Jacopo Tagliabue  00:28:29.840  00:28:54.900  00:00:25.060   \n",
       "79      79  Jacopo Tagliabue  00:28:54.900  00:29:23.550  00:00:28.650   \n",
       "80      80  Jacopo Tagliabue  00:29:24.630  00:30:00.400  00:00:35.770   \n",
       "81      81     Simba Khadder  00:30:00.400  00:30:23.670  00:00:23.270   \n",
       "82      82  Jacopo Tagliabue  00:30:23.670  00:30:27.950  00:00:04.280   \n",
       "\n",
       "                                                 Text  \\\n",
       "0   Hey everyone. Simba Khadder here and you are l...   \n",
       "1   Hey, everyone, my name is Mikiko Bazeley and I...   \n",
       "2   I've also worked in a ton of different industr...   \n",
       "3   I know we've been talking about doing this for...   \n",
       "4   Yeah, it's funny because when I've talked to o...   \n",
       "..                                                ...   \n",
       "78  My suggestion is always the same is, A, it's a...   \n",
       "79  The second point that I would say the second t...   \n",
       "80  Metaflow is always the same. I see a lot of pe...   \n",
       "81  To expand on that, I bet you if you think, wou...   \n",
       "82  Oh, thanks so much for having me and thanks fo...   \n",
       "\n",
       "                             filename  \n",
       "0   MLOps Weekly - 04-03-2023.mp3.csv  \n",
       "1   MLOps Weekly - 04-03-2023.mp3.csv  \n",
       "2   MLOps Weekly - 04-03-2023.mp3.csv  \n",
       "3   MLOps Weekly - 04-03-2023.mp3.csv  \n",
       "4   MLOps Weekly - 04-03-2023.mp3.csv  \n",
       "..                                ...  \n",
       "78      MLOps Weekly - Jacopo.mp3.csv  \n",
       "79      MLOps Weekly - Jacopo.mp3.csv  \n",
       "80      MLOps Weekly - Jacopo.mp3.csv  \n",
       "81      MLOps Weekly - Jacopo.mp3.csv  \n",
       "82      MLOps Weekly - Jacopo.mp3.csv  \n",
       "\n",
       "[1482 rows x 7 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.dataframe(process_episode_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f30003a8-3f57-4fda-b8d1-87a6edb6ca17",
   "metadata": {},
   "outputs": [],
   "source": [
    "@local.df_transformation(inputs=[process_episode_files])\n",
    "def speaker_primary_key(episodes_df):\n",
    "    episodes_df[\"PK\"] = episodes_df.apply(lambda row: f\"{row['Speaker']}_{row['Start time']}_{row['filename']}\", axis=1)\n",
    "    \n",
    "    return episodes_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "269a1212-a0d5-4d6d-8180-ef3c96a008a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying Run: webinar\n",
      "Creating provider local-mode \n",
      "Creating source speaker_primary_key  webinar\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Number</th>\n",
       "      <th>Speaker</th>\n",
       "      <th>Start time</th>\n",
       "      <th>End time</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Text</th>\n",
       "      <th>filename</th>\n",
       "      <th>PK</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Simba Khadder</td>\n",
       "      <td>00:00:06.170</td>\n",
       "      <td>00:00:22.730</td>\n",
       "      <td>00:00:16.560</td>\n",
       "      <td>Hey everyone. Simba Khadder here and you are l...</td>\n",
       "      <td>MLOps Weekly - 04-03-2023.mp3.csv</td>\n",
       "      <td>Simba Khadder_00:00:06.170_MLOps Weekly - 04-0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Mikiko Bazeley</td>\n",
       "      <td>00:00:22.930</td>\n",
       "      <td>00:00:51.720</td>\n",
       "      <td>00:00:28.790</td>\n",
       "      <td>Hey, everyone, my name is Mikiko Bazeley and I...</td>\n",
       "      <td>MLOps Weekly - 04-03-2023.mp3.csv</td>\n",
       "      <td>Mikiko Bazeley_00:00:22.930_MLOps Weekly - 04-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Mikiko Bazeley</td>\n",
       "      <td>00:00:51.780</td>\n",
       "      <td>00:01:16.460</td>\n",
       "      <td>00:00:24.680</td>\n",
       "      <td>I've also worked in a ton of different industr...</td>\n",
       "      <td>MLOps Weekly - 04-03-2023.mp3.csv</td>\n",
       "      <td>Mikiko Bazeley_00:00:51.780_MLOps Weekly - 04-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Simba Khadder</td>\n",
       "      <td>00:01:16.610</td>\n",
       "      <td>00:01:43.330</td>\n",
       "      <td>00:00:26.720</td>\n",
       "      <td>I know we've been talking about doing this for...</td>\n",
       "      <td>MLOps Weekly - 04-03-2023.mp3.csv</td>\n",
       "      <td>Simba Khadder_00:01:16.610_MLOps Weekly - 04-0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Mikiko Bazeley</td>\n",
       "      <td>00:01:43.430</td>\n",
       "      <td>00:01:58.780</td>\n",
       "      <td>00:00:15.350</td>\n",
       "      <td>Yeah, it's funny because when I've talked to o...</td>\n",
       "      <td>MLOps Weekly - 04-03-2023.mp3.csv</td>\n",
       "      <td>Mikiko Bazeley_00:01:43.430_MLOps Weekly - 04-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>78</td>\n",
       "      <td>Jacopo Tagliabue</td>\n",
       "      <td>00:28:29.840</td>\n",
       "      <td>00:28:54.900</td>\n",
       "      <td>00:00:25.060</td>\n",
       "      <td>My suggestion is always the same is, A, it's a...</td>\n",
       "      <td>MLOps Weekly - Jacopo.mp3.csv</td>\n",
       "      <td>Jacopo Tagliabue_00:28:29.840_MLOps Weekly - J...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>79</td>\n",
       "      <td>Jacopo Tagliabue</td>\n",
       "      <td>00:28:54.900</td>\n",
       "      <td>00:29:23.550</td>\n",
       "      <td>00:00:28.650</td>\n",
       "      <td>The second point that I would say the second t...</td>\n",
       "      <td>MLOps Weekly - Jacopo.mp3.csv</td>\n",
       "      <td>Jacopo Tagliabue_00:28:54.900_MLOps Weekly - J...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>80</td>\n",
       "      <td>Jacopo Tagliabue</td>\n",
       "      <td>00:29:24.630</td>\n",
       "      <td>00:30:00.400</td>\n",
       "      <td>00:00:35.770</td>\n",
       "      <td>Metaflow is always the same. I see a lot of pe...</td>\n",
       "      <td>MLOps Weekly - Jacopo.mp3.csv</td>\n",
       "      <td>Jacopo Tagliabue_00:29:24.630_MLOps Weekly - J...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>81</td>\n",
       "      <td>Simba Khadder</td>\n",
       "      <td>00:30:00.400</td>\n",
       "      <td>00:30:23.670</td>\n",
       "      <td>00:00:23.270</td>\n",
       "      <td>To expand on that, I bet you if you think, wou...</td>\n",
       "      <td>MLOps Weekly - Jacopo.mp3.csv</td>\n",
       "      <td>Simba Khadder_00:30:00.400_MLOps Weekly - Jaco...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>82</td>\n",
       "      <td>Jacopo Tagliabue</td>\n",
       "      <td>00:30:23.670</td>\n",
       "      <td>00:30:27.950</td>\n",
       "      <td>00:00:04.280</td>\n",
       "      <td>Oh, thanks so much for having me and thanks fo...</td>\n",
       "      <td>MLOps Weekly - Jacopo.mp3.csv</td>\n",
       "      <td>Jacopo Tagliabue_00:30:23.670_MLOps Weekly - J...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1482 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Number           Speaker    Start time      End time      Duration  \\\n",
       "0        0     Simba Khadder  00:00:06.170  00:00:22.730  00:00:16.560   \n",
       "1        1    Mikiko Bazeley  00:00:22.930  00:00:51.720  00:00:28.790   \n",
       "2        2    Mikiko Bazeley  00:00:51.780  00:01:16.460  00:00:24.680   \n",
       "3        3     Simba Khadder  00:01:16.610  00:01:43.330  00:00:26.720   \n",
       "4        4    Mikiko Bazeley  00:01:43.430  00:01:58.780  00:00:15.350   \n",
       "..     ...               ...           ...           ...           ...   \n",
       "78      78  Jacopo Tagliabue  00:28:29.840  00:28:54.900  00:00:25.060   \n",
       "79      79  Jacopo Tagliabue  00:28:54.900  00:29:23.550  00:00:28.650   \n",
       "80      80  Jacopo Tagliabue  00:29:24.630  00:30:00.400  00:00:35.770   \n",
       "81      81     Simba Khadder  00:30:00.400  00:30:23.670  00:00:23.270   \n",
       "82      82  Jacopo Tagliabue  00:30:23.670  00:30:27.950  00:00:04.280   \n",
       "\n",
       "                                                 Text  \\\n",
       "0   Hey everyone. Simba Khadder here and you are l...   \n",
       "1   Hey, everyone, my name is Mikiko Bazeley and I...   \n",
       "2   I've also worked in a ton of different industr...   \n",
       "3   I know we've been talking about doing this for...   \n",
       "4   Yeah, it's funny because when I've talked to o...   \n",
       "..                                                ...   \n",
       "78  My suggestion is always the same is, A, it's a...   \n",
       "79  The second point that I would say the second t...   \n",
       "80  Metaflow is always the same. I see a lot of pe...   \n",
       "81  To expand on that, I bet you if you think, wou...   \n",
       "82  Oh, thanks so much for having me and thanks fo...   \n",
       "\n",
       "                             filename  \\\n",
       "0   MLOps Weekly - 04-03-2023.mp3.csv   \n",
       "1   MLOps Weekly - 04-03-2023.mp3.csv   \n",
       "2   MLOps Weekly - 04-03-2023.mp3.csv   \n",
       "3   MLOps Weekly - 04-03-2023.mp3.csv   \n",
       "4   MLOps Weekly - 04-03-2023.mp3.csv   \n",
       "..                                ...   \n",
       "78      MLOps Weekly - Jacopo.mp3.csv   \n",
       "79      MLOps Weekly - Jacopo.mp3.csv   \n",
       "80      MLOps Weekly - Jacopo.mp3.csv   \n",
       "81      MLOps Weekly - Jacopo.mp3.csv   \n",
       "82      MLOps Weekly - Jacopo.mp3.csv   \n",
       "\n",
       "                                                   PK  \n",
       "0   Simba Khadder_00:00:06.170_MLOps Weekly - 04-0...  \n",
       "1   Mikiko Bazeley_00:00:22.930_MLOps Weekly - 04-...  \n",
       "2   Mikiko Bazeley_00:00:51.780_MLOps Weekly - 04-...  \n",
       "3   Simba Khadder_00:01:16.610_MLOps Weekly - 04-0...  \n",
       "4   Mikiko Bazeley_00:01:43.430_MLOps Weekly - 04-...  \n",
       "..                                                ...  \n",
       "78  Jacopo Tagliabue_00:28:29.840_MLOps Weekly - J...  \n",
       "79  Jacopo Tagliabue_00:28:54.900_MLOps Weekly - J...  \n",
       "80  Jacopo Tagliabue_00:29:24.630_MLOps Weekly - J...  \n",
       "81  Simba Khadder_00:30:00.400_MLOps Weekly - Jaco...  \n",
       "82  Jacopo Tagliabue_00:30:23.670_MLOps Weekly - J...  \n",
       "\n",
       "[1482 rows x 8 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.dataframe(speaker_primary_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e5322225-9886-4f58-a0c4-cb4f5c71180e",
   "metadata": {},
   "outputs": [],
   "source": [
    "@local.df_transformation(inputs=[speaker_primary_key])\n",
    "def vectorize_comments(episodes_df):\n",
    "    from sentence_transformers import SentenceTransformer\n",
    "\n",
    "    model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",
    "    embeddings = model.encode(episodes_df[\"Text\"].tolist())\n",
    "    episodes_df[\"Vector\"] = embeddings.tolist()\n",
    "    \n",
    "    return episodes_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "376352a3-9927-4b9b-a56e-85a782fc8ddc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying Run: webinar\n",
      "Creating provider local-mode \n",
      "Creating provider weaviate-chatbot \n",
      "Creating source vectorize_comments  webinar\n"
     ]
    }
   ],
   "source": [
    "weaviate = ff.register_weaviate(\n",
    "    name=\"weaviate-chatbot\",\n",
    "    url=weaviate_cluster,\n",
    "    api_key=weaviate_apikey,\n",
    "    description=\"A Weaviate project for using embeddings in Featureform\"\n",
    ")\n",
    "\n",
    "client.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a09f2303-a0f7-417d-a491-ed39f5cb4633",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying Run: webinar\n",
      "Creating provider local-mode \n",
      "Creating entity speaker \n",
      "Creating feature comment_embeddings  webinar\n",
      "Creating feature comments  webinar\n",
      "Updating Feature Table: |--------------------------------------------------| 0% Complete"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updating Feature Table: |##################################################| 100% Complete\n",
      "\n",
      "Updating Feature Table: |##################################################| 100% Complete\n",
      "\n"
     ]
    }
   ],
   "source": [
    "@ff.entity\n",
    "class Speaker:\n",
    "    comment_embeddings = ff.Embedding(\n",
    "        vectorize_comments[[\"PK\", \"Vector\"]],\n",
    "        dims=384,\n",
    "        vector_db=weaviate,\n",
    "        description=\"Embeddings created from speakers' comments in episodes\"    )\n",
    "    comments = ff.Feature(\n",
    "        speaker_primary_key[[\"PK\", \"Text\"]],\n",
    "        type=ff.String,\n",
    "        description=\"Speakers' original comments\"\n",
    "    )\n",
    "\n",
    "client.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2ffc8b89-3728-480b-a170-2e0257156022",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying Run: webinar\n",
      "Creating provider local-mode \n",
      "Creating ondemand_feature relevent_comments  webinar\n"
     ]
    }
   ],
   "source": [
    "@ff.ondemand_feature()\n",
    "def relevent_comments(client, params, entity):\n",
    "    from sentence_transformers import SentenceTransformer\n",
    "\n",
    "    model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",
    "    search_vector = model.encode(params[\"query\"])\n",
    "    res = client.nearest(\"comment_embeddings\", \"webinar\", search_vector, k=3)\n",
    "    return res\n",
    "\n",
    "client.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d569ce3e-49f9-464c-8c5a-9ff25f7e111d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Sam Ramji_00:06:20.590_MLOps Weekly - Sam.mp3.csv',\n",
       "        'Simba Khadder_00:13:31.630_MLOps Weekly - Liran Hason.mp3.csv',\n",
       "        'Simba Khadder_00:07:02.820_MLOps Weekly - Liran Hason.mp3.csv']],\n",
       "      dtype='<U61')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.features([(\"relevent_comments\", \"webinar\")], {}, params={\"query\": \"enterprise MLOps\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "52eb94de-5095-44f9-a03b-9a5104eb18d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "@ff.ondemand_feature(variant=\"final\")\n",
    "def contextualized_prompt(client, params, entity):\n",
    "    pks = client.features([(\"relevent_comments\", \"webinar\")], {}, params=params)\n",
    "    prompt = \"Use the following snippets from our podcast to answer the following question\\n\"\n",
    "    for pk in pks[0]:\n",
    "        prompt += \"```\"\n",
    "        prompt += client.features([(\"comments\", \"webinar\")], {\"speaker\": pk})[0]\n",
    "        prompt += \"```\\n\"\n",
    "    prompt += \"Question: \"\n",
    "    prompt += params[\"query\"]\n",
    "    prompt += \"?\"\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c0c36075-a755-40b3-b916-50789168c062",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying Run: tender_davinci\n",
      "Creating provider local-mode \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([\"Use the following snippets from our podcast to answer the following question\\n```The second thing is that MLOps is incredibly valuable because it does create an environment where you can predictably make better decision engines faster. Every business is basically a decision factory.```\\n```I want to zoom out a little bit. You mentioned a bit about the creation of MLOps as a category. I have a similar story where I built a whole MLOps platform for my last company. It wasn't called MLOps. I don't even know what we called it.```\\n```I think a lot of what I've seen, especially in ML, is everyone has such different pain points. If you're doing your vision and you're a large enterprise or you're fraud detection at a startup, problems you face are different. The MLOps vendors you need, or let's say, Observability versus Feature Store, based on what you're doing, you might eventually need both.```\\nQuestion: enterprise MLOps?\"],\n",
       "      dtype='<U927')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.apply()\n",
    "client.features([(\"contextualized_prompt\", \"final\")], {}, params={\"query\": \"enterprise MLOps\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cb3e0871-1227-4211-bd0f-117ff5566129",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying Run: tender_davinci\n",
      "Creating provider local-mode \n",
      "\n",
      "\n",
      "Answer: MLOps testing involves making sure that data inputs are consistent throughout the model's lifecycle. This includes tracking data during experimentation, ingestion, training, and deployment. It also involves making sure that the inputs remain consistent over time.\n"
     ]
    }
   ],
   "source": [
    "client.apply()\n",
    "q = \"What should I know about MLOps testing\"\n",
    "prompt = client.features([(\"contextualized_prompt\", \"final\")], {}, params={\"query\": q})[0]\n",
    "import openai\n",
    "openai.organization = None\n",
    "openai.api_key = openai_key\n",
    "print(openai.Completion.create(\n",
    "    model=\"text-davinci-003\",\n",
    "    prompt=prompt,\n",
    "    max_tokens=1000, # The max number of tokens to generate\n",
    "    temperature=1.0 # A measure of randomness\n",
    ")[\"choices\"][0][\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2aa98d5b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}