Lesson_3_Recommender_Systems.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "V100",
      "authorship_tag": "ABX9TyOZJJF+X8MmfsvYYMpu0/ks"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Lesson 3 - Basic Recommendation Systems #"
      ],
      "metadata": {
        "id": "I-r1Pt90IwlP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -qU \"pinecone-client[grpc]\" openai tqdm langchain pandas\n"
      ],
      "metadata": {
        "id": "lF_IqYtfbGFG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter\n",
        "from google.colab import userdata\n",
        "from openai import OpenAI\n",
        "\n",
        "import pandas as pd\n",
        "import pinecone\n",
        "import time\n",
        "from tqdm import tqdm\n"
      ],
      "metadata": {
        "id": "kiVchTjjbD74"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Download a sample article dataset. ###\n",
        "* The [dataset](https://components.one/datasets/all-the-news-2-news-articles-dataset/) used throughout this example contains 2.7 million news articles and essays from 27 American publications.\n",
        "* The link to the data is [here](https://www.dropbox.com/s/cn2utnr5ipathhh/all-the-news-2-1.zip)"
      ],
      "metadata": {
        "id": "fxDB1yE6I0j8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://www.dropbox.com/s/cn2utnr5ipathhh/all-the-news-2-1.zip -q --show-progress\n"
      ],
      "metadata": {
        "id": "a2rme2ZtYXqm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Let's prepare the data ###\n",
        "* unzip\n",
        "* examine the shape/format of the data"
      ],
      "metadata": {
        "id": "YKNNyHrhJTIJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip all-the-news-2-1.zip"
      ],
      "metadata": {
        "id": "zQmm28vpZ61S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "* Let's look at the headers"
      ],
      "metadata": {
        "id": "WOJjbsDlulHo"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xpnLE0bFYSRK"
      },
      "outputs": [],
      "source": [
        "with open('all-the-news-2-1.csv', 'r') as f:\n",
        "  header = f.readline()\n",
        "  print(header)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* Actually let's look at the data itself use a Dataframe"
      ],
      "metadata": {
        "id": "7yac_HbAum0g"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('all-the-news-2-1.csv', nrows=99)\n",
        "df.head()"
      ],
      "metadata": {
        "id": "ex8G_xaPuhE3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Prepare Pinecone ###\n",
        "* Get our API keys\n",
        "* Prepare an index\n",
        "* Connect to Pinecone\n",
        "* Note to keep things clean across subsequent runs, let's delete and recreate the index"
      ],
      "metadata": {
        "id": "VovzaSQmJbND"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# get api key from app.pinecone.io\n",
        "PINECONE_API_KEY = userdata.get('PINECONE_API_KEY')\n",
        "OPENAI_API_KEY = userdata.get('OPENAI_API_KEY')\n",
        "\n",
        "openai_client = OpenAI(api_key=OPENAI_API_KEY)\n",
        "\n",
        "pinecone.init(\n",
        "    api_key=PINECONE_API_KEY\n",
        ")\n",
        "\n",
        "index_name = 'lesson3'\n",
        "\n",
        "pinecone.delete_index(name=index_name)\n",
        "pinecone.create_index(name=index_name, dimension=1536)\n",
        "time.sleep(1)\n",
        "\n",
        "index = pinecone.GRPCIndex(index_name)\n",
        "\n",
        "index"
      ],
      "metadata": {
        "id": "ABd5qX7Va7a5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_embeddings(articles, model=\"text-embedding-ada-002\"):\n",
        "   return openai_client.embeddings.create(input = articles, model=model)\n"
      ],
      "metadata": {
        "id": "zvc0m36y7VtC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Prepare and insert data ###\n",
        "* Read as a dataframe, CHUNK_SIZE rows at a time\n",
        "* Extract the article title, author and article itself\n",
        "* Build embeddings from the titles only\n",
        "* Insert into Pinecone"
      ],
      "metadata": {
        "id": "dUgGZ3vtKB-b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "CHUNK_SIZE=500\n",
        "TOTAL_ROWS=10000\n",
        "progress_bar = tqdm(total=TOTAL_ROWS)\n",
        "chunks = pd.read_csv('all-the-news-2-1.csv', chunksize=CHUNK_SIZE, nrows=TOTAL_ROWS)\n",
        "chunk_num = 0\n",
        "for chunk in chunks:\n",
        "    chunk = chunk.dropna()\n",
        "    articles = chunk['article'].tolist()\n",
        "    titles = chunk['title'].tolist()\n",
        "    embeddings = get_embeddings(titles)\n",
        "    prepped = [{'id':str(chunk_num*CHUNK_SIZE+i), 'values':embeddings.data[i].embedding,\n",
        "                'metadata':{'title':titles[i]},} for i in range(0,len(titles))]\n",
        "    chunk_num = chunk_num + 1\n",
        "    index.upsert(prepped)\n",
        "    progress_bar.update(len(chunk))\n",
        "\n",
        "print('DONE')\n"
      ],
      "metadata": {
        "id": "FJMx-wsK3My1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_embeddings(articles, model=\"text-embedding-ada-002\"):\n",
        "   return openai_client.embeddings.create(input = articles, model=model)\n"
      ],
      "metadata": {
        "id": "Jmw8HkrGpeAC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Fetch results from Pinecone ###\n",
        "* Get the embedding for *search_term*\n",
        "* Query pinecone, return result and format\n"
      ],
      "metadata": {
        "id": "eP6auiyHpHz0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_recommendations(pinecone_index, search_term, top_k=10):\n",
        "  embed = get_embeddings([search_term]).data[0].embedding\n",
        "  res = pinecone_index.query(vector=embed, top_k=top_k, include_metadata=True)\n",
        "  return res\n",
        "\n",
        "reco = get_recommendations(index, 'tennis')\n",
        "for r in reco.matches:\n",
        "  print(f'{r.score} : {r.metadata[\"title\"]}')\n"
      ],
      "metadata": {
        "id": "UgGb98Y7qsa6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Can we do better? ###\n",
        "* We were embedding article titles before\n",
        "* Let's try embedding the article itself"
      ],
      "metadata": {
        "id": "bFgycX7X4muK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "articles_index_name = 'lesson3articles'\n",
        "pinecone.delete_index(name=articles_index_name)\n",
        "pinecone.create_index(name=articles_index_name, dimension=1536)\n",
        "time.sleep(1)\n",
        "articles_index = pinecone.GRPCIndex(articles_index_name)\n",
        "articles_index"
      ],
      "metadata": {
        "id": "8MKX_tBOj9Y_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Chunk up the articles and generate embeddings ###\n",
        "* Use Langchain RecursiveCharacterTextSplitter to chunk\n",
        "* Read the file into chunks of 1,000 rows each\n",
        "* For each chunk, get the articles out\n",
        "* Then for each article, generate one or more embeddings per article (depending on length)"
      ],
      "metadata": {
        "id": "GTxjSyj049H8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from tqdm import trange\n",
        "CHUNK_SIZE=500\n",
        "chunks = pd.read_csv('all-the-news-2-1.csv', chunksize=CHUNK_SIZE, nrows=9999)\n",
        "pd.options.display.max_rows = 999\n",
        "embed_num = 0\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=20)\n",
        "for chunk in chunks:\n",
        "    chunk = chunk.dropna()\n",
        "    articles = chunk['article'].tolist()\n",
        "    titles = chunk['title'].tolist()\n",
        "    for art_idx in trange(0, len(articles)):\n",
        "      art = articles[art_idx]\n",
        "      texts = text_splitter.split_text(art)\n",
        "      embeddings = get_embeddings(texts)\n",
        "      prepped = []\n",
        "      for embedding in embeddings.data:\n",
        "        prepped.append({'id':str(embed_num), 'values':embedding.embedding, 'metadata':{'title':titles[art_idx]}})\n",
        "        embed_num += 1\n",
        "      articles_index.upsert(prepped)\n",
        "\n",
        "print('DONE')"
      ],
      "metadata": {
        "id": "K5GtrlQdkRqc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "reco = get_recommendations(articles_index, 'obama', top_k=100)\n",
        "seen = {}\n",
        "for r in reco.matches:\n",
        "  title = r.metadata['title']\n",
        "  if title not in seen:\n",
        "    print(f'{r.score} : {title}')\n",
        "    seen[title] = '.'"
      ],
      "metadata": {
        "id": "ojHajkrImcrh"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}