MDB_emedded_datasets.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Load the dataset from Hugging Face (AIatMongoDB/)\n",
        "Link: https://huggingface.co/datasets/AIatMongoDB/..."
      ],
      "metadata": {
        "id": "TIsBaLX0TN3_"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yG2dbTbwq0-d"
      },
      "outputs": [],
      "source": [
        "!pip install pymongo\n",
        "!pip install datasets"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Update user and password in the URI"
      ],
      "metadata": {
        "id": "eOC5HDjoUX4B"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ['MONGODB_ATLAS_URI'] = 'mongodb+srv://userid:password@cluster0.ozciyn7.mongodb.net/?retryWrites=true&w=majority'"
      ],
      "metadata": {
        "id": "Jvq1brSA0p6_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pymongo import MongoClient\n",
        "import datasets\n",
        "from datasets import load_dataset\n",
        "from bson import json_util"
      ],
      "metadata": {
        "id": "bBMZJlWOrR-b"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "uri = os.environ.get('MONGODB_ATLAS_URI')\n",
        "client = MongoClient(uri)\n",
        "\n",
        "#db_name = 'whatscooking'\n",
        "#collection_name = 'restaurants'\n",
        "#hf_dataset = 'AIatMongoDB/whatscooking.restaurants'\n",
        "\n",
        "db_name = 'sample_mflix'\n",
        "collection_name = 'embedded_items'\n",
        "hf_dataset = 'AIatMongoDB/embedded_movies'"
      ],
      "metadata": {
        "id": "iEXD37PDr29k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embedded_items_collection = client[db_name][collection_name]"
      ],
      "metadata": {
        "id": "T9nGB8uc7VNF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Clear out the collection if you want to start from scratch"
      ],
      "metadata": {
        "id": "uYPz4W2DGaZy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "doc_count = embedded_items_collection.count_documents (filter = {})\n",
        "print (f\"Document count before delete : {doc_count:,}\")\n",
        "\n",
        "result = embedded_items_collection.delete_many(filter= {})\n",
        "print (f\"Deleted docs : {result.deleted_count}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rGOvb-QDCi2K",
        "outputId": "6aa3f069-bb77-4af0-9c51-1ec7b4e7cd37"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Document count before delete : 0\n",
            "Deleted docs : 0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = load_dataset(hf_dataset)"
      ],
      "metadata": {
        "id": "WLo8HbFU7awh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's tske a look at the huggingface dataset"
      ],
      "metadata": {
        "id": "abfK19HHFqBq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "insert_data = []"
      ],
      "metadata": {
        "id": "VEqpUoKs7k3J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Process 1k at a time"
      ],
      "metadata": {
        "id": "ckkJdvhxWfhq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for item in dataset['train']:\n",
        "    doc_items = json_util.loads(json_util.dumps(item))\n",
        "    insert_data.append(doc_items)\n",
        "    if len(insert_data) == 1000:\n",
        "        embedded_items_collection.insert_many(insert_data)\n",
        "        print(\"1000 records ingested\")\n",
        "        insert_data = []\n",
        "\n",
        "if len(insert_data) > 0:\n",
        "    embedded_items_collection.insert_many(insert_data)\n",
        "    insert_data = []\n",
        "\n",
        "print(\"Data Ingested\")"
      ],
      "metadata": {
        "id": "oEv-VViv7pQj"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}