Run.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lib.robots import check_robots_txt\n",
    "from lib.shingling import *\n",
    "from lib.renderer import *\n",
    "from lib.crawler import crawl_url\n",
    "from lib.data import *\n",
    "from lib.normalize import normalize_corpus\n",
    "from engine import *\n",
    "\n",
    "import json\n",
    "\n",
    "from web.site_generator import build_site_data, build_sites\n",
    "from web.publish import Publish\n",
    "\n",
    "import config as cfg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Configuration File:\n",
      "\n",
      "bert_weight : 0.7\n",
      "browser_user_agent : Googlebot\n",
      "crawler_seed : https://data-science-blog.github.io/Customer-Data-Platform/\n",
      "embedding_size : 100\n",
      "i_type : bm25\n",
      "per_sec_crawl_rate : 1\n",
      "pr_weight : 0.7\n",
      "sg_gh_user : jroakes-locomotive\n",
      "sg_page_template : ---\n",
      "layout: post\n",
      "title:  {title}\n",
      "categories: [{topic}]\n",
      "---\n",
      "\n",
      "{content}\n",
      "\n",
      "\n",
      "sg_save_folder : files\n",
      "sg_sites : [{'topic': 'python software', 'org_name': 'python-software'}, {'topic': 'data science', 'org_name': 'data-science-blog'}, {'topic': 'search engine optimization', 'org_name': 'search-engine-optimization-blog'}]\n",
      "sim_weight : 0.7\n",
      "title_boost : 3\n",
      "transformer_model : distilbert-base-uncased\n"
     ]
    }
   ],
   "source": [
    "print('Configuration File:\\n')\n",
    "_ = [print(i,':', vars(cfg)[i]) for i in list(dir(cfg)) if '__' not in i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Crawling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Crawling: https://data-science-blog.github.io/Customer-Data-Platform/\n",
      "Crawling: https://data-science-blog.github.io/Consistency-Database-Systems/\n",
      "Crawling: https://data-science-blog.github.io/about\n",
      "Crawling: https://data-science-blog.github.io/Big-Data\n",
      "Crawling: https://data-science-blog.github.io/\n",
      "Crawling: https://python-software.github.io/Eric-Software\n",
      "Crawling: https://data-science-blog.github.io/Data\n",
      "Crawling: https://data-science-blog.github.io/categories\n",
      "Crawling: https://data-science-blog.github.io/Data/\n",
      "Crawling: https://data-science-blog.github.io/Committee-On-Data-For-Science-And-Technology/\n",
      "Crawling: https://python-software.github.io/Core-Python-Programming\n",
      "Crawling: https://python-software.github.io/Intel-Parallel-Studio\n",
      "Crawling: https://data-science-blog.github.io/Berkeley-Institute-For-Data-Science\n",
      "Crawling: https://python-software.github.io/History-Of-Python\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Audio-Search-Engine\n",
      "Crawling: https://data-science-blog.github.io/Berkeley-Institute-For-Data-Science/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Google-Custom-Search\n",
      "Crawling: https://python-software.github.io/Anaconda-Python-Distribution\n",
      "Crawling: https://python-software.github.io/Comparison-Of-Integrated-Development-Environments\n",
      "Crawling: https://data-science-blog.github.io/Black-Swan-Data/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Contextual-Advertising\n",
      "Crawling: https://data-science-blog.github.io/Committee-On-Data-For-Science-And-Technology\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Google-Search\n",
      "Crawling: https://data-science-blog.github.io/Customer-Data-Platform\n",
      "Crawling: https://data-science-blog.github.io/page2/\n",
      "Crawling: https://data-science-blog.github.io/Chief-Data-Officer/\n",
      "Crawling: https://data-science-blog.github.io/Coding-Bootcamp/\n",
      "Crawling: https://data-science-blog.github.io/Dataintensive-Computing/\n",
      "Crawling: https://data-science-blog.github.io/Big-Data/\n",
      "Crawling: https://python-software.github.io/History-Of-Python/\n",
      "Crawling: https://python-software.github.io/about\n",
      "Crawling: https://python-software.github.io/\n",
      "Crawling: https://python-software.github.io/categories\n",
      "Crawling: https://python-software.github.io/Core-Python-Programming/\n",
      "Crawling: https://python-software.github.io/Eric-Software/\n",
      "Crawling: https://python-software.github.io/Comparison-Of-Integrated-Development-Environments/\n",
      "Crawling: https://python-software.github.io/Idle/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Clean-Url\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Archie-Search-Engine/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Barry-Schwartz-Technologist/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/about\n",
      "Crawling: https://search-engine-optimization-blog.github.io/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/categories\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Google-Search/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Fulltext-Search/\n",
      "Crawling: https://python-software.github.io/Activestate/\n",
      "Crawling: https://python-software.github.io/Biopython/\n",
      "Crawling: https://python-software.github.io/Circuitpython/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Clean-Url/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Danny-Sullivan-Technologist/\n",
      "Crawling: https://data-science-blog.github.io/Black-Swan-Data\n",
      "Crawling: https://data-science-blog.github.io/Consistency-Database-Systems\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Google-Custom-Search/\n",
      "Crawling: https://python-software.github.io/page2/\n",
      "Crawling: https://python-software.github.io/Intel-Parallel-Studio/\n",
      "Crawling: https://python-software.github.io/Anaconda-Python-Distribution/\n",
      "Crawling: https://python-software.github.io/Ironpython/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Contextual-Advertising/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/Audio-Search-Engine/\n",
      "Crawling: https://search-engine-optimization-blog.github.io/page2/\n",
      "Crawling: https://data-science-blog.github.io/Chief-Data-Officer\n"
     ]
    }
   ],
   "source": [
    "crawler = Crawler()\n",
    "crawler.crawl('https://data-science-blog.github.io/Customer-Data-Platform/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rendering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I:pyppeteer.launcher] Browser listening on: ws://127.0.0.1:54834/devtools/browser/96ca47b5-f2df-4962-9944-b9e520c41ad4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rendering: https://data-science-blog.github.io/Customer-Data-Platform/\n",
      "Rendering: https://data-science-blog.github.io/Consistency-Database-Systems/\n",
      "Rendering: https://data-science-blog.github.io/about\n",
      "Rendering: https://data-science-blog.github.io/Big-Data\n",
      "Rendering: https://data-science-blog.github.io/\n",
      "Rendering: https://python-software.github.io/Eric-Software\n",
      "Rendering: https://data-science-blog.github.io/Data\n",
      "Rendering: https://data-science-blog.github.io/categories\n",
      "Rendering: https://data-science-blog.github.io/Data/\n",
      "Rendering: https://data-science-blog.github.io/Committee-On-Data-For-Science-And-Technology/\n",
      "Rendering: https://python-software.github.io/Core-Python-Programming\n",
      "Rendering: https://python-software.github.io/Intel-Parallel-Studio\n",
      "Rendering: https://data-science-blog.github.io/Berkeley-Institute-For-Data-Science\n",
      "Rendering: https://python-software.github.io/History-Of-Python\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Audio-Search-Engine\n",
      "Rendering: https://data-science-blog.github.io/Berkeley-Institute-For-Data-Science/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Google-Custom-Search\n",
      "Rendering: https://python-software.github.io/Anaconda-Python-Distribution\n",
      "Rendering: https://python-software.github.io/Comparison-Of-Integrated-Development-Environments\n",
      "Rendering: https://data-science-blog.github.io/Black-Swan-Data/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Contextual-Advertising\n",
      "Rendering: https://data-science-blog.github.io/Committee-On-Data-For-Science-And-Technology\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Google-Search\n",
      "Rendering: https://data-science-blog.github.io/Customer-Data-Platform\n",
      "Rendering: https://data-science-blog.github.io/page2/\n",
      "Rendering: https://data-science-blog.github.io/Chief-Data-Officer/\n",
      "Rendering: https://data-science-blog.github.io/Coding-Bootcamp/\n",
      "Rendering: https://data-science-blog.github.io/Dataintensive-Computing/\n",
      "Rendering: https://data-science-blog.github.io/Big-Data/\n",
      "Rendering: https://python-software.github.io/History-Of-Python/\n",
      "Rendering: https://python-software.github.io/about\n",
      "Rendering: https://python-software.github.io/\n",
      "Rendering: https://python-software.github.io/categories\n",
      "Rendering: https://python-software.github.io/Core-Python-Programming/\n",
      "Rendering: https://python-software.github.io/Eric-Software/\n",
      "Rendering: https://python-software.github.io/Comparison-Of-Integrated-Development-Environments/\n",
      "Rendering: https://python-software.github.io/Idle/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Clean-Url\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Archie-Search-Engine/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Barry-Schwartz-Technologist/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/about\n",
      "Rendering: https://search-engine-optimization-blog.github.io/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/categories\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Google-Search/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Fulltext-Search/\n",
      "Rendering: https://python-software.github.io/Activestate/\n",
      "Rendering: https://python-software.github.io/Biopython/\n",
      "Rendering: https://python-software.github.io/Circuitpython/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Clean-Url/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Danny-Sullivan-Technologist/\n",
      "Rendering: https://data-science-blog.github.io/Black-Swan-Data\n",
      "Rendering: https://data-science-blog.github.io/Consistency-Database-Systems\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Google-Custom-Search/\n",
      "Rendering: https://python-software.github.io/page2/\n",
      "Rendering: https://python-software.github.io/Intel-Parallel-Studio/\n",
      "Rendering: https://python-software.github.io/Anaconda-Python-Distribution/\n",
      "Rendering: https://python-software.github.io/Ironpython/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Contextual-Advertising/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/Audio-Search-Engine/\n",
      "Rendering: https://search-engine-optimization-blog.github.io/page2/\n",
      "Rendering: https://data-science-blog.github.io/Chief-Data-Officer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I:pyppeteer.connection] connection closed\n"
     ]
    }
   ],
   "source": [
    "crawler.render()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json from cache at C:\\Users\\jroak\\.cache\\torch\\transformers\\a41e817d5c0743e29e86ff85edc8c257e61bc8d88e4271bb1b243b6e7614c633.1ccd1a11c9ff276830e114ea477ea2407100f4a3be7bdc45d37be9e37fa71c7e\n",
      "Model config {\n",
      "  \"activation\": \"gelu\",\n",
      "  \"attention_dropout\": 0.1,\n",
      "  \"dim\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_dim\": 3072,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"n_heads\": 12,\n",
      "  \"n_layers\": 6,\n",
      "  \"num_labels\": 2,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pruned_heads\": {},\n",
      "  \"qa_dropout\": 0.1,\n",
      "  \"seq_classif_dropout\": 0.2,\n",
      "  \"sinusoidal_pos_embds\": false,\n",
      "  \"tie_weights_\": true,\n",
      "  \"torchscript\": false,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-pytorch_model.bin from cache at C:\\Users\\jroak\\.cache\\torch\\transformers\\7b8a8f0b21c4e7f6962451c9370a5d9af90372a5f64637a251f2de154d0fc72c.c2015533705b9dff680ae707e205a35e2860e8d148b45d35085419d74fe57ac5\n",
      "loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at C:\\Users\\jroak\\.cache\\torch\\transformers\\26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "Building BERT Embeddings: 100%|██████████| 20207/20207 [15:01<00:00, 22.42it/s]\n"
     ]
    }
   ],
   "source": [
    "indexer = Indexer(crawler)\n",
    "indexer.build_index()\n",
    "indexer.build_bert_embeddings()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Searching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>67fb3a56f4a5b4ada650888415692c1c</th>\n",
       "      <td>Demo Site | Demo site generated from Wikipedia...</td>\n",
       "      <td>https://python-software.github.io/</td>\n",
       "      <td>Demo site generated from Wikipedia topics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ccb6272667b7c94201d9f774c6ed2d47</th>\n",
       "      <td>Core Python Programming Python Software | Demo...</td>\n",
       "      <td>https://python-software.github.io/Core-Python-...</td>\n",
       "      <td>Core Python Programming is a textbook on the P...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>983c83a5a8b3496d96406d4917e3df92</th>\n",
       "      <td>Eric Software Python Software | Demo Site</td>\n",
       "      <td>https://python-software.github.io/Eric-Software</td>\n",
       "      <td>eric is a free integrated development environm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6e1416433f99a49c8f4b2830708db8e6</th>\n",
       "      <td>History Of Python Python Software | Demo Site</td>\n",
       "      <td>https://python-software.github.io/History-Of-P...</td>\n",
       "      <td>The programming language Python was conceived ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8495394a15fb395351dd4bfd823fecb7</th>\n",
       "      <td>Circuitpython Python Software | Demo Site</td>\n",
       "      <td>https://python-software.github.io/Circuitpython/</td>\n",
       "      <td>CircuitPythonis an open source derivative of t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e68978a1b854f773f72b4b0b23a48fa7</th>\n",
       "      <td>Comparison Of Integrated Development Environme...</td>\n",
       "      <td>https://python-software.github.io/Comparison-O...</td>\n",
       "      <td>The following tables list notable software pac...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798b6e9b7b0bcfdeb5fd901e0d14754f</th>\n",
       "      <td>Activestate Python Software | Demo Site</td>\n",
       "      <td>https://python-software.github.io/Activestate/</td>\n",
       "      <td>ActiveState Software Inc. is a Canadian softwa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0b6ffb38999298c6c1de11ca9fd6e70f</th>\n",
       "      <td>Biopython Python Software | Demo Site</td>\n",
       "      <td>https://python-software.github.io/Biopython/</td>\n",
       "      <td>The Biopython Project is an open-source collec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eed0f1a092770413483ff8bab22e17cb</th>\n",
       "      <td>Anaconda Python Distribution Python Software |...</td>\n",
       "      <td>https://python-software.github.io/Anaconda-Pyt...</td>\n",
       "      <td>Anaconda is a free and open-sourcedistribution...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0d12d0e7e810721289a525936b9722d0</th>\n",
       "      <td>Intel Parallel Studio Python Software | Demo Site</td>\n",
       "      <td>https://python-software.github.io/Intel-Parall...</td>\n",
       "      <td>Intel Parallel Studio XE is a software develop...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1217d635fe4743542b132907316aee55</th>\n",
       "      <td>Ironpython Python Software | Demo Site</td>\n",
       "      <td>https://python-software.github.io/Ironpython/</td>\n",
       "      <td>IronPython is an implementation of the Python ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27d57f0a975a7d3a6386dfcc12e94c24</th>\n",
       "      <td>Idle Python Software | Demo Site</td>\n",
       "      <td>https://python-software.github.io/Idle/</td>\n",
       "      <td>IDLE (short for Integrated DeveLopment Environ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d25ec09ee5cc9c9b8bf02e826301466e</th>\n",
       "      <td>Big Data Data Science | Demo Site</td>\n",
       "      <td>https://data-science-blog.github.io/Big-Data</td>\n",
       "      <td>Information assets characterized by such a hig...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>beb45c4bdc37cb73e91cf2ea2b961f84</th>\n",
       "      <td>Dataintensive Computing Data Science | Demo Site</td>\n",
       "      <td>https://data-science-blog.github.io/Dataintens...</td>\n",
       "      <td>Data-intensive computing is a class of paralle...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381fefb838959736f69c34b52f776844</th>\n",
       "      <td>Consistency Database Systems Data Science | De...</td>\n",
       "      <td>https://data-science-blog.github.io/Consistenc...</td>\n",
       "      <td>Consistency in database systems refers to the ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                              title  \\\n",
       "67fb3a56f4a5b4ada650888415692c1c  Demo Site | Demo site generated from Wikipedia...   \n",
       "ccb6272667b7c94201d9f774c6ed2d47  Core Python Programming Python Software | Demo...   \n",
       "983c83a5a8b3496d96406d4917e3df92          Eric Software Python Software | Demo Site   \n",
       "6e1416433f99a49c8f4b2830708db8e6      History Of Python Python Software | Demo Site   \n",
       "8495394a15fb395351dd4bfd823fecb7          Circuitpython Python Software | Demo Site   \n",
       "e68978a1b854f773f72b4b0b23a48fa7  Comparison Of Integrated Development Environme...   \n",
       "798b6e9b7b0bcfdeb5fd901e0d14754f            Activestate Python Software | Demo Site   \n",
       "0b6ffb38999298c6c1de11ca9fd6e70f              Biopython Python Software | Demo Site   \n",
       "eed0f1a092770413483ff8bab22e17cb  Anaconda Python Distribution Python Software |...   \n",
       "0d12d0e7e810721289a525936b9722d0  Intel Parallel Studio Python Software | Demo Site   \n",
       "1217d635fe4743542b132907316aee55             Ironpython Python Software | Demo Site   \n",
       "27d57f0a975a7d3a6386dfcc12e94c24                   Idle Python Software | Demo Site   \n",
       "d25ec09ee5cc9c9b8bf02e826301466e                  Big Data Data Science | Demo Site   \n",
       "beb45c4bdc37cb73e91cf2ea2b961f84   Dataintensive Computing Data Science | Demo Site   \n",
       "381fefb838959736f69c34b52f776844  Consistency Database Systems Data Science | De...   \n",
       "\n",
       "                                                                                url  \\\n",
       "67fb3a56f4a5b4ada650888415692c1c                 https://python-software.github.io/   \n",
       "ccb6272667b7c94201d9f774c6ed2d47  https://python-software.github.io/Core-Python-...   \n",
       "983c83a5a8b3496d96406d4917e3df92    https://python-software.github.io/Eric-Software   \n",
       "6e1416433f99a49c8f4b2830708db8e6  https://python-software.github.io/History-Of-P...   \n",
       "8495394a15fb395351dd4bfd823fecb7   https://python-software.github.io/Circuitpython/   \n",
       "e68978a1b854f773f72b4b0b23a48fa7  https://python-software.github.io/Comparison-O...   \n",
       "798b6e9b7b0bcfdeb5fd901e0d14754f     https://python-software.github.io/Activestate/   \n",
       "0b6ffb38999298c6c1de11ca9fd6e70f       https://python-software.github.io/Biopython/   \n",
       "eed0f1a092770413483ff8bab22e17cb  https://python-software.github.io/Anaconda-Pyt...   \n",
       "0d12d0e7e810721289a525936b9722d0  https://python-software.github.io/Intel-Parall...   \n",
       "1217d635fe4743542b132907316aee55      https://python-software.github.io/Ironpython/   \n",
       "27d57f0a975a7d3a6386dfcc12e94c24            https://python-software.github.io/Idle/   \n",
       "d25ec09ee5cc9c9b8bf02e826301466e       https://data-science-blog.github.io/Big-Data   \n",
       "beb45c4bdc37cb73e91cf2ea2b961f84  https://data-science-blog.github.io/Dataintens...   \n",
       "381fefb838959736f69c34b52f776844  https://data-science-blog.github.io/Consistenc...   \n",
       "\n",
       "                                                                        description  \n",
       "67fb3a56f4a5b4ada650888415692c1c          Demo site generated from Wikipedia topics  \n",
       "ccb6272667b7c94201d9f774c6ed2d47  Core Python Programming is a textbook on the P...  \n",
       "983c83a5a8b3496d96406d4917e3df92  eric is a free integrated development environm...  \n",
       "6e1416433f99a49c8f4b2830708db8e6  The programming language Python was conceived ...  \n",
       "8495394a15fb395351dd4bfd823fecb7  CircuitPythonis an open source derivative of t...  \n",
       "e68978a1b854f773f72b4b0b23a48fa7  The following tables list notable software pac...  \n",
       "798b6e9b7b0bcfdeb5fd901e0d14754f  ActiveState Software Inc. is a Canadian softwa...  \n",
       "0b6ffb38999298c6c1de11ca9fd6e70f  The Biopython Project is an open-source collec...  \n",
       "eed0f1a092770413483ff8bab22e17cb  Anaconda is a free and open-sourcedistribution...  \n",
       "0d12d0e7e810721289a525936b9722d0  Intel Parallel Studio XE is a software develop...  \n",
       "1217d635fe4743542b132907316aee55  IronPython is an implementation of the Python ...  \n",
       "27d57f0a975a7d3a6386dfcc12e94c24  IDLE (short for Integrated DeveLopment Environ...  \n",
       "d25ec09ee5cc9c9b8bf02e826301466e  Information assets characterized by such a hig...  \n",
       "beb45c4bdc37cb73e91cf2ea2b961f84  Data-intensive computing is a class of paralle...  \n",
       "381fefb838959736f69c34b52f776844  Consistency in database systems refers to the ...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_term = 'python programming'\n",
    "df = indexer.search_index(search_term)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Individual Components"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Crawl Page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "## Status ####################\n",
      "\n",
      "200\n",
      "\n",
      "## Meta ####################\n",
      "\n",
      "{\n",
      "    \"description\": \"LOCOMOTIVE\\u00ae - 2019 U.S. Search Awards \\\"Best SEO Agency\\\". We are an agency team of enterprise technical, and on-page SEO specialists: Moving you forward.\",\n",
      "    \"lang\": \"en\",\n",
      "    \"keywords\": \"\",\n",
      "    \"favicon\": \"\",\n",
      "    \"canonical\": \"https://locomotive.agency/\",\n",
      "    \"encoding\": \"utf-8\",\n",
      "    \"robots\": [\n",
      "        \"max-snippet:-1\",\n",
      "        \"max-image-preview:large\",\n",
      "        \"max-video-preview:-1\"\n",
      "    ]\n",
      "}\n",
      "\n",
      "## Content ####################\n",
      "\n",
      "Our team was humbled to be a finalist for 3 U.S. Search Awards, and receive one of the top awards of the night for SEO agencies. All awards are judged upon results for clients, as well as agency operations, by a panel of SEO industry veteran judges.\n"
     ]
    }
   ],
   "source": [
    "url = \"https://locomotive.agency/\"\n",
    "page_data = crawl_url(url)\n",
    "print(\"## Status \" + \"#\"*20 + \"\\n\")\n",
    "print(page_data['status'])\n",
    "print(\"\\n## Meta \" + \"#\"*20 + \"\\n\")\n",
    "print(json.dumps(page_data['meta'], indent=4))\n",
    "print(\"\\n## Content \" + \"#\"*20 + \"\\n\")\n",
    "print(page_data['content'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shingling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "## Hashes ####################\n",
      "\n",
      "[-2145608475, -2092676559, -2100324990, -2106062289, -2101729913]\n",
      "[-2145608475, -2092676559, -2100324990, -2106062289, -2101729913]\n",
      "[-2091386696, -2135102216, -2102983285, -2091906514, -2113156462]\n",
      "\n",
      "## Length ####################\n",
      "\n",
      "3\n",
      "\n",
      "## Similarity ####################\n",
      "\n",
      "    urlhash       sim\n",
      "1  content2  1.000000\n",
      "0  content1  0.886792\n",
      "2  content3  0.000000\n"
     ]
    }
   ],
   "source": [
    "content1 = \"\"\"Many packages don't create a build for every single release which forces your pip to build from source. If you're happy to use the latest pre-compiled binary version, use --only-binary :all: to allow pip to use an older binary version.\"\"\"\n",
    "content2 = \"\"\"Most packages don't create a build for every single release which forces your pip to build from source. If you're happy to use the latest pre-compiled binary version, use --only-binary :all: to allow pip to use an older binary version.\"\"\"\n",
    "content3 = \"\"\"The C++ Build Tools allow you to build C++ libraries and applications targeting Windows desktop. They are the same tools that you find in Visual Studio 2019, Visual Studio 2017, and Visual Studio 2015 in a scriptable standalone installer. Now you only need to download the MSVC compiler toolset you need to build C++ projects on your build servers.\"\"\"\n",
    "\n",
    "\n",
    "hashdb = HashLookup()\n",
    "\n",
    "hashdb.add_hash('content1', content1)\n",
    "hashdb.add_hash('content2', content2)\n",
    "hashdb.add_hash('content3', content3)\n",
    "\n",
    "print(\"## Hashes \" + \"#\"*20 + \"\\n\")\n",
    "print(hashdb.get_hash('content1')[:5])\n",
    "print(hashdb.get_hash('content2')[:5])\n",
    "print(hashdb.get_hash('content3')[:5])\n",
    "print(\"\\n## Length \" + \"#\"*20 + \"\\n\")\n",
    "print(len(hashdb))\n",
    "print(\"\\n## Similarity \" + \"#\"*20 + \"\\n\")\n",
    "print(hashdb.get_similarity_df(content2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PageRank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "## PageRank ####################\n",
      "\n",
      "            score\n",
      "urlhash          \n",
      "urlb     0.362550\n",
      "urle     0.254421\n",
      "urla     0.199763\n",
      "urld     0.091633\n",
      "urlc     0.091633\n"
     ]
    }
   ],
   "source": [
    "url_list = ['urla', 'urlb', 'urlc', 'urld', 'urle']\n",
    "link_tuples = [('urla','urlb'), ('urlc','urlb'), ('urla','urle'), ('urle','urla'), ('urlc','urlb'), ('urld','urle'), ('urle','urlb')]\n",
    "\n",
    "pr_valid = {'url': {0: 'urlb', 1: 'urle', 2: 'urla', 3: 'urld', 4: 'urlc'}, 'score': {0: 0.3625498007448575, 1: 0.2544205750109898, 2: 0.19976269190396267, 3: 0.09163346617009499, 4: 0.09163346617009499}}\n",
    "df = build_pagerank_df(url_list, link_tuples)\n",
    "\n",
    "print(\"## PageRank \" + \"#\"*20 + \"\\n\")\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json from cache at C:\\Users\\jroak\\.cache\\torch\\transformers\\a41e817d5c0743e29e86ff85edc8c257e61bc8d88e4271bb1b243b6e7614c633.1ccd1a11c9ff276830e114ea477ea2407100f4a3be7bdc45d37be9e37fa71c7e\n",
      "Model config {\n",
      "  \"activation\": \"gelu\",\n",
      "  \"attention_dropout\": 0.1,\n",
      "  \"dim\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_dim\": 3072,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"n_heads\": 12,\n",
      "  \"n_layers\": 6,\n",
      "  \"num_labels\": 2,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pruned_heads\": {},\n",
      "  \"qa_dropout\": 0.1,\n",
      "  \"seq_classif_dropout\": 0.2,\n",
      "  \"sinusoidal_pos_embds\": false,\n",
      "  \"tie_weights_\": true,\n",
      "  \"torchscript\": false,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-pytorch_model.bin from cache at C:\\Users\\jroak\\.cache\\torch\\transformers\\7b8a8f0b21c4e7f6962451c9370a5d9af90372a5f64637a251f2de154d0fc72c.c2015533705b9dff680ae707e205a35e2860e8d148b45d35085419d74fe57ac5\n",
      "loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at C:\\Users\\jroak\\.cache\\torch\\transformers\\26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query: trim a chrismas tree       ===> Best Guess: decorate a tree (0.8258)\n",
      "Query: jog on a path              ===> Best Guess: run on a road (0.8056)\n",
      "Query: kindle ebook               ===> Best Guess: electric book (0.6704)\n",
      "Query: italian restaurant         ===> Best Guess: cafe in italy (0.7463)\n",
      "Query: internet landing page      ===> Best Guess: website homepage (0.7787)\n"
     ]
    }
   ],
   "source": [
    "from lib.bert import *\n",
    "\n",
    "queries = ['trim a chrismas tree', 'jog on a path', 'kindle ebook', 'italian restaurant', 'internet landing page']\n",
    "ngrams = ['decorate a tree', 'run on a road', 'electric book', 'cafe in italy', 'website homepage']\n",
    "\n",
    "bert = BERT(dims=None)\n",
    "bert.add_terms(ngrams)\n",
    "\n",
    "for q in queries:\n",
    "    best, sim = bert.get_most_similar(q)\n",
    "    print(\"Query: {} {} ===> Best Guess: {} ({})\".format(q, ' '*(25-len(q)), best, round(sim,4)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}