github_scraping_topics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f37fe7f7",
   "metadata": {},
   "source": [
    "# Scraping Popular Github Topics\n",
    "\n",
    "Scrape the webpage https://github.com/topics and get the topics titles, its description, and the link towards it directs the users "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c8934025",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "<!DOCTYPE html>\n",
      "<html lang=\"en\" data-color-mode=\"auto\" data-light-theme=\"light\" data-dark-theme=\"dark\"  data-a11y-animated-images=\"system\" data-a11y-link-underlines=\"false\">\n",
      "  <head>\n",
      "    <meta charset=\"utf-8\">\n",
      "  <link rel=\"dns-prefetch\" href=\"https://github.githubassets.com\">\n",
      "  <link rel=\"dns-prefetch\" href=\"https://avatars.githubusercontent.com\">\n",
      "  <link rel=\"dns-prefetch\" href=\"https://github-cloud.s3.amazonaws.com\">\n",
      "  <link rel=\"dns-prefetch\" href=\"https://user-images.githubusercontent.com/\">\n",
      "  <link rel=\"preconnect\" href=\"https://github.githubassets.com\" crossorigin>\n",
      "  <link rel=\"preconnect\" href=\"https://avatars.githubusercontent.com\">\n",
      "\n",
      "  \n",
      "\n",
      "  <link crossorigin=\"anonymous\" media=\"all\" rel=\"stylesheet\" href=\"https://github.githubassets.com/assets/light-8cafbcbd78f4.css\" /><link crossorigin=\"anonymous\" media=\"all\" rel=\"stylesheet\" href=\"https://github.githubassets.com/assets/dark-31dc14e38457.css\" /><link data-color-theme=\"dark_dimmed\" crossorigin=\"anonymous\" media=\"all\" rel=\"style\n"
     ]
    }
   ],
   "source": [
    "# get the url webpage \n",
    "url = 'https://github.com/topics'\n",
    "\n",
    "# import the necessary library and read the webpage as a html file \n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "\n",
    "page = requests.get(url)\n",
    "\n",
    "webpage = BeautifulSoup(page.text, 'html')\n",
    "\n",
    "# print the first 1000 characters of the webpage, to have an overview...\n",
    "print(page.text[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "57a19e0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['3D', 'Ajax', 'Algorithm', 'Amp', 'Android', 'Angular', 'Ansible', 'API', 'Arduino', 'ASP.NET', 'Atom', 'Awesome Lists', 'Amazon Web Services', 'Azure', 'Babel', 'Bash', 'Bitcoin', 'Bootstrap', 'Bot', 'C', 'Chrome', 'Chrome extension', 'Command line interface', 'Clojure', 'Code quality', 'Code review', 'Compiler', 'Continuous integration', 'COVID-19', 'C++']\n"
     ]
    }
   ],
   "source": [
    "# find and print each topic titles from the webpage in a list called topic_titles_list\n",
    "\n",
    "topic_titles = webpage.find_all('p', class_= \"f3 lh-condensed mb-0 mt-1 Link--primary\")\n",
    "\n",
    "topic_titles_list = [item.text for item in topic_titles]\n",
    "\n",
    "print(topic_titles_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "fab567da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['3D refers to the use of three-dimensional graphics, modeling, and animation in various industries.', 'Ajax is a technique for creating interactive web applications.', 'Algorithms are self-contained sequences that carry out a variety of tasks.', 'Amp is a non-blocking concurrency library for PHP.', 'Android is an operating system built by Google designed for mobile devices.', 'Angular is an open source web application platform.', 'Ansible is a simple and powerful automation engine.', 'An API (Application Programming Interface) is a collection of protocols and subroutines for building software.', 'Arduino is an open source platform for building electronic devices.', 'ASP.NET is a web framework for building modern web apps and services.', 'Atom is a open source text editor built with web technologies.', 'An awesome list is a list of awesome things curated by the community.', 'Amazon Web Services provides on-demand cloud computing platforms on a subscription basis.', 'Azure is a cloud computing service created by Microsoft.', 'Babel is a compiler for writing next generation JavaScript, today.', 'Bash is a shell and command language interpreter for the GNU operating system.', 'Bitcoin is a cryptocurrency developed by Satoshi Nakamoto.', 'Bootstrap is an HTML, CSS, and JavaScript framework.', 'A bot is an application that runs automated tasks over the Internet.', 'C is a general purpose programming language that first appeared in 1972.', 'Chrome is a web browser from the tech company Google.', 'Chrome extensions enable users to customize the Chrome browsing experience.', 'A CLI, or command-line interface, is a console that helps users issue commands to a program.', 'Clojure is a dynamic, general-purpose programming language.', 'Automate your code review with style, quality, security, and test‑coverage checks when you need them.', 'Ensure your code meets quality standards and ship with confidence.', 'Compilers are software that translate higher-level programming languages to lower-level languages (e.g. machine code).', 'Automatically build and test your code as you push it upstream, preventing bugs from being deployed to production.', 'The coronavirus disease 2019 (COVID-19) is an infectious disease caused by SARS-CoV-2.', 'C++ is a general purpose and object-oriented programming language.']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find extract the topic descriptions \n",
    "\n",
    "topic_descriptions = webpage.find_all('p', class_= \"f5 color-fg-muted mb-0 mt-1\")\n",
    "\n",
    "topic_descriptions_list = [item.text.strip() for item in topic_descriptions]\n",
    "\n",
    "print(topic_descriptions_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9dda07b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['https://github.com/topics/topics/3d', 'https://github.com/topics/topics/ajax', 'https://github.com/topics/topics/algorithm', 'https://github.com/topics/topics/amphp', 'https://github.com/topics/topics/android', 'https://github.com/topics/topics/angular', 'https://github.com/topics/topics/ansible', 'https://github.com/topics/topics/api', 'https://github.com/topics/topics/arduino', 'https://github.com/topics/topics/aspnet', 'https://github.com/topics/topics/atom', 'https://github.com/topics/topics/awesome', 'https://github.com/topics/topics/aws', 'https://github.com/topics/topics/azure', 'https://github.com/topics/topics/babel', 'https://github.com/topics/topics/bash', 'https://github.com/topics/topics/bitcoin', 'https://github.com/topics/topics/bootstrap', 'https://github.com/topics/topics/bot', 'https://github.com/topics/topics/c', 'https://github.com/topics/topics/chrome', 'https://github.com/topics/topics/chrome-extension', 'https://github.com/topics/topics/cli', 'https://github.com/topics/topics/clojure', 'https://github.com/topics/topics/code-quality', 'https://github.com/topics/topics/code-review', 'https://github.com/topics/topics/compiler', 'https://github.com/topics/topics/continuous-integration', 'https://github.com/topics/topics/covid-19', 'https://github.com/topics/topics/cpp']\n"
     ]
    }
   ],
   "source": [
    "# find extract the topic url\n",
    "\n",
    "topic_url = webpage.find_all('a', class_= \"no-underline flex-1 d-flex flex-column\")\n",
    "\n",
    "index = len(topic_url)\n",
    "#print(topic_url[0]['href'])\n",
    "\n",
    "topic_url_list = [ url+topic_url[i]['href'] for i in range(index)]\n",
    "\n",
    "print(topic_url_list)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "45bf7a6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Topic_titles</th>\n",
       "      <th>Topic_descriptions</th>\n",
       "      <th>Topic_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3D</td>\n",
       "      <td>3D refers to the use of three-dimensional grap...</td>\n",
       "      <td>https://github.com/topics/topics/3d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ajax</td>\n",
       "      <td>Ajax is a technique for creating interactive w...</td>\n",
       "      <td>https://github.com/topics/topics/ajax</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Algorithm</td>\n",
       "      <td>Algorithms are self-contained sequences that c...</td>\n",
       "      <td>https://github.com/topics/topics/algorithm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Amp</td>\n",
       "      <td>Amp is a non-blocking concurrency library for ...</td>\n",
       "      <td>https://github.com/topics/topics/amphp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Android</td>\n",
       "      <td>Android is an operating system built by Google...</td>\n",
       "      <td>https://github.com/topics/topics/android</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Angular</td>\n",
       "      <td>Angular is an open source web application plat...</td>\n",
       "      <td>https://github.com/topics/topics/angular</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Ansible</td>\n",
       "      <td>Ansible is a simple and powerful automation en...</td>\n",
       "      <td>https://github.com/topics/topics/ansible</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>API</td>\n",
       "      <td>An API (Application Programming Interface) is ...</td>\n",
       "      <td>https://github.com/topics/topics/api</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Arduino</td>\n",
       "      <td>Arduino is an open source platform for buildin...</td>\n",
       "      <td>https://github.com/topics/topics/arduino</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ASP.NET</td>\n",
       "      <td>ASP.NET is a web framework for building modern...</td>\n",
       "      <td>https://github.com/topics/topics/aspnet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Atom</td>\n",
       "      <td>Atom is a open source text editor built with w...</td>\n",
       "      <td>https://github.com/topics/topics/atom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Awesome Lists</td>\n",
       "      <td>An awesome list is a list of awesome things cu...</td>\n",
       "      <td>https://github.com/topics/topics/awesome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Amazon Web Services</td>\n",
       "      <td>Amazon Web Services provides on-demand cloud c...</td>\n",
       "      <td>https://github.com/topics/topics/aws</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Azure</td>\n",
       "      <td>Azure is a cloud computing service created by ...</td>\n",
       "      <td>https://github.com/topics/topics/azure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Babel</td>\n",
       "      <td>Babel is a compiler for writing next generatio...</td>\n",
       "      <td>https://github.com/topics/topics/babel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Bash</td>\n",
       "      <td>Bash is a shell and command language interpret...</td>\n",
       "      <td>https://github.com/topics/topics/bash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Bitcoin</td>\n",
       "      <td>Bitcoin is a cryptocurrency developed by Satos...</td>\n",
       "      <td>https://github.com/topics/topics/bitcoin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Bootstrap</td>\n",
       "      <td>Bootstrap is an HTML, CSS, and JavaScript fram...</td>\n",
       "      <td>https://github.com/topics/topics/bootstrap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Bot</td>\n",
       "      <td>A bot is an application that runs automated ta...</td>\n",
       "      <td>https://github.com/topics/topics/bot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>C</td>\n",
       "      <td>C is a general purpose programming language th...</td>\n",
       "      <td>https://github.com/topics/topics/c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Chrome</td>\n",
       "      <td>Chrome is a web browser from the tech company ...</td>\n",
       "      <td>https://github.com/topics/topics/chrome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Chrome extension</td>\n",
       "      <td>Chrome extensions enable users to customize th...</td>\n",
       "      <td>https://github.com/topics/topics/chrome-extension</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Command line interface</td>\n",
       "      <td>A CLI, or command-line interface, is a console...</td>\n",
       "      <td>https://github.com/topics/topics/cli</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Clojure</td>\n",
       "      <td>Clojure is a dynamic, general-purpose programm...</td>\n",
       "      <td>https://github.com/topics/topics/clojure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Code quality</td>\n",
       "      <td>Automate your code review with style, quality,...</td>\n",
       "      <td>https://github.com/topics/topics/code-quality</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Code review</td>\n",
       "      <td>Ensure your code meets quality standards and s...</td>\n",
       "      <td>https://github.com/topics/topics/code-review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Compiler</td>\n",
       "      <td>Compilers are software that translate higher-l...</td>\n",
       "      <td>https://github.com/topics/topics/compiler</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Continuous integration</td>\n",
       "      <td>Automatically build and test your code as you ...</td>\n",
       "      <td>https://github.com/topics/topics/continuous-in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>COVID-19</td>\n",
       "      <td>The coronavirus disease 2019 (COVID-19) is an ...</td>\n",
       "      <td>https://github.com/topics/topics/covid-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>C++</td>\n",
       "      <td>C++ is a general purpose and object-oriented p...</td>\n",
       "      <td>https://github.com/topics/topics/cpp</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Topic_titles                                 Topic_descriptions  \\\n",
       "0                       3D  3D refers to the use of three-dimensional grap...   \n",
       "1                     Ajax  Ajax is a technique for creating interactive w...   \n",
       "2                Algorithm  Algorithms are self-contained sequences that c...   \n",
       "3                      Amp  Amp is a non-blocking concurrency library for ...   \n",
       "4                  Android  Android is an operating system built by Google...   \n",
       "5                  Angular  Angular is an open source web application plat...   \n",
       "6                  Ansible  Ansible is a simple and powerful automation en...   \n",
       "7                      API  An API (Application Programming Interface) is ...   \n",
       "8                  Arduino  Arduino is an open source platform for buildin...   \n",
       "9                  ASP.NET  ASP.NET is a web framework for building modern...   \n",
       "10                    Atom  Atom is a open source text editor built with w...   \n",
       "11           Awesome Lists  An awesome list is a list of awesome things cu...   \n",
       "12     Amazon Web Services  Amazon Web Services provides on-demand cloud c...   \n",
       "13                   Azure  Azure is a cloud computing service created by ...   \n",
       "14                   Babel  Babel is a compiler for writing next generatio...   \n",
       "15                    Bash  Bash is a shell and command language interpret...   \n",
       "16                 Bitcoin  Bitcoin is a cryptocurrency developed by Satos...   \n",
       "17               Bootstrap  Bootstrap is an HTML, CSS, and JavaScript fram...   \n",
       "18                     Bot  A bot is an application that runs automated ta...   \n",
       "19                       C  C is a general purpose programming language th...   \n",
       "20                  Chrome  Chrome is a web browser from the tech company ...   \n",
       "21        Chrome extension  Chrome extensions enable users to customize th...   \n",
       "22  Command line interface  A CLI, or command-line interface, is a console...   \n",
       "23                 Clojure  Clojure is a dynamic, general-purpose programm...   \n",
       "24            Code quality  Automate your code review with style, quality,...   \n",
       "25             Code review  Ensure your code meets quality standards and s...   \n",
       "26                Compiler  Compilers are software that translate higher-l...   \n",
       "27  Continuous integration  Automatically build and test your code as you ...   \n",
       "28                COVID-19  The coronavirus disease 2019 (COVID-19) is an ...   \n",
       "29                     C++  C++ is a general purpose and object-oriented p...   \n",
       "\n",
       "                                            Topic_url  \n",
       "0                 https://github.com/topics/topics/3d  \n",
       "1               https://github.com/topics/topics/ajax  \n",
       "2          https://github.com/topics/topics/algorithm  \n",
       "3              https://github.com/topics/topics/amphp  \n",
       "4            https://github.com/topics/topics/android  \n",
       "5            https://github.com/topics/topics/angular  \n",
       "6            https://github.com/topics/topics/ansible  \n",
       "7                https://github.com/topics/topics/api  \n",
       "8            https://github.com/topics/topics/arduino  \n",
       "9             https://github.com/topics/topics/aspnet  \n",
       "10              https://github.com/topics/topics/atom  \n",
       "11           https://github.com/topics/topics/awesome  \n",
       "12               https://github.com/topics/topics/aws  \n",
       "13             https://github.com/topics/topics/azure  \n",
       "14             https://github.com/topics/topics/babel  \n",
       "15              https://github.com/topics/topics/bash  \n",
       "16           https://github.com/topics/topics/bitcoin  \n",
       "17         https://github.com/topics/topics/bootstrap  \n",
       "18               https://github.com/topics/topics/bot  \n",
       "19                 https://github.com/topics/topics/c  \n",
       "20            https://github.com/topics/topics/chrome  \n",
       "21  https://github.com/topics/topics/chrome-extension  \n",
       "22               https://github.com/topics/topics/cli  \n",
       "23           https://github.com/topics/topics/clojure  \n",
       "24      https://github.com/topics/topics/code-quality  \n",
       "25       https://github.com/topics/topics/code-review  \n",
       "26          https://github.com/topics/topics/compiler  \n",
       "27  https://github.com/topics/topics/continuous-in...  \n",
       "28          https://github.com/topics/topics/covid-19  \n",
       "29               https://github.com/topics/topics/cpp  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# create a dataframe containing the extracted elements\n",
    "dictionnary = { 'Topic_titles': topic_titles_list,\n",
    "              'Topic_descriptions': topic_descriptions_list,\n",
    "              'Topic_url': topic_url_list }\n",
    "\n",
    "import pandas as pd\n",
    "df = pd.DataFrame(dictionnary)\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a3ee37ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save this table as a csv file \n",
    "df.to_csv(r'C:\\Users\\Meli Imelda\\Documents\\first_web_scraping_output\\git_topics.csv', index = False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76bd810c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}