mars_data_challenge_part_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c36e8a0d",
   "metadata": {},
   "source": [
    "# Deliverable 1: web scraping the Mars News\n",
    "- Scrape **titles** and **preview texts** from Mars news articles.\n",
    "- Optionally export the data into a JSON file or a MongoDB database. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bb657996",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import dependencies\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from splinter import Browser\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from bs4 import BeautifulSoup as soup\n",
    "# import below when using Chrome browser\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from webdriver_manager.chrome import ChromeDriverManager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0dbf4940",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import json\n",
    "import os\n",
    "# set local chromedriver\n",
    "chromedrv = os.path.expanduser('~/ucbDA/chromedriver_win32/chromedriver')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "884cf120",
   "metadata": {},
   "source": [
    "## Method 1: splinter's executable_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d40511cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set some default options for chrome browser\n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument(\"--lang=en\")\n",
    "options.add_argument(\"--start-maximized\")\n",
    "options.add_argument(\"--disable-notifications\")\n",
    "# Set up splinter (PS: executable_path has deprecated)\n",
    "executable_path = {'executable_path': ChromeDriverManager().install()}\n",
    "browser = Browser('chrome', **executable_path, headless=False, options=options)\n",
    "# Visit a site defined in the url\n",
    "def geturl(url):\n",
    "    browser.visit(url)\n",
    "    # Optional delay for loading the page (unit: seconds)\n",
    "    browser.is_element_present_by_css('div.list_text', wait_time=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9d700339",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visit the Mars NASA news site\n",
    "geturl('https://redplanetscience.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bc144174",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse the HTML and select all news articles\n",
    "html = browser.html\n",
    "news_soup = soup(html, 'html.parser')\n",
    "slide_elems = news_soup.select('div.list_text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "edac772a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find and store all news article titles and preview texts\n",
    "news_list = []\n",
    "for elem in slide_elems:\n",
    "    # Use the parent element to find the news article title\n",
    "    title = elem.find('div', class_='content_title').text\n",
    "    # Use the parent element to find the paragraph text\n",
    "    preview = elem.find('div', class_='article_teaser_body').text\n",
    "    # Append each key-value pair to a list/dict\n",
    "    news_list.append({'title': title, 'preview': preview})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9b41e310",
   "metadata": {},
   "outputs": [],
   "source": [
    "# export the Python list/dict into a JSON file\n",
    "outfile = './Data/mars_data_method1.json'\n",
    "with open(outfile, 'w', encoding='utf-8') as f:\n",
    "    json.dump(news_list, f, ensure_ascii=False, indent=4)\n",
    "# close file (optional)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ddf3f9d6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Verify the json file\n",
    "infile = open(outfile, 'r', encoding='utf-8')\n",
    "mars_data = json.load(infile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a84d49c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "browser.quit()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fef2e35c",
   "metadata": {},
   "source": [
    "## Method 2: selenium's webdriver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e0a76dbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set some default options for chrome browser\n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument(\"--lang=en\")\n",
    "options.add_argument(\"--start-maximized\")\n",
    "options.add_argument(\"--disable-notifications\")\n",
    "# Set up splinter (selenium 4)\n",
    "driver = webdriver.Chrome(service=Service(chromedrv), options=options)\n",
    "# Visit a site defined in the url\n",
    "def geturl(url):\n",
    "    driver.get(url)\n",
    "    # Optional delay for loading the page (unit: seconds)\n",
    "    driver.implicitly_wait(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7aef1979",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visit the Mars NASA news site\n",
    "geturl('https://redplanetscience.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "bd95b717",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse the HTML (selenium 4)\n",
    "html = driver.page_source\n",
    "news_soup = soup(html, 'html.parser')\n",
    "slide_elems = news_soup.select('div.list_text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8f760645",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find and store all news article titles and preview texts\n",
    "news_list = []\n",
    "for elem in slide_elems:\n",
    "    # Use the parent element to find the news article title\n",
    "    title = elem.find('div', class_='content_title').text\n",
    "    # Use the parent element to find the paragraph text\n",
    "    preview = elem.find('div', class_='article_teaser_body').text\n",
    "    # Append each key-value pair to a list/dict\n",
    "    news_list.append({'title': title, 'preview': preview})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "eef06d95",
   "metadata": {},
   "outputs": [],
   "source": [
    "# export the Python list/dict into a JSON file\n",
    "outfile = './Data/mars_data.json'\n",
    "with open(outfile, 'w', encoding='utf-8') as f:\n",
    "    json.dump(news_list, f, ensure_ascii=False, indent=4)\n",
    "# close file (optional)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8e7b48dd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'title': \"NASA Wins 4 Webbys, 4 People's Voice Awards\",\n",
       "  'preview': 'Winners include the JPL-managed \"Send Your Name to Mars\" campaign, NASA\\'s Global Climate Change website and Solar System Interactive.'},\n",
       " {'title': 'Heat and Dust Help Launch Martian Water Into Space, Scientists Find',\n",
       "  'preview': 'Scientists using an instrument aboard NASA’s Mars Atmosphere and Volatile EvolutioN, or MAVEN, spacecraft have discovered that water vapor near the surface of the Red Planet is lofted higher into the atmosphere than anyone expected was possible. '},\n",
       " {'title': 'Mars 2020 Unwrapped and Ready for More Testing',\n",
       "  'preview': \"In time-lapse video, bunny-suited engineers remove the inner layer of protective foil on NASA's Mars 2020 rover after it was relocated for testing.\"},\n",
       " {'title': \"NASA's Briefcase-Size MarCO Satellite Picks Up Honors\",\n",
       "  'preview': 'The twin spacecraft, the first of their kind to fly into deep space, earn a Laureate from Aviation Week & Space Technology.'},\n",
       " {'title': \"8 Martian Postcards to Celebrate Curiosity's Landing Anniversary\",\n",
       "  'preview': 'The NASA rover touched down eight years ago, on Aug. 5, 2012, and will soon be joined by a second rover, Perseverance.'},\n",
       " {'title': \"NASA Readies Perseverance Mars Rover's Earthly Twin \",\n",
       "  'preview': \"Did you know NASA's next Mars rover has a nearly identical sibling on Earth for testing? Even better, it's about to roll for the first time through a replica Martian landscape.\"},\n",
       " {'title': \"The Detective Aboard NASA's Perseverance Rover\",\n",
       "  'preview': 'An instrument called SHERLOC will, with the help of its partner WATSON, hunt for signs of ancient life by detecting organic molecules and minerals.'},\n",
       " {'title': 'NASA Moves Forward With Campaign to Return Mars Samples to Earth',\n",
       "  'preview': 'During this next phase, the program will mature critical technologies and make critical design decisions as well as assess industry partnerships.'},\n",
       " {'title': 'Join NASA for the Launch of the Mars 2020 Perseverance Rover',\n",
       "  'preview': 'No matter where you live, choose from a menu of activities to join NASA as we \"Countdown to Mars\" and launch the Perseverance rover to the Red Planet.'},\n",
       " {'title': 'Naming a NASA Mars Rover Can Change Your Life',\n",
       "  'preview': 'Want to name the robotic scientist NASA is sending to Mars in 2020? The student who named Curiosity — the rover currently exploring Mars — will tell you this is an opportunity worth taking.'},\n",
       " {'title': \"NASA's Perseverance Rover Will Carry First Spacesuit Materials to Mars\",\n",
       "  'preview': 'In a Q&A, spacesuit designer Amy Ross explains how five samples, including a piece of helmet visor, will be tested aboard the rover, which is targeting a July 30 launch. '},\n",
       " {'title': 'Sensors on Mars 2020 Spacecraft Answer Long-Distance Call From Earth',\n",
       "  'preview': \"Instruments tailored to collect data during the descent of NASA's next rover through the Red Planet's atmosphere have been checked in flight.\"},\n",
       " {'title': \"How NASA's Perseverance Mars Team Adjusted to Work in the Time of Coronavirus \",\n",
       "  'preview': 'Like much of the rest of the world, the Mars rover team is pushing forward with its mission-critical work while putting the health and safety of their colleagues and community first.'},\n",
       " {'title': \"NASA's Perseverance Rover Attached to Atlas V Rocket\",\n",
       "  'preview': 'Ready for its launch later in the month, the Mars-bound rover will touch terra firma no more.'},\n",
       " {'title': 'NASA to Broadcast Mars 2020 Perseverance Launch, Prelaunch Activities',\n",
       "  'preview': 'Starting July 27, news activities will cover everything from mission engineering and science to returning samples from Mars to, of course, the launch itself.'}]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Verify the json file\n",
    "infile = open(outfile, 'r', encoding='utf-8')\n",
    "mars_data = json.load(infile)\n",
    "mars_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c333cb09",
   "metadata": {},
   "source": [
    "### Create a MongoDB Database\n",
    "```\n",
    "mongoimport --type json -d mars_data -c news_list --drop --jsonArray mars_data.json\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "dde661f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['admin', 'config', 'local', 'mars_data', 'my_db1', 'petsitly_marketing']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pymongo import MongoClient\n",
    "# Create an instance of MongoClient\n",
    "mongo = MongoClient(port=27017)\n",
    "mongo.list_database_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a3ba6c6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['news_list', 'customer_list']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mars_db = mongo['mars_data']\n",
    "mars_db.list_collection_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e14d5769",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 Martian Postcards to Celebrate Curiosity's Landing Anniversary\n",
      "Heat and Dust Help Launch Martian Water Into Space, Scientists Find\n",
      "How NASA's Perseverance Mars Team Adjusted to Work in the Time of Coronavirus \n",
      "Join NASA for the Launch of the Mars 2020 Perseverance Rover\n",
      "Mars 2020 Unwrapped and Ready for More Testing\n",
      "NASA Moves Forward With Campaign to Return Mars Samples to Earth\n",
      "NASA Readies Perseverance Mars Rover's Earthly Twin \n",
      "NASA Wins 4 Webbys, 4 People's Voice Awards\n",
      "NASA to Broadcast Mars 2020 Perseverance Launch, Prelaunch Activities\n",
      "NASA's Briefcase-Size MarCO Satellite Picks Up Honors\n",
      "NASA's Perseverance Rover Attached to Atlas V Rocket\n",
      "NASA's Perseverance Rover Will Carry First Spacesuit Materials to Mars\n",
      "Naming a NASA Mars Rover Can Change Your Life\n",
      "Sensors on Mars 2020 Spacecraft Answer Long-Distance Call From Earth\n",
      "The Detective Aboard NASA's Perseverance Rover\n"
     ]
    }
   ],
   "source": [
    "titles = mars_db['news_list'].find().distinct('title')\n",
    "for title in titles:\n",
    "    print(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0dcfad77",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An instrument called SHERLOC will, with the help of its partner WATSON, hunt for signs of ancient life by detecting organic molecules and minerals.\n",
      "Did you know NASA's next Mars rover has a nearly identical sibling on Earth for testing? Even better, it's about to roll for the first time through a replica Martian landscape.\n",
      "During this next phase, the program will mature critical technologies and make critical design decisions as well as assess industry partnerships.\n",
      "In a Q&A, spacesuit designer Amy Ross explains how five samples, including a piece of helmet visor, will be tested aboard the rover, which is targeting a July 30 launch. \n",
      "In time-lapse video, bunny-suited engineers remove the inner layer of protective foil on NASA's Mars 2020 rover after it was relocated for testing.\n",
      "Instruments tailored to collect data during the descent of NASA's next rover through the Red Planet's atmosphere have been checked in flight.\n",
      "Like much of the rest of the world, the Mars rover team is pushing forward with its mission-critical work while putting the health and safety of their colleagues and community first.\n",
      "No matter where you live, choose from a menu of activities to join NASA as we \"Countdown to Mars\" and launch the Perseverance rover to the Red Planet.\n",
      "Ready for its launch later in the month, the Mars-bound rover will touch terra firma no more.\n",
      "Scientists using an instrument aboard NASA’s Mars Atmosphere and Volatile EvolutioN, or MAVEN, spacecraft have discovered that water vapor near the surface of the Red Planet is lofted higher into the atmosphere than anyone expected was possible. \n",
      "Starting July 27, news activities will cover everything from mission engineering and science to returning samples from Mars to, of course, the launch itself.\n",
      "The NASA rover touched down eight years ago, on Aug. 5, 2012, and will soon be joined by a second rover, Perseverance.\n",
      "The twin spacecraft, the first of their kind to fly into deep space, earn a Laureate from Aviation Week & Space Technology.\n",
      "Want to name the robotic scientist NASA is sending to Mars in 2020? The student who named Curiosity — the rover currently exploring Mars — will tell you this is an opportunity worth taking.\n",
      "Winners include the JPL-managed \"Send Your Name to Mars\" campaign, NASA's Global Climate Change website and Solar System Interactive.\n"
     ]
    }
   ],
   "source": [
    "previews = mars_db['news_list'].find().distinct('preview')\n",
    "for preview in previews:\n",
    "    print(preview)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5e4a05be",
   "metadata": {},
   "outputs": [],
   "source": [
    "mongo.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "98652626",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.quit()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py39env",
   "language": "python",
   "name": "py39env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}