mission_to_mars.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dependencies\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import requests as requests\n",
    "import pandas as pd\n",
    "from splinter import Browser\n",
    "import time\n",
    "import pymongo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NASA Mars News"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n\\nOpportunity Hunkers Down During Dust Storm\\n\\n'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mars news url \n",
    "url = \"https://mars.nasa.gov/news/\"\n",
    "\n",
    "# Retrieve page using request\n",
    "html =  requests.get(url)\n",
    "\n",
    "# create Beautiful soup object and parse with html.parser\n",
    "soup = bs(html.text, 'html.parser')\n",
    "\n",
    "#to retrieve title and parapraph text\n",
    "news_title = soup.find('div', 'content_title', 'a').text\n",
    "news_title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nIt's the beginning of the end for the planet-encircling dust storm on Mars. But it could still be weeks, or even months, before skies are clear enough for NASA's Opportunity rover to recharge its batteries and phone home. \\n\""
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news_p = soup.find('div','rollover_description_inner').text\n",
    "news_p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# JPL Mars Space Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# URL\n",
    "# Using Chromedriver and Setting up Splinter and\n",
    "url = \"https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars\"\n",
    "executable_path = {\"executable_path\": \"chromedriver.exe\"}\n",
    "browser = Browser(\"chrome\", **executable_path, headless=False)\n",
    "browser.visit(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/spaceimages/images/mediumsize/PIA15883_ip.jpg'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# scrape the browser into soup\n",
    "#save the image url\n",
    "html = browser.html\n",
    "url = bs(html, 'html.parser')\n",
    "image_url = url.find('a', {'id': 'full_image', 'data-fancybox-href': True}).get('data-fancybox-href')\n",
    "image_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<div class=\"jpl_logo\">\n",
      "<a href=\"//www.jpl.nasa.gov/\" id=\"jpl_logo\" title=\"Jet Propulsion Laboratory\">Jet Propulsion Laboratory</a>\n",
      "</div>, <div class=\"jpl_logo\">\n",
      "<a class=\"\" href=\"\" id=\"jpl_logo\" title=\"\">Jet Propulsion Laboratory</a>\n",
      "</div>]\n"
     ]
    }
   ],
   "source": [
    "# Get the base url from the href of the website\n",
    "jpl_logo = url.find_all('div', class_ = 'jpl_logo')\n",
    "print(jpl_logo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating bs object and parse it with 'html.parser'\n",
    "\n",
    "html_page = browser.html\n",
    "jpl_url_soup = bs(html_page, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['http://www.nasa.gov', '//www.jpl.nasa.gov/', 'http://www.caltech.edu/', '#main', 'javascript:void(0);', 'http://www.nasa.gov', '', '', None, 'javascript:void(0);', '/about', '/about', '/about/exec.php', '/about/history.php', '/about/reports.php', '/contact_JPL.php', '/opportunities/', '/events', '/events', '/events/tours/views', '/events/lectures.php', '/events/speakers-bureau.php', '/events/team-competitions.php', '/events/special-events.php', '/edu/', '/edu/intern/', '/edu/learn/', '/edu/teach/', '/edu/news/', '/edu/events/', '/news', '/news', '/news/presskits.php', '/news/factsheets.php', '/news/mediainformation.php', 'http://blogs.jpl.nasa.gov', '/missions/', '/missions/?type=current', '/missions/?type=past', '/missions/?type=future', '/missions/?type=proposed', '/missions', '/spaceimages', '/spaceimages', '/videos', '/infographics', '/multimedia/audio.php', '/apps/', '/social', 'http://www.facebook.com/NASAJPL', '//twitter.com/NASAJPL', 'http://www.youtube.com/user/JPLnews?sub_confirmation=1', 'http://instagram.com/nasajpl', '/social', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, '', 'http://photojournal.jpl.nasa.gov/', 'http://photojournal.jpl.nasa.gov/', 'http://www.nasa.gov/multimedia/imagegallery/', 'http://www.nasa.gov/multimedia/imagegallery/', '//www.jpl.nasa.gov/news/news.php?feature=7216', '//www.jpl.nasa.gov/news/news.php?feature=7215', '//www.jpl.nasa.gov/news/news.php?feature=7211', '/news', 'http://www.facebook.com/NASAJPL', '//twitter.com/NASAJPL', 'http://www.youtube.com/user/JPLnews?sub_confirmation=1', 'http://instagram.com/nasajpl', '/social', '/about/', 'https://www.jpl.nasa.gov/jpl2025/vision/', '/about/exec.php', '/about/history.php', '/about/reports.php', '/contact_JPL.php', '/opportunities/', 'https://thejplstore.com', '/acquisition/', '/missions/?type=current', '/missions/?type=past', '/missions/?type=future', '/missions/?type=proposed', '/missions', '/edu/intern/', '/edu/learn/', '/edu/teach/', '/edu/news/', '/edu/events/', '/news', '/news/presskits.php', '/news/factsheets.php', '/news/mediainformation.php', '/universe/', '/events/', '/events/tours/views/', '/events/lectures.php', '/events/speakers-bureau.php', '/events/team-competitions.php', '/events/special-events.php', '/asteroidwatch/', 'http://saturn.jpl.nasa.gov/index.cfm', 'http://climate.nasa.gov', 'http://planetquest.jpl.nasa.gov', '/missions/juno/', 'http://marsprogram.jpl.nasa.gov/', 'http://marsprogram.jpl.nasa.gov/msl/', 'http://rosetta.jpl.nasa.gov/', 'http://scienceandtechnology.jpl.nasa.gov/', 'http://solarsystem.nasa.gov/', 'https://eyes.nasa.gov/', 'http://www.spitzer.caltech.edu/', '/spaceimages/', '/videos/', '/infographics/', 'https://photojournal.jpl.nasa.gov/', 'http://www.nasaimages.org/', '/apps/', '/signup/', 'https://www.facebook.com/NASAJPL', 'http://twitter.com/NASAJPL', 'http://www.youtube.com/user/JPLnews', 'http://www.flickr.com/photos/nasa-jpl', 'http://instagram.com/nasajpl', 'https://www.linkedin.com/company/2004/', 'http://itunes.apple.com/podcast/hd-nasas-jet-propulsion-laboratory/id262254981', 'http://www.ustream.tv/nasajpl', '/rss/', 'http://blogs.jpl.nasa.gov', '/onthego/', '/social/', 'http://jplwater.nasa.gov', 'http://www.hq.nasa.gov/office/pao/FOIA/agency/', 'http://www.nasa.gov/', 'http://www.caltech.edu/', '/copyrights.php', '/imagepolicy', '/faq.php', '/contact_JPL.php']\n"
     ]
    }
   ],
   "source": [
    "# all hrefs of url \n",
    "links = []\n",
    "for link in jpl_url_soup.find_all('a'):\n",
    "    links.append(link.get('href'))\n",
    "    \n",
    "print(links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "www.jpl.nasa.gov\n"
     ]
    }
   ],
   "source": [
    "# to save the 2nd href on the list of href\n",
    "jpl_link = links[1].strip('/')\n",
    "print(jpl_link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.jpl.nasa.gov/spaceimages/images/mediumsize/PIA15883_ip.jpg\n"
     ]
    }
   ],
   "source": [
    "featured_image_url = 'https://'+ jpl_link + image_url\n",
    "print(featured_image_url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mars Weather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using chromedirver to open Mars weather twitter page\n",
    "twitter_url = \"https://twitter.com/marswxreport?lang=en\"\n",
    "browser.visit(twitter_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = browser.html\n",
    "twitter_news = bs(html, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Sol 2146 (2018-08-20), high -10C/14F, low -67C/-88F, pressure at 8.70 hPa, daylight 05:29-17:43'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weather = twitter_news.find('p', class_='TweetTextSize TweetTextSize--normal js-tweet-text tweet-text').text.strip()\n",
    "weather"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mars Facts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using chromedriver\n",
    "mars_facts_url = \"https://space-facts.com/mars/\"\n",
    "browser.visit(mars_facts_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert to pandas df\n",
    "mars_dataframe = pd.read_html(mars_facts_url)\n",
    "mars_facts_df = pd.DataFrame(mars_dataframe[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Characteristic</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Equatorial Diameter:</th>\n",
       "      <td>6,792 km</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Polar Diameter:</th>\n",
       "      <td>6,752 km</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mass:</th>\n",
       "      <td>6.42 x 10^23 kg (10.7% Earth)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Moons:</th>\n",
       "      <td>2 (Phobos &amp; Deimos)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Orbit Distance:</th>\n",
       "      <td>227,943,824 km (1.52 AU)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Orbit Period:</th>\n",
       "      <td>687 days (1.9 years)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Surface Temperature:</th>\n",
       "      <td>-153 to 20 °C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>First Record:</th>\n",
       "      <td>2nd millennium BC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recorded By:</th>\n",
       "      <td>Egyptian astronomers</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Data\n",
       "Characteristic                                     \n",
       "Equatorial Diameter:                       6,792 km\n",
       "Polar Diameter:                            6,752 km\n",
       "Mass:                 6.42 x 10^23 kg (10.7% Earth)\n",
       "Moons:                          2 (Phobos & Deimos)\n",
       "Orbit Distance:            227,943,824 km (1.52 AU)\n",
       "Orbit Period:                  687 days (1.9 years)\n",
       "Surface Temperature:                  -153 to 20 °C\n",
       "First Record:                     2nd millennium BC\n",
       "Recorded By:                   Egyptian astronomers"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the columns and set the index.\n",
    "mars_facts_df.columns = ['Characteristic','Data']\n",
    "mars_df_table = mars_facts_df.set_index('Characteristic')\n",
    "mars_df_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<table border=\"1\" class=\"dataframe marsdata\">  <thead>    <tr style=\"text-align: right;\">      <th></th>      <th>Data</th>    </tr>    <tr>      <th>Characteristic</th>      <th></th>    </tr>  </thead>  <tbody>    <tr>      <th>Equatorial Diameter:</th>      <td>6,792 km</td>    </tr>    <tr>      <th>Polar Diameter:</th>      <td>6,752 km</td>    </tr>    <tr>      <th>Mass:</th>      <td>6.42 x 10^23 kg (10.7% Earth)</td>    </tr>    <tr>      <th>Moons:</th>      <td>2 (Phobos &amp; Deimos)</td>    </tr>    <tr>      <th>Orbit Distance:</th>      <td>227,943,824 km (1.52 AU)</td>    </tr>    <tr>      <th>Orbit Period:</th>      <td>687 days (1.9 years)</td>    </tr>    <tr>      <th>Surface Temperature:</th>      <td>-153 to 20 °C</td>    </tr>    <tr>      <th>First Record:</th>      <td>2nd millennium BC</td>    </tr>    <tr>      <th>Recorded By:</th>      <td>Egyptian astronomers</td>    </tr>  </tbody></table>'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert the pandas dataframe to html to clean up ( use to_html)\n",
    "mars_htmlTable = mars_df_table.to_html(classes='marsdata')\n",
    "mars_table = mars_htmlTable.replace('\\n', '')\n",
    "mars_table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mars Hemispheres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Usin Chromedriver to open up USGS website\n",
    "browser = Browser('chrome', headless=False)\n",
    "Mars_hemisphere_url = \"https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars\"\n",
    "browser.visit(Mars_hemisphere_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = browser.html\n",
    "Mars_hemisphere = bs(html, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Cerberus Hemisphere Enhanced',\n",
       " 'Schiaparelli Hemisphere Enhanced',\n",
       " 'Syrtis Major Hemisphere Enhanced',\n",
       " 'Valles Marineris Hemisphere Enhanced']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find all description without the text\n",
    "a = Mars_hemisphere.find_all('h3')\n",
    "descriptions = [h3.text.strip() for h3 in a]\n",
    "descriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "link1 = 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/cerberus_enhanced.tif/full.jpg'\n",
    "link2 = 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/schiaparelli_enhanced.tif/full.jpg'\n",
    "link3 = 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/syrtis_major_enhanced.tif/full.jpg'\n",
    "link4 = 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/valles_marineris_enhanced.tif/full.jpg'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "links = [link1,link2,link3,link4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'img_url': 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/cerberus_enhanced.tif/full.jpg',\n",
       "  'title': 'Cerberus Hemisphere Enhanced'},\n",
       " {'img_url': 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/schiaparelli_enhanced.tif/full.jpg',\n",
       "  'title': 'Schiaparelli Hemisphere Enhanced'},\n",
       " {'img_url': 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/syrtis_major_enhanced.tif/full.jpg',\n",
       "  'title': 'Syrtis Major Hemisphere Enhanced'},\n",
       " {'img_url': 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/valles_marineris_enhanced.tif/full.jpg',\n",
       "  'title': 'Valles Marineris Hemisphere Enhanced'}]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hemisphere_image_url = [{'title' : description, 'img_url': link} for description, link in zip(descriptions, links)]\n",
    "hemisphere_image_url"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}