improving warning messages and typing hints

.github/workflows/documentation.yml

@@ -46,6 +46,9 @@ jobs:
  run: poetry install
  - name: Build docs
  run: poetry run bash scripts/build-docs.sh
+ env:
+ EARTHDATA_USERNAME: ${{ secrets.EDL_USERNAME }}
+ EARTHDATA_PASSWORD: ${{ secrets.EDL_PASSWORD }}
 
  - name: Deploy
  if: |

docs/CONTRIBUTING.md

@@ -0,0 +1 @@
+../CONTRIBUTING.md

docs/tutorials/demo.ipynb → docs/tutorials/queries.ipynb

@@ -8,7 +8,7 @@
  },
  "source": [
  "\n",
- "## Overview\n",
+ "# Overview\n",
  "\n",
  "\n",
  "# <img src=\"https://logos-world.net/wp-content/uploads/2020/05/NASA-Logo-1959-present.png\" width=\"100px\" align=\"middle\" /> Introducing NASA earthaccess 🌍\n",
@@ -38,14 +38,41 @@
  "\n",
  "Earthdata Login provides free and immediate access to thousands of EOSDIS data products covering all Earth science disciplines and topic areas for researchers, applied science users, application developers, and the general public.\n",
  "\n",
- "Once we have our NASA EDL login credentials we can start accessing NASA data in a programmatic way.\n"
+ "Once we have our NASA EDL login credentials we can start accessing NASA data in a programmatic way.\n",
+ "\n",
+ "\n",
+ "## Querying CMR using earthaccess\n",
+ "\n",
+ "This short tutorial uses the `collection_query()` and `granule_query()` methods, these methods return a lower level Query Builder instance that can be used to query NASA's CMR.\n",
+ "For convenience the top level API also offers the `dataset_search(**kwargs)` and `data_search(**kwargs)` methods that map what these query builders do. \n",
+ "\n",
+ "For instance \n",
+ "\n",
+ "```python\n",
+ "query = earthaccess.granule_query().doi(\"some_doi\").temporal(\"1990-01-01\", \"2020-12-31\").cloud_hosted(True)\n",
+ "granules = query.get(10)\n",
+ "\n",
+ "```\n",
+ "\n",
+ "is equivalent to\n",
+ "\n",
+ "```python\n",
+ "granules = earthaccess.search_data(\n",
+ " doi=\"some_doi\",\n",
+ " temporal = (\"1990-01-01\",\"2020-12-31\"),\n",
+ " cloud_hosted=True,\n",
+ " limit=10\n",
+ ")\n",
+ "```"
  ]
  },
  {
  "cell_type": "code",
  "execution_count": null,
  "id": "10f6c9ed-fe58-4e03-b29b-c6c447061f84",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "import earthaccess\n",
@@ -85,12 +112,14 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "caab3b4b-80cc-4790-9417-1dd12503aa55",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "# are we authenticated?\n",
  "\n",
- "auth = earthaccess.login(strategy=\"netrc\")\n"
+ "auth = earthaccess.login()\n"
  ]
  },
  {
@@ -107,7 +136,9 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "8d5bf4c9-571b-4c93-af94-e66bd51cb584",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "# The first step is to create a DataCollections query \n",
@@ -139,7 +170,9 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "8cb5154c-f131-44ad-a68f-cf0fa21ce18f",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "collections[0][\"umm\"][\"ShortName\"]"
@@ -171,11 +204,13 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "48cdcd74-dfe3-4b83-93f4-7378a0d981df",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "# We can now search for collections using a pythonic API client for CMR.\n",
- "Query = earthaccess.collection_query().daac(\"PODAAC\")\n",
+ "Query = earthaccess.collection_query().daac(\"ASF\")\n",
  "\n",
  "print(f'Collections found: {Query.hits()}')\n",
  "collections = Query.fields(['ShortName']).get(10)\n",
@@ -187,11 +222,13 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "63792353-ab3e-4f0b-963d-7750e4b89113",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "# What if we want cloud collections\n",
- "Query = earthaccess.collection_query().daac(\"PODAAC\").cloud_hosted(True)\n",
+ "Query = earthaccess.collection_query().daac(\"ASF\").cloud_hosted(True)\n",
  "\n",
  "print(f'Collections found: {Query.hits()}')\n",
  "collections = Query.fields(['ShortName']).get(10)\n",
@@ -203,7 +240,9 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "c4c5a34a-e808-4cc9-b34d-353d091a8242",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "# Printing the concept-id for the first 10 collections\n",
@@ -230,17 +269,18 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "9364d737-5a79-4089-853f-76d2ad1c85a7",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "from pprint import pprint\n",
  "\n",
  "# We build our query\n",
  "\n",
- "Query = earthaccess.granule_query().short_name('ATL06').version(\"005\").bounding_box(-134.7,58.9,-133.9,59.2)\n",
+ "Query = earthaccess.granule_query().short_name('HLSL30').bounding_box(-134.7,58.9,-133.9,59.2)\n",
  "# We get 5 metadata records\n",
- "granules = Query.get(5)\n",
- "granules"
+ "granules = Query.get(5)"
  ]
  },
  {
@@ -257,12 +297,13 @@
  {
  "cell_type": "code",
  "execution_count": null,
- "id": "66cd5f5c-a854-4a72-a831-33b8bd7ce9d2",
- "metadata": {},
+ "id": "0b56b119-ec9b-4922-911a-f37501597451",
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
- "# printing 2 granules using display\n",
- "[display(granule) for granule in granules]"
+ "[display(g) for g in granules]"
  ]
  },
  {
@@ -280,7 +321,9 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "00aa39ec-e2fb-49d1-bc54-8d8a2f0655aa",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "Query = earthaccess.granule_query().short_name(\"ATL06\").temporal(\"2020-03-01\", \"2020-03-30\").bounding_box(-134.7,58.9,-133.9,59.2).version(\"005\")\n",
@@ -292,7 +335,9 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "8c493585-0d48-41bb-8815-6c83ad20ae80",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "# Now we can print some info about these granules using the built-in methods\n",
@@ -325,7 +370,9 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "910e4b90-f0e0-42e5-a4e2-d5444089161f",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "import earthaccess\n",
@@ -354,11 +401,13 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "434466a3-602b-4dff-a260-f7db6901514a",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "%%time\n",
- "files = earthaccess.download(granules[0:4], \"./data/C1972955240-PODAAC/\")"
+ "files = earthaccess.download(granules[0:2], \"./data/C1972955240-PODAAC/\")"
  ]
  },
  {
@@ -381,7 +430,9 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "44403d51-0aa3-423c-8fff-e40d4969aa9d",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "\n",
@@ -396,7 +447,9 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "5e59ca3e-b5d5-490f-b967-01d1c7b3fdf0",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "# Let's pretty print this\n",
@@ -407,7 +460,9 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "b2a294f1-b1f9-4cd4-8751-dfc32feacec1",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "%%time\n",
@@ -436,7 +491,9 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "aecdb529-5961-4fa6-b7e0-70bbd0d85041",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "import warnings\n",
@@ -451,8 +508,23 @@
  "\n",
  "for granule in results:\n",
  " https_links.extend(granule.data_links(access=\"on_prem\"))\n",
- " s3_links.extend(granule.data_links(access=\"direct\"))\n",
- "s3_links"
+ " s3_links.extend(granule.data_links(access=\"direct\"))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "50e6f01e-86f0-4e29-869b-d6d437c8b130",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "fileset = earthaccess.open(results[0:3])\n",
+ "\n",
+ "# test that we can read data from the files\n",
+ "with fileset[0] as f:\n",
+ " print(f.read(100))"
  ]
  },
  {
@@ -467,16 +539,21 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "e693af6a-a80e-4ca2-a034-8da194c18aaf",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
  "outputs": [],
  "source": [
  "%%time\n",
  "\n",
+ "# Note: this will be slow if we do it out of AWS\n",
+ "\n",
  "ds_L3 = xr.open_mfdataset(\n",
- " earthaccess.open(results[0:3]),\n",
+ " fileset,\n",
  " combine='nested',\n",
  " concat_dim='time',\n",
  " coords='minimal',\n",
+ " engine=\"h5netcdf\"\n",
  " )\n",
  "ds_L3"
  ]
@@ -548,7 +625,7 @@
  "name": "python",
  "nbconvert_exporter": "python",
  "pygments_lexer": "ipython3",
- "version": "3.9.14"
+ "version": "3.9.16"
  }
  },
  "nbformat": 4,

earthaccess/api.py

@@ -1,10 +1,11 @@
 from typing import Any, Dict, List, Optional, Type, Union
 
-import earthaccess
 import requests
 import s3fs
 from fsspec import AbstractFileSystem
 
+import earthaccess
+
 from .auth import Auth
 from .search import CollectionQuery, DataCollections, DataGranules, GranuleQuery
 from .store import Store
@@ -168,8 +169,7 @@ def download(
  except AttributeError as err:
  print(err)
  print("You must call earthaccess.login() before you can download data")
- return None
-
+ return []
  return results
 
 

earthaccess/formatters.py

@@ -23,7 +23,7 @@ def _repr_granule_html(granule: Any) -> str:
  css_inline = f"""<div id="{uuid4()}" style="height: 0px; display: none">
  {''.join([f"<style>{style}</style>" for style in css_styles])}
  </div>"""
- style = "max-height: 140px;"
+ style = "max-height: 120px;"
  dataviz_img = "".join(
  [
  f'<a href="{link}"><img style="{style}" src="{link}" alt="Data Preview"/></a>'
@@ -47,7 +47,7 @@ def _repr_granule_html(granule: Any) -> str:
  <div class="col-6">
  <p><b>Data</b>: {data_links}<p/>
  <p><b>Size</b>: {granule_size} MB</p>
- <p><b>Spatial</b>: <span>{granule["umm"]["SpatialExtent"]}</span></p>
+ <p><b>Cloud Hosted</b>: <span>{granule.cloud_hosted}</span></p>
  </div>
  <div class="col-2 offset-sm-3 pull-right">
  {dataviz_img}