A1_get_hls_metadata_associated_to_dswx.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4433a8ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "from cmr import GranuleQuery\n",
    "import datetime\n",
    "from shapely.geometry import Polygon\n",
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "from tqdm import tqdm\n",
    "from osgeo import gdal\n",
    "import rasterio\n",
    "import concurrent.futures\n",
    "import backoff\n",
    "from rasterio.errors import RasterioIOError\n",
    "import requests\n",
    "import boto3\n",
    "from rasterio.session import AWSSession\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38369d8a",
   "metadata": {},
   "source": [
    "Source: https://git.earthdata.nasa.gov/projects/LPDUR/repos/lpdaac_cloud_data_access/browse/notebooks/Topic_2__Cloud_Optimized_Data.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "750dcb88",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<rasterio.env.Env at 0x7fca555a86d0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_temp_creds():\n",
    "    temp_creds_url = 'https://data.lpdaac.earthdatacloud.nasa.gov/s3credentials'\n",
    "    return requests.get(temp_creds_url).json()\n",
    "temp_creds_req = get_temp_creds()\n",
    "\n",
    "session = boto3.Session(aws_access_key_id=temp_creds_req['accessKeyId'], \n",
    "                        aws_secret_access_key=temp_creds_req['secretAccessKey'],\n",
    "                        aws_session_token=temp_creds_req['sessionToken'],\n",
    "                        region_name='us-west-2')\n",
    "rio_env = rasterio.Env(AWSSession(session),\n",
    "                       GDAL_DISABLE_READDIR_ON_OPEN='TRUE',\n",
    "                       CPL_VSIL_CURL_ALLOWED_EXTENSIONS='tif',\n",
    "                       VSI_CACHE=True,\n",
    "                       region_name='us-west-2',\n",
    "                       GDAL_HTTP_COOKIEFILE=os.path.expanduser('~/cookies.txt'),\n",
    "                       GDAL_HTTP_COOKIEJAR=os.path.expanduser('~/cookies.txt'))\n",
    "rio_env.__enter__()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6443080",
   "metadata": {},
   "source": [
    "# Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9ce76a0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "HLS_S30_CONCEPT_ID = 'C2021957295-LPCLOUD'\n",
    "HLS_L30_CONCEPT_ID = 'C2021957657-LPCLOUD'\n",
    "DSWX_CONCEPT_ID = 'C2617126679-POCLOUD'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1be95d4e",
   "metadata": {},
   "source": [
    "# Read DSWx metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0ad2d2a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dswx = gpd.read_file('dswx_metadata_with_tags.geojson')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "360482c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def hls_url_generator(hls_id: str, band='B04') -> str:\n",
    "    base_url = 'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected'\n",
    "    tokens = hls_id.split('.')\n",
    "    prefix = f'{tokens[0]}{tokens[1]}.020'\n",
    "    return f'{base_url}/{prefix}/{hls_id}/{hls_id}.{band}.tif'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d23b72a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/HLSL30.020/HLS.L30.T32RMT.2023124T095600.v2.0/HLS.L30.T32RMT.2023124T095600.v2.0.B04.tif\n"
     ]
    }
   ],
   "source": [
    "hls_id = 'HLS.L30.T32RMT.2023124T095600.v2.0'\n",
    "print(hls_url_generator(hls_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3656f0e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = df_dswx.hls_dataset.map(hls_url_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3a1a701a",
   "metadata": {},
   "outputs": [],
   "source": [
    "@backoff.on_exception(backoff.expo,\n",
    "                      Exception,\n",
    "                      max_tries=10)\n",
    "def get_hls_tag_data(url: str) -> dict:\n",
    "    with rasterio.open(url) as ds:\n",
    "        t = ds.tags()\n",
    "    out = {key.lower() + '_hls': val for key, val in t.items()}\n",
    "    return out"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51f2a01b",
   "metadata": {},
   "source": [
    "Sequentially:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4fec9995",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:03<00:00,  1.26s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'accode_hls': 'LaSRC',\n",
       "  'add_offset_hls': '0.0',\n",
       "  'area_or_point_hls': 'Area',\n",
       "  'arop_ave_xshift(meters)_hls': '0',\n",
       "  'arop_ave_yshift(meters)_hls': '0',\n",
       "  'arop_ncp_hls': '0',\n",
       "  'arop_rmse(meters)_hls': '0',\n",
       "  'arop_s2_refimg_hls': 'NONE',\n",
       "  'cloud_coverage_hls': '5',\n",
       "  'datastrip_id_hls': 'S2B_OPER_MSI_L1C_DS_2BPS_20230504T140720_S20230504T123257_N05.09',\n",
       "  'hls_processing_time_hls': '2023-05-06T08:18:05Z',\n",
       "  'horizontal_cs_code_hls': 'EPSG:32725',\n",
       "  'horizontal_cs_name_hls': 'WGS84 / UTM zone 25S',\n",
       "  'l1c_image_quality_hls': 'NONE',\n",
       "  'l1_processing_time_hls': '2023-05-04T14:22:41.440202Z',\n",
       "  'long_name_hls': 'Red',\n",
       "  'mean_sun_azimuth_angle_hls': '48.3220864006816',\n",
       "  'mean_sun_zenith_angle_hls': '30.5477928719829',\n",
       "  'mean_view_azimuth_angle_hls': '103.938965586048',\n",
       "  'mean_view_zenith_angle_hls': '8.74447582209279',\n",
       "  'msi band 01 bandpass adjustment slope and offset_hls': '0.995900, -0.000200',\n",
       "  'msi band 02 bandpass adjustment slope and offset_hls': '0.977800, -0.004000',\n",
       "  'msi band 03 bandpass adjustment slope and offset_hls': '1.007500, -0.000800',\n",
       "  'msi band 04 bandpass adjustment slope and offset_hls': '0.976100, 0.001000',\n",
       "  'msi band 11 bandpass adjustment slope and offset_hls': '1.000000, -0.000300',\n",
       "  'msi band 12 bandpass adjustment slope and offset_hls': '0.986700, 0.000400',\n",
       "  'msi band 8a bandpass adjustment slope and offset_hls': '0.996600, 0.000000',\n",
       "  'nbar_solar_zenith_hls': '31.6621002873495',\n",
       "  'ncols_hls': '3660',\n",
       "  'nrows_hls': '3660',\n",
       "  'ovr_resampling_alg_hls': 'NEAREST',\n",
       "  'processing_baseline_hls': '05.09',\n",
       "  'product_uri_hls': 'S2B_MSIL1C_20230504T123259_N0509_R109_T25MER_20230504T140720.SAFE',\n",
       "  'scale_factor_hls': '0.0001',\n",
       "  'sensing_time_hls': '2023-05-04T12:33:08.84582Z',\n",
       "  'spacecraft_name_hls': 'Sentinel-2B',\n",
       "  'spatial_coverage_hls': '59',\n",
       "  'spatial_resolution_hls': '30',\n",
       "  'tile_id_hls': 'S2B_OPER_MSI_L1C_TL_2BPS_20230504T140720_A032167_T25MER_N05.09',\n",
       "  'ulx_hls': '499980',\n",
       "  'uly_hls': '-399960',\n",
       "  '_fillvalue_hls': '-9999'},\n",
       " {'accode_hls': 'Lasrc',\n",
       "  'add_offset_hls': '0.0',\n",
       "  'area_or_point_hls': 'Area',\n",
       "  'arop_ave_xshift(meters)_hls': '0',\n",
       "  'arop_ave_yshift(meters)_hls': '0',\n",
       "  'arop_ncp_hls': '0',\n",
       "  'arop_rmse(meters)_hls': '0',\n",
       "  'arop_s2_refimg_hls': 'NONE',\n",
       "  'cloud_coverage_hls': '13',\n",
       "  'hls_processing_time_hls': '2023-05-06T07:21:25Z',\n",
       "  'horizontal_cs_name_hls': 'UTM, WGS84, UTM ZONE 42',\n",
       "  'l1_processing_time_hls': '2023-05-04T16:04:35Z',\n",
       "  'landsat_product_id_hls': 'LC08_L1TP_207244_20230504_20230504_02_RT',\n",
       "  'landsat_scene_id_hls': 'LC82072442023124LGN00',\n",
       "  'long_name_hls': 'Red',\n",
       "  'mean_sun_azimuth_angle_hls': '264.560107823245',\n",
       "  'mean_sun_zenith_angle_hls': '72.9258968299621',\n",
       "  'mean_view_azimuth_angle_hls': '16.4119240911807',\n",
       "  'mean_view_zenith_angle_hls': '6.03167516491508',\n",
       "  'nbar_solar_zenith_hls': '66.8335586216939',\n",
       "  'ncols_hls': '3660',\n",
       "  'nrows_hls': '3660',\n",
       "  'ovr_resampling_alg_hls': 'NEAREST',\n",
       "  'processing_level_hls': 'L1TP',\n",
       "  'scale_factor_hls': '0.0001',\n",
       "  'sensing_time_hls': '2023-05-04T12:56:39.1871320Z',\n",
       "  'sensor_hls': 'OLI_TIRS',\n",
       "  'sentinel2_tileid_hls': '42XVQ',\n",
       "  'spatial_coverage_hls': '46',\n",
       "  'spatial_resolution_hls': '30',\n",
       "  'tirs_ssm_model_hls': 'PRELIMINARY',\n",
       "  'tirs_ssm_position_status_hls': 'ESTIMATED',\n",
       "  'ulx_hls': '399960',\n",
       "  'uly_hls': '9000000',\n",
       "  'usgs_software_hls': 'LPGS_16.2.0',\n",
       "  '_fillvalue_hls': '-9999'},\n",
       " {'accode_hls': 'Lasrc',\n",
       "  'add_offset_hls': '0.0',\n",
       "  'area_or_point_hls': 'Area',\n",
       "  'arop_ave_xshift(meters)_hls': '0',\n",
       "  'arop_ave_yshift(meters)_hls': '0',\n",
       "  'arop_ncp_hls': '0',\n",
       "  'arop_rmse(meters)_hls': '0',\n",
       "  'arop_s2_refimg_hls': 'NONE',\n",
       "  'cloud_coverage_hls': '16',\n",
       "  'hls_processing_time_hls': '2023-05-06T07:16:09Z',\n",
       "  'horizontal_cs_name_hls': 'UTM, WGS84, UTM ZONE 42',\n",
       "  'l1_processing_time_hls': '2023-05-04T16:04:35Z',\n",
       "  'landsat_product_id_hls': 'LC08_L1TP_207244_20230504_20230504_02_RT',\n",
       "  'landsat_scene_id_hls': 'LC82072442023124LGN00',\n",
       "  'long_name_hls': 'Red',\n",
       "  'mean_sun_azimuth_angle_hls': '264.057858414389',\n",
       "  'mean_sun_zenith_angle_hls': '72.8473788332777',\n",
       "  'mean_view_azimuth_angle_hls': '16.2212830062955',\n",
       "  'mean_view_zenith_angle_hls': '6.15860630470203',\n",
       "  'nbar_solar_zenith_hls': '66.8246518395416',\n",
       "  'ncols_hls': '3660',\n",
       "  'nrows_hls': '3660',\n",
       "  'ovr_resampling_alg_hls': 'NEAREST',\n",
       "  'processing_level_hls': 'L1TP',\n",
       "  'scale_factor_hls': '0.0001',\n",
       "  'sensing_time_hls': '2023-05-04T12:56:39.1871320Z',\n",
       "  'sensor_hls': 'OLI_TIRS',\n",
       "  'sentinel2_tileid_hls': '41XNK',\n",
       "  'spatial_coverage_hls': '43',\n",
       "  'spatial_resolution_hls': '30',\n",
       "  'tirs_ssm_model_hls': 'PRELIMINARY',\n",
       "  'tirs_ssm_position_status_hls': 'ESTIMATED',\n",
       "  'ulx_hls': '499980',\n",
       "  'uly_hls': '9000000',\n",
       "  'usgs_software_hls': 'LPGS_16.2.0',\n",
       "  '_fillvalue_hls': '-9999'}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hls_tags = list(map(get_hls_tag_data, tqdm(urls[:3])))\n",
    "hls_tags"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b87b1772",
   "metadata": {},
   "source": [
    "Multithread:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "168d739f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████| 50596/50596 [22:36<00:00, 37.29it/s]\n"
     ]
    }
   ],
   "source": [
    "n = len(urls)\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:\n",
    "    hls_tags = list(tqdm(executor.map(get_hls_tag_data, urls[:]), total=n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a13fb3b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accode_hls</th>\n",
       "      <th>add_offset_hls</th>\n",
       "      <th>area_or_point_hls</th>\n",
       "      <th>arop_ave_xshift(meters)_hls</th>\n",
       "      <th>arop_ave_yshift(meters)_hls</th>\n",
       "      <th>arop_ncp_hls</th>\n",
       "      <th>arop_rmse(meters)_hls</th>\n",
       "      <th>arop_s2_refimg_hls</th>\n",
       "      <th>cloud_coverage_hls</th>\n",
       "      <th>datastrip_id_hls</th>\n",
       "      <th>...</th>\n",
       "      <th>uly_hls</th>\n",
       "      <th>_fillvalue_hls</th>\n",
       "      <th>landsat_product_id_hls</th>\n",
       "      <th>landsat_scene_id_hls</th>\n",
       "      <th>processing_level_hls</th>\n",
       "      <th>sensor_hls</th>\n",
       "      <th>sentinel2_tileid_hls</th>\n",
       "      <th>tirs_ssm_model_hls</th>\n",
       "      <th>tirs_ssm_position_status_hls</th>\n",
       "      <th>usgs_software_hls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LaSRC</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Area</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NONE</td>\n",
       "      <td>5</td>\n",
       "      <td>S2B_OPER_MSI_L1C_DS_2BPS_20230504T140720_S2023...</td>\n",
       "      <td>...</td>\n",
       "      <td>-399960</td>\n",
       "      <td>-9999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Lasrc</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Area</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NONE</td>\n",
       "      <td>13</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>9000000</td>\n",
       "      <td>-9999</td>\n",
       "      <td>LC08_L1TP_207244_20230504_20230504_02_RT</td>\n",
       "      <td>LC82072442023124LGN00</td>\n",
       "      <td>L1TP</td>\n",
       "      <td>OLI_TIRS</td>\n",
       "      <td>42XVQ</td>\n",
       "      <td>PRELIMINARY</td>\n",
       "      <td>ESTIMATED</td>\n",
       "      <td>LPGS_16.2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lasrc</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Area</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NONE</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>9000000</td>\n",
       "      <td>-9999</td>\n",
       "      <td>LC08_L1TP_207244_20230504_20230504_02_RT</td>\n",
       "      <td>LC82072442023124LGN00</td>\n",
       "      <td>L1TP</td>\n",
       "      <td>OLI_TIRS</td>\n",
       "      <td>41XNK</td>\n",
       "      <td>PRELIMINARY</td>\n",
       "      <td>ESTIMATED</td>\n",
       "      <td>LPGS_16.2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Lasrc</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Area</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NONE</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>9100020</td>\n",
       "      <td>-9999</td>\n",
       "      <td>LC08_L1TP_207244_20230504_20230504_02_RT</td>\n",
       "      <td>LC82072442023124LGN00</td>\n",
       "      <td>L1TP</td>\n",
       "      <td>OLI_TIRS</td>\n",
       "      <td>42XVR</td>\n",
       "      <td>PRELIMINARY</td>\n",
       "      <td>ESTIMATED</td>\n",
       "      <td>LPGS_16.2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Lasrc</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Area</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NONE</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>9100020</td>\n",
       "      <td>-9999</td>\n",
       "      <td>LC08_L1TP_207244_20230504_20230504_02_RT</td>\n",
       "      <td>LC82072442023124LGN00</td>\n",
       "      <td>L1TP</td>\n",
       "      <td>OLI_TIRS</td>\n",
       "      <td>40XER</td>\n",
       "      <td>PRELIMINARY</td>\n",
       "      <td>ESTIMATED</td>\n",
       "      <td>LPGS_16.2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  accode_hls add_offset_hls area_or_point_hls arop_ave_xshift(meters)_hls   \n",
       "0      LaSRC            0.0              Area                           0  \\\n",
       "1      Lasrc            0.0              Area                           0   \n",
       "2      Lasrc            0.0              Area                           0   \n",
       "3      Lasrc            0.0              Area                           0   \n",
       "4      Lasrc            0.0              Area                           0   \n",
       "\n",
       "  arop_ave_yshift(meters)_hls arop_ncp_hls arop_rmse(meters)_hls   \n",
       "0                           0            0                     0  \\\n",
       "1                           0            0                     0   \n",
       "2                           0            0                     0   \n",
       "3                           0            0                     0   \n",
       "4                           0            0                     0   \n",
       "\n",
       "  arop_s2_refimg_hls cloud_coverage_hls   \n",
       "0               NONE                  5  \\\n",
       "1               NONE                 13   \n",
       "2               NONE                 16   \n",
       "3               NONE                  8   \n",
       "4               NONE                  3   \n",
       "\n",
       "                                    datastrip_id_hls  ...  uly_hls   \n",
       "0  S2B_OPER_MSI_L1C_DS_2BPS_20230504T140720_S2023...  ...  -399960  \\\n",
       "1                                                NaN  ...  9000000   \n",
       "2                                                NaN  ...  9000000   \n",
       "3                                                NaN  ...  9100020   \n",
       "4                                                NaN  ...  9100020   \n",
       "\n",
       "  _fillvalue_hls                    landsat_product_id_hls   \n",
       "0          -9999                                       NaN  \\\n",
       "1          -9999  LC08_L1TP_207244_20230504_20230504_02_RT   \n",
       "2          -9999  LC08_L1TP_207244_20230504_20230504_02_RT   \n",
       "3          -9999  LC08_L1TP_207244_20230504_20230504_02_RT   \n",
       "4          -9999  LC08_L1TP_207244_20230504_20230504_02_RT   \n",
       "\n",
       "    landsat_scene_id_hls processing_level_hls sensor_hls sentinel2_tileid_hls   \n",
       "0                    NaN                  NaN        NaN                  NaN  \\\n",
       "1  LC82072442023124LGN00                 L1TP   OLI_TIRS                42XVQ   \n",
       "2  LC82072442023124LGN00                 L1TP   OLI_TIRS                41XNK   \n",
       "3  LC82072442023124LGN00                 L1TP   OLI_TIRS                42XVR   \n",
       "4  LC82072442023124LGN00                 L1TP   OLI_TIRS                40XER   \n",
       "\n",
       "  tirs_ssm_model_hls tirs_ssm_position_status_hls usgs_software_hls  \n",
       "0                NaN                          NaN               NaN  \n",
       "1        PRELIMINARY                    ESTIMATED       LPGS_16.2.0  \n",
       "2        PRELIMINARY                    ESTIMATED       LPGS_16.2.0  \n",
       "3        PRELIMINARY                    ESTIMATED       LPGS_16.2.0  \n",
       "4        PRELIMINARY                    ESTIMATED       LPGS_16.2.0  \n",
       "\n",
       "[5 rows x 50 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hls = pd.DataFrame(hls_tags)\n",
    "df_hls.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "fd1e43a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>granule_id</th>\n",
       "      <th>time_acquired</th>\n",
       "      <th>time_updated</th>\n",
       "      <th>B01_WTR_link</th>\n",
       "      <th>time_acq_str</th>\n",
       "      <th>mgrs_tile_id</th>\n",
       "      <th>accode</th>\n",
       "      <th>aerosol_class_remapping_enabled</th>\n",
       "      <th>aerosol_not_water_to_high_conf_water_fmask_values</th>\n",
       "      <th>aerosol_partial_surface_aggressive_to_high_conf_water_fmask_values</th>\n",
       "      <th>...</th>\n",
       "      <th>_fillvalue_hls</th>\n",
       "      <th>landsat_product_id_hls</th>\n",
       "      <th>landsat_scene_id_hls</th>\n",
       "      <th>processing_level_hls</th>\n",
       "      <th>sensor_hls</th>\n",
       "      <th>sentinel2_tileid_hls</th>\n",
       "      <th>tirs_ssm_model_hls</th>\n",
       "      <th>tirs_ssm_position_status_hls</th>\n",
       "      <th>usgs_software_hls</th>\n",
       "      <th>hls_url_B04</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>OPERA_L3_DSWx-HLS_T25MER_20230504T123259Z_2023...</td>\n",
       "      <td>2023-05-04T12:33:08.845000+00:00</td>\n",
       "      <td>2023-05-06T16:51:17.371000+00:00</td>\n",
       "      <td>https://archive.podaac.earthdata.nasa.gov/poda...</td>\n",
       "      <td>20230504T123259Z</td>\n",
       "      <td>T25MER</td>\n",
       "      <td>LaSRC</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>224,160,96</td>\n",
       "      <td>224,192,160,128,96</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://data.lpdaac.earthdatacloud.nasa.gov/lp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>OPERA_L3_DSWx-HLS_T42XVQ_20230504T125639Z_2023...</td>\n",
       "      <td>2023-05-04T12:56:39.187000+00:00</td>\n",
       "      <td>2023-05-06T08:15:15.550000+00:00</td>\n",
       "      <td>https://archive.podaac.earthdata.nasa.gov/poda...</td>\n",
       "      <td>20230504T125639Z</td>\n",
       "      <td>T42XVQ</td>\n",
       "      <td>Lasrc</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>224,160,96</td>\n",
       "      <td>224,192,160,128,96</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999</td>\n",
       "      <td>LC08_L1TP_207244_20230504_20230504_02_RT</td>\n",
       "      <td>LC82072442023124LGN00</td>\n",
       "      <td>L1TP</td>\n",
       "      <td>OLI_TIRS</td>\n",
       "      <td>42XVQ</td>\n",
       "      <td>PRELIMINARY</td>\n",
       "      <td>ESTIMATED</td>\n",
       "      <td>LPGS_16.2.0</td>\n",
       "      <td>https://data.lpdaac.earthdatacloud.nasa.gov/lp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>OPERA_L3_DSWx-HLS_T41XNK_20230504T125639Z_2023...</td>\n",
       "      <td>2023-05-04T12:56:39.187000+00:00</td>\n",
       "      <td>2023-05-06T08:15:16.575001+00:00</td>\n",
       "      <td>https://archive.podaac.earthdata.nasa.gov/poda...</td>\n",
       "      <td>20230504T125639Z</td>\n",
       "      <td>T41XNK</td>\n",
       "      <td>Lasrc</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>224,160,96</td>\n",
       "      <td>224,192,160,128,96</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999</td>\n",
       "      <td>LC08_L1TP_207244_20230504_20230504_02_RT</td>\n",
       "      <td>LC82072442023124LGN00</td>\n",
       "      <td>L1TP</td>\n",
       "      <td>OLI_TIRS</td>\n",
       "      <td>41XNK</td>\n",
       "      <td>PRELIMINARY</td>\n",
       "      <td>ESTIMATED</td>\n",
       "      <td>LPGS_16.2.0</td>\n",
       "      <td>https://data.lpdaac.earthdatacloud.nasa.gov/lp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OPERA_L3_DSWx-HLS_T42XVR_20230504T125639Z_2023...</td>\n",
       "      <td>2023-05-04T12:56:39.187000+00:00</td>\n",
       "      <td>2023-05-08T02:12:16.707001+00:00</td>\n",
       "      <td>https://archive.podaac.earthdata.nasa.gov/poda...</td>\n",
       "      <td>20230504T125639Z</td>\n",
       "      <td>T42XVR</td>\n",
       "      <td>Lasrc</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>224,160,96</td>\n",
       "      <td>224,192,160,128,96</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999</td>\n",
       "      <td>LC08_L1TP_207244_20230504_20230504_02_RT</td>\n",
       "      <td>LC82072442023124LGN00</td>\n",
       "      <td>L1TP</td>\n",
       "      <td>OLI_TIRS</td>\n",
       "      <td>42XVR</td>\n",
       "      <td>PRELIMINARY</td>\n",
       "      <td>ESTIMATED</td>\n",
       "      <td>LPGS_16.2.0</td>\n",
       "      <td>https://data.lpdaac.earthdatacloud.nasa.gov/lp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OPERA_L3_DSWx-HLS_T40XER_20230504T125639Z_2023...</td>\n",
       "      <td>2023-05-04T12:56:39.187000+00:00</td>\n",
       "      <td>2023-05-08T02:13:14.809000+00:00</td>\n",
       "      <td>https://archive.podaac.earthdata.nasa.gov/poda...</td>\n",
       "      <td>20230504T125639Z</td>\n",
       "      <td>T40XER</td>\n",
       "      <td>Lasrc</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>224,160,96</td>\n",
       "      <td>224,192,160,128,96</td>\n",
       "      <td>...</td>\n",
       "      <td>-9999</td>\n",
       "      <td>LC08_L1TP_207244_20230504_20230504_02_RT</td>\n",
       "      <td>LC82072442023124LGN00</td>\n",
       "      <td>L1TP</td>\n",
       "      <td>OLI_TIRS</td>\n",
       "      <td>40XER</td>\n",
       "      <td>PRELIMINARY</td>\n",
       "      <td>ESTIMATED</td>\n",
       "      <td>LPGS_16.2.0</td>\n",
       "      <td>https://data.lpdaac.earthdatacloud.nasa.gov/lp...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 102 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          granule_id   \n",
       "0  OPERA_L3_DSWx-HLS_T25MER_20230504T123259Z_2023...  \\\n",
       "1  OPERA_L3_DSWx-HLS_T42XVQ_20230504T125639Z_2023...   \n",
       "2  OPERA_L3_DSWx-HLS_T41XNK_20230504T125639Z_2023...   \n",
       "3  OPERA_L3_DSWx-HLS_T42XVR_20230504T125639Z_2023...   \n",
       "4  OPERA_L3_DSWx-HLS_T40XER_20230504T125639Z_2023...   \n",
       "\n",
       "                      time_acquired                      time_updated   \n",
       "0  2023-05-04T12:33:08.845000+00:00  2023-05-06T16:51:17.371000+00:00  \\\n",
       "1  2023-05-04T12:56:39.187000+00:00  2023-05-06T08:15:15.550000+00:00   \n",
       "2  2023-05-04T12:56:39.187000+00:00  2023-05-06T08:15:16.575001+00:00   \n",
       "3  2023-05-04T12:56:39.187000+00:00  2023-05-08T02:12:16.707001+00:00   \n",
       "4  2023-05-04T12:56:39.187000+00:00  2023-05-08T02:13:14.809000+00:00   \n",
       "\n",
       "                                        B01_WTR_link      time_acq_str   \n",
       "0  https://archive.podaac.earthdata.nasa.gov/poda...  20230504T123259Z  \\\n",
       "1  https://archive.podaac.earthdata.nasa.gov/poda...  20230504T125639Z   \n",
       "2  https://archive.podaac.earthdata.nasa.gov/poda...  20230504T125639Z   \n",
       "3  https://archive.podaac.earthdata.nasa.gov/poda...  20230504T125639Z   \n",
       "4  https://archive.podaac.earthdata.nasa.gov/poda...  20230504T125639Z   \n",
       "\n",
       "  mgrs_tile_id accode aerosol_class_remapping_enabled   \n",
       "0       T25MER  LaSRC                            TRUE  \\\n",
       "1       T42XVQ  Lasrc                            TRUE   \n",
       "2       T41XNK  Lasrc                            TRUE   \n",
       "3       T42XVR  Lasrc                            TRUE   \n",
       "4       T40XER  Lasrc                            TRUE   \n",
       "\n",
       "  aerosol_not_water_to_high_conf_water_fmask_values   \n",
       "0                                        224,160,96  \\\n",
       "1                                        224,160,96   \n",
       "2                                        224,160,96   \n",
       "3                                        224,160,96   \n",
       "4                                        224,160,96   \n",
       "\n",
       "  aerosol_partial_surface_aggressive_to_high_conf_water_fmask_values  ...   \n",
       "0                                 224,192,160,128,96                  ...  \\\n",
       "1                                 224,192,160,128,96                  ...   \n",
       "2                                 224,192,160,128,96                  ...   \n",
       "3                                 224,192,160,128,96                  ...   \n",
       "4                                 224,192,160,128,96                  ...   \n",
       "\n",
       "  _fillvalue_hls                    landsat_product_id_hls   \n",
       "0          -9999                                       NaN  \\\n",
       "1          -9999  LC08_L1TP_207244_20230504_20230504_02_RT   \n",
       "2          -9999  LC08_L1TP_207244_20230504_20230504_02_RT   \n",
       "3          -9999  LC08_L1TP_207244_20230504_20230504_02_RT   \n",
       "4          -9999  LC08_L1TP_207244_20230504_20230504_02_RT   \n",
       "\n",
       "    landsat_scene_id_hls processing_level_hls sensor_hls sentinel2_tileid_hls   \n",
       "0                    NaN                  NaN        NaN                  NaN  \\\n",
       "1  LC82072442023124LGN00                 L1TP   OLI_TIRS                42XVQ   \n",
       "2  LC82072442023124LGN00                 L1TP   OLI_TIRS                41XNK   \n",
       "3  LC82072442023124LGN00                 L1TP   OLI_TIRS                42XVR   \n",
       "4  LC82072442023124LGN00                 L1TP   OLI_TIRS                40XER   \n",
       "\n",
       "  tirs_ssm_model_hls tirs_ssm_position_status_hls usgs_software_hls   \n",
       "0                NaN                          NaN               NaN  \\\n",
       "1        PRELIMINARY                    ESTIMATED       LPGS_16.2.0   \n",
       "2        PRELIMINARY                    ESTIMATED       LPGS_16.2.0   \n",
       "3        PRELIMINARY                    ESTIMATED       LPGS_16.2.0   \n",
       "4        PRELIMINARY                    ESTIMATED       LPGS_16.2.0   \n",
       "\n",
       "                                         hls_url_B04  \n",
       "0  https://data.lpdaac.earthdatacloud.nasa.gov/lp...  \n",
       "1  https://data.lpdaac.earthdatacloud.nasa.gov/lp...  \n",
       "2  https://data.lpdaac.earthdatacloud.nasa.gov/lp...  \n",
       "3  https://data.lpdaac.earthdatacloud.nasa.gov/lp...  \n",
       "4  https://data.lpdaac.earthdatacloud.nasa.gov/lp...  \n",
       "\n",
       "[5 rows x 102 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all = pd.concat([df_dswx, df_hls], axis=1)\n",
    "df_all['hls_url_B04'] = urls\n",
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b4e38837",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all.to_file('dswx_metadata_linked_with_hls.geojson', driver='GeoJSON')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dswx_val",
   "language": "python",
   "name": "dswx_val"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}