src/notebooks/save_huc2_comids.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save HUC2 02 COMIDs to a JSON file on S3\n",
    "\n",
    "These will be used to create a subset of NWM for testing purposes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from os.path import basename, join\n",
    "\n",
    "from smart_open import open as smart_open\n",
    "import boto3\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Code from https://alexwlchan.net/2017/07/listing-s3-keys/\n",
    "def get_matching_s3_objects(bucket, prefix='', suffix='',\n",
    "                            request_payer='None'):\n",
    "    \"\"\"\n",
    "    Generate objects in an S3 bucket.\n",
    "    :param bucket: Name of the S3 bucket.\n",
    "    :param prefix: Only fetch objects whose key starts with\n",
    "        this prefix (optional).\n",
    "    :param suffix: Only fetch objects whose keys end with\n",
    "        this suffix (optional).\n",
    "    \"\"\"\n",
    "    s3 = boto3.client('s3')\n",
    "    kwargs = {'Bucket': bucket, 'RequestPayer': request_payer}\n",
    "\n",
    "    # If the prefix is a single string (not a tuple of strings), we can\n",
    "    # do the filtering directly in the S3 API.\n",
    "    if isinstance(prefix, str):\n",
    "        kwargs['Prefix'] = prefix\n",
    "\n",
    "    while True:\n",
    "\n",
    "        # The S3 API response is a large blob of metadata.\n",
    "        # 'Contents' contains information about the listed objects.\n",
    "        resp = s3.list_objects_v2(**kwargs)\n",
    "\n",
    "        try:\n",
    "            contents = resp['Contents']\n",
    "        except KeyError:\n",
    "            return\n",
    "\n",
    "        for obj in contents:\n",
    "            key = obj['Key']\n",
    "            if key.startswith(prefix) and key.endswith(suffix):\n",
    "                yield obj\n",
    "\n",
    "        # The S3 API is paginated, returning up to 1000 keys at a time.\n",
    "        # Pass the continuation token into the next response, until we\n",
    "        # reach the final page (when this field is missing).\n",
    "        try:\n",
    "            kwargs['ContinuationToken'] = resp['NextContinuationToken']\n",
    "        except KeyError:\n",
    "            break\n",
    "\n",
    "def get_json(uri):\n",
    "    with smart_open(uri) as fd:\n",
    "        return json.load(fd)\n",
    "\n",
    "def save_json(json_dict, uri):\n",
    "    with smart_open(uri, 'w') as fd:\n",
    "        json.dump(json_dict, fd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 88/88 [00:33<00:00,  2.65it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1748535,\n",
       " 1748537,\n",
       " 1748539,\n",
       " 1748541,\n",
       " 1748543,\n",
       " 1748545,\n",
       " 1748547,\n",
       " 1748549,\n",
       " 1748551,\n",
       " 1748553,\n",
       " 1748555,\n",
       " 1748557,\n",
       " 1748559,\n",
       " 1748561,\n",
       " 1748563,\n",
       " 1748565,\n",
       " 1748567,\n",
       " 1748569,\n",
       " 1748571,\n",
       " 1748573,\n",
       " 1748575,\n",
       " 1748577,\n",
       " 1748579,\n",
       " 1748581,\n",
       " 1748583,\n",
       " 1748585,\n",
       " 1748587,\n",
       " 1748589,\n",
       " 1748591,\n",
       " 1748593,\n",
       " 1748595,\n",
       " 1748597,\n",
       " 1748599,\n",
       " 1748601,\n",
       " 1748603,\n",
       " 1748605,\n",
       " 1748607,\n",
       " 1748609,\n",
       " 1748611,\n",
       " 1748613,\n",
       " 1748615,\n",
       " 1748617,\n",
       " 1748619,\n",
       " 1748621,\n",
       " 1748623,\n",
       " 1748625,\n",
       " 1748627,\n",
       " 1748629,\n",
       " 1748631,\n",
       " 1748633,\n",
       " 1748635,\n",
       " 1748637,\n",
       " 1748639,\n",
       " 1748641,\n",
       " 1748643,\n",
       " 1748645,\n",
       " 1748647,\n",
       " 1748649,\n",
       " 1748651,\n",
       " 1748707,\n",
       " 1748709,\n",
       " 1748711,\n",
       " 1748713,\n",
       " 1748715,\n",
       " 1748717,\n",
       " 1748719,\n",
       " 1748723,\n",
       " 1748725,\n",
       " 1748727,\n",
       " 1748729,\n",
       " 1748731,\n",
       " 1748733,\n",
       " 1748735,\n",
       " 1748737,\n",
       " 1748739,\n",
       " 1748741,\n",
       " 1748743,\n",
       " 1748745,\n",
       " 1748747,\n",
       " 1748749,\n",
       " 1748751,\n",
       " 1748753,\n",
       " 1748755,\n",
       " 1748757,\n",
       " 1748759,\n",
       " 1748761,\n",
       " 1748763,\n",
       " 1748765,\n",
       " 1748767,\n",
       " 1748769,\n",
       " 1748771,\n",
       " 1748773,\n",
       " 1748775,\n",
       " 1748777,\n",
       " 1748779,\n",
       " 1748781,\n",
       " 1748783,\n",
       " 1748785,\n",
       " 1748787,\n",
       " 1749159,\n",
       " 1749161,\n",
       " 1749163,\n",
       " 1749165,\n",
       " 1749169,\n",
       " 1749171,\n",
       " 1749175,\n",
       " 1749177,\n",
       " 1749183,\n",
       " 1749185,\n",
       " 1749187,\n",
       " 1749189,\n",
       " 1749191,\n",
       " 1749193,\n",
       " 1749195,\n",
       " 1749197,\n",
       " 1749199,\n",
       " 1749201,\n",
       " 1749203,\n",
       " 1749205,\n",
       " 1749207,\n",
       " 1749209,\n",
       " 1749211,\n",
       " 1749213,\n",
       " 1749215,\n",
       " 1749217,\n",
       " 1749219,\n",
       " 1749221,\n",
       " 1749223,\n",
       " 1749225,\n",
       " 1749227,\n",
       " 1749229,\n",
       " 1749231,\n",
       " 1749233,\n",
       " 1749235,\n",
       " 1749237,\n",
       " 1749239,\n",
       " 1749241,\n",
       " 1749243,\n",
       " 1749245,\n",
       " 1749247,\n",
       " 1749249,\n",
       " 1749251,\n",
       " 1749253,\n",
       " 1749255,\n",
       " 1749257,\n",
       " 1749259,\n",
       " 1749261,\n",
       " 1749263,\n",
       " 1749265,\n",
       " 1749267,\n",
       " 1749269,\n",
       " 1749271,\n",
       " 1749273,\n",
       " 1749275,\n",
       " 1749277,\n",
       " 1749279,\n",
       " 1749281,\n",
       " 1749283,\n",
       " 1749285,\n",
       " 1749287,\n",
       " 1749289,\n",
       " 1749291,\n",
       " 1749293,\n",
       " 1749295,\n",
       " 1749297,\n",
       " 1749299,\n",
       " 1749301,\n",
       " 1749303,\n",
       " 1749305,\n",
       " 1749307,\n",
       " 1749309,\n",
       " 1749311,\n",
       " 1749313,\n",
       " 1749315,\n",
       " 1749317,\n",
       " 1749319,\n",
       " 1749321,\n",
       " 1749323,\n",
       " 1749325,\n",
       " 1749377,\n",
       " 1749379,\n",
       " 1749381,\n",
       " 1749383,\n",
       " 1749385,\n",
       " 1749387,\n",
       " 1749389,\n",
       " 1749391,\n",
       " 1749393,\n",
       " 1749395,\n",
       " 1749397,\n",
       " 1749399,\n",
       " 1749401,\n",
       " 1749403,\n",
       " 1749409,\n",
       " 1749411,\n",
       " 1749413,\n",
       " 1749415,\n",
       " 1749417,\n",
       " 1749419,\n",
       " 1750339,\n",
       " 1750341,\n",
       " 1750343,\n",
       " 1750345,\n",
       " 1750347,\n",
       " 1750349,\n",
       " 1750351,\n",
       " 1750353,\n",
       " 1750355,\n",
       " 1750357,\n",
       " 1750359,\n",
       " 1750361,\n",
       " 1750363,\n",
       " 1750365,\n",
       " 1750367,\n",
       " 1750369,\n",
       " 1750371,\n",
       " 1750373,\n",
       " 1750375,\n",
       " 1750377,\n",
       " 1750379,\n",
       " 1750381,\n",
       " 1750383,\n",
       " 1750385,\n",
       " 1750387,\n",
       " 1750389,\n",
       " 1750391,\n",
       " 1750393,\n",
       " 1750395,\n",
       " 1750397,\n",
       " 1750399,\n",
       " 1750401,\n",
       " 1750403,\n",
       " 1750405,\n",
       " 1750407,\n",
       " 1750409,\n",
       " 1750411,\n",
       " 1750413,\n",
       " 1750415,\n",
       " 1750417,\n",
       " 1750419,\n",
       " 1750421,\n",
       " 1750423,\n",
       " 1750425,\n",
       " 1750427,\n",
       " 1750429,\n",
       " 1750431,\n",
       " 1750433,\n",
       " 1750435,\n",
       " 1750437,\n",
       " 1750439,\n",
       " 1750441,\n",
       " 1750443,\n",
       " 1750445,\n",
       " 1750447,\n",
       " 1750449,\n",
       " 1750451,\n",
       " 1750453,\n",
       " 1750455,\n",
       " 1750457,\n",
       " 1750459,\n",
       " 1750461,\n",
       " 1750463,\n",
       " 1750465,\n",
       " 1750467,\n",
       " 1750469,\n",
       " 1750471,\n",
       " 1750473,\n",
       " 1750475,\n",
       " 1750477,\n",
       " 1750479,\n",
       " 1750481,\n",
       " 1750483,\n",
       " 1750485,\n",
       " 1750487,\n",
       " 1750489,\n",
       " 1750491,\n",
       " 1750495,\n",
       " 1750497,\n",
       " 1750499,\n",
       " 1750501,\n",
       " 1750503,\n",
       " 1750505,\n",
       " 1750507,\n",
       " 1750509,\n",
       " 1750511,\n",
       " 1750513,\n",
       " 1750515,\n",
       " 1750517,\n",
       " 1750519,\n",
       " 1750521,\n",
       " 1750525,\n",
       " 1750527,\n",
       " 1750529,\n",
       " 1750531,\n",
       " 1750533,\n",
       " 1750535,\n",
       " 1750573,\n",
       " 1750575,\n",
       " 1750577,\n",
       " 1750579,\n",
       " 1750581,\n",
       " 1750583,\n",
       " 1750585,\n",
       " 1750587,\n",
       " 1750589,\n",
       " 1750591,\n",
       " 1750593,\n",
       " 1750595,\n",
       " 1750597,\n",
       " 1750599,\n",
       " 1750601,\n",
       " 1750603,\n",
       " 1750605,\n",
       " 1750605,\n",
       " 1750605,\n",
       " 1750607,\n",
       " 1750609,\n",
       " 1750609,\n",
       " 1750609,\n",
       " 1750611,\n",
       " 1750613,\n",
       " 1750617,\n",
       " 1750619,\n",
       " 1750621,\n",
       " 1750623,\n",
       " 1750625,\n",
       " 1750627,\n",
       " 1750629,\n",
       " 1750631,\n",
       " 1750633,\n",
       " 1750635,\n",
       " 1750637,\n",
       " 1750639,\n",
       " 1750641,\n",
       " 1750643,\n",
       " 1750645,\n",
       " 1750647,\n",
       " 1750649,\n",
       " 1750651,\n",
       " 1750653,\n",
       " 1750655,\n",
       " 1750657,\n",
       " 1750659,\n",
       " 1750661,\n",
       " 1750663,\n",
       " 1750665,\n",
       " 1750667,\n",
       " 1750669,\n",
       " 1750671,\n",
       " 1750673,\n",
       " 1750675,\n",
       " 1750677,\n",
       " 1750679,\n",
       " 1750681,\n",
       " 1750683,\n",
       " 1750685,\n",
       " 1750687,\n",
       " 1750689,\n",
       " 1750691,\n",
       " 1750693,\n",
       " 1750695,\n",
       " 1750697,\n",
       " 1750699,\n",
       " 1750701,\n",
       " 1750703,\n",
       " 1750705,\n",
       " 1750707,\n",
       " 1750709,\n",
       " 1750711,\n",
       " 1750713,\n",
       " 1750715,\n",
       " 1750717,\n",
       " 1750719,\n",
       " 1750721,\n",
       " 1750723,\n",
       " 1750725,\n",
       " 1750727,\n",
       " 1750729,\n",
       " 1752001,\n",
       " 1752003,\n",
       " 1752005,\n",
       " 1752007,\n",
       " 1752009,\n",
       " 1752011,\n",
       " 1752013,\n",
       " 1752015,\n",
       " 1752017,\n",
       " 1752019,\n",
       " 1752021,\n",
       " 1752023,\n",
       " 1752025,\n",
       " 1752027,\n",
       " 1752029,\n",
       " 1752031,\n",
       " 1752033,\n",
       " 1752035,\n",
       " 1752037,\n",
       " 1752039,\n",
       " 1752041,\n",
       " 1752043,\n",
       " 1752045,\n",
       " 1752047,\n",
       " 1752049,\n",
       " 1752051,\n",
       " 1752053,\n",
       " 1752055,\n",
       " 1752057,\n",
       " 1752059,\n",
       " 1752061,\n",
       " 1752063,\n",
       " 1752065,\n",
       " 1752067,\n",
       " 1752069,\n",
       " 1752071,\n",
       " 1752073,\n",
       " 1752093,\n",
       " 1752095,\n",
       " 1752105,\n",
       " 1752107,\n",
       " 1752109,\n",
       " 1752111,\n",
       " 1752113,\n",
       " 1752115,\n",
       " 1752117,\n",
       " 1752119,\n",
       " 1752121,\n",
       " 1752123,\n",
       " 1752125,\n",
       " 1752127,\n",
       " 1752129,\n",
       " 1752131,\n",
       " 1752133,\n",
       " 1752135,\n",
       " 1752137,\n",
       " 1752139,\n",
       " 1752141,\n",
       " 1752143,\n",
       " 1752145,\n",
       " 1752147,\n",
       " 1752149,\n",
       " 1752151,\n",
       " 1752153,\n",
       " 1752155,\n",
       " 1752157,\n",
       " 1752159,\n",
       " 1752159,\n",
       " 1752161,\n",
       " 1752163,\n",
       " 1752165,\n",
       " 1752167,\n",
       " 1752169,\n",
       " 1752171,\n",
       " 1752173,\n",
       " 1752833,\n",
       " 1752843,\n",
       " 2583185,\n",
       " 2583187,\n",
       " 2583189,\n",
       " 2583191,\n",
       " 2583193,\n",
       " 2583195,\n",
       " 2583197,\n",
       " 2583199,\n",
       " 2583201,\n",
       " 2583203,\n",
       " 2583205,\n",
       " 2583207,\n",
       " 2583209,\n",
       " 2583211,\n",
       " 2583213,\n",
       " 2583215,\n",
       " 2583217,\n",
       " 2583219,\n",
       " 2583221,\n",
       " 2583223,\n",
       " 2583225,\n",
       " 2583227,\n",
       " 2583229,\n",
       " 2583231,\n",
       " 2583233,\n",
       " 2583235,\n",
       " 2583237,\n",
       " 2583239,\n",
       " 2583241,\n",
       " 2583243,\n",
       " 2583245,\n",
       " 2583247,\n",
       " 2583249,\n",
       " 2583251,\n",
       " 2583253,\n",
       " 2583255,\n",
       " 2583257,\n",
       " 2583259,\n",
       " 2583261,\n",
       " 2583263,\n",
       " 2583265,\n",
       " 2583267,\n",
       " 2583269,\n",
       " 2583271,\n",
       " 2583273,\n",
       " 2583275,\n",
       " 2583277,\n",
       " 2583279,\n",
       " 2583281,\n",
       " 2583283,\n",
       " 2583285,\n",
       " 2583287,\n",
       " 2583289,\n",
       " 2583291,\n",
       " 2583293,\n",
       " 2583295,\n",
       " 2583297,\n",
       " 2583299,\n",
       " 2583301,\n",
       " 2583303,\n",
       " 2583305,\n",
       " 2583307,\n",
       " 2583309,\n",
       " 2583311,\n",
       " 2583313,\n",
       " 2583315,\n",
       " 2583317,\n",
       " 2583319,\n",
       " 2583321,\n",
       " 2583323,\n",
       " 2583325,\n",
       " 2583327,\n",
       " 2583329,\n",
       " 2583331,\n",
       " 2583333,\n",
       " 2583335,\n",
       " 2583337,\n",
       " 2583339,\n",
       " 2583341,\n",
       " 2583343,\n",
       " 2583345,\n",
       " 2583347,\n",
       " 2583351,\n",
       " 2583353,\n",
       " 2583355,\n",
       " 2583357,\n",
       " 2583359,\n",
       " 2583361,\n",
       " 2583363,\n",
       " 2583365,\n",
       " 2583367,\n",
       " 2583369,\n",
       " 2583371,\n",
       " 2583373,\n",
       " 2583375,\n",
       " 2583377,\n",
       " 2583379,\n",
       " 2583385,\n",
       " 2583387,\n",
       " 2583389,\n",
       " 2583391,\n",
       " 2583393,\n",
       " 2583395,\n",
       " 2583397,\n",
       " 2583399,\n",
       " 2583401,\n",
       " 2583403,\n",
       " 2583405,\n",
       " 2583407,\n",
       " 2583409,\n",
       " 2583411,\n",
       " 2583413,\n",
       " 2583415,\n",
       " 2583417,\n",
       " 2583419,\n",
       " 2583421,\n",
       " 2583423,\n",
       " 2583425,\n",
       " 2583427,\n",
       " 2583429,\n",
       " 2583431,\n",
       " 2583433,\n",
       " 2583435,\n",
       " 2583437,\n",
       " 2583439,\n",
       " 2583441,\n",
       " 2583443,\n",
       " 2583445,\n",
       " 2583447,\n",
       " 2583449,\n",
       " 2583451,\n",
       " 2583453,\n",
       " 2583455,\n",
       " 2583457,\n",
       " 2583459,\n",
       " 2583461,\n",
       " 2583463,\n",
       " 2583465,\n",
       " 2583467,\n",
       " 2583469,\n",
       " 2583471,\n",
       " 2583473,\n",
       " 2583475,\n",
       " 2583479,\n",
       " 2583481,\n",
       " 2583483,\n",
       " 2583485,\n",
       " 2583487,\n",
       " 2583489,\n",
       " 2583491,\n",
       " 2583493,\n",
       " 2583495,\n",
       " 2583497,\n",
       " 2583499,\n",
       " 2583501,\n",
       " 2583503,\n",
       " 2583505,\n",
       " 2583507,\n",
       " 2583509,\n",
       " 2583511,\n",
       " 2583513,\n",
       " 2583515,\n",
       " 2583517,\n",
       " 2583519,\n",
       " 2583521,\n",
       " 2583523,\n",
       " 2583525,\n",
       " 2583527,\n",
       " 2583529,\n",
       " 2583531,\n",
       " 2583533,\n",
       " 2583535,\n",
       " 2583537,\n",
       " 2583539,\n",
       " 2583541,\n",
       " 2583543,\n",
       " 2583545,\n",
       " 2583547,\n",
       " 2583613,\n",
       " 2583615,\n",
       " 2583617,\n",
       " 2583619,\n",
       " 2583621,\n",
       " 2583623,\n",
       " 2583625,\n",
       " 2583627,\n",
       " 2583635,\n",
       " 2583637,\n",
       " 2583639,\n",
       " 2583641,\n",
       " 2583643,\n",
       " 2583645,\n",
       " 2583647,\n",
       " 2583649,\n",
       " 2583651,\n",
       " 2583653,\n",
       " 2583655,\n",
       " 2583657,\n",
       " 2583659,\n",
       " 2583661,\n",
       " 2583663,\n",
       " 2583665,\n",
       " 2583667,\n",
       " 2583669,\n",
       " 2583671,\n",
       " 2583673,\n",
       " 2583675,\n",
       " 2583677,\n",
       " 2583679,\n",
       " 2583681,\n",
       " 2583683,\n",
       " 2583685,\n",
       " 2583687,\n",
       " 2583689,\n",
       " 2583691,\n",
       " 2583693,\n",
       " 2583695,\n",
       " 2583697,\n",
       " 2583699,\n",
       " 2583701,\n",
       " 2583703,\n",
       " 2583705,\n",
       " 2583707,\n",
       " 2583709,\n",
       " 2583711,\n",
       " 2583713,\n",
       " 2583715,\n",
       " 2583717,\n",
       " 2583719,\n",
       " 2583721,\n",
       " 2583723,\n",
       " 2583725,\n",
       " 2583727,\n",
       " 2583729,\n",
       " 2583731,\n",
       " 2583733,\n",
       " 2583735,\n",
       " 2583737,\n",
       " 2583739,\n",
       " 2583741,\n",
       " 2583743,\n",
       " 2583745,\n",
       " 2583747,\n",
       " 2583749,\n",
       " 2583751,\n",
       " 2583753,\n",
       " 2583755,\n",
       " 2583757,\n",
       " 2583759,\n",
       " 2583761,\n",
       " 2584979,\n",
       " 2584981,\n",
       " 2584983,\n",
       " 2584985,\n",
       " 2584987,\n",
       " 2584989,\n",
       " 2585049,\n",
       " 2585051,\n",
       " 2585053,\n",
       " 2586087,\n",
       " 2586089,\n",
       " 2586091,\n",
       " 2586093,\n",
       " 2586095,\n",
       " 2586097,\n",
       " 2586099,\n",
       " 2586101,\n",
       " 2586103,\n",
       " 2586105,\n",
       " 2586107,\n",
       " 2586109,\n",
       " 2586111,\n",
       " 2586113,\n",
       " 2586117,\n",
       " 2586119,\n",
       " 2586121,\n",
       " 2586123,\n",
       " 2586125,\n",
       " 2586127,\n",
       " 2586129,\n",
       " 2586131,\n",
       " 2586133,\n",
       " 2586135,\n",
       " 2586137,\n",
       " 2586139,\n",
       " 2586141,\n",
       " 2586143,\n",
       " 2586145,\n",
       " 2586147,\n",
       " 2586149,\n",
       " 2586153,\n",
       " 2586157,\n",
       " 2586159,\n",
       " 2586161,\n",
       " 2586163,\n",
       " 2586165,\n",
       " 2586167,\n",
       " 2586169,\n",
       " 2586173,\n",
       " 2586175,\n",
       " 2586177,\n",
       " 2586181,\n",
       " 2586183,\n",
       " 2586185,\n",
       " 2586187,\n",
       " 2586189,\n",
       " 2586191,\n",
       " 2586193,\n",
       " 2586195,\n",
       " 2586197,\n",
       " 2586199,\n",
       " 2586201,\n",
       " 2586203,\n",
       " 2586205,\n",
       " 2586207,\n",
       " 2586209,\n",
       " 2586211,\n",
       " 2586213,\n",
       " 2586215,\n",
       " 2586217,\n",
       " 2586219,\n",
       " 2586221,\n",
       " 2586223,\n",
       " 2586225,\n",
       " 2586227,\n",
       " 2586229,\n",
       " 2586231,\n",
       " 2586233,\n",
       " 2586235,\n",
       " 2586237,\n",
       " 2586239,\n",
       " 2586241,\n",
       " 2586243,\n",
       " 2586245,\n",
       " 2586247,\n",
       " 2586249,\n",
       " 2586251,\n",
       " 2586253,\n",
       " 2586255,\n",
       " 2586257,\n",
       " 2586259,\n",
       " 2586261,\n",
       " 2586263,\n",
       " 2586265,\n",
       " 2586267,\n",
       " 2586269,\n",
       " 2586271,\n",
       " 2586273,\n",
       " 2586275,\n",
       " 2586277,\n",
       " 2586279,\n",
       " 2586281,\n",
       " 2586283,\n",
       " 2586285,\n",
       " 2586287,\n",
       " 2586289,\n",
       " 2586291,\n",
       " 2586293,\n",
       " 2586295,\n",
       " 2586297,\n",
       " 2586299,\n",
       " 2586301,\n",
       " 2586303,\n",
       " 2586305,\n",
       " 2586307,\n",
       " 2586309,\n",
       " 2586311,\n",
       " 2586313,\n",
       " 2586315,\n",
       " 2586317,\n",
       " 2586319,\n",
       " 2586321,\n",
       " 2586323,\n",
       " 2586325,\n",
       " 2586327,\n",
       " 2586329,\n",
       " 2586331,\n",
       " 2586333,\n",
       " 2586335,\n",
       " 2586337,\n",
       " 2586339,\n",
       " 2586341,\n",
       " 2586343,\n",
       " 2586345,\n",
       " 2586347,\n",
       " 2586349,\n",
       " 2586351,\n",
       " 2586353,\n",
       " 2586355,\n",
       " 2586357,\n",
       " 2586359,\n",
       " 2586361,\n",
       " 2586363,\n",
       " 2586365,\n",
       " 2586367,\n",
       " 2586369,\n",
       " 2586371,\n",
       " 2586373,\n",
       " 2586375,\n",
       " 2586377,\n",
       " 2586379,\n",
       " 2586381,\n",
       " 2586383,\n",
       " 2586385,\n",
       " 2586387,\n",
       " 2586389,\n",
       " 2586391,\n",
       " 2586393,\n",
       " 2586395,\n",
       " 2586397,\n",
       " 2586399,\n",
       " 2586401,\n",
       " 2586403,\n",
       " 2586405,\n",
       " 2586407,\n",
       " 2586409,\n",
       " 2586411,\n",
       " 2586413,\n",
       " 2586415,\n",
       " 2586417,\n",
       " 2586419,\n",
       " 2586421,\n",
       " 2586423,\n",
       " 2586425,\n",
       " 2586427,\n",
       " 2586429,\n",
       " 2586431,\n",
       " 2586433,\n",
       " 2586435,\n",
       " 2586437,\n",
       " 2586439,\n",
       " 2586441,\n",
       " 2586443,\n",
       " 2586445,\n",
       " 2586447,\n",
       " 2586449,\n",
       " 2586451,\n",
       " 2586453,\n",
       " 2586455,\n",
       " 2586457,\n",
       " 2586459,\n",
       " 2586461,\n",
       " 2586463,\n",
       " 2586465,\n",
       " 2586471,\n",
       " 2586473,\n",
       " 2586475,\n",
       " 2586477,\n",
       " 2586479,\n",
       " 2586481,\n",
       " 2586483,\n",
       " 2586487,\n",
       " 2586489,\n",
       " 2586491,\n",
       " 2586493,\n",
       " 2586495,\n",
       " 2586497,\n",
       " 2586499,\n",
       " 2586501,\n",
       " 2586503,\n",
       " 2586517,\n",
       " 2586523,\n",
       " 2586525,\n",
       " 2586535,\n",
       " 2586537,\n",
       " 2586547,\n",
       " 2586607,\n",
       " 2586641,\n",
       " 2586657,\n",
       " 2586659,\n",
       " 2586665,\n",
       " 2586667,\n",
       " 2586671,\n",
       " 2586675,\n",
       " 2586679,\n",
       " 2586681,\n",
       " 2586685,\n",
       " 2586695,\n",
       " 2586697,\n",
       " 2586699,\n",
       " 2586701,\n",
       " 2586765,\n",
       " 2586767,\n",
       " 2586769,\n",
       " 2586771,\n",
       " 2586773,\n",
       " 2586775,\n",
       " 2586777,\n",
       " 2586779,\n",
       " 2586781,\n",
       " 2586783,\n",
       " 2586785,\n",
       " 2586787,\n",
       " 2586789,\n",
       " 2586791,\n",
       " 2586797,\n",
       " 2586799,\n",
       " 2586801,\n",
       " 2586803,\n",
       " 2586805,\n",
       " 2586807,\n",
       " 2586809,\n",
       " 2586815,\n",
       " 2586865,\n",
       " 2586867,\n",
       " 2586869,\n",
       " 2586871,\n",
       " 2586873,\n",
       " 2586875,\n",
       " 2586877,\n",
       " 2586879,\n",
       " 2586881,\n",
       " 2586883,\n",
       " 2586885,\n",
       " 2586887,\n",
       " 2586889,\n",
       " 2586891,\n",
       " 2586893,\n",
       " 2586895,\n",
       " 2586897,\n",
       " 2586899,\n",
       " 2586901,\n",
       " 2586903,\n",
       " 2586905,\n",
       " 2586907,\n",
       " 2586909,\n",
       " 2586911,\n",
       " 2586913,\n",
       " 2586915,\n",
       " 2586917,\n",
       " 2586919,\n",
       " 2586921,\n",
       " 2586923,\n",
       " 2586925,\n",
       " 2586927,\n",
       " 2586929,\n",
       " 2586931,\n",
       " 2586933,\n",
       " 2586935,\n",
       " 2586937,\n",
       " 2586939,\n",
       " 2586941,\n",
       " 2586943,\n",
       " 2586945,\n",
       " 2586947,\n",
       " ...]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get all huc8 extract files on s3 with 02 in prefix.\n",
    "# this is the set that lies within huc2 02 (which is in themid-atlantic region)\n",
    "\n",
    "huc8_root_uri = 's3://azavea-noaa-hydro-data/noaa/huc8-extracts/transformed/'\n",
    "huc8_fns = [\n",
    "    basename(x['Key'])\n",
    "    for x in list(get_matching_s3_objects('azavea-noaa-hydro-data', 'noaa/huc8-extracts/transformed/02', 'json'))]\n",
    "\n",
    "reach_ids = []\n",
    "for huc8_fn in tqdm(huc8_fns):\n",
    "    huc8_dict = get_json(join(huc8_root_uri, huc8_fn))\n",
    "    reach_ids.extend(huc8_dict['features'][0]['properties']['comids'])\n",
    "\n",
    "reach_ids.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_uri = 's3://azavea-noaa-hydro-data/noaa/huc2-comids.json'\n",
    "save_json({'comids': reach_ids}, out_uri)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('notebook')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "33388988d2b1cb540819d28e8f67ee9e7e7397cf80503e5dd9c105f19f832c00"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}