From c38ceb9efe10b647f5fec7bf2d934f11f65906bc Mon Sep 17 00:00:00 2001 From: Brian Blaylock Date: Wed, 13 Mar 2024 23:02:37 -0700 Subject: [PATCH] update --- .../multiple_variables_and_merge.ipynb | 995 +++++++++++++++++- 1 file changed, 971 insertions(+), 24 deletions(-) diff --git a/docs/user_guide/_bonus_notebooks/multiple_variables_and_merge.ipynb b/docs/user_guide/_bonus_notebooks/multiple_variables_and_merge.ipynb index de79f15e..1efff80d 100644 --- a/docs/user_guide/_bonus_notebooks/multiple_variables_and_merge.ipynb +++ b/docs/user_guide/_bonus_notebooks/multiple_variables_and_merge.ipynb @@ -6,12 +6,12 @@ "source": [ "# Merge Multiple Variables into Single Dataset\n", "\n", - "Often users want to get multiple files across several files (across time or forecast hours). Here is an example of how to get those data and merge them into a single xarray Dataset." + "Often users want to get multiple files across several files (across time or forecast hours). Here is an example of how to get those data and merge them into a single xarray Dataset.\n" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -40,7 +40,32 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([\u001b[48;2;255;255;255m\u001b[38;2;136;33;27m▌\u001b[0m\u001b[38;2;12;53;118m\u001b[48;2;240;234;210m▌\u001b[38;2;0;0;0m\u001b[1mHerbie\u001b[0m RAP model \u001b[3mawp130pgrb\u001b[0m product initialized \u001b[38;2;41;130;13m2024-Jan-01 00:00 UTC\u001b[92m F00\u001b[0m ┊ \u001b[38;2;255;153;0m\u001b[3msource=local\u001b[0m,\n", + " \u001b[48;2;255;255;255m\u001b[38;2;136;33;27m▌\u001b[0m\u001b[38;2;12;53;118m\u001b[48;2;240;234;210m▌\u001b[38;2;0;0;0m\u001b[1mHerbie\u001b[0m RAP model \u001b[3mawp130pgrb\u001b[0m product initialized \u001b[38;2;41;130;13m2024-Jan-01 01:00 UTC\u001b[92m F00\u001b[0m ┊ \u001b[38;2;255;153;0m\u001b[3msource=aws\u001b[0m,\n", + " \u001b[48;2;255;255;255m\u001b[38;2;136;33;27m▌\u001b[0m\u001b[38;2;12;53;118m\u001b[48;2;240;234;210m▌\u001b[38;2;0;0;0m\u001b[1mHerbie\u001b[0m RAP model \u001b[3mawp130pgrb\u001b[0m product initialized \u001b[38;2;41;130;13m2024-Jan-01 02:00 UTC\u001b[92m F00\u001b[0m ┊ \u001b[38;2;255;153;0m\u001b[3msource=aws\u001b[0m],\n", + " [\u001b[48;2;255;255;255m\u001b[38;2;136;33;27m▌\u001b[0m\u001b[38;2;12;53;118m\u001b[48;2;240;234;210m▌\u001b[38;2;0;0;0m\u001b[1mHerbie\u001b[0m RAP model \u001b[3mawp130bgrb\u001b[0m product initialized \u001b[38;2;41;130;13m2024-Jan-01 00:00 UTC\u001b[92m F00\u001b[0m ┊ \u001b[38;2;255;153;0m\u001b[3msource=aws\u001b[0m,\n", + " \u001b[48;2;255;255;255m\u001b[38;2;136;33;27m▌\u001b[0m\u001b[38;2;12;53;118m\u001b[48;2;240;234;210m▌\u001b[38;2;0;0;0m\u001b[1mHerbie\u001b[0m RAP model \u001b[3mawp130bgrb\u001b[0m product initialized \u001b[38;2;41;130;13m2024-Jan-01 01:00 UTC\u001b[92m F00\u001b[0m ┊ \u001b[38;2;255;153;0m\u001b[3msource=aws\u001b[0m,\n", + " \u001b[48;2;255;255;255m\u001b[38;2;136;33;27m▌\u001b[0m\u001b[38;2;12;53;118m\u001b[48;2;240;234;210m▌\u001b[38;2;0;0;0m\u001b[1mHerbie\u001b[0m RAP model \u001b[3mawp130bgrb\u001b[0m product initialized \u001b[38;2;41;130;13m2024-Jan-01 02:00 UTC\u001b[92m F00\u001b[0m ┊ \u001b[38;2;255;153;0m\u001b[3msource=aws\u001b[0m])" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "FH_prs.file_exists, FH_nat.file_exists" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -49,7 +74,7 @@ "12" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -68,7 +93,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -77,7 +102,7 @@ "6" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -93,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -128,7 +153,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -533,8 +558,8 @@ " description: Rapid Refresh (RAP) from NOMADS and Big Data Pro...\n", " remote_grib: /home/blaylock/data/rap/20240101/rap.t00z.awp130...\n", " local_grib: /home/blaylock/data/rap/20240101/subset_6bef9e80...\n", - " searchString: (?:TMP:2 m|GRD:10 m|DPT:2 m|GUST|TMP:1000 mb|TMP...
  • GRIB_edition :
    2
    GRIB_centre :
    kwbc
    GRIB_centreDescription :
    US National Weather Service - NCEP
    GRIB_subCentre :
    0
    Conventions :
    CF-1.7
    institution :
    US National Weather Service - NCEP
    model :
    rap
    product :
    awp130pgrb
    description :
    Rapid Refresh (RAP) from NOMADS and Big Data Program
    remote_grib :
    /home/blaylock/data/rap/20240101/rap.t00z.awp130pgrbf00.grib2
    local_grib :
    /home/blaylock/data/rap/20240101/subset_6bef9e80__rap.t00z.awp130pgrbf00.grib2
    searchString :
    (?:TMP:2 m|GRD:10 m|DPT:2 m|GUST|TMP:1000 mb|TMP:500 mb)
    • " ], "text/plain": [ "\n", @@ -946,16 +971,938 @@ " searchString: (?:TMP:2 m|GRD:10 m|DPT:2 m|GUST|TMP:1000 mb|TMP..." ] }, - "execution_count": 6, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Merge prs and nat data into one dataframe\n", + "# (NOTE: I'm not 100% convinced I did the merge correct. Why are there\n", + "# so many NaN values?)\n", "ds = merge_datasets(ds_prs + ds_nat)\n", "ds" ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset>\n",
+       "Dimensions:              (time: 3, heightAboveGround: 2, isobaricInhPa: 2,\n",
+       "                          depthBelowLandLayer: 9)\n",
+       "Coordinates:\n",
+       "  * time                 (time) datetime64[ns] 2024-01-01 ... 2024-01-01T02:0...\n",
+       "    step                 timedelta64[ns] 00:00:00\n",
+       "  * heightAboveGround    (heightAboveGround) float64 2.0 10.0\n",
+       "    latitude             float64 30.52\n",
+       "    longitude            float64 244.4\n",
+       "    valid_time           datetime64[ns] 2024-01-01\n",
+       "  * isobaricInhPa        (isobaricInhPa) float64 1e+03 500.0\n",
+       "    surface              float64 0.0\n",
+       "  * depthBelowLandLayer  (depthBelowLandLayer) float64 0.0 0.01 0.04 ... 1.6 3.0\n",
+       "Data variables:\n",
+       "    u10                  (heightAboveGround, time) float32 nan nan ... nan nan\n",
+       "    v10                  (heightAboveGround, time) float32 nan nan ... nan nan\n",
+       "    t2m                  (heightAboveGround, time) float32 287.2 nan ... nan nan\n",
+       "    d2m                  (heightAboveGround, time) float32 280.6 nan ... nan nan\n",
+       "    t                    (time, isobaricInhPa) float32 289.6 255.8 ... nan nan\n",
+       "    gust                 (time) float32 2.868 nan nan\n",
+       "    st                   (time, depthBelowLandLayer) float32 287.9 288.2 ... nan\n",
+       "    soilw                (time, depthBelowLandLayer) float32 0.081 ... nan\n",
+       "    gppbfas              (time) float32 7.0 nan nan\n",
+       "Attributes:\n",
+       "    GRIB_edition:            2\n",
+       "    GRIB_centre:             kwbc\n",
+       "    GRIB_centreDescription:  US National Weather Service - NCEP\n",
+       "    GRIB_subCentre:          0\n",
+       "    Conventions:             CF-1.7\n",
+       "    institution:             US National Weather Service - NCEP\n",
+       "    model:                   rap\n",
+       "    product:                 awp130pgrb\n",
+       "    description:             Rapid Refresh (RAP) from NOMADS and Big Data Pro...\n",
+       "    remote_grib:             /home/blaylock/data/rap/20240101/rap.t00z.awp130...\n",
+       "    local_grib:              /home/blaylock/data/rap/20240101/subset_6bef9e80...\n",
+       "    searchString:            (?:TMP:2 m|GRD:10 m|DPT:2 m|GUST|TMP:1000 mb|TMP...
    " + ], + "text/plain": [ + "\n", + "Dimensions: (time: 3, heightAboveGround: 2, isobaricInhPa: 2,\n", + " depthBelowLandLayer: 9)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 2024-01-01 ... 2024-01-01T02:0...\n", + " step timedelta64[ns] 00:00:00\n", + " * heightAboveGround (heightAboveGround) float64 2.0 10.0\n", + " latitude float64 30.52\n", + " longitude float64 244.4\n", + " valid_time datetime64[ns] 2024-01-01\n", + " * isobaricInhPa (isobaricInhPa) float64 1e+03 500.0\n", + " surface float64 0.0\n", + " * depthBelowLandLayer (depthBelowLandLayer) float64 0.0 0.01 0.04 ... 1.6 3.0\n", + "Data variables:\n", + " u10 (heightAboveGround, time) float32 nan nan ... nan nan\n", + " v10 (heightAboveGround, time) float32 nan nan ... nan nan\n", + " t2m (heightAboveGround, time) float32 287.2 nan ... nan nan\n", + " d2m (heightAboveGround, time) float32 280.6 nan ... nan nan\n", + " t (time, isobaricInhPa) float32 289.6 255.8 ... nan nan\n", + " gust (time) float32 2.868 nan nan\n", + " st (time, depthBelowLandLayer) float32 287.9 288.2 ... nan\n", + " soilw (time, depthBelowLandLayer) float32 0.081 ... nan\n", + " gppbfas (time) float32 7.0 nan nan\n", + "Attributes:\n", + " GRIB_edition: 2\n", + " GRIB_centre: kwbc\n", + " GRIB_centreDescription: US National Weather Service - NCEP\n", + " GRIB_subCentre: 0\n", + " Conventions: CF-1.7\n", + " institution: US National Weather Service - NCEP\n", + " model: rap\n", + " product: awp130pgrb\n", + " description: Rapid Refresh (RAP) from NOMADS and Big Data Pro...\n", + " remote_grib: /home/blaylock/data/rap/20240101/rap.t00z.awp130...\n", + " local_grib: /home/blaylock/data/rap/20240101/subset_6bef9e80...\n", + " searchString: (?:TMP:2 m|GRD:10 m|DPT:2 m|GUST|TMP:1000 mb|TMP..." + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get data at a single point\n", + "point = ds.sel(x=100, y=100).squeeze()\n", + "point\n", + "\n", + "# (Note: to get data nearest a specific lat/lon point, you would have to\n", + "# do a nearest neighbor search on the latitude/longitude coordinates\n", + "# get the index, and then query the specific x/y location. Maybe a Ball\n", + "# Tree algorithm could be used for that.)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.DataArray 'u10' (heightAboveGround: 2, time: 3)>\n",
+       "array([[      nan,       nan,       nan],\n",
+       "       [1.8895483,       nan,       nan]], dtype=float32)\n",
+       "Coordinates:\n",
+       "  * time               (time) datetime64[ns] 2024-01-01 ... 2024-01-01T02:00:00\n",
+       "    step               timedelta64[ns] 00:00:00\n",
+       "  * heightAboveGround  (heightAboveGround) float64 2.0 10.0\n",
+       "    latitude           float64 30.52\n",
+       "    longitude          float64 244.4\n",
+       "    valid_time         datetime64[ns] 2024-01-01\n",
+       "    surface            float64 0.0\n",
+       "Attributes: (12/37)\n",
+       "    GRIB_paramId:                             165\n",
+       "    GRIB_dataType:                            fc\n",
+       "    GRIB_numberOfPoints:                      151987\n",
+       "    GRIB_typeOfLevel:                         heightAboveGround\n",
+       "    GRIB_stepUnits:                           1\n",
+       "    GRIB_stepType:                            instant\n",
+       "    ...                                       ...\n",
+       "    GRIB_stepRange:                           0\n",
+       "    GRIB_units:                               m s**-1\n",
+       "    long_name:                                10 metre U wind component\n",
+       "    units:                                    m s**-1\n",
+       "    standard_name:                            eastward_wind\n",
+       "    grid_mapping:                             gribfile_projection
    " + ], + "text/plain": [ + "\n", + "array([[ nan, nan, nan],\n", + " [1.8895483, nan, nan]], dtype=float32)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 2024-01-01 ... 2024-01-01T02:00:00\n", + " step timedelta64[ns] 00:00:00\n", + " * heightAboveGround (heightAboveGround) float64 2.0 10.0\n", + " latitude float64 30.52\n", + " longitude float64 244.4\n", + " valid_time datetime64[ns] 2024-01-01\n", + " surface float64 0.0\n", + "Attributes: (12/37)\n", + " GRIB_paramId: 165\n", + " GRIB_dataType: fc\n", + " GRIB_numberOfPoints: 151987\n", + " GRIB_typeOfLevel: heightAboveGround\n", + " GRIB_stepUnits: 1\n", + " GRIB_stepType: instant\n", + " ... ...\n", + " GRIB_stepRange: 0\n", + " GRIB_units: m s**-1\n", + " long_name: 10 metre U wind component\n", + " units: m s**-1\n", + " standard_name: eastward_wind\n", + " grid_mapping: gribfile_projection" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "point.u10" + ] } ], "metadata": {