feat: statewide stats

src/wmrc/config.py

@@ -22,7 +22,8 @@
 AGOL_ORG = "https://utahdeq.maps.arcgis.com"
 SENDGRID_SETTINGS = {  #: Settings for SendGridHandler
     "from_address": "noreply@utah.gov",
-    "to_addresses": ["jdadams@utah.gov", "stevienorcross@utah.gov", "gerardorodriguez@utah.gov"],
+    "to_addresses": ["jdadams@utah.gov"],
+    # "to_addresses": ["jdadams@utah.gov", "stevienorcross@utah.gov", "gerardorodriguez@utah.gov"],
     "prefix": f"{SKID_NAME} on {HOST_NAME}: ",
 }
 LOG_LEVEL = logging.DEBUG
@@ -39,5 +40,6 @@
 
 MATERIALS_LAYER_ITEMID = "7e13c71b32284915bad47983aca3602b"
 COMPOSTING_LAYER_ITEMID = "9b543c20de704839b90502a49ce1b8f3"
+STATEWIDE_LAYER_ITEMID = "a5abaa46850e46ff88ee6024ce21f52e"
 
 YEAR = 2023

src/wmrc/helpers.py

@@ -79,11 +79,11 @@ def _build_field_mapping(self):
             "Total Material managed by AD/C",
             "Municipal Waste In-State (in Tons)",
             "Facility Name",
-            # "Solid Waste Facility ID Number",
+            # "Solid Waste Facility ID Number",  #: From nested Facility object
             "Combined Total of Material Recycled",
             "Total Materials recycled",
             "Total Materials sent to composting",
-            # "Combined Total Material for Composting",
+            # "Combined Total Material for Composting",  #: Typo in field name
             "Total Material managed by AD/C",
             "Combined Total Material for Combustion",
             "Total Materials combusted",
@@ -124,6 +124,7 @@ def _build_field_mapping(self):
             "Total Drywall received",
             "Total Other CM received",
             "Calendar Year",
+            "Annual Recycling Contamination Rate",
         ]
         missing_fields = []
         for alias in aliases:
@@ -309,3 +310,36 @@ def rates_per_material(year_df: pd.DataFrame, classification: str, fields: list[
     )
 
     return sum_df
+
+
+def statewide_yearly_metrics(county_year_df: pd.DataFrame) -> pd.DataFrame:
+    """Calculate statewide yearly metrics for recycling, composting, digestion, and landfilling (RCDL).
+
+    Args:
+        county_year_df (pd.DataFrame): Dataframe of county summaries for a given year with the RCDL metrics (can be
+            applied to a groupby (year) object).
+
+    Returns:
+        pd.DataFrame: Statewide yearly metrics.
+    """
+    statewide_series = pd.Series()
+    statewide_series["statewide_msw_recycled"] = county_year_df["county_wide_msw_recycled"].sum()
+    statewide_series["statewide_msw_composted"] = county_year_df["county_wide_msw_composted"].sum()
+    statewide_series["statewide_msw_digested"] = county_year_df["county_wide_msw_digested"].sum()
+    statewide_series["statewide_msw_landfilled"] = county_year_df["county_wide_msw_landfilled"].sum()
+    statewide_series["statewide_msw_recycling_rate"] = (
+        (
+            statewide_series["statewide_msw_recycled"]
+            + statewide_series["statewide_msw_composted"]
+            + statewide_series["statewide_msw_digested"]
+        )
+        / (
+            statewide_series["statewide_msw_recycled"]
+            + statewide_series["statewide_msw_composted"]
+            + statewide_series["statewide_msw_digested"]
+            + statewide_series["statewide_msw_landfilled"]
+        )
+        * 100
+    )
+
+    return statewide_series

src/wmrc/main.py

@@ -175,6 +175,16 @@ def process(self):
         composting_loader = load.FeatureServiceUpdater(gis, config.COMPOSTING_LAYER_ITEMID, self.tempdir_path)
         composting_count = composting_loader.truncate_and_load_features(composting_spatial)
 
+        #: Statewide metrics
+        self.skid_logger.info("Updating statewide metrics...")
+        statewide_totals_df = county_summary_df.groupby("data_year").apply(helpers.statewide_yearly_metrics)
+        contamination_rates_df = self._contamination_rates_by_tonnage(records)
+        # contamination_rates_df = self._contamination_rates_by_facility(records)
+        statewide_metrics = pd.concat([statewide_totals_df, contamination_rates_df], axis=1)
+        statewide_spatial = self._add_bogus_geometries(statewide_metrics)
+        statewide_loader = load.FeatureServiceUpdater(gis, config.STATEWIDE_LAYER_ITEMID, self.tempdir_path)
+        statewide_count = statewide_loader.truncate_and_load_features(statewide_spatial)
+
         end = datetime.now()
 
         summary_message = MessageDetails()
@@ -191,6 +201,7 @@ def process(self):
             f"County rows loaded: {counties_update_count}",
             f"Materials recycled rows loaded: {materials_count}",
             f"Materials composted rows loaded: {composting_count}",
+            f"Statewide metrics rows loaded: {statewide_count}",
         ]
 
         summary_message.message = "\n".join(summary_rows)
@@ -452,6 +463,50 @@ def _materials_composted(records: helpers.SalesForceRecords) -> pd.DataFrame:
 
         return materials_composted
 
+    @staticmethod
+    def _contamination_rates_by_tonnage(records: helpers.SalesForceRecords) -> pd.DataFrame:
+        records.df["recycling_tons_contaminated"] = (
+            records.df["Annual_Recycling_Contamination_Rate__c"]
+            / 100
+            * records.df["Combined_Total_of_Material_Recycled__c"]
+        )
+        records.df["recycling_tons_report_contamination_total"] = pd.NA
+        records.df.loc[
+            ~records.df["recycling_tons_contaminated"].isnull(), "recycling_tons_report_contamination_total"
+        ] = records.df["Combined_Total_of_Material_Recycled__c"]
+        records.df[
+            [
+                "Combined_Total_of_Material_Recycled__c",
+                "Annual_Recycling_Contamination_Rate__c",
+                "recycling_tons_contaminated",
+                "recycling_tons_report_contamination_total",
+            ]
+        ].sort_values("Combined_Total_of_Material_Recycled__c", ascending=False).head(20)
+
+        contamination_rates = records.df.groupby("Calendar_Year__c").apply(
+            lambda year_df: (
+                1
+                - (
+                    year_df["recycling_tons_contaminated"].sum()
+                    / year_df["recycling_tons_report_contamination_total"].sum()
+                )
+            )
+            * 100
+        )
+        contamination_rates.name = "annual_recycling_uncontaminated_rate"
+        contamination_rates.index.name = "data_year"
+        contamination_rates.index = contamination_rates.index.map(helpers.convert_to_int)
+
+        return contamination_rates
+
+    def _contamination_rates_by_facility(records: helpers.SalesForceRecords) -> pd.DataFrame:
+
+        records.df["annual_recycling_uncontaminated_rate"] = 100 - records.df["Annual_Recycling_Contamination_Rate__c"]
+        yearly_stats = records.df.groupby("Calendar_Year__c").describe()
+        yearly_stats.index = yearly_stats.index.map(helpers.convert_to_int)
+        yearly_stats.index.name = "data_year"
+        return yearly_stats[["count", "mean", "std"]]
+
     @staticmethod
     def _add_bogus_geometries(input_dataframe: pd.DataFrame) -> pd.DataFrame:
         """Add a bogus geometry (point in downtown Malad City, ID) to a dataframe in WKID 4326.