diff --git a/src/wmrc/config.py b/src/wmrc/config.py index 937bb50..e33a091 100644 --- a/src/wmrc/config.py +++ b/src/wmrc/config.py @@ -22,7 +22,8 @@ AGOL_ORG = "https://utahdeq.maps.arcgis.com" SENDGRID_SETTINGS = { #: Settings for SendGridHandler "from_address": "noreply@utah.gov", - "to_addresses": ["jdadams@utah.gov", "stevienorcross@utah.gov", "gerardorodriguez@utah.gov"], + "to_addresses": ["jdadams@utah.gov"], + # "to_addresses": ["jdadams@utah.gov", "stevienorcross@utah.gov", "gerardorodriguez@utah.gov"], "prefix": f"{SKID_NAME} on {HOST_NAME}: ", } LOG_LEVEL = logging.DEBUG @@ -39,5 +40,6 @@ MATERIALS_LAYER_ITEMID = "7e13c71b32284915bad47983aca3602b" COMPOSTING_LAYER_ITEMID = "9b543c20de704839b90502a49ce1b8f3" +STATEWIDE_LAYER_ITEMID = "a5abaa46850e46ff88ee6024ce21f52e" YEAR = 2023 diff --git a/src/wmrc/helpers.py b/src/wmrc/helpers.py index 36b2d64..f748f9a 100644 --- a/src/wmrc/helpers.py +++ b/src/wmrc/helpers.py @@ -79,11 +79,11 @@ def _build_field_mapping(self): "Total Material managed by AD/C", "Municipal Waste In-State (in Tons)", "Facility Name", - # "Solid Waste Facility ID Number", + # "Solid Waste Facility ID Number", #: From nested Facility object "Combined Total of Material Recycled", "Total Materials recycled", "Total Materials sent to composting", - # "Combined Total Material for Composting", + # "Combined Total Material for Composting", #: Typo in field name "Total Material managed by AD/C", "Combined Total Material for Combustion", "Total Materials combusted", @@ -124,6 +124,7 @@ def _build_field_mapping(self): "Total Drywall received", "Total Other CM received", "Calendar Year", + "Annual Recycling Contamination Rate", ] missing_fields = [] for alias in aliases: @@ -309,3 +310,36 @@ def rates_per_material(year_df: pd.DataFrame, classification: str, fields: list[ ) return sum_df + + +def statewide_yearly_metrics(county_year_df: pd.DataFrame) -> pd.DataFrame: + """Calculate statewide yearly metrics for recycling, composting, digestion, and landfilling (RCDL). + + Args: + county_year_df (pd.DataFrame): Dataframe of county summaries for a given year with the RCDL metrics (can be + applied to a groupby (year) object). + + Returns: + pd.DataFrame: Statewide yearly metrics. + """ + statewide_series = pd.Series() + statewide_series["statewide_msw_recycled"] = county_year_df["county_wide_msw_recycled"].sum() + statewide_series["statewide_msw_composted"] = county_year_df["county_wide_msw_composted"].sum() + statewide_series["statewide_msw_digested"] = county_year_df["county_wide_msw_digested"].sum() + statewide_series["statewide_msw_landfilled"] = county_year_df["county_wide_msw_landfilled"].sum() + statewide_series["statewide_msw_recycling_rate"] = ( + ( + statewide_series["statewide_msw_recycled"] + + statewide_series["statewide_msw_composted"] + + statewide_series["statewide_msw_digested"] + ) + / ( + statewide_series["statewide_msw_recycled"] + + statewide_series["statewide_msw_composted"] + + statewide_series["statewide_msw_digested"] + + statewide_series["statewide_msw_landfilled"] + ) + * 100 + ) + + return statewide_series diff --git a/src/wmrc/main.py b/src/wmrc/main.py index f7bad21..80297b3 100644 --- a/src/wmrc/main.py +++ b/src/wmrc/main.py @@ -175,6 +175,16 @@ def process(self): composting_loader = load.FeatureServiceUpdater(gis, config.COMPOSTING_LAYER_ITEMID, self.tempdir_path) composting_count = composting_loader.truncate_and_load_features(composting_spatial) + #: Statewide metrics + self.skid_logger.info("Updating statewide metrics...") + statewide_totals_df = county_summary_df.groupby("data_year").apply(helpers.statewide_yearly_metrics) + contamination_rates_df = self._contamination_rates_by_tonnage(records) + # contamination_rates_df = self._contamination_rates_by_facility(records) + statewide_metrics = pd.concat([statewide_totals_df, contamination_rates_df], axis=1) + statewide_spatial = self._add_bogus_geometries(statewide_metrics) + statewide_loader = load.FeatureServiceUpdater(gis, config.STATEWIDE_LAYER_ITEMID, self.tempdir_path) + statewide_count = statewide_loader.truncate_and_load_features(statewide_spatial) + end = datetime.now() summary_message = MessageDetails() @@ -191,6 +201,7 @@ def process(self): f"County rows loaded: {counties_update_count}", f"Materials recycled rows loaded: {materials_count}", f"Materials composted rows loaded: {composting_count}", + f"Statewide metrics rows loaded: {statewide_count}", ] summary_message.message = "\n".join(summary_rows) @@ -452,6 +463,50 @@ def _materials_composted(records: helpers.SalesForceRecords) -> pd.DataFrame: return materials_composted + @staticmethod + def _contamination_rates_by_tonnage(records: helpers.SalesForceRecords) -> pd.DataFrame: + records.df["recycling_tons_contaminated"] = ( + records.df["Annual_Recycling_Contamination_Rate__c"] + / 100 + * records.df["Combined_Total_of_Material_Recycled__c"] + ) + records.df["recycling_tons_report_contamination_total"] = pd.NA + records.df.loc[ + ~records.df["recycling_tons_contaminated"].isnull(), "recycling_tons_report_contamination_total" + ] = records.df["Combined_Total_of_Material_Recycled__c"] + records.df[ + [ + "Combined_Total_of_Material_Recycled__c", + "Annual_Recycling_Contamination_Rate__c", + "recycling_tons_contaminated", + "recycling_tons_report_contamination_total", + ] + ].sort_values("Combined_Total_of_Material_Recycled__c", ascending=False).head(20) + + contamination_rates = records.df.groupby("Calendar_Year__c").apply( + lambda year_df: ( + 1 + - ( + year_df["recycling_tons_contaminated"].sum() + / year_df["recycling_tons_report_contamination_total"].sum() + ) + ) + * 100 + ) + contamination_rates.name = "annual_recycling_uncontaminated_rate" + contamination_rates.index.name = "data_year" + contamination_rates.index = contamination_rates.index.map(helpers.convert_to_int) + + return contamination_rates + + def _contamination_rates_by_facility(records: helpers.SalesForceRecords) -> pd.DataFrame: + + records.df["annual_recycling_uncontaminated_rate"] = 100 - records.df["Annual_Recycling_Contamination_Rate__c"] + yearly_stats = records.df.groupby("Calendar_Year__c").describe() + yearly_stats.index = yearly_stats.index.map(helpers.convert_to_int) + yearly_stats.index.name = "data_year" + return yearly_stats[["count", "mean", "std"]] + @staticmethod def _add_bogus_geometries(input_dataframe: pd.DataFrame) -> pd.DataFrame: """Add a bogus geometry (point in downtown Malad City, ID) to a dataframe in WKID 4326.