diff --git a/src/wmrc/config.py b/src/wmrc/config.py index 540e7d9..937bb50 100644 --- a/src/wmrc/config.py +++ b/src/wmrc/config.py @@ -37,6 +37,7 @@ COUNTY_LAYER_ITEMID = "ea80c3c34bb147fba462ea100179bf09" COUNTY_BOUNDARIES_ITEMID = "4d0671aac09347d3a03415632f19ddb3" #: Must include a feature with the name "Out of State" -MATERIALS_LAYER_ITEM_ID = "43c5fe6a798346b6955f549404348f56" +MATERIALS_LAYER_ITEMID = "7e13c71b32284915bad47983aca3602b" +COMPOSTING_LAYER_ITEMID = "9b543c20de704839b90502a49ce1b8f3" YEAR = 2023 diff --git a/src/wmrc/main.py b/src/wmrc/main.py index c274196..f7bad21 100644 --- a/src/wmrc/main.py +++ b/src/wmrc/main.py @@ -149,19 +149,31 @@ def process(self): self.skid_logger.info("Loading records from Salesforce...") records = self._load_salesforce_data() facility_summary_df = self._facility_summaries(records).query("data_year == @config.YEAR") - county_summary_df = self._county_summaries(records) #.query("data_year == @config.YEAR") - # materials_recycled_df = self._materials_recycled(records) - # materials_composted_df = self._materials_composted(records) + county_summary_df = self._county_summaries(records) # .query("data_year == @config.YEAR") + materials_recycled_df = self._materials_recycled(records) + materials_composted_df = self._materials_composted(records) #: Facilities on map + self.skid_logger.info("Updating facility info...") facilities_load_count = self._update_facilities(gis, facility_summary_df) #: county summaries on map, dashboard: + self.skid_logger.info("Updating county info...") counties_update_count = self._update_counties(gis, county_summary_df) #: Materials recycled on dashboard: - #: Truncate and load live data with sf analyses - + self.skid_logger.info("Updating materials recycled...") + materials_spatial = self._add_bogus_geometries(materials_recycled_df) + materials_spatial.rename(columns={"percent": "percent_"}, inplace=True) + materials_loader = load.FeatureServiceUpdater(gis, config.MATERIALS_LAYER_ITEMID, self.tempdir_path) + materials_count = materials_loader.truncate_and_load_features(materials_spatial) + + #: Materials composted on dashboard: + self.skid_logger.info("Updating materials composted...") + composting_spatial = self._add_bogus_geometries(materials_composted_df) + composting_spatial.rename(columns={"percent": "percent_"}, inplace=True) + composting_loader = load.FeatureServiceUpdater(gis, config.COMPOSTING_LAYER_ITEMID, self.tempdir_path) + composting_count = composting_loader.truncate_and_load_features(composting_spatial) end = datetime.now() @@ -177,6 +189,8 @@ def process(self): "", f"Facility rows loaded: {facilities_load_count}", f"County rows loaded: {counties_update_count}", + f"Materials recycled rows loaded: {materials_count}", + f"Materials composted rows loaded: {composting_count}", ] summary_message.message = "\n".join(summary_rows) @@ -189,12 +203,12 @@ def process(self): self._remove_log_file_handlers(loggers) def _update_counties(self, gis, county_summary_df): - # existing_county_data = transform.FeatureServiceMerging.get_live_dataframe(gis, config.COUNTY_LAYER_ITEMID) - county_geoms = transform.FeatureServiceMerging.get_live_dataframe(gis, config.COUNTY_BOUNDARIES_ITEMID)[["name", "SHAPE"]].set_index("name") + county_geoms = transform.FeatureServiceMerging.get_live_dataframe(gis, config.COUNTY_BOUNDARIES_ITEMID)[ + ["name", "SHAPE"] + ].set_index("name") new_data = county_summary_df.merge(county_geoms, left_index=True, right_index=True, how="left") - # existing_county_data.update(county_summary_df) new_data.reset_index(inplace=True) new_data.spatial.project(4326) new_data.spatial.sr = {"wkid": 4326} @@ -339,7 +353,9 @@ def _county_summaries(records: helpers.SalesForceRecords) -> pd.DataFrame: ) county_df.index.names = ["data_year", "name"] county_df.reset_index(level="data_year", inplace=True) - county_df.rename(index={name: name.replace("__c", "").replace("_", " ") for name in county_df.index}, inplace=True) + county_df.rename( + index={name: name.replace("__c", "").replace("_", " ") for name in county_df.index}, inplace=True + ) # county_df.index = county_df.index.apply(lambda x: x.replace("__c", "").replace("_", " ")) county_df["data_year"] = county_df["data_year"].apply(helpers.convert_to_int) @@ -395,9 +411,9 @@ def _materials_recycled(records: helpers.SalesForceRecords) -> pd.DataFrame: ) .droplevel(1) .reset_index() - .rename(columns={"Calendar_Year__c": "year"}) + .rename(columns={"Calendar_Year__c": "year_"}) ) - materials_recycled["year"] = materials_recycled["year"].apply(helpers.convert_to_int) + materials_recycled["year_"] = materials_recycled["year_"].apply(helpers.convert_to_int) return materials_recycled @@ -430,12 +446,32 @@ def _materials_composted(records: helpers.SalesForceRecords) -> pd.DataFrame: ) .droplevel(1) .reset_index() - .rename(columns={"Calendar_Year__c": "year"}) + .rename(columns={"Calendar_Year__c": "year_"}) ) - materials_composted["year"] = materials_composted["year"].apply(helpers.convert_to_int) + materials_composted["year_"] = materials_composted["year_"].apply(helpers.convert_to_int) return materials_composted + @staticmethod + def _add_bogus_geometries(input_dataframe: pd.DataFrame) -> pd.DataFrame: + """Add a bogus geometry (point in downtown Malad City, ID) to a dataframe in WKID 4326. + + Args: + input_dataframe (pd.DataFrame): Non-spatial dataframe to add geometry to + + Returns: + pd.DataFrame: Spatially-enabled dataframe version of input input_dataframe with geometry added to every row + """ + + input_dataframe["x"] = 12_495_000 + input_dataframe["y"] = 5_188_000 + + spatial_dataframe = pd.DataFrame.spatial.from_xy(input_dataframe, "x", "y", sr=4326) + + spatial_dataframe.drop(columns=["x", "y"], inplace=True) + + return spatial_dataframe + def main(event, context): # pylint: disable=unused-argument """Entry point for Google Cloud Function triggered by pub/sub event diff --git a/tests/test_wmrc.py b/tests/test_wmrc.py index 005b830..28b7a05 100644 --- a/tests/test_wmrc.py +++ b/tests/test_wmrc.py @@ -1,5 +1,4 @@ import pandas as pd - from wmrc import main @@ -65,6 +64,7 @@ def test_update_counties_merges_data_and_shapes(self, mocker): pd.testing.assert_frame_equal(updater_mock.truncate_and_load_features.call_args[0][0], test_df) + class TestSummaryMethods: def test_county_summaries_happy_path(self, mocker): @@ -101,3 +101,18 @@ def test_county_summaries_happy_path(self, mocker): test_df.index.name = "name" pd.testing.assert_frame_equal(result_df, test_df) + + +class TestSmallMethods: + + def test_add_bogus_geometries_happy_path(self): + input_df = pd.DataFrame( + { + "a": [1, 2], + "b": [3, 4], + } + ) + + result_df = main.Skid._add_bogus_geometries(input_df) + + assert result_df.spatial.validate()