From 58c442c965e66934a39ca69f15fd6d8ae3280f93 Mon Sep 17 00:00:00 2001 From: Luiz Felipe Zafra Saggioro Date: Wed, 1 Jan 2020 13:44:52 -0500 Subject: [PATCH 1/2] Add HTTPError handling when Reimbursement URL doesn't exist The issue happened when running Rosie on 2020-01-01. By the way the adapter.py works, it tries to fetch reimbursement files up to the year in which the application is running .e.g. 2020. Since there is no such URL (at the time of this commit), the whole process fails due to an HTTPError 404. By adding this handling, the reimbursements that cannot be found are skipped from the download process and the application runs without any issues. --- rosie/rosie/chamber_of_deputies/adapter.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rosie/rosie/chamber_of_deputies/adapter.py b/rosie/rosie/chamber_of_deputies/adapter.py index 566fc1bbb..8d5059304 100644 --- a/rosie/rosie/chamber_of_deputies/adapter.py +++ b/rosie/rosie/chamber_of_deputies/adapter.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +from urllib.error import HTTPError from serenata_toolbox.chamber_of_deputies.reimbursements import Reimbursements from serenata_toolbox.datasets import fetch @@ -85,7 +86,10 @@ def update_reimbursements(self, years=None): for year in years: self.log.info(f'Updating reimbursements from {year}') - Reimbursements(year, self.path)() + try: + Reimbursements(year, self.path)() + except HTTPError as e: + self.log.error(f'Could not update Reimbursement from year {year}: {e} - {e.filename}') def prepare_dataset(self, df): self.rename_categories(df) From 6cd58b3fe42c6d93245ab023f205b57a0f84590e Mon Sep 17 00:00:00 2001 From: Luiz Felipe Zafra Saggioro Date: Wed, 1 Jan 2020 13:56:59 -0500 Subject: [PATCH 2/2] Update 'Reimbursement' to 'Reimbursements' on the log message --- rosie/rosie/chamber_of_deputies/adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rosie/rosie/chamber_of_deputies/adapter.py b/rosie/rosie/chamber_of_deputies/adapter.py index 8d5059304..204721237 100644 --- a/rosie/rosie/chamber_of_deputies/adapter.py +++ b/rosie/rosie/chamber_of_deputies/adapter.py @@ -89,7 +89,7 @@ def update_reimbursements(self, years=None): try: Reimbursements(year, self.path)() except HTTPError as e: - self.log.error(f'Could not update Reimbursement from year {year}: {e} - {e.filename}') + self.log.error(f'Could not update Reimbursements from year {year}: {e} - {e.filename}') def prepare_dataset(self, df): self.rename_categories(df)