Merge pull request #22 from ShorensteinCenter/devel

[Bugfix, N/A] Added exponential backoff to async api requests for other error types

README.md

@@ -36,7 +36,8 @@ These instructions will get you a copy of the project up and running on your loc
 * `SES_REGION_NAME` - AWS Simple Email Service region.
 * `SES_DEFAULT_EMAIL_SOURCE` - The default email address to send from. This email needs to be verified by SES and active outside the SES sandbox.
 * `SES_CONFIGURATION_SET` - SES Configuration set for tracking opens/clicks/etc. Optional.
-* `NO_PROXY` - We use proxies to distribute our MailChimp requests across IP addresses. Set this variable to `True` in order to disable proxying, or modify the `enable_proxy` method in `app/lists.py` according to your proxy setup.
+* `NO_PROXY` - We use proxies to distribute our MailChimp requests across IP addresses. Set this variable to `True` in order to disable proxying, or modify the `enable_proxy` method in `app/lists.py` according to your proxy configuration.
+* `NO_EMAIL` - Suppresses sending of emails. Note that SES is still required, emails will just be logged rather than sent. Optional.
 * `ADMIN_EMAIL` - Email address to send error emails to. Optional.
 
 ##### Upgrade the database

app/emails.py

@@ -1,4 +1,6 @@
 """This module contains functions associated with sending email."""
+import os
+import logging
 import boto3
 from flask import render_template
 from app import app
@@ -32,6 +34,13 @@ def send_email(subject, recipients, template_name, template_context, # pylint: d
 
  with app.app_context():
  html = render_template(template_name, **template_context)
+ if os.environ.get('NO_EMAIL'):
+ logger = logging.getLogger(__name__)
+ logger.warning('NO_EMAIL environment variable set. '
+ 'Suppressing an email with the following params: '
+ 'Sender: %s. Recipients: %s. Subject: %s.',
+ sender, recipients, subject)
+ return
  ses.send_email(
  Source=sender,
  Destination={'ToAddresses': recipients},

app/lists.py

@@ -12,8 +12,6 @@
 from pandas.io.json import json_normalize
 import numpy as np
 from aiohttp import ClientSession, BasicAuth
-from aiohttp.client_exceptions import (
- ClientHttpProxyError, ServerDisconnectedError)
 import iso8601
 from celery.utils.log import get_task_logger
 
@@ -247,9 +245,7 @@ async def make_async_request(self, url, params, session, retry=0):
  error_details)
 
  # Catch proxy problems as well as potential asyncio timeouts/disconnects
- except (asyncio.TimeoutError, # pylint: disable=invalid-name
- ClientHttpProxyError,
- ServerDisconnectedError) as e:
+ except Exception as e: # pylint: disable=invalid-name
 
  exception_type = type(e).__name__
 
@@ -261,10 +257,15 @@ async def make_async_request(self, url, params, session, retry=0):
  self.logger.warning('Server disconnected! URL: %s. API key: '
  '%s.', url, self.api_key)
 
- else:
+ elif exception_type == 'TimeoutError':
  self.logger.warning('Asyncio request timed out! URL: %s. '
  'API key: %s.', url, self.api_key)
 
+ else:
+ self.logger.warning('An unforseen error type occurred. '
+ 'Error type: %s. URL: %s. API Key: %s.',
+ exception_type, url, self.api_key)
+
  # Retry if we haven't already retried a few times
  if retry < self.MAX_RETRIES: