diff --git a/cool_finance/constants.py b/cool_finance/constants.py index 1edd7ab..c263e3f 100644 --- a/cool_finance/constants.py +++ b/cool_finance/constants.py @@ -9,9 +9,9 @@ LOG_FORMAT = '%(asctime)s - %(name)s - %(threadName)s - ' \ '%(levelname)s - %(message)s' # DEBUG_LOG_LEVEL must stricter than LOG_LEVEL -# if DEBUG_LOG_FILE is not None, DEBUG_LOG_LEVEL must be set -# if if DEBUG_LOG_FILE is None, DEBUG_LOG_LEVEL would be ignored -DEBUG_LOG_FILE = None +# if DEBUG_LOG_DIR is not None, DEBUG_LOG_LEVEL must be set +# if if DEBUG_LOG_DIR is None, DEBUG_LOG_LEVEL would be ignored +DEBUG_LOG_DIR = None DEBUG_LOG_LEVEL = "DEBUG" # set START_NOW will override the START_HOUR_MIN_SEC @@ -22,3 +22,11 @@ TIMEZONE = 'US/Eastern' # Sat, Sun are closed market day CLOSED_WEEKDAYS = [5, 6] +# Only one notification will be generated +# every NOTIFICATION_INTERVAL_S seconds for same stock. +NOTIFICATION_INTERVAL_S = 300 +# Guarantee query result is within QUERY_PRECISION_S seconds up to date. +# A significant large number (10x, 100x) can reduce query amount to +# date source server. Tweak it if you have a daily query limit. +# Google data source doesn't seem to have a limit. +QUERY_PRECISION_S = 0.1 diff --git a/cool_finance/data_sources/constants.py b/cool_finance/data_sources/constants.py index a75c0b5..b42db08 100644 --- a/cool_finance/data_sources/constants.py +++ b/cool_finance/data_sources/constants.py @@ -1,8 +1,9 @@ # datasource vendor names BASE_VENDOR = "BASE" GOOGLE_FINANCE_VENDOR = "google_finance" +GOOGLE_FINANCE_BATCH_VENDOR = "google_finance_batch" -DEFAULT_DATASOURCE = GOOGLE_FINANCE_VENDOR +DEFAULT_DATASOURCE = GOOGLE_FINANCE_BATCH_VENDOR INDEX = "index" # NASDAQ, NYSE... LAST_TRADE_PRICE = "last_trade_price" diff --git a/cool_finance/data_sources/manager.py b/cool_finance/data_sources/manager.py index 200d103..9486a11 100644 --- a/cool_finance/data_sources/manager.py +++ b/cool_finance/data_sources/manager.py @@ -1,12 +1,20 @@ from cool_finance.data_sources import constants as const -from cool_finance.data_sources.vendors import (common, google_finance) +from cool_finance.data_sources.vendors import (common, google_finance, + google_finance_batch) class DataSourceManager(object): _supported_vendors = { const.BASE_VENDOR: common.BaseSource, - const.GOOGLE_FINANCE_VENDOR: google_finance.GoogleFinance + const.GOOGLE_FINANCE_VENDOR: google_finance.GoogleFinance, + const.GOOGLE_FINANCE_BATCH_VENDOR: + google_finance_batch.GoogleFinanceBatch + } + + _support_batch_query_vendor = { + const.GOOGLE_FINANCE_BATCH_VENDOR: + google_finance_batch.GoogleFinanceBatch } def __init__(self, default_vendor=const.DEFAULT_DATASOURCE): diff --git a/cool_finance/data_sources/vendors/common.py b/cool_finance/data_sources/vendors/common.py index ac39b5f..b01a9f7 100644 --- a/cool_finance/data_sources/vendors/common.py +++ b/cool_finance/data_sources/vendors/common.py @@ -38,6 +38,7 @@ class BaseSource(object): ] def __init__(self, stock_symbol, *args, **kwargs): + stock_symbol = stock_symbol.encode('utf-8').upper() self.stock_symbol = stock_symbol if self._support_data_json: self._data_json = None diff --git a/cool_finance/data_sources/vendors/google_finance.py b/cool_finance/data_sources/vendors/google_finance.py index 005cf2a..65f0e25 100644 --- a/cool_finance/data_sources/vendors/google_finance.py +++ b/cool_finance/data_sources/vendors/google_finance.py @@ -28,5 +28,4 @@ class GoogleFinance(BaseSource): def _fetch(self, *args, **kwargs): # googlefinance has a bug if the getQuote() input is unicode # change the type to str explicitly - stock_symbol = self.stock_symbol.encode('utf-8') - return getQuotes(stock_symbol)[0] + return getQuotes(self.stock_symbol)[0] diff --git a/cool_finance/data_sources/vendors/google_finance_batch.py b/cool_finance/data_sources/vendors/google_finance_batch.py new file mode 100644 index 0000000..1fc2cb8 --- /dev/null +++ b/cool_finance/data_sources/vendors/google_finance_batch.py @@ -0,0 +1,83 @@ +import datetime +from threading import Lock + +from googlefinance import getQuotes + +from cool_finance.constants import QUERY_PRECISION_S +from cool_finance.data_sources import constants as const +from cool_finance.data_sources.vendors.google_finance import GoogleFinance +from cool_finance.log import logger + +LAST_UPDATE = "last_update" +STOCKS_DATA = "stocks_data" + + +class GoogleFinanceBatchHandler(object): + + _symbol_key = GoogleFinance._data_json_keys[const.STOCK_SYMBOL] + + def __init__(self): + self._stocks_list = [] + # self._stocks_data = {"last_update":datetime, + # "stocks_data": { + # "ABC":{...}}, + # "BCD":{...}} } } + self._stocks_data = {LAST_UPDATE: None, + STOCKS_DATA: {}} + self._data_access_lock = Lock() + + def add_stock(self, stock_symbol): + # googlefinance has a bug if the getQuote() input is unicode + # change the type to str explicitly + self._stocks_list.append(stock_symbol) + + def fetch(self, stock_symbol): + # Multiple works could call fetch() as well as _should_update() + # and _fetch_batch(). Must use the _data_access_lock to guard them. + with self._data_access_lock: + if self._should_update(): + self._stocks_data = self._fetch_batch(self._stocks_list) + logger.debug("Look for %s, new request sent to Google", + stock_symbol) + return dict(self._stocks_data[STOCKS_DATA][stock_symbol]) + + def _should_update(self): + if self._stocks_data[LAST_UPDATE]: + now = datetime.datetime.now() + last_update = self._stocks_data[LAST_UPDATE] + delta = now - last_update + if delta.total_seconds() <= QUERY_PRECISION_S: + return False + return True + + def _fetch_batch(self, stocks_list): + if not stocks_list: + stocks_list = self._stocks_list + quotes_list = getQuotes(stocks_list) + now = datetime.datetime.now() + self._stocks_data[LAST_UPDATE] = now + self._stocks_data[STOCKS_DATA].clear() + for item in quotes_list: + stock_symbol = item[self._symbol_key] + self._stocks_data[STOCKS_DATA][stock_symbol] = item + return self._stocks_data + + +batch_handler = GoogleFinanceBatchHandler() + + +class GoogleFinanceBatch(GoogleFinance): + + vendor_name = const.GOOGLE_FINANCE_BATCH_VENDOR + + def __init__(self, stock_symbol): + super(GoogleFinanceBatch, self).__init__(stock_symbol) + global batch_handler + self.batch_handler = batch_handler + # googlefinance has a bug if the getQuote() input is unicode + # change the type to str explicitly + stock_symbol = stock_symbol.encode('utf-8') + self.batch_handler.add_stock(stock_symbol) + + def _fetch(self, *args, **kwargs): + return self.batch_handler.fetch(self.stock_symbol)