From 6784d288cc01890ec86692ce6f91d88bd5ec35e4 Mon Sep 17 00:00:00 2001 From: maoyang Date: Sat, 22 Nov 2014 08:33:47 +0800 Subject: [PATCH 1/2] move get_raw_rows_name from SimpleAnalytics to FetchData , since __raw_rows_name cannot be accessed in SimpleAnalytics class --- grs/fetch_data.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/grs/fetch_data.py b/grs/fetch_data.py index f8dec7e..7c14a49 100644 --- a/grs/fetch_data.py +++ b/grs/fetch_data.py @@ -79,7 +79,24 @@ def info(self): :returns: (股票代碼, 股票名稱) """ return self.__info + @property + def get_raw_rows_name(self): + """ 原始檔案的欄位名稱 + + 0. 日期 + 1. 成交股數 + 2. 成交金額 + 3. 開盤價 + 4. 最高價（續） + 5. 最低價 + 6. 收盤價 + 7. 漲跌價差 + 8. 成交筆數 + :rtype: list + """ + result = [i.decode('cp950') for i in self.__raw_rows_name] + return result def to_list(self, csv_file): """ 串接每日資料 舊→新 @@ -242,24 +259,7 @@ def get_raw_rows(self, rows=6): """ return self.__serial_price(rows) - @property - def get_raw_rows_name(self): - """ 原始檔案的欄位名稱 - - 0. 日期 - 1. 成交股數 - 2. 成交金額 - 3. 開盤價 - 4. 最高價（續） - 5. 最低價 - 6. 收盤價 - 7. 漲跌價差 - 8. 成交筆數 - - :rtype: list - """ - result = [i.decode('cp950') for i in self.__raw_rows_name] - return result + def plus_mons(self, month): """ 新增擴充月份資料 From b2876665cee7684fa85549cd645cae38dee7fc19 Mon Sep 17 00:00:00 2001 From: "maoyang.chien" Date: Mon, 24 Nov 2014 05:18:07 +0800 Subject: [PATCH 2/2] for Python3 porting to python3 --- grs/best_buy_or_sell.py | 16 ++++---- grs/fetch_data.py | 35 ++++++++-------- grs/realtime.py | 8 ++-- grs/realtime2.py | 4 +- grs/twseno.py | 16 ++++---- setup.py | 2 +- test_unittest.py | 10 ++--- tools/make_otc_list.py | 82 ++++++++++++++++++------------------- tools/make_twse_list.py | 90 ++++++++++++++++++++--------------------- 9 files changed, 133 insertions(+), 130 deletions(-) diff --git a/grs/best_buy_or_sell.py b/grs/best_buy_or_sell.py index 5a55021..dcc29f4 100644 --- a/grs/best_buy_or_sell.py +++ b/grs/best_buy_or_sell.py @@ -124,13 +124,13 @@ def best_four_point_to_buy(self): (self.best_buy_1() or self.best_buy_2() or self.best_buy_3() or \ self.best_buy_4()): if self.best_buy_1(): - result.append(self.best_buy_1.__doc__.strip().decode('utf-8')) + result.append(self.best_buy_1.__doc__.strip()) if self.best_buy_2(): - result.append(self.best_buy_2.__doc__.strip().decode('utf-8')) + result.append(self.best_buy_2.__doc__.strip()) if self.best_buy_3(): - result.append(self.best_buy_3.__doc__.strip().decode('utf-8')) + result.append(self.best_buy_3.__doc__.strip()) if self.best_buy_4(): - result.append(self.best_buy_4.__doc__.strip().decode('utf-8')) + result.append(self.best_buy_4.__doc__.strip()) result = ', '.join(result) else: result = False @@ -146,13 +146,13 @@ def best_four_point_to_sell(self): (self.best_sell_1() or self.best_sell_2() or self.best_sell_3() or \ self.best_sell_4()): if self.best_sell_1(): - result.append(self.best_sell_1.__doc__.strip().decode('utf-8')) + result.append(self.best_sell_1.__doc__.strip()) if self.best_sell_2(): - result.append(self.best_sell_2.__doc__.strip().decode('utf-8')) + result.append(self.best_sell_2.__doc__.strip()) if self.best_sell_3(): - result.append(self.best_sell_3.__doc__.strip().decode('utf-8')) + result.append(self.best_sell_3.__doc__.strip()) if self.best_sell_4(): - result.append(self.best_sell_4.__doc__.strip().decode('utf-8')) + result.append(self.best_sell_4.__doc__.strip()) result = ', '.join(result) else: result = False diff --git a/grs/fetch_data.py b/grs/fetch_data.py index 7c14a49..93c8779 100644 --- a/grs/fetch_data.py +++ b/grs/fetch_data.py @@ -24,11 +24,12 @@ import logging import random import urllib3 +import types from .error import ConnectionError from .error import StockNoError from .twseno import OTCNo from .twseno import TWSENo -from cStringIO import StringIO +from io import StringIO from datetime import datetime from dateutil.relativedelta import relativedelta @@ -95,7 +96,7 @@ def get_raw_rows_name(self): :rtype: list """ - result = [i.decode('cp950') for i in self.__raw_rows_name] + result = [i for i in self.__raw_rows_name] return result def to_list(self, csv_file): """ 串接每日資料 舊→新 @@ -115,7 +116,7 @@ def to_list(self, csv_file): if self._twse: if tolist: self.__info = (tolist[0][0].split(' ')[1], - tolist[0][0].split(' ')[2].decode('cp950')) + tolist[0][0].split(' ')[2]) self.__raw_rows_name = tolist[1] return tuple(tolist[2:]) return tuple([]) @@ -181,7 +182,8 @@ def fetch_data(self, stock_no, nowdatetime): logging.info(url) result = GRETAI_CONNECTIONS.urlopen('GET', url) - csv_files = csv.reader(StringIO(result.data)) + resultdataStr = str(result.data,'cp950') + csv_files = csv.reader(StringIO(resultdataStr)) self.__url.append(GRETAI_HOST + url) return csv_files @@ -221,7 +223,8 @@ def fetch_data(self, stock_no, nowdatetime): 'rand': random.randrange(1, 1000000)} logging.info(url) result = TWSE_CONNECTIONS.urlopen('GET', url) - csv_files = csv.reader(StringIO(result.data)) + resultdataStr = str(result.data,'cp950') + csv_files = csv.reader(StringIO(resultdataStr)) self.__url.append(TWSE_HOST + url) return csv_files @@ -474,31 +477,31 @@ def __init__(self, stock_no, mons=3, twse=False, otc=False): pass def __new__(cls, stock_no, mons=3, twse=False, otc=False): - assert isinstance(stock_no, basestring), '`stock_no` must be a string' + assert isinstance(stock_no, str), '`stock_no` must be a string' assert not twse == otc == True, 'Only `twse` or `otc` to be True' if twse and not otc: - stock_proxy = type('Stock', (TWSEFetch, SimpleAnalytics), {})() + stock_proxy = types.new_class('Stock', (TWSEFetch, SimpleAnalytics)) twse = True elif not twse and otc: - stock_proxy = type('Stock', (OTCFetch, SimpleAnalytics), {})() + stock_proxy = types.new_class('Stock', (OTCFetch, SimpleAnalytics)) twse = False elif stock_no in TWSENo().all_stock_no: - stock_proxy = type('Stock', (TWSEFetch, SimpleAnalytics), {})() + stock_proxy = types.new_class('Stock', (TWSEFetch, SimpleAnalytics)) twse = True elif stock_no in OTCNo().all_stock_no: - stock_proxy = type('Stock', (OTCFetch, SimpleAnalytics), {})() + stock_proxy = types.new_class('Stock', (OTCFetch, SimpleAnalytics)) twse = False else: raise StockNoError() - stock_proxy.__init__() + stockproxy = stock_proxy() try: - cls.__raw_data = stock_proxy.serial_fetch(stock_no, mons, twse) - stock_proxy._load_data(cls.__raw_data) + stockproxy.__raw_data = stockproxy.serial_fetch(stock_no, mons, twse) + stockproxy._load_data(stockproxy.__raw_data) except urllib3.exceptions.HTTPError: - raise ConnectionError(), u'IN OFFLINE, NO DATA FETCH.' + raise ConnectionError()('IN OFFLINE, NO DATA FETCH.') except Exception as e: - print e + print(e) - return stock_proxy + return stockproxy diff --git a/grs/realtime.py b/grs/realtime.py index 2e70f21..4e60b96 100644 --- a/grs/realtime.py +++ b/grs/realtime.py @@ -47,13 +47,13 @@ class RealtimeStock(object): :rtype: dict """ def __init__(self, no): - assert isinstance(no, basestring), '`no` must be a string' + assert isinstance(no, str), '`no` must be a string' self.__raw = '' try: page = TSE_CONNECTIONS.urlopen('GET', '/data/%s.csv?r=%s' % (no, random.randrange(1, 10000))).data except urllib3.exceptions.HTTPError: - raise ConnectionError(), u'IN OFFLINE, NO DATA FETCH.' + raise ConnectionError()('IN OFFLINE, NO DATA FETCH.') logging.info('twsk no %s', no) reader = csv.reader(page.split('\r\n')) @@ -99,7 +99,7 @@ def real(self): try: unch = sum([covstr(self.__raw[3]), covstr(self.__raw[4])]) / 2 result = { - 'name': unicode(self.__raw[36].replace(' ', ''), 'cp950'), + 'name': str(self.__raw[36].replace(' ', ''), 'cp950'), 'no': self.__raw[0], 'range': self.__raw[1], # 漲跌價 'time': self.__raw[2], # 取得時間 @@ -162,7 +162,7 @@ def __init__(self): page = TSE_CONNECTIONS.urlopen('GET', '/data/TSEIndex.csv?r=%s' % random.randrange(1, 10000)).data except urllib3.exceptions.HTTPError: - raise ConnectionError(), u'IN OFFLINE, NO DATA FETCH.' + raise ConnectionError()('IN OFFLINE, NO DATA FETCH.') reader = csv.reader(page.split('\r\n')) for i in reader: diff --git a/grs/realtime2.py b/grs/realtime2.py index cc235fa..fb343f2 100644 --- a/grs/realtime2.py +++ b/grs/realtime2.py @@ -88,8 +88,8 @@ def make_format(raw): best_ask_volume = [int(v) for v in i['f'].split('_')[:-1]] best_bid_volume = [int(v) for v in i['g'].split('_')[:-1]] - data[i['c']]['best_ask_list'] = zip(best_ask_price, best_ask_volume) - data[i['c']]['best_bid_list'] = zip(best_bid_price, best_bid_volume) + data[i['c']]['best_ask_list'] = list(zip(best_ask_price, best_ask_volume)) + data[i['c']]['best_bid_list'] = list(zip(best_bid_price, best_bid_volume)) data[i['c']]['best_ask_price'] = best_ask_price[0] data[i['c']]['best_ask_volume'] = best_ask_volume[0] data[i['c']]['best_bid_price'] = best_bid_price[0] diff --git a/grs/twseno.py b/grs/twseno.py index 22b3f8d..7596e7f 100644 --- a/grs/twseno.py +++ b/grs/twseno.py @@ -44,10 +44,10 @@ def importcsv(self): result = {} for i in csv_data: try: - result[i[0]] = str(i[1]).decode('utf-8') + result[i[0]] = str(i[1]) except ValueError: if i[0] == 'UPDATE': - self.last_update = str(i[1]).decode('utf-8') + self.last_update = str(i[1]) else: pass return result @@ -60,7 +60,7 @@ def __industry_code(self): csv_data = csv.reader(csv_file) result = {} for i in csv_data: - result[i[0]] = i[1].decode('utf-8') + result[i[0]] = i[1] return result def __loadindcomps(self): @@ -73,10 +73,10 @@ def __loadindcomps(self): for i in csv_data: if check_words.match(i[2]): try: - result[i[2]].append(i[0].decode('utf-8')) + result[i[2]].append(i[0]) except (ValueError, KeyError): try: - result[i[2]] = [i[0].decode('utf-8')] + result[i[2]] = [i[0]] except KeyError: pass return result @@ -125,7 +125,7 @@ def all_stock_no(self): :rtype: list """ - return self.__allstockno.keys() + return list(self.__allstockno.keys()) @property def all_stock_name(self): @@ -133,7 +133,7 @@ def all_stock_name(self): :rtype: list """ - return self.__allstockno.values() + return list(self.__allstockno.values()) @property def industry_code(self): @@ -200,4 +200,4 @@ def __init__(self): t = TWSENo() #t = OTCNo() t_list = t.get_stock_list() - print t_list + print(t_list) diff --git a/setup.py b/setup.py index 4ac2ab1..068e346 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ license=grs.__license__, keywords="Taiwan Stock Exchange taipei twse otc gretai " + \ "台灣 台北 股市 即時 上市 上櫃", - install_requires=['python-dateutil==1.5', 'ujson', 'urllib3'], + install_requires=['python-dateutil==2.2', 'ujson', 'urllib3'], classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', diff --git a/test_unittest.py b/test_unittest.py index ca6a411..87c62b3 100644 --- a/test_unittest.py +++ b/test_unittest.py @@ -65,7 +65,7 @@ def test_stock_value(self): def test_twse_no(): twse_no = grs.TWSENo() assert isinstance(twse_no.all_stock, dict) - result = twse_no.search(u'中') + result = twse_no.search('中') # 1701 中化 assert '1701' in result result = twse_no.searchbyno(17) @@ -95,7 +95,7 @@ def test_realtime(): real_time = grs.RealtimeStock('0050') assert real_time.real['no'] == '0050' try: - real_time = grs.RealtimeStock(0050) + real_time = grs.RealtimeStock(0o050) except AssertionError: pass @@ -107,9 +107,9 @@ def test_countdown(): @staticmethod def test_taiwan_50(): stock = grs.Stock('0050') - assert u'台灣50' == stock.info[1] + assert '台灣50' == stock.info[1] try: - stock = grs.Stock(0050) + stock = grs.Stock(0o050) except AssertionError: pass @@ -162,7 +162,7 @@ def test_stock_value(self): def test_otc_no(): otc_no = grs.OTCNo() assert isinstance(otc_no.all_stock, dict) - result = otc_no.search(u'華') + result = otc_no.search('華') # 8446 華研 assert '8446' in result result = otc_no.searchbyno(46) diff --git a/tools/make_otc_list.py b/tools/make_otc_list.py index 7648a7f..650cf54 100644 --- a/tools/make_otc_list.py +++ b/tools/make_otc_list.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import csv import re -import urllib2 +import urllib.request, urllib.error, urllib.parse from datetime import datetime @@ -15,45 +15,45 @@ 'day': NOW.day,} OTCCLS = { - '01': u'水泥工業', - '02': u'食品工業', - '03': u'塑膠工業', - '04': u'紡織纖維', - '05': u'電機機械', - '06': u'電器電纜', - '07': u'化學生技醫療', - '08': u'玻璃陶瓷', - '09': u'造紙工業', - '10': u'鋼鐵工業', - '11': u'橡膠工業', - '12': u'汽車工業', - '13': u'電子工業', - '14': u'建材營造', - '15': u'航運業', - '16': u'觀光事業', - '17': u'金融保險', - '18': u'貿易百貨', - '19': u'綜合', - '20': u'其他', - '21': u'化學工業', - '22': u'生技醫療業', - '23': u'油電燃氣業', - '24': u'半導體業', - '25': u'電腦及週邊設備業', - '26': u'光電業', - '27': u'通信網路業', - '28': u'電子零組件業', - '29': u'電子通路業', - '30': u'資訊服務業', - '31': u'其他電子業', - '32': u'文化創意業', - '80': u'管理股票', - 'AA': u'受益證券', - 'EE': u'上櫃指數股票型基金(ETF)', - 'TD': u'台灣存託憑證(TDR)', - 'WW': u'認購售權證', - 'GG': u'認股權憑證', - 'BC': u'牛證熊證', + '01': '水泥工業', + '02': '食品工業', + '03': '塑膠工業', + '04': '紡織纖維', + '05': '電機機械', + '06': '電器電纜', + '07': '化學生技醫療', + '08': '玻璃陶瓷', + '09': '造紙工業', + '10': '鋼鐵工業', + '11': '橡膠工業', + '12': '汽車工業', + '13': '電子工業', + '14': '建材營造', + '15': '航運業', + '16': '觀光事業', + '17': '金融保險', + '18': '貿易百貨', + '19': '綜合', + '20': '其他', + '21': '化學工業', + '22': '生技醫療業', + '23': '油電燃氣業', + '24': '半導體業', + '25': '電腦及週邊設備業', + '26': '光電業', + '27': '通信網路業', + '28': '電子零組件業', + '29': '電子通路業', + '30': '資訊服務業', + '31': '其他電子業', + '32': '文化創意業', + '80': '管理股票', + 'AA': '受益證券', + 'EE': '上櫃指數股票型基金(ETF)', + 'TD': '台灣存託憑證(TDR)', + 'WW': '認購售權證', + 'GG': '認股權憑證', + 'BC': '牛證熊證', #'EW': u'所有證券(不含權證、牛熊證)', #'AL': u'所有證券 ', #'OR': u'委託及成交資訊(16:05提供)', @@ -66,7 +66,7 @@ def fetch_otc_list(): re_sub = re.compile(r'[^\w\d]') for no in OTCCLS: - for i in csv.reader(urllib2.urlopen(OTCURL % no).readlines()): + for i in csv.reader(urllib.request.urlopen(OTCURL % no).readlines()): if len(i) >= 3 and re_pattern.match(i[0]): pass else: diff --git a/tools/make_twse_list.py b/tools/make_twse_list.py index a1376ed..5094c26 100644 --- a/tools/make_twse_list.py +++ b/tools/make_twse_list.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import csv import re -import urllib2 +import urllib.request, urllib.error, urllib.parse from datetime import datetime @@ -10,49 +10,49 @@ INDUSTRYCODE = '../grs/industry_code.csv' TWSEURL = 'http://www.twse.com.tw/ch/trading/exchange/MI_INDEX/MI_INDEX2_print.php?genpage=genpage/Report%(year)s%(mon)02d/A112%(year)s%(mon)02d%(day)02d%%s.php&type=csv' % {'year': NOW.year, 'mon': NOW.month, 'day': NOW.day} -TWSECLS = {'0049': u'封閉式基金', - '0099P': u'ETF', - '019919T': u'受益證券', - '0999': u'認購權證', #(不含牛證) - '0999P': u'認售權證', #(不含熊證) - '0999C': u'牛證', - '0999B': u'熊證', - '0999GA': u'附認股權特別股', - '0999GD': u'附認股權公司債', - '0999G9': u'認股權憑證', - '01': u'水泥工業', - '02': u'食品工業', - '03': u'塑膠工業', - '04': u'紡織纖維', - '05': u'電機機械', - '06': u'電器電纜', - '07': u'化學生技醫療', - '21': u'化學工業', - '22': u'生技醫療業', - '08': u'玻璃陶瓷', - '09': u'造紙工業', - '10': u'鋼鐵工業', - '11': u'橡膠工業', - '12': u'汽車工業', - '13': u'電子工業', - '24': u'半導體業', - '25': u'電腦及週邊設備業', - '26': u'光電業', - '27': u'通信網路業', - '28': u'電子零組件業', - '29': u'電子通路業', - '30': u'資訊服務業', - '31': u'其他電子業', - '14': u'建材營造', - '15': u'航運業', - '16': u'觀光事業', - '17': u'金融保險', - '18': u'貿易百貨', - '9299': u'存託憑證', - '23': u'油電燃氣業', - '19': u'綜合', - '20': u'其他', - 'CB': u'可轉換公司債',} +TWSECLS = {'0049': '封閉式基金', + '0099P': 'ETF', + '019919T': '受益證券', + '0999': '認購權證', #(不含牛證) + '0999P': '認售權證', #(不含熊證) + '0999C': '牛證', + '0999B': '熊證', + '0999GA': '附認股權特別股', + '0999GD': '附認股權公司債', + '0999G9': '認股權憑證', + '01': '水泥工業', + '02': '食品工業', + '03': '塑膠工業', + '04': '紡織纖維', + '05': '電機機械', + '06': '電器電纜', + '07': '化學生技醫療', + '21': '化學工業', + '22': '生技醫療業', + '08': '玻璃陶瓷', + '09': '造紙工業', + '10': '鋼鐵工業', + '11': '橡膠工業', + '12': '汽車工業', + '13': '電子工業', + '24': '半導體業', + '25': '電腦及週邊設備業', + '26': '光電業', + '27': '通信網路業', + '28': '電子零組件業', + '29': '電子通路業', + '30': '資訊服務業', + '31': '其他電子業', + '14': '建材營造', + '15': '航運業', + '16': '觀光事業', + '17': '金融保險', + '18': '貿易百貨', + '9299': '存託憑證', + '23': '油電燃氣業', + '19': '綜合', + '20': '其他', + 'CB': '可轉換公司債',} #'ALL_1': u'全部'} def fetch_twse_list(): @@ -62,7 +62,7 @@ def fetch_twse_list(): re_sub = re.compile(r'[^\w\d]') for no in TWSECLS: - for i in csv.reader(urllib2.urlopen(TWSEURL % no).readlines()): + for i in csv.reader(urllib.request.urlopen(TWSEURL % no).readlines()): if len(i) >= 3 and re_pattern.match(i[0]): pass else: