-
Notifications
You must be signed in to change notification settings - Fork 4
/
utils.py
369 lines (325 loc) · 15.2 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
'''
Created on 5 Mar 2017
@author: af
'''
import matplotlib as mpl
mpl.use('Agg')
from collections import Counter
from collections import defaultdict
import json
import logging
import pdb
import pickle
from matplotlib import ticker
from matplotlib.mlab import griddata
from matplotlib.patches import Polygon as MplPolygon
from mpl_toolkits.basemap import Basemap, cm, maskoceans
from scipy import stats
from scipy.interpolate import griddata as gd
from scipy.spatial import ConvexHull
import shapefile
import shapely
from shapely.geometry import MultiPoint, Point, Polygon, asShape, shape
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
sns.set(style="white")
#from matplotlib.patches import Polygon
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO)
short_state_names = {
# 'AK': 'Alaska',
'AL': 'Alabama',
'AR': 'Arkansas',
# 'AS': 'American Samoa',
'AZ': 'Arizona',
'CA': 'California',
'CO': 'Colorado',
'CT': 'Connecticut',
'DC': 'District of Columbia',
'DE': 'Delaware',
'FL': 'Florida',
'GA': 'Georgia',
#'GU': 'Guam',
# 'HI': 'Hawaii',
'IA': 'Iowa',
'ID': 'Idaho',
'IL': 'Illinois',
'IN': 'Indiana',
'KS': 'Kansas',
'KY': 'Kentucky',
'LA': 'Louisiana',
'MA': 'Massachusetts',
'MD': 'Maryland',
'ME': 'Maine',
'MI': 'Michigan',
'MN': 'Minnesota',
'MO': 'Missouri',
'MP': 'Northern Mariana Islands',
'MS': 'Mississippi',
'MT': 'Montana',
# 'NA': 'National',
'NC': 'North Carolina',
'ND': 'North Dakota',
'NE': 'Nebraska',
'NH': 'New Hampshire',
'NJ': 'New Jersey',
'NM': 'New Mexico',
'NV': 'Nevada',
'NY': 'New York',
'OH': 'Ohio',
'OK': 'Oklahoma',
'OR': 'Oregon',
'PA': 'Pennsylvania',
#'PR': 'Puerto Rico',
'RI': 'Rhode Island',
'SC': 'South Carolina',
'SD': 'South Dakota',
'TN': 'Tennessee',
'TX': 'Texas',
'UT': 'Utah',
'VA': 'Virginia',
'VI': 'Virgin Islands',
'VT': 'Vermont',
'WA': 'Washington',
'WI': 'Wisconsin',
'WV': 'West Virginia',
'WY': 'Wyoming'
}
stop_words = ['the', 'of', 'and', 'to', 'a', 'in', 'for', 'is', 'on', 'that', 'by', 'this', 'with', 'i', 'you', 'it', 'not', 'or', 'be', 'are', 'from', 'at', 'as', 'your', 'all', 'have', 'new', 'more', 'an', 'was', 'we', 'will', 'home', 'can', 'us', 'about', 'if', 'page', 'my', 'has', 'search', 'free', 'but', 'our', 'one', 'other', 'do', 'no', 'information', 'time', 'they', 'site', 'he', 'up', 'may', 'what', 'which', 'their', 'news', 'out', 'use', 'any', 'there', 'see', 'only', 'so', 'his', 'when', 'contact', 'here', 'business', 'who', 'web', 'also', 'now', 'help', 'get', 'pm', 'view', 'online', 'c', 'e', 'first', 'am', 'been', 'would', 'how', 'were', 'me', 's', 'services', 'some', 'these', 'click', 'its', 'like', 'service', 'x', 'than', 'find', 'price', 'date', 'back', 'top', 'people', 'had', 'list', 'name', 'just', 'over', 'state', 'year', 'day', 'into', 'email', 'two', 'health', 'n', 'world', 're', 'next', 'used', 'go', 'b', 'work', 'last', 'most', 'products', 'music', 'buy', 'data', 'make', 'them', 'should', 'product', 'system', 'post', 'her', 'city', 't', 'add', 'policy', 'number', 'such', 'please', 'available', 'copyright', 'support', 'message', 'after', 'best', 'software', 'then', 'jan', 'good', 'video', 'well', 'd', 'where', 'info', 'rights', 'public', 'books', 'high', 'school', 'through', 'm', 'each', 'links', 'she', 'review', 'years', 'order', 'very', 'privacy', 'book', 'items', 'company', 'r', 'read', 'group', 'sex', 'need', 'many', 'user', 'said', 'de', 'does', 'set', 'under', 'general', 'research', 'university', 'january', 'mail', 'full', 'map', 'reviews', 'program', 'life']
dialect_state = {
'atlantic':["Connecticut", "Delaware", "Florida", "Georgia", "Maine", "Maryland", "Massachusetts", "New Hampshire", "New Jersey", "New York", "North Carolina", "Pennsylvania", "Rhode Island", "South Carolina", "Vermont", "Virginia", "Washington, DC"],
'central':["Arkansas", "Kansas", "Missouri", "Nebraska", "Oklahoma"],
'central atlantic':["Delaware", "Washington, DC"],
'delmarva':["Delaware"],
'desert southwest':["Arizona", "New Mexico"],
'great lakes':['michigan', 'minnesota', 'wisconsin'],
'golf states':['alabama', 'florida', 'louisiana', 'mississippi'],
'inland north':['michigan', 'montana', 'new york', 'washington', 'minnesota', 'north dakota', 'oregon', 'wisconsin'],
'inland south':["Alabama", "Kentucky", "Mississippi", "Tennessee"],
'lower mississippi valley':['arkansas', 'mississippi', 'louisiana'],
'middle atlantic':['maryland', 'south carolina', 'washington, dc', 'north carolina', 'virginia'],
'midland':['kentucky', 'nebraska', 'tennessee'],
'mississippi valley':["Arkansas", "Illinois", "Iowa", "Louisiana", "Minnesota", "Mississippi", "Missouri", "Wisconsin"],
'mississippi-ohio valley':["Illinois", "Indiana", "Iowa", "Kentucky", "Minnesota", "Missouri", "Ohio", "Wisconsin"],
'new england':["Connecticut", "Maine", "Massachusetts", "New Hampshire", "Rhode Island", "Vermont"],
'north':["Connecticut", "Maine", "Massachusetts", "Michigan", "Minnesota", "Montana", "New Hampshire", "New York", "North Dakota", "Oregon", "Rhode Island", "Vermont", "Washington", "Wisconsin"],
'north atlantic':["Connecticut", "Maine", "Massachusetts", "New Hampshir", "Rhode Island", "Vermont"],
'north central':["Illinois", "Indiana", "Kentucky", "Michigan", "Ohio", "Wisconsin"],
'north midland':['nebraska'],
'northeast':["Connecticut", "Maine", "Massachusetts", "New Hampshire", "New Jersey", "New York", "Rhode Island", "Vermont"],
'northwest':["Idaho", "Oregon", "Washington", "Montana", "Wyoming"],
'ohio valley':['kentucky'],
'pacific':['california', 'washington', 'oregon'],
'pacific northwest':['washington', 'oregon'],
'plains states':['nebraska', 'kansas'],
'rocky mountains':['montana', 'utah', 'idaho', 'nevada', 'wyoming'],
'south':['florida', 'washington, dc', 'alabama', 'georgia', 'louisiana', 'mississippi', 'north carolina', 'south carolina'],
'south atlantic':['florida', 'georgia', 'north carolina', 'south carolina'],
'south midland':['kentucky', 'arkansas', 'tennessee', 'washington, dc', 'west virginia'],
'southeast':['alabama', 'georgia', 'north carolina', 'tennessee', 'north carolina', 'mississippi', 'florida'],
'southwest':['arizona', 'new mexico', 'texas', 'oklohama'],
'upper midwest':['iowa', 'nebraska', 'south dakota', 'north dakota', 'minnesota'],
'upper mississippi valley':['iowa', 'minnesota', 'wisconsin', 'illinois', 'missouri'],
'west':["Arizona", "California", "Colorado", "Idaho", "Montana", "Nevada", "New Mexico", "Oregon", "Utah", "Washington", "Wyoming"],
'west midland':['iowa', 'ohio', 'arkansas', 'tennessee', 'west virginia', 'illinois', 'indiana', 'kentucky', 'nebraska', ]
}
def get_us_city_name():
#we might exclude words in city names
all_us_city_names = set()
with open('./data/us_cities.txt', 'r') as fin:
for line in fin:
words = set(line.strip().lower().split())
for word in words:
all_us_city_names.add(word)
return all_us_city_names
def retrieve_location_from_coordinates():
points = []
#read points from a file
with open('./data/latlon_world.txt', 'r') as fin:
for line in fin:
line = line.strip()
lat, lon = line.split('\t')
lat, lon = float(lat), float(lon)
point = (lat, lon)
points.append(point)
#read point city-countries from http://people.eng.unimelb.edu.au/tbaldwin/resources/jair2014-geoloc/
latlon_country = {}
with open('./data/han_cook_baldwin.geo', 'r') as fin:
for line in fin:
line = line.strip()
fields = line.split('\t')
country = fields[0].split('-')[-1].upper()
lat = float(fields[2])
lon = float(fields[3])
latlon_country[(lat, lon)] = country
country_count = Counter()
for point in points:
country = latlon_country[point]
country_count[country] += 1
countries = [c for c, count in country_count.iteritems() if count>100]
with open('./data/country_count.txt', 'w') as fout:
json.dump(countries, fout)
def get_state_from_coordinates(coordnates):
#coordinates = np.array([(34, -118), (40.7, -74)])
sf = shapefile.Reader('./data/us_states_st99/st99_d00')
#sf = shapefile.Reader("./data/states/cb_2015_us_state_20m")
shapes = sf.shapes()
#shapes[i].points
fields = sf.fields
records = sf.records()
state_polygons = defaultdict(list)
for i, record in enumerate(records):
state = record[5]
points = shapes[i].points
poly = shape(shapes[i])
state_polygons[state].append(poly)
coor_state = {}
for i in range(coordnates.shape[0]):
lat, lon = coordnates[i]
for state, polies in state_polygons.iteritems():
for poly in polies:
point = Point(lon, lat)
if poly.contains(point):
coor_state[(lat, lon)] = state.lower()
return coor_state
def contour(coordinates, scores, world=False, filename="contour", do_contour = False, **kwargs):
from matplotlib import rc
rc('font',**{'family':'sans-serif','sans-serif':['Helvetica']})
## for Palatino and other serif fonts use:
#rc('font',**{'family':'serif','serif':['Palatino']})
rc('text', usetex=True)
scores = np.array(scores)
lllat = 24.396308
lllon = -124.848974
urlat = 49.384358
urlon = -66.885444
if world:
lllat = -90
lllon = -180
urlat = 90
urlon = 180
fig = plt.figure(figsize=(2.5, 2))
ax = fig.add_subplot(111, axisbg='w', frame_on=False)
grid_interpolation_method = 'nearest'
#scores = np.log(scores)
m = Basemap(llcrnrlat=lllat,
urcrnrlat=urlat,
llcrnrlon=lllon,
urcrnrlon=urlon,
resolution='i', projection='cyl')
m.drawmapboundary(fill_color = 'white')
#m.drawcoastlines(linewidth=0.2)
m.drawcountries(linewidth=0.2)
if world:
m.drawstates(linewidth=0.2, color='lightgray')
#m.fillcontinents(color='white', lake_color='#0000ff', zorder=2)
#m.drawrivers(color='#0000ff')
#m.drawlsmask(land_color='gray',ocean_color="#b0c4de", lakes=True)
#m.drawcounties()
shp_info = m.readshapefile('./data/us_states_st99/st99_d00','states',drawbounds=True, zorder=0)
printed_names = []
ax = plt.gca()
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
for spine in ax.spines.itervalues():
spine.set_visible(False)
state_names_set = set(short_state_names.values())
mi_index = 0
wi_index = 0
for shapedict,state in zip(m.states_info, m.states):
if world: break
draw_state_name = True
if shapedict['NAME'] not in state_names_set: continue
short_name = short_state_names.keys()[short_state_names.values().index(shapedict['NAME'])]
if short_name in printed_names and short_name not in ['MI', 'WI']:
continue
if short_name == 'MI':
if mi_index != 3:
draw_state_name = False
mi_index += 1
if short_name == 'WI':
if wi_index != 2:
draw_state_name = False
wi_index += 1
# center of polygon
x, y = np.array(state).mean(axis=0)
hull = ConvexHull(state)
hull_points = np.array(state)[hull.vertices]
x, y = hull_points.mean(axis=0)
if short_name == 'MD':
y = y - 0.5
x = x + 0.5
elif short_name == 'DC':
y = y + 0.1
elif short_name == 'MI':
x = x - 1
elif short_name == 'RI':
x = x + 1
y = y - 1
#poly = MplPolygon(state,facecolor='lightgray',edgecolor='black')
#x, y = np.median(np.array(state), axis=0)
# You have to align x,y manually to avoid overlapping for little states
if draw_state_name:
plt.text(x+.1, y, short_name, ha="center", fontsize=4)
#ax.add_patch(poly)
#pdb.set_trace()
printed_names += [short_name,]
mlon, mlat = m(*(coordinates[:,1], coordinates[:,0]))
# grid data
if do_contour:
numcols, numrows = 2000, 2000
xi = np.linspace(mlon.min(), mlon.max(), numcols)
yi = np.linspace(mlat.min(), mlat.max(), numrows)
xi, yi = np.meshgrid(xi, yi)
# interpolate
x, y, z = mlon, mlat, scores
#pdb.set_trace()
#zi = griddata(x, y, z, xi, yi)
zi = gd(
(mlon, mlat),
scores,
(xi, yi),
method=grid_interpolation_method, rescale=False)
#Remove the lakes and oceans
data = maskoceans(xi, yi, zi)
con = m.contourf(xi, yi, data, cmap=plt.get_cmap('YlOrRd'))
else:
cmap=plt.get_cmap('YlOrRd')
con = m.scatter(mlon, mlat, c=scores, s=3, cmap=cmap )
#con = m.contour(xi, yi, data, 3, cmap=plt.get_cmap('YlOrRd'), linewidths=1)
#con = m.contour(x, y, z, 3, cmap=plt.get_cmap('YlOrRd'), tri=True, linewidths=1)
#conf = m.contourf(x, y, z, 3, cmap=plt.get_cmap('coolwarm'), tri=True)
cbar = m.colorbar(con,location='right',pad="3%")
#plt.setp(cbar.ax.get_yticklabels(), visible=False)
#cbar.ax.tick_params(axis=u'both', which=u'both',length=0)
#cbar.ax.set_yticklabels(['low', 'high'])
#tick_locator = ticker.MaxNLocator(nbins=9)
#cbar.locator = tick_locator
#cbar.update_ticks()
cbar.ax.tick_params(labelsize=6)
cbar.ax.xaxis.set_tick_params(pad=0)
cbar.ax.yaxis.set_tick_params(pad=0)
cbar.set_label('error in km', size=8, labelpad=1)
for line in cbar.lines:
line.set_linewidth(20)
#read countries for world dataset with more than 100 number of users
with open('./data/country_count.json', 'r') as fin:
top_countries = set(json.load(fin))
world_shp_info = m.readshapefile('./data/CNTR_2014_10M_SH/Data/CNTR_RG_10M_2014','world',drawbounds=False, zorder=100)
for shapedict,state in zip(m.world_info, m.world):
if not world:
if shapedict['CNTR_ID'] not in ['CA', 'MX']: continue
else:
if shapedict['CNTR_ID'] in top_countries: continue
poly = MplPolygon(state,facecolor='gray',edgecolor='gray')
ax.add_patch(poly)
#plt.title('term: ' + word )
plt.tight_layout()
plt.savefig('./maps/' + filename + '.pdf', bbox_inches='tight')
plt.close()
del m
if __name__ == '__main__':
#retrieve_location_from_coordinates()
#get_state_from_coordinates(coordnates=None)
contour(coordinates=None, scores=None, world=False, filename='errormap', do_contour=True)