-
Notifications
You must be signed in to change notification settings - Fork 2
/
amazon_data_multiprocessing.py
49 lines (42 loc) · 1.32 KB
/
amazon_data_multiprocessing.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
import pandas as pd
import numpy as np
import multiprocessing
from multiprocessing import Pool
num_partitions = multiprocessing.cpu_count()
num_cores = multiprocessing.cpu_count()
df_product = pd.read_pickle('data/product_intersection.pkl')
product_asin = df_product.index.values
print("data loading finished...")
def also_viewed(x):
if 'also_viewed' in x.keys():
temp = []
for asin in x['also_viewed']:
if asin in product_asin:
temp.append(asin)
return temp
else:
return []
def also_bought(x):
if 'also_bought' in x.keys():
temp = []
for asin in x['also_bought']:
if asin in product_asin:
temp.append(asin)
return temp
else:
return []
def parallelize_dataframe(df, func):
df_split = np.array_split(df, num_partitions)
pool = Pool(num_cores)
df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df
def multiply_columns(data):
data['also_viewed'] = data['related'].apply(lambda x: also_viewed(x))
data['also_bought'] = data['related'].apply(lambda x: also_bought(x))
return data
print("start running...")
df_product = parallelize_dataframe(df_product, multiply_columns)
df_product.to_pickle('data/product_also_viewed_bought.pkl')
print("ALL DONE!")