-
Notifications
You must be signed in to change notification settings - Fork 0
/
permutation_importance.py
246 lines (182 loc) · 10 KB
/
permutation_importance.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
"""
Module to perform permutation importance
Author: Son Gyo Jung
Email: sgj13@cam.ac.uk
"""
import os
import pandas as pd
import joblib
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
from sklearn.inspection import permutation_importance
from sklearn import metrics
from sklearn.preprocessing import MinMaxScaler
from lightgbm.sklearn import LGBMClassifier, LGBMRegressor
from xgboost import XGBClassifier, XGBRegressor
class permutation_importance_of_features():
"""
Class to perform permutation importance analysis
args:
(1) path_to_file (type:str) - location of the data file with features
(2) path_to_save (type:str) - location to save new data files
(3) target (type:str) - name of target variable
(4) features (list) - list of exploratory features (e.g. those with multicollinearity reduced)
(5) scaled (bool) - whether the features are scaled in the training dataset
(5) problem (type:str) - whether it is a 'classification' or 'regression' problem
return:
(1) list of features obtained by applying RFE
"""
def __init__(self, path_to_file, path_to_save, target, features, scaled, problem, *args, **kwargs):
self.path_to_save = path_to_save
self.sample_train = joblib.load(path_to_file)
# Define input and target variables
if isinstance(features, list):
self.features = features
else:
self.features = joblib.load(features)
self.target = target
self.problem = problem
print('Target:', self.target)
print('No. of features:', len(self.features))
if scaled is False:
# Scale the features
scaling = MinMaxScaler(feature_range=(0, 1))
self.sample_train[self.features] = pd.DataFrame(
scaling.fit_transform(self.sample_train[self.features].values),
columns=self.sample_train[self.features].columns,
index=self.sample_train[self.features].index
)
def base_model(self, boosting_method, *args, **kwargs):
"""
Select the baseline model
Note:
For classification, multi-class models are defined as shown below
This can be changed into a binary problem by changing the 'objective' to 'binary' for LGBMClassifier, or to 'binary:logistic' or 'binary:logitraw' for XGBClassifier (see description in links below)
https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMClassifier.html
https://xgboost.readthedocs.io/en/latest/parameter.html
https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMRegressor.html
args:
(1) boosting_method (type:str) - either 'lightGBM' or 'XGBoost'
(2) objective (type:str) - For classification,'binary', 'multiclass', 'multi:softprob'
return:
(1)baseline model
"""
objective = kwargs.get('objective')
if self.problem == 'classification':
if boosting_method == 'lightGBM':
self.estimator = LGBMClassifier(
boosting_type='gbdt',
objective=objective,
importance_type='gain',
max_depth=-1,
verbose=-1
)
elif boosting_method == 'XGBoost':
self.estimator = XGBClassifier(
objective=objective,
booster='gbtree',
importance_type='total_gain'
)
elif self.problem == 'regression':
if boosting_method == 'lightGBM':
self.estimator = LGBMRegressor(
boosting_type ='gbdt',
importance_type='gain',
max_depth=-1,
verbose=-1
)
elif boosting_method == 'XGBoost':
self.estimator = XGBClassifier(
objective='reg:squarederror',
booster='gbtree',
random_state=42,
importance_type='total_gain'
)
return self.estimator
def perform(self, cv_fold=10, save=True):
"""
Perform RFE
"""
# Define metric to use
if self.problem == 'classification':
self.scoring = 'f1_weighted'
elif self.problem == 'regression':
self.scoring = 'neg_root_mean_squared_error'
# Fit estimator to training data
self.estimator = self.estimator.fit(self.sample_train[self.features],self.sample_train[self.target].values.ravel())
self.result = permutation_importance(
estimator = self.estimator,
X = self.sample_train[self.features],
y = self.sample_train[self.target].values.ravel(),
scoring = self.scoring,
n_repeats = cv_fold,
random_state = 42
)
if save:
joblib.dump(self.result, self.path_to_save + 'permutation_importance.pkl')
print('Saved as ' + str(self.path_to_save) + 'permutation_importance.pkl')
return self.result
def plot(self, top_n=5, *args, **kwargs):
# Custom list of x
x_list = kwargs.get('x_list')
x_unit = kwargs.get('x_unit')
tight_plot = kwargs.get('tight_plot')
perm_sorted_idx = self.result.importances_mean.argsort()[-top_n:]
tree_importance_sorted_idx = np.argsort(self.estimator.feature_importances_)[-top_n:]
tree_indices = np.arange(0, len(self.estimator.feature_importances_))[:top_n] + 0.5
fontsize = 10
if x_list is None:
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 10))
ax1.barh(tree_indices, self.estimator.feature_importances_[tree_importance_sorted_idx], height=0.7)
ax1.set_yticks(tree_indices)
ax1.set_yticklabels(np.array(self.features)[tree_importance_sorted_idx], fontsize = fontsize)
ax1.set_ylim((0, top_n))
ax1.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
ax2.boxplot(
self.result.importances[perm_sorted_idx].T,
vert=False,
labels=np.array(self.features)[perm_sorted_idx],
)
ax1.barh(tree_indices, self.estimator.feature_importances_[tree_importance_sorted_idx], height=0.7)
ax1.set_yticks(tree_indices)
ax1.set_yticklabels(np.array(self.features)[tree_importance_sorted_idx], fontsize = fontsize)
ax1.set_ylim((0, top_n))
ax1.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
ax2.boxplot(
self.result.importances[perm_sorted_idx].T,
vert=False,
labels=np.array(self.features)[perm_sorted_idx],
)
else:
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 10))
ax1.barh(tree_indices, self.estimator.feature_importances_[tree_importance_sorted_idx], height=0.7)
ax1.set_yticks(tree_indices)
ax1.set_yticklabels(np.array(x_list)[tree_importance_sorted_idx], fontsize = fontsize)
ax1.set_ylim((0, top_n))
ax1.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
ax2.boxplot(
self.result.importances[perm_sorted_idx].T,
vert=False,
labels=np.array(x_list)[perm_sorted_idx],
)
fontsize = 25
ax1.set_ylabel('Feature number in order of their relevance', fontsize=fontsize)
ax1.set_xlabel('Total loss reduction', fontsize=fontsize)
ax2.set_ylabel('Feature number in order of \n permutation importance', fontsize=fontsize)
# Define metric to use
if self.problem == 'classification':
ax2.set_xlabel('Reduction in F1-score', fontsize=fontsize)
elif self.problem == 'regression':
if x_unit is not None:
ax2.set_xlabel('Increase in RMSE (' + str(x_unit) + ')', fontsize=fontsize)
else:
ax2.set_xlabel('Increase in RMSE ' , fontsize=fontsize)
ax1.tick_params(axis='both', which='major', labelsize=fontsize, direction = 'in')
ax2.tick_params(axis='both', which='major', labelsize=fontsize, direction = 'in')
if tight_plot:
fig.tight_layout()
plt.show()
fig.savefig(os.path.join(self.path_to_save, r'permutation_importance_plot_' + self.target + '.png'), dpi = 300, bbox_inches="tight")
print('Figure saved as: permutation_importance_plot_' + self.target + '.png')
return np.array(self.features)[perm_sorted_idx][::-1], np.mean(self.result.importances[perm_sorted_idx].T, axis=0)[::-1]