-
Notifications
You must be signed in to change notification settings - Fork 18
/
check_grad.py
82 lines (60 loc) · 2.38 KB
/
check_grad.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
import numpy as np
import functools
def check_backprop(model, data, labels, d=1e-6):
"Check backprop implementation."
original_params_vec = get_params_vec(model)
f = functools.partial(loss, model=model, data=data, labels=labels)
grad = functools.partial(backprop_grad, model=model, labels=labels)
try:
result = check_grad(f, grad, original_params_vec, d=d)
finally:
use_params_vec(original_params_vec, model)
return result
def loss(params_vec, model, data, labels):
"Compute loss with given parameter vector."
use_params_vec(params_vec, model)
probs, loss = model.forwardprop(data, labels)
return loss
def backprop_grad(params_vec, model, labels):
use_params_vec(params_vec, model)
model.backprop(labels)
return get_grad_params_vec(model)
def compute_fd_grad(f, params_vec, d):
fd_grad = np.zeros_like(params_vec)
for i in xrange(params_vec.shape[0]):
d_v = np.zeros_like(params_vec)
d_v[i] = d
fd_grad[i] = (f(params_vec+d_v) - f(params_vec-d_v)) / (2*d)
return fd_grad
def check_grad(f, grad, params_vec, d=1e-6):
fd_grad = compute_fd_grad(f, params_vec, d)
computed_grad = grad(params_vec)
return (np.linalg.norm(fd_grad - computed_grad)
/ max(np.linalg.norm(fd_grad),
np.linalg.norm(computed_grad)))
def get_params_vec(model):
return np.hstack([layer.w.flatten() for layer in model.layers]
+ [layer.b.flatten() for layer in model.layers])
def get_grad_params_vec(model):
result = np.hstack([layer.d_w.flatten() for layer in model.layers]
+ [layer.d_b.flatten() for layer in model.layers])
return result
def use_params_vec(params_vec, model):
for layer, (w, b) in zip(model.layers, get_w_b_list(params_vec, model)):
layer.w = w
layer.b = b
def get_w_b_list(params_vec, model):
w_list = []
pointer = 0
for layer in model.layers:
shape = layer.w.shape
flat_w = params_vec[pointer:pointer + shape[0] * shape[1]]
w_list.append(flat_w.reshape(shape))
pointer += shape[0] * shape[1]
b_list = []
for layer in model.layers:
shape = layer.b.shape
flat_w = params_vec[pointer:pointer + shape[1]]
b_list.append(flat_w.reshape(shape))
pointer += shape[0] * shape[1]
return zip(w_list, b_list)