forked from dcferreira/deep_architect
-
Notifications
You must be signed in to change notification settings - Fork 0
/
search_spaces.py
148 lines (130 loc) · 5.94 KB
/
search_spaces.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
from darch.modules import *
from darch.initializers import *
# this search space is used for exploration of the different search methods.
# 5 repetitions are to be run, with 15 minutes per model. 100 models total.
# 2 days per repetition per method, divided by the number of gpus used.
# *** has 2200 different models
def tfref_convnet_ss0(num_classes):
""" Naive conv net approximately as described in
https://www.tensorflow.org/tutorials/mnist/pros/ . See docstring for
tfref_convnet_single. Search space built around that.
"""
conv_initers = [ kaiming2015delving_initializer_conv(1.0) ]
aff_initers = [ invsqrt_size_gaussian_initializer_affine( np.sqrt(2.0) )]
# some parameters.
filter_ns = [32, 48, 64, 96, 128]
filter_ls = [1, 3, 5]
aff_ns = [256, 512, 1024]
keep_ps = [0.25, 0.5, 0.75]
b_search = Concat([
Conv2D(filter_ns, filter_ls, [1], ["SAME"], conv_initers),
ReLU(),
MaxPooling2D([2], [2], ["SAME"]),
Conv2D(filter_ns, filter_ls, [1], ["SAME"], conv_initers),
ReLU(),
MaxPooling2D([2], [2], ["SAME"]),
Affine(aff_ns, aff_initers),
ReLU(),
Dropout(keep_ps),
Affine([num_classes], aff_initers)
])
return b_search
# this search space is used for searching for models with a larger time (e.g.
# 3h). run on 15 models; then take the 3 most promising ones and train them
# for 10 tries with a larger computational budget, but still with the stopping
# condition 12 hours at most.
# try for random.
# these search spaces are much bigger.
def deepconv_ss0(num_classes):
conv_initers = [ kaiming2015delving_initializer_conv(1.0) ]
aff_initers = [ xavier_initializer_affine( 1.0 )]
def Module_fn(filter_ns, filter_ls, keep_ps, repeat_ns):
b = RepeatTied(
Concat([
Conv2D(filter_ns, filter_ls, [1], ["SAME"], conv_initers),
MaybeSwap_fn( ReLU(), BatchNormalization() ),
Optional_fn( Dropout(keep_ps) )
]), repeat_ns)
return b
filter_numbers_min = range(48, 129, 16)
repeat_numbers = [2 ** i for i in range(6)]
mult_fn = lambda ls, alpha: list(alpha * np.array(ls))
b_search = Concat([
# this reduction layer wasn't here before, but it is convenient.
Conv2D(filter_numbers_min, [3, 5, 7], [2], ["SAME"], conv_initers),
Module_fn(filter_numbers_min, [3, 5], [0.5, 0.9], repeat_numbers),
Conv2D(filter_numbers_min, [3, 5, 7], [2], ["SAME"], conv_initers),
Module_fn(mult_fn(filter_numbers_min, 2), [3, 5], [0.5, 0.9], repeat_numbers),
Affine([num_classes], aff_initers)
])
return b_search
# use the same type of experimental setting as the deepconv model.
# this search space is much bigger.
def resnet_ss0(num_classes):
gains = [1.0]
aff_initers = tuple([ xavier_initializer_affine(g) for g in gains ])
conv2d_initers = [kaiming2015delving_initializer_conv(g) for g in gains]
def Res_fn(filter_ns, filter_ls, keep_ps, repeat_inner_ns):
return Concat([
Residual(
Concat([
RepeatTied(
Concat([
Conv2D(filter_ns, filter_ls, [1], ["SAME"], conv2d_initers),
MaybeSwap_fn( BatchNormalization(), ReLU() )
]),
repeat_inner_ns),
Conv2D(filter_ns, filter_ls, filter_ls, ["SAME"], conv2d_initers)
])
),
Optional_fn( Dropout(keep_ps) )
])
filter_ns = range(48, 129, 16)
filter_ls = [1, 3, 5]
repeat_numbers = [2 ** i for i in range(6)]
mult_fn = lambda ls, alpha: list(alpha * np.array(ls))
repeat_inner_ns = [1, 2, 4]
repeat_outer_ns = [1, 2, 4, 8]
keep_ps = [0.5, 0.9]
b_search = Concat([
Repeat(
Res_fn(filter_ns, filter_ls, keep_ps, repeat_inner_ns),
repeat_outer_ns),
Optional_fn(
Concat([
AvgPooling2D([3, 5], [2], ["SAME"]),
Repeat(
Res_fn(mult_fn(filter_ns, 2), filter_ls, keep_ps, repeat_inner_ns),
repeat_outer_ns)
])
),
Affine([num_classes], aff_initers)
])
return b_search
def deepconv_ss1(num_classes):
conv_initers = [ kaiming2015delving_initializer_conv(1.0) ]
aff_initers = [ xavier_initializer_affine( 1.0 )]
def Module_fn(filter_ns, filter_ls, keep_ps, repeat_ns):
b = Repeat(
Concat([
Conv2D(filter_ns, filter_ls, [1], ["SAME"], conv_initers),
BatchNormalization(),
ReLU(),
Optional_fn( Dropout(keep_ps) )
]), repeat_ns)
return b
# filter_numbers_min = range(48, 129, 16)
filter_numbers_min = [32, 64, 128]
repeat_numbers = [2 ** i for i in range(4)]
mult_fn = lambda ls, alpha: list(alpha * np.array(ls))
b_search = Concat([
# this reduction layer wasn't here before, but it is convenient.
Conv2D(filter_numbers_min, [1, 3, 5, 7], [1], ["SAME"], conv_initers),
Module_fn(filter_numbers_min, [1, 3, 5, 7], [0.5, 0.9], repeat_numbers),
Optional_fn(AvgPooling2D([2], [2], ["SAME"])),
# Conv2D(filter_numbers_min, [1, 3, 5, 7], [2], ["SAME"], conv_initers),
Module_fn(mult_fn(filter_numbers_min, 2), [1, 3, 5, 7], [0.5, 0.9], repeat_numbers),
Optional_fn(AvgPooling2D([2], [2], ["SAME"])),
Affine([num_classes], aff_initers)
])
return b_search