-
Notifications
You must be signed in to change notification settings - Fork 1
/
cnn_cifar_init_mnin.m
197 lines (166 loc) · 8.69 KB
/
cnn_cifar_init_mnin.m
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
function net = cnn_cifar_init_mnin4(varargin)
% CIFAR-10 model from
opts.networkType = 'simplenn' ;
opts = vl_argparse(opts, varargin) ;
net.layers = {} ;
b=0 ;
f=0.01;
g=0.05;
% Block 1
unit0 = 192;
piece0 = 1;
net.layers{end+1} = struct('type', 'conv', ...
'name', 'conv1', ...
'weights', {{single(orthonorm(5,5,3,unit0)), b*ones(1,unit0*piece0,'single')}}, ... %32
'stride', 1, ...
'learningRate', [.1 1], ...
'weightDecay', [1 0], ...
'pad', 2) ;
net.layers{end+1} = struct('type', 'bnorm', 'name', 'bn1', ...
'weights', {{ones(unit0*piece0, 1, 'single'), zeros(unit0*piece0, 1, 'single')}},'learningRate', [1 1 .5],'weightDecay', [0 0]) ;
%% maxout layer 1
unit1=160;
piece1 = 5;
net.layers{end+1} = struct('type', 'conv', ...
'name', 'maxoutconv1', ...
'weights', {{single(orthonorm(1,1,unit0,unit1*piece1)), b*ones(1,unit1*piece1,'single')}}, ...
'stride', 1, ...
'learningRate', [.1 1], ...
'weightDecay', [1 0], ...
'pad', 0) ;
net.layers{end+1} = struct('type', 'bnorm', 'name', 'bn2', ...
'weights', {{ones(unit1*piece1, 1, 'single'), zeros(unit1*piece1, 1, 'single')}},'learningRate', [1 1 .5],'weightDecay', [0 0]) ;
net.layers{end+1} = struct('type', 'maxout','numunit',unit1,'numpiece',piece1) ;
%% maxout layer 2
unit2=96;
piece2 = 5;
net.layers{end+1} = struct('type', 'conv', ...
'name', 'maxoutconv2', ...
'weights', {{single(orthonorm(1,1,unit1,unit2*piece2)), b*ones(1,unit2*piece2,'single')}}, ...
'stride', 1, ...
'learningRate', [.1 1], ...
'weightDecay', [1 0], ...
'pad', 0) ;
net.layers{end+1} = struct('type', 'bnorm', 'name', 'bn3', ...
'weights', {{ones(unit2*piece2, 1, 'single'), zeros(unit2*piece2, 1, 'single')}},'learningRate', [1 1 .5],'weightDecay', [0 0]);
net.layers{end+1} = struct('type', 'maxout','numunit',unit2,'numpiece',piece2) ;
net.layers{end+1} = struct('name', 'pool1', ... %15
'type', 'pool', ...
'method', 'avg', ...
'pool', [3 3], ...
'stride', 2, ...
'pad', [0 1 0 1]) ;
net.layers{end+1} = struct('type', 'dropout', 'name', 'dropout1', 'rate', 0.5) ;
% Block 2
unit0 = 192;
piece0 = 1;
net.layers{end+1} = struct('type', 'conv', ...
'name', 'conv2', ...
'weights', {{single(orthonorm(5,5,unit2,unit0)), b*ones(1,unit0*piece0,'single')}}, ... %16
'stride', 1, ...
'learningRate', [.1 1], ...
'weightDecay', [1 0], ...
'pad', 2) ;
net.layers{end+1} = struct('type', 'bnorm', 'name', 'bn4', ...
'weights', {{ones(unit0*piece0, 1, 'single'), zeros(unit0*piece0, 1, 'single')}},'learningRate', [1 1 .5],'weightDecay', [0 0]) ;
%% maxout layer 1
unit1=192;
piece1 = 5;
net.layers{end+1} = struct('type', 'conv', ...
'name', 'maxoutconv3', ...
'weights', {{single(orthonorm(1,1,unit0,unit1*piece1)), b*ones(1,unit1*piece1,'single')}}, ...
'stride', 1, ...
'learningRate', [.1 1], ...
'weightDecay', [1 0], ...
'pad', 0) ;
net.layers{end+1} = struct('type', 'bnorm', 'name', 'bn5', ...
'weights', {{ones(unit1*piece1, 1, 'single'), zeros(unit1*piece1, 1, 'single')}},'learningRate', [1 1 .5],'weightDecay', [0 0]) ;
net.layers{end+1} = struct('type', 'maxout','numunit',unit1,'numpiece',piece1) ;
%% maxout layer 2
unit2=192;
piece2 = 5;
net.layers{end+1} = struct('type', 'conv', ...
'name', 'maxoutconv4', ...
'weights', {{single(orthonorm(1,1,unit1,unit2*piece2)), b*ones(1,unit2*piece2,'single')}}, ...
'stride', 1, ...
'learningRate', [.1 1], ...
'weightDecay', [1 0], ...
'pad', 0) ;
net.layers{end+1} = struct('type', 'bnorm', 'name', 'bn6', ...
'weights', {{ones(unit2*piece2, 1, 'single'), zeros(unit2*piece2, 1, 'single')}},'learningRate', [1 1 .5],'weightDecay', [0 0]) ;
net.layers{end+1} = struct('type', 'maxout','numunit',unit2,'numpiece',piece2) ;
net.layers{end+1} = struct('name', 'pool2', ... %7
'type', 'pool', ...
'method', 'avg', ...
'pool', [3 3], ...
'stride', 2, ...
'pad', [0 1 0 1]) ;
net.layers{end+1} = struct('type', 'dropout', 'name', 'dropout2', 'rate', 0.5) ;
% Block 3
unit0 = 192;
piece0 = 1;
net.layers{end+1} = struct('type', 'conv', ...
'name', 'conv3', ...
'weights', {{single(orthonorm(3,3,unit2,unit0)), b*ones(1, unit0*piece0, 'single')}}, ... %4
'stride', 1, ...
'learningRate', [.1 1], ...
'weightDecay', [1 0], ...
'pad', 1) ;
net.layers{end+1} = struct('type', 'bnorm', 'name', 'bn7', ...
'weights', {{ones(unit0*piece0, 1, 'single'), zeros(unit0*piece0, 1, 'single')}},'learningRate', [1 1 .5],'weightDecay', [0 0]) ;
%% maxout layer 1
unit1=192;
piece1 = 5;
net.layers{end+1} = struct('type', 'conv', ...
'name', 'maxoutconv5', ...
'weights', {{single(orthonorm(1,1,unit0,unit1*piece1)), b*ones(1,unit1*piece1,'single')}}, ...
'stride', 1, ...
'learningRate', [.1 1], ...
'weightDecay', [1 0], ...
'pad', 0) ;
net.layers{end+1} = struct('type', 'bnorm', 'name', 'bn8', ...
'weights', {{ones(unit1*piece1, 1, 'single'), zeros(unit1*piece1, 1, 'single')}},'learningRate', [1 1 .5],'weightDecay', [0 0]) ;
net.layers{end+1} = struct('type', 'maxout','numunit',unit1,'numpiece',piece1) ;
%% maxout layer 2
unit2=10;
piece2 = 5;
net.layers{end+1} = struct('type', 'conv', ...
'name', 'maxoutconv6', ...
'weights', {{single(orthonorm(1,1,unit1,unit2*piece2)), b*ones(1,unit2*piece2,'single')}}, ...
'stride', 1, ...
'learningRate', 0.1*[.1 1], ...
'weightDecay', [1 0], ...
'pad', 0) ;
net.layers{end+1} = struct('type', 'bnorm', 'name', 'bn9', ...
'weights', {{ones(unit2*piece2, 1, 'single'), zeros(unit2*piece2, 1, 'single')}},'learningRate', [1 1 .5],'weightDecay', [0 0]) ;
net.layers{end+1} = struct('type', 'maxout','numunit',unit2,'numpiece',piece2) ;
%%
net.layers{end+1} = struct('type', 'pool', ...
'name', 'pool3', ...
'method', 'avg', ...
'pool', [8 8] ,...
'stride',1, ...
'pad', 0) ;
% Loss layer
net.layers{end+1} = struct('type', 'softmaxloss') ;
% Fill in default values
net = vl_simplenn_tidy(net) ;
vl_simplenn_display(net,'inputSize', [32 32 3 100])
% Meta parameters
net.meta.inputSize = [32 32 3] ;
net.meta.trainOpts.learningRate = [0.5*ones(1,80) 0.5:-0.005:0.005 0.005:-0.0001:0.00005];
net.meta.trainOpts.weightDecay = 0.0005;
net.meta.trainOpts.batchSize = 100 ;
net.meta.trainOpts.numEpochs = 222;
net.meta.trainOpts.augmentation = false;
% Switch to DagNN if requested
switch lower(opts.networkType)
case 'simplenn'
% done
case 'dagnn'
net = dagnn.DagNN.fromSimpleNN(net, 'canonicalNames', true) ;
net.addLayer('error', dagnn.Loss('loss', 'classerror'), ...
{'prediction','label'}, 'error') ;
otherwise
assert(false) ;
end