fs_weights.py

import numpy as np

# Time: 2020-02-24 19:16:20.352338 Loss: 0.012150533497333527 (v:0.015619919635355473,z:11.051372528076172 )
# best in fs_results: roughly 83% without fs sigmoid, 81.?% with. 
swish_h = [0.7844085, 1.7550646, 1.4212719, 1.9244663,
           1.4400164, 1.9249276, 1.6100428, 1.29061, 0.83985287,
           3.0827985, 0.43503317, 0.5155389, 0.3269445, 7.618048,
           -14.419675, 0]
swish_d = [0.40088242, 1.8856316, 1.4915457, 1.947229, 1.4717816, 1.9147849,
           1.6389303, 1.2934641, 0.8703124, 0.68460804, 0.44262582, 0.32832766,
           0.13907616, -0.26217145, 0.23736191, -0.14830568]
swish_T = [0.05946945, 1.6541206, 1.3066754, 1.3267198, 1.1000695, 1.2308974,
           1.1188028, 0.9472501, 0.5447531, 1.0236325, 0.11921431, -0.12855051,
           -0.44731247, -2.1383545, 1., -4.109145]

# Time: 2020-02-28 21:48:52.037026 Loss: 0.00011577547411434352 (v:6.222464435268193e-05,z:2.737070083618164 )
# best in sigmoid_results_2.txt
sigmoid_h = [1.6558512, 1.0989026, 10.7739, 1.3579437, 1.0146959,
             0.8972655, 2.1207616, 1.6949341, 3.6397197, -2.1483946, 5.1177883, 0]
sigmoid_d = [0.37689912, 0.21977554, 0.40271342, 0.17005783, 0.09584032, 0.06293014,
             0.04360681, -0.0443096, 0.01999909, 0.07107325, 0.01502249, 0.01021793]
sigmoid_T = [-0.00824843, -0.9319625, 1.0080122, -1.2074932, -1.7885877, -2.7065408,
             -3.2715735, 0.07132628, -3.719372, -0.543918, -3.7693157, -5.5447803]

relu_K = 10
alpha = 25
relu_h = alpha * 2**(-relu_K) * np.array([float(2 ** (relu_K - i)) for i in range(1, relu_K + 1)]).astype(np.float32)
relu_d = alpha * 2**(-relu_K) * np.array([float(2 ** (relu_K - i)) for i in range(1, relu_K + 1)]).astype(np.float32)
relu_T = alpha * 2**(-relu_K) * np.array([float(2 ** (relu_K - i)) for i in range(1, relu_K + 1)]).astype(np.float32)