made ESN work with new initilaizers, started separation of different …

…models

README.md

@@ -51,14 +51,15 @@ test = data[:, (shift + train_len):(shift + train_len + predict_len - 1)]
 Now that we have the data we can initialize the ESN with the chosen parameters. Given that this is a quick example we are going to change the least amount of possible parameters. For more detailed examples and explanations of the functions please refer to the documentation.
 
 ```julia
+input_size = 3
 res_size = 300
-esn = ESN(input_data;
-    reservoir = RandSparseReservoir(res_size, radius = 1.2, sparsity = 6 / res_size),
-    input_layer = WeightedLayer(),
+esn = ESN(input_data, input_size, res_size;
+    reservoir = rand_sparse(;radius = 1.2, sparsity = 6 / res_size),
+    input_layer = weighted_init,
     nla_type = NLAT2())
 ```
 
-The echo state network can now be trained and tested. If not specified, the training will always be Ordinary Least Squares regression. The full range of training methods is detailed in the documentation.
+The echo state network can now be trained and tested. If not specified, the training will always be ordinary least squares regression. The full range of training methods is detailed in the documentation.
 
 ```julia
 output_layer = train(esn, target_data)

src/ReservoirComputing.jl

@@ -19,10 +19,11 @@ export NLADefault, NLAT1, NLAT2, NLAT3
 export StandardStates, ExtendedStates, PaddedStates, PaddedExtendedStates
 export StandardRidge, LinearModel
 export AbstractLayer, create_layer
-export scaled_rand
+export scaled_rand, weighted_init
 export rand_sparse, delay_line
 export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal
-export ESN, Default, Hybrid, train
+export ESN, train
+export DeepESN, HybridESN
 export RECA, train
 export RandomMapping, RandomMaps
 export Generative, Predictive, OutputLayer
@@ -73,7 +74,7 @@ function Predictive(prediction_data)
 end
 
 #fallbacks for initializers
-for initializer in (:rand_sparse, :delay_line, :scaled_rand)
+for initializer in (:rand_sparse, :delay_line, :scaled_rand, :weighted_init)
     NType = ifelse(initializer === :rand_sparse, Real, Number)
     @eval function ($initializer)(dims::Integer...; kwargs...)
         return $initializer(_default_rng(), Float32, dims...; kwargs...)
@@ -107,7 +108,9 @@ include("train/supportvector_regression.jl")
 include("esn/esn_input_layers.jl")
 include("esn/esn_reservoirs.jl")
 include("esn/esn_reservoir_drivers.jl")
-include("esn/echostatenetwork.jl")
+include("esn/esn.jl")
+include("esn/deepesn.jl")
+include("esn/hybridesn.jl")
 include("esn/esn_predict.jl")
 
 #reca

src/esn/deepesn.jl

@@ -0,0 +1,84 @@
+struct DeepESN{I, S, V, N, T, O, M, B, ST, W, IS} <: AbstractEchoStateNetwork
+    res_size::I
+    train_data::S
+    variation::V
+    nla_type::N
+    input_matrix::T
+    reservoir_driver::O
+    reservoir_matrix::M
+    bias_vector::B
+    states_type::ST
+    washout::W
+    states::IS
+end
+
+function DeepESN(
+    train_data,
+    in_size::Int,
+    res_size::AbstractArray;
+    input_layer = scaled_rand,
+    reservoir = rand_sparse,
+    bias = zeros64,
+    reservoir_driver = RNN(),
+    nla_type = NLADefault(),
+    states_type = StandardStates(),
+    washout = 0,
+    rng = _default_rng(),
+    T=Float64,
+    matrix_type = typeof(train_data)
+)
+
+    if states_type isa AbstractPaddedStates
+        in_size = size(train_data, 1) + 1
+        train_data = vcat(Adapt.adapt(matrix_type, ones(1, size(train_data, 2))),
+            train_data)
+    end
+
+    reservoir_matrix = reservoir(rng, T, res_size, res_size)
+    input_matrix = input_layer(rng, T, res_size, in_size)
+    bias_vector = bias(rng, T, res_size)
+    inner_res_driver = reservoir_driver_params(reservoir_driver, res_size, in_size)
+    states = create_states(inner_res_driver, train_data, washout, reservoir_matrix,
+        input_matrix, bias_vector)
+    train_data = train_data[:, (washout + 1):end]
+
+    ESN(sum(res_size), train_data, variation, nla_type, input_matrix,
+        inner_res_driver, reservoir_matrix, bias_vector, states_type, washout,
+        states)
+end
+
+function obtain_layers(in_size,
+    input_layer,
+    reservoir::Vector,
+    bias;
+    matrix_type = Matrix{Float64})
+esn_depth = length(reservoir)
+input_res_sizes = [get_ressize(reservoir[i]) for i in 1:esn_depth]
+in_sizes = zeros(Int, esn_depth)
+in_sizes[2:end] = input_res_sizes[1:(end - 1)]
+in_sizes[1] = in_size
+
+if input_layer isa Array
+    input_matrix = [create_layer(input_layer[j], input_res_sizes[j], in_sizes[j],
+        matrix_type = matrix_type) for j in 1:esn_depth]
+else
+    _input_layer = fill(input_layer, esn_depth)
+    input_matrix = [create_layer(_input_layer[k], input_res_sizes[k], in_sizes[k],
+        matrix_type = matrix_type) for k in 1:esn_depth]
+end
+
+res_sizes = [get_ressize(input_matrix[j]) for j in 1:esn_depth]
+reservoir_matrix = [create_reservoir(reservoir[k], res_sizes[k],
+    matrix_type = matrix_type) for k in 1:esn_depth]
+
+if bias isa Array
+    bias_vector = [create_layer(bias[j], res_sizes[j], 1, matrix_type = matrix_type)
+                   for j in 1:esn_depth]
+else
+    _bias = fill(bias, esn_depth)
+    bias_vector = [create_layer(_bias[k], res_sizes[k], 1, matrix_type = matrix_type)
+                   for k in 1:esn_depth]
+end
+
+return input_matrix, reservoir_matrix, bias_vector, res_sizes
+end