single-stage accelerated stochastic approximation (AC-SA) for stochas…

…tic optimization

AC-SA/Algorithm_Selecting.m

@@ -0,0 +1,25 @@
+mtitle = 'Algorithm Selecting';
+method_selection = menu (mtitle, 'Single-stage AC-SA', 'Multi-stage AC-SA', 'Shrinking single-stage AC-SA' , 'Shrinking multi-stage AC-SA','Classic SA' , 'Batch-learning', 'Change algorithm parameters','Change initial sapmle' ,'Quit');
+if method_selection == 1
+    single_stage;
+elseif  method_selection == 2
+    multi_stage;
+elseif method_selection == 3
+    single_stage_shrinkage;
+elseif  method_selection == 4
+    multi_stage_shrinkage;
+elseif method_selection == 5
+    classic_SA;
+elseif  method_selection == 6
+    deterministic;
+elseif  method_selection == 7
+    fprintf('  \n');
+    Input_Parameters;
+    Estimating_Parameters;
+elseif  method_selection == 8
+    running = 0;
+    Sample_generation;
+    Estimating_Parameters;
+elseif  method_selection == 9
+    indicator = 1;
+end

AC-SA/Computing_Data.m

@@ -0,0 +1,5 @@
+objective_final = mean(objective_value);
+run_time = mean(r_time);
+final_data = [run_time , objective_final];
+fprintf ('Runtime:% 4.2f     Final objective value:% 6.2f\n', run_time , objective_final);
+

AC-SA/Data_generation.m

@@ -0,0 +1,3 @@
+er(index) = normrnd(0,st);
+x(index,:) = rand(1,d);
+y(index) = dot(x(index, :), z_sample) + er(index);

AC-SA/Data_generation2.m

@@ -0,0 +1,3 @@
+x = rand(N_iter,d);
+er = normrnd(0,st,1,N_iter)';
+y = x*z_sample + er;

AC-SA/Estimating_Parameters.m

@@ -0,0 +1,9 @@
+load (filename);
+N = N_initial;
+z_sample = z_initial;
+initial_solution = R_ini * z_ini;
+A = (2/N)*x_initial'*x_initial + 2*lamda*eye(d);
+% L is the lipschitz parameter
+L = max(eig(A));
+% mu is the strong convexity parameter
+mu = min(eig(A));

AC-SA/Input_Parameters.m

@@ -0,0 +1,32 @@
+% fprintf('  \n');
+% running = 0;
+% while running == 0
+% mtitle = 'Algorithm Parameters';
+% awr = menu(mtitle, ['Number of iterations: ',num2str(N_iter)] , ['Regularization parameter(rho): ', num2str(lamda)] , ['Validation samples: ', num2str(N_vali)] , ['Run times: ', num2str(Run_times)] , ['Magnitude initial point: ' , num2str(R_ini)], 'Run Algorithms','Quit');
+%     if awr ==1
+%         N_iter = input('Enter number of iterations: ');
+%     elseif  awr ==2
+%         lamda = input('Enter the regularization parameter (rho) : ');
+%     elseif  awr ==3
+%         N_vali = input('Enter the validation samples : ');
+%     elseif  awr ==4
+%         Run_times = input('Enter run times : ');
+%     elseif  awr ==5
+%         R_ini = input('Enter magnitude initial point: ');
+%     elseif awr ==6
+%         running = 1;
+%     elseif  awr == 7
+%         return;
+%     end
+% end
+
+
+% N_iter = input('Enter number of iterations: ');
+% lamda = input('Enter the regularization parameter (rho) : ');
+% N_vali = input('Enter the validation samples : ');
+% Run_times = input('Enter run times : ');
+% R_ini = input('Enter magnitude initial point: ');
+
+
+mtitle = 'Algorithm Parameters';
+awr = menu(mtitle, ['Number of iterations: ',num2str(N_iter)] , ['Regularization parameter(rho): ', num2str(lamda)] , ['Validation samples: ', num2str(N_vali)] , ['Run times: ', num2str(Run_times)] , ['Magnitude initial point: ' , num2str(R_ini)], 'Run Algorithms','Quit');

AC-SA/Main_Program.m

@@ -0,0 +1,7 @@
+% Main Program
+initializer;
+% Generating initial sample for estimating parameters
+common_prog;
+while indicator == 0
+    Algorithm_Selecting;
+end

AC-SA/N200-d50-st2.txt

@@ -0,0 +1,8 @@
+
+*** Running Single-stage AC-SA: lambda =    0.1
+
+       Runtime: 0.15     Final objective value:  5.63
+
+*** Running Multi-stage AC-SA: lambda =    0.1
+
+       Runtime: 0.14     Final objective value:  6.07

AC-SA/Results.txt

@@ -0,0 +1,88 @@
+
+**************************** Data: N200-d50-st2 , rho:    0.1
+
+*** Running Single-stage AC-SA
+
+       Runtime: 0.15     Final objective value:  5.63
+
+*** Running Single-stage AC-SA
+
+       Runtime: 0.14     Final objective value:  5.65
+
+**************************** Data: N200-d50-st2 , rho:    0.1
+
+*** Running Single-stage AC-SA
+
+       Runtime: 0.14     Final objective value:  5.70
+
+**************************** Data: N200-d20-st2 , rho:    0.1
+
+*** Running Multi-stage AC-SA
+
+       Runtime: 0.14     Final objective value:  5.39
+
+**************************** Data: N200-d20-st2 , rho:    0.1
+
+*** Running Single-stage AC-SA
+
+       Runtime: 0.13     Final objective value:  4.94
+
+**************************** Data: N200-d20-st2 , rho:    0.1
+
+*** Running Single-stage AC-SA
+
+       Runtime: 0.13     Final objective value:  4.94
+
+**********Data: N200-d20-st2 , rho:    0.1
+
+*** Running Single-stage AC-SA
+
+       Runtime: 0.17     Final objective value:  4.94
+
+*** Running Multi-stage AC-SA
+
+       Runtime: 0.17     Final objective value:  5.34
+
+*** Running Shrinking single-stage AC-SA
+
+       Runtime: 0.18     Final objective value:  4.90
+
+*** Running Shrinking multi-stage AC-SA
+
+       Runtime: 0.17     Final objective value:  6.10
+
+*** Running Classic SA
+
+       Runtime: 0.16     Final objective value: 182.28
+
+*** Running Batch-learning
+
+       Runtime: 0.00     Final objective value:  4.95
+
+********** Data: N200-d20-st2 , rho:   0.01
+
+*** Running Classic SA
+
+       Runtime: 0.17     Final objective value: 449005.88
+
+********** Data: N200-d20-st2 , rho:   0.01
+
+*** Running Classic SA
+
+       Runtime: 0.17     Final objective value: 99943157626646136000000000000000000000000000000000000000000.00
+
+********** Data: N200-d20-st2 , rho:   0.01
+
+*** Running Classic SA
+
+       Runtime: 0.17     Final objective value: 21.87
+
+*** Running Single-stage AC-SA
+
+       Runtime: 0.17     Final objective value:  4.40
+
+**********Data: N100-d30-st3.mat , rho:    0.1
+
+*** Running Single-stage AC-SA
+
+       Runtime: 0.26     Final objective value: 10.41

AC-SA/SGradient.m

@@ -0,0 +1,2 @@
+%function SG = SGradient(x,y,z_md,lamda)
+GR = -2*(y(index) - dot(z_md, x(index,:)))*x(index,:)'+ 2*lamda*z_md;

AC-SA/Sample_generation.m

@@ -0,0 +1,35 @@
+% Generating or Loading initila sample
+while  running == 0
+    data_name = ['Initial Sample : ', filename];
+    mtitle = data_name;
+    awr = menu(mtitle, 'Load sample ' , 'Generating new sample' , 'Run Algorithms' , ' Quit');
+    if awr == 1
+        [filename,PathName,FilterIndex] = uigetfile('*.mat');
+        load (filename);
+    elseif awr == 2
+        fprintf('  \n');
+        N_initial = input('Enter Number of initial samples : ');
+        d = input('Enter the dimension of problem : ');
+        st = input('Enter the standard variation of noise : ');
+        er_initial = normrnd(0,st,1,N_initial)';
+        x_initial = rand(N_initial,d);
+        % z is the substitution for x in the algorithm
+        z_initial = rand(1,d)';
+        z_ini = rand(1,d)';
+        y_initial = x_initial*z_initial + er_initial;
+        M = 0;
+        epsln = 0.01;
+        filename = ['N', num2str(N_initial),'-d',num2str(d),'-st', num2str(st)];
+        save (filename, 'N_initial','d', 'st', 'er_initial', 'x_initial', 'z_initial', 'z_ini', 'y_initial', 'M' ,'epsln'); 
+    elseif  awr == 3
+        if filename == ' '
+            error('You should choose an initial sample');
+        end
+        running = 1;
+    elseif  awr == 4
+        break;
+    end
+end
+
+
+

AC-SA/common_prog.m

@@ -0,0 +1,5 @@
+Sample_generation;
+Input_Parameters;
+Estimating_Parameters;
+count_alg = 0;
+indicator = 0;

AC-SA/data_showing.m

@@ -0,0 +1,9 @@
+% data1 = [mean_data{1}, '     ', mean_data{3}, '     ', mean_data{5}, '   ', mean_data{7}];
+% blnk = '                                                                            ';   
+% data2 = [num2str(mean_data{2}), '      ' , num2str(mean_data{4}), '              ' ,num2str(mean_data{6}), '              ' ,num2str(mean_data{8})];
+% data_show = {blnk; data1 ; blnk; blnk; data2; blnk; blnk; blnk};
+data1 = [mean_data{1}, '     ', mean_data{3},'     ', mean_data{5}];
+blnk = '                                                          ';   
+data2 = [num2str(mean_data{2}), '      ' , num2str(mean_data{4}),'      ' , num2str(mean_data{6})];
+data_show = {blnk; data1 ; blnk; blnk; data2; blnk; blnk};
+msgbox(data_show);

AC-SA/deterministic.m

@@ -0,0 +1,25 @@
+fprintf('     \n');
+fprintf('*** Running Batch-learning\n');
+for j = 1 : Run_times
+    tic
+    er = normrnd(0,st,1,N_iter)';
+    x = rand(N_iter,d);
+    y = x*z_sample + er;
+    B = inv(N*lamda*eye(d)+x'*x);
+    final_solution = (B*x'*y)';
+    objective_computation;
+end
+Computing_Data;
+% file_name = [filename,'.txt'];
+% frep=fopen(file_name,'a');
+frep=fopen('Results.txt','a');
+count_alg = count_alg +1;
+if count_alg == 1
+    fprintf(frep,'     \n');
+    fprintf(frep,'********** Data: %s , rho: %6.5g\n',filename, lamda);
+end
+fprintf(frep,'     \n');
+fprintf(frep,'*** Running Batch-learning\n');
+fprintf(frep,'     \n');
+fprintf(frep,'       Runtime:% 4.2f     Final objective value:% 6.2f\n', final_data);
+fclose(frep);

AC-SA/initializer.m

@@ -0,0 +1,9 @@
+clear
+clc
+filename = ' ';
+running = 0;
+N_iter = 1000;
+lamda = 0.1;
+N_vali = 10000;
+Run_times = 50;
+R_ini = 10;

AC-SA/mean_objective_computation.m

@@ -0,0 +1,5 @@
+avereage_runtime = mean ( run_time);
+average_objfun_initial = mean(objfun_initial);
+average_objfun_final = mean(objfun_final);
+average_optimlaity_gap_percent = mean(percent_optimlaity_gap);
+

AC-SA/objective_computation.m

@@ -0,0 +1,6 @@
+clear x y er
+r_time(j) = toc;
+x_vali = rand(N_vali,d);
+y_vali = x_vali*z_sample + normrnd(0,st,1,N_vali)';
+objective_value(j) = (1/N_vali)*(y_vali - x_vali*final_solution')'*(y_vali - x_vali*final_solution')+ lamda*norm(final_solution)^2;
+

AC-SA/prep_Excel.m

@@ -0,0 +1,7 @@
+objective_final = mean(objective_value);
+var_objective_final = var (objective_value);
+run_time = mean(r_time);
+mean_data = {'runtime' ,'objfun_final' , ' variance'; run_time , objective_final, var_objective_final}
+mean_data2 = ['runtime' ,'objfun_final' , ' variance'];
+sheet_name = ['lamda-', num2str(lamda)];
+%data_showing;

AC-SA/single_stage.m

@@ -0,0 +1,39 @@
+fprintf('     \n');
+fprintf('*** Running Single-stage AC-SA\n');
+for j = 1: Run_times
+    tic
+    z = initial_solution;
+    z_ag = z;
+    t = 1;
+    while  t <= N_iter
+        % Step1
+        alfa = 2/(t+1);
+        beta = 4 * L / (t * (t+1));
+        cx_p1 = (1-alfa)*(mu + beta) /( beta + (1 - alfa^2) * mu);
+        z_md = cx_p1 * z_ag + (1.0-cx_p1) *z;
+        cx_p2 = alfa * mu / (mu + beta);
+        z_ps = cx_p2 * z_md + (1.0 - cx_p2) * z;
+        index = t;
+        Data_generation;
+        SGradient;
+        z = (-alfa/(mu + beta))*GR + z_ps;
+        z_ag = alfa*z + (1-alfa)*z_ag;
+        t = t+1;
+    end
+    final_solution = z_ag';
+    objective_computation;
+end
+Computing_Data;
+% file_name = [filename,'.txt'];
+% frep=fopen(file_name,'a');
+frep=fopen('Results.txt','a');
+count_alg = count_alg +1;
+if count_alg == 1
+    fprintf(frep,'     \n');
+    fprintf(frep,'**********Data: %s , rho: %6.5g\n',filename, lamda);
+end
+fprintf(frep,'     \n');
+fprintf(frep,'*** Running Single-stage AC-SA\n');
+fprintf(frep,'     \n');
+fprintf(frep,'       Runtime:% 4.2f     Final objective value:% 6.2f\n', final_data);
+fclose(frep);