From a8b5ed02f47dee7383a51785d9cdcfb1334f8499 Mon Sep 17 00:00:00 2001 From: Neea Rusch <2580981+nkrusch@users.noreply.github.com> Date: Fri, 24 Nov 2023 02:08:45 -0500 Subject: [PATCH] -perf variants (#69) --- .../__bar_acc_PT-1_ref_result_perf_darwin.pdf | Bin 14379 -> 0 bytes .../__bar_acc_PT-1_ref_result_perf_debian.pdf | Bin 14369 -> 0 bytes ...__bar_acc_PT-1_ref_result_perf_windows.pdf | Bin 14349 -> 0 bytes .../__bar_acc_PT-2_ref_result_perf_darwin.pdf | Bin 14311 -> 0 bytes .../__bar_acc_PT-2_ref_result_perf_debian.pdf | Bin 14295 -> 0 bytes ...__bar_acc_PT-2_ref_result_perf_windows.pdf | Bin 14304 -> 0 bytes .../__cls_acc_ref_result_perf_darwin.txt | 7 - .../__cls_acc_ref_result_perf_debian.txt | 7 - .../__cls_acc_ref_result_perf_windows.txt | 7 - .../__config_ref_result_perf_darwin.txt | 7 - .../__config_ref_result_perf_debian.txt | 7 - .../__config_ref_result_perf_windows.txt | 7 - ref_result/__plot_ref_result_perf_darwin.txt | 19 -- ref_result/__plot_ref_result_perf_debian.txt | 19 -- ref_result/__plot_ref_result_perf_windows.txt | 19 -- ref_result/__time_ref_result_perf_darwin.txt | 19 -- ref_result/__time_ref_result_perf_debian.txt | 19 -- ref_result/__time_ref_result_perf_windows.txt | 19 -- .../darwin/PT-1-03_dnn_cpgd_auto_988.yaml | 113 ------------ .../perf/darwin/PT-1-03_dnn_pgd_40_853.yaml | 111 ------------ .../darwin/PT-1-06_dnn_cpgd_auto_053.yaml | 128 -------------- .../perf/darwin/PT-1-06_dnn_pgd_40_831.yaml | 126 -------------- .../darwin/PT-1-12_dnn_cpgd_auto_769.yaml | 158 ----------------- .../perf/darwin/PT-1-12_dnn_pgd_40_624.yaml | 156 ----------------- .../darwin/PT-2-03_dnn_cpgd_auto_195.yaml | 113 ------------ .../perf/darwin/PT-2-03_dnn_pgd_40_278.yaml | 111 ------------ .../darwin/PT-2-06_dnn_cpgd_auto_453.yaml | 128 -------------- .../perf/darwin/PT-2-06_dnn_pgd_40_213.yaml | 126 -------------- .../darwin/PT-2-12_dnn_cpgd_auto_275.yaml | 158 ----------------- .../perf/darwin/PT-2-12_dnn_pgd_40_110.yaml | 156 ----------------- .../darwin/REG_PT-1-03_dnn_pgd_40_346.yaml | 111 ------------ .../darwin/REG_PT-1-06_dnn_pgd_40_325.yaml | 126 -------------- .../darwin/REG_PT-1-12_dnn_pgd_40_775.yaml | 156 ----------------- .../darwin/REG_PT-2-03_dnn_pgd_40_256.yaml | 111 ------------ .../darwin/REG_PT-2-06_dnn_pgd_40_002.yaml | 126 -------------- .../darwin/REG_PT-2-12_dnn_pgd_40_898.yaml | 156 ----------------- .../debian/PT-1-03_dnn_cpgd_auto_118.yaml | 115 ------------- .../perf/debian/PT-1-03_dnn_pgd_40_926.yaml | 113 ------------ .../debian/PT-1-06_dnn_cpgd_auto_124.yaml | 130 -------------- .../perf/debian/PT-1-06_dnn_pgd_40_650.yaml | 128 -------------- .../debian/PT-1-12_dnn_cpgd_auto_094.yaml | 160 ----------------- .../perf/debian/PT-1-12_dnn_pgd_40_647.yaml | 158 ----------------- .../debian/PT-2-03_dnn_cpgd_auto_490.yaml | 115 ------------- .../perf/debian/PT-2-03_dnn_pgd_40_707.yaml | 113 ------------ .../debian/PT-2-06_dnn_cpgd_auto_019.yaml | 130 -------------- .../perf/debian/PT-2-06_dnn_pgd_40_181.yaml | 128 -------------- .../debian/PT-2-12_dnn_cpgd_auto_735.yaml | 160 ----------------- .../perf/debian/PT-2-12_dnn_pgd_40_899.yaml | 158 ----------------- .../debian/REG_PT-1-03_dnn_pgd_40_873.yaml | 113 ------------ .../debian/REG_PT-1-06_dnn_pgd_40_683.yaml | 128 -------------- .../debian/REG_PT-1-12_dnn_pgd_40_458.yaml | 158 ----------------- .../debian/REG_PT-2-03_dnn_pgd_40_818.yaml | 113 ------------ .../debian/REG_PT-2-06_dnn_pgd_40_576.yaml | 128 -------------- .../debian/REG_PT-2-12_dnn_pgd_40_163.yaml | 158 ----------------- .../windows/PT-1-03_dnn_cpgd_auto_668.yaml | 117 ------------- .../perf/windows/PT-1-03_dnn_pgd_40_514.yaml | 115 ------------- .../windows/PT-1-06_dnn_cpgd_auto_074.yaml | 132 -------------- .../perf/windows/PT-1-06_dnn_pgd_40_307.yaml | 130 -------------- .../windows/PT-1-12_dnn_cpgd_auto_972.yaml | 162 ------------------ .../perf/windows/PT-1-12_dnn_pgd_40_240.yaml | 160 ----------------- .../windows/PT-2-03_dnn_cpgd_auto_294.yaml | 117 ------------- .../perf/windows/PT-2-03_dnn_pgd_40_556.yaml | 115 ------------- .../windows/PT-2-06_dnn_cpgd_auto_404.yaml | 132 -------------- .../perf/windows/PT-2-06_dnn_pgd_40_130.yaml | 130 -------------- .../windows/PT-2-12_dnn_cpgd_auto_069.yaml | 162 ------------------ .../perf/windows/PT-2-12_dnn_pgd_40_338.yaml | 160 ----------------- .../windows/REG_PT-1-03_dnn_pgd_40_519.yaml | 115 ------------- .../windows/REG_PT-1-06_dnn_pgd_40_484.yaml | 130 -------------- .../windows/REG_PT-1-12_dnn_pgd_40_978.yaml | 160 ----------------- .../windows/REG_PT-2-03_dnn_pgd_40_714.yaml | 115 ------------- .../windows/REG_PT-2-06_dnn_pgd_40_611.yaml | 130 -------------- .../windows/REG_PT-2-12_dnn_pgd_40_474.yaml | 160 ----------------- 72 files changed, 7374 deletions(-) delete mode 100644 ref_result/__bar_acc_PT-1_ref_result_perf_darwin.pdf delete mode 100644 ref_result/__bar_acc_PT-1_ref_result_perf_debian.pdf delete mode 100644 ref_result/__bar_acc_PT-1_ref_result_perf_windows.pdf delete mode 100644 ref_result/__bar_acc_PT-2_ref_result_perf_darwin.pdf delete mode 100644 ref_result/__bar_acc_PT-2_ref_result_perf_debian.pdf delete mode 100644 ref_result/__bar_acc_PT-2_ref_result_perf_windows.pdf delete mode 100644 ref_result/__cls_acc_ref_result_perf_darwin.txt delete mode 100644 ref_result/__cls_acc_ref_result_perf_debian.txt delete mode 100644 ref_result/__cls_acc_ref_result_perf_windows.txt delete mode 100644 ref_result/__config_ref_result_perf_darwin.txt delete mode 100644 ref_result/__config_ref_result_perf_debian.txt delete mode 100644 ref_result/__config_ref_result_perf_windows.txt delete mode 100644 ref_result/__plot_ref_result_perf_darwin.txt delete mode 100644 ref_result/__plot_ref_result_perf_debian.txt delete mode 100644 ref_result/__plot_ref_result_perf_windows.txt delete mode 100644 ref_result/__time_ref_result_perf_darwin.txt delete mode 100644 ref_result/__time_ref_result_perf_debian.txt delete mode 100644 ref_result/__time_ref_result_perf_windows.txt delete mode 100644 ref_result/perf/darwin/PT-1-03_dnn_cpgd_auto_988.yaml delete mode 100644 ref_result/perf/darwin/PT-1-03_dnn_pgd_40_853.yaml delete mode 100644 ref_result/perf/darwin/PT-1-06_dnn_cpgd_auto_053.yaml delete mode 100644 ref_result/perf/darwin/PT-1-06_dnn_pgd_40_831.yaml delete mode 100644 ref_result/perf/darwin/PT-1-12_dnn_cpgd_auto_769.yaml delete mode 100644 ref_result/perf/darwin/PT-1-12_dnn_pgd_40_624.yaml delete mode 100644 ref_result/perf/darwin/PT-2-03_dnn_cpgd_auto_195.yaml delete mode 100644 ref_result/perf/darwin/PT-2-03_dnn_pgd_40_278.yaml delete mode 100644 ref_result/perf/darwin/PT-2-06_dnn_cpgd_auto_453.yaml delete mode 100644 ref_result/perf/darwin/PT-2-06_dnn_pgd_40_213.yaml delete mode 100644 ref_result/perf/darwin/PT-2-12_dnn_cpgd_auto_275.yaml delete mode 100644 ref_result/perf/darwin/PT-2-12_dnn_pgd_40_110.yaml delete mode 100644 ref_result/perf/darwin/REG_PT-1-03_dnn_pgd_40_346.yaml delete mode 100644 ref_result/perf/darwin/REG_PT-1-06_dnn_pgd_40_325.yaml delete mode 100644 ref_result/perf/darwin/REG_PT-1-12_dnn_pgd_40_775.yaml delete mode 100644 ref_result/perf/darwin/REG_PT-2-03_dnn_pgd_40_256.yaml delete mode 100644 ref_result/perf/darwin/REG_PT-2-06_dnn_pgd_40_002.yaml delete mode 100644 ref_result/perf/darwin/REG_PT-2-12_dnn_pgd_40_898.yaml delete mode 100644 ref_result/perf/debian/PT-1-03_dnn_cpgd_auto_118.yaml delete mode 100644 ref_result/perf/debian/PT-1-03_dnn_pgd_40_926.yaml delete mode 100644 ref_result/perf/debian/PT-1-06_dnn_cpgd_auto_124.yaml delete mode 100644 ref_result/perf/debian/PT-1-06_dnn_pgd_40_650.yaml delete mode 100644 ref_result/perf/debian/PT-1-12_dnn_cpgd_auto_094.yaml delete mode 100644 ref_result/perf/debian/PT-1-12_dnn_pgd_40_647.yaml delete mode 100644 ref_result/perf/debian/PT-2-03_dnn_cpgd_auto_490.yaml delete mode 100644 ref_result/perf/debian/PT-2-03_dnn_pgd_40_707.yaml delete mode 100644 ref_result/perf/debian/PT-2-06_dnn_cpgd_auto_019.yaml delete mode 100644 ref_result/perf/debian/PT-2-06_dnn_pgd_40_181.yaml delete mode 100644 ref_result/perf/debian/PT-2-12_dnn_cpgd_auto_735.yaml delete mode 100644 ref_result/perf/debian/PT-2-12_dnn_pgd_40_899.yaml delete mode 100644 ref_result/perf/debian/REG_PT-1-03_dnn_pgd_40_873.yaml delete mode 100644 ref_result/perf/debian/REG_PT-1-06_dnn_pgd_40_683.yaml delete mode 100644 ref_result/perf/debian/REG_PT-1-12_dnn_pgd_40_458.yaml delete mode 100644 ref_result/perf/debian/REG_PT-2-03_dnn_pgd_40_818.yaml delete mode 100644 ref_result/perf/debian/REG_PT-2-06_dnn_pgd_40_576.yaml delete mode 100644 ref_result/perf/debian/REG_PT-2-12_dnn_pgd_40_163.yaml delete mode 100644 ref_result/perf/windows/PT-1-03_dnn_cpgd_auto_668.yaml delete mode 100644 ref_result/perf/windows/PT-1-03_dnn_pgd_40_514.yaml delete mode 100644 ref_result/perf/windows/PT-1-06_dnn_cpgd_auto_074.yaml delete mode 100644 ref_result/perf/windows/PT-1-06_dnn_pgd_40_307.yaml delete mode 100644 ref_result/perf/windows/PT-1-12_dnn_cpgd_auto_972.yaml delete mode 100644 ref_result/perf/windows/PT-1-12_dnn_pgd_40_240.yaml delete mode 100644 ref_result/perf/windows/PT-2-03_dnn_cpgd_auto_294.yaml delete mode 100644 ref_result/perf/windows/PT-2-03_dnn_pgd_40_556.yaml delete mode 100644 ref_result/perf/windows/PT-2-06_dnn_cpgd_auto_404.yaml delete mode 100644 ref_result/perf/windows/PT-2-06_dnn_pgd_40_130.yaml delete mode 100644 ref_result/perf/windows/PT-2-12_dnn_cpgd_auto_069.yaml delete mode 100644 ref_result/perf/windows/PT-2-12_dnn_pgd_40_338.yaml delete mode 100644 ref_result/perf/windows/REG_PT-1-03_dnn_pgd_40_519.yaml delete mode 100644 ref_result/perf/windows/REG_PT-1-06_dnn_pgd_40_484.yaml delete mode 100644 ref_result/perf/windows/REG_PT-1-12_dnn_pgd_40_978.yaml delete mode 100644 ref_result/perf/windows/REG_PT-2-03_dnn_pgd_40_714.yaml delete mode 100644 ref_result/perf/windows/REG_PT-2-06_dnn_pgd_40_611.yaml delete mode 100644 ref_result/perf/windows/REG_PT-2-12_dnn_pgd_40_474.yaml diff --git a/ref_result/__bar_acc_PT-1_ref_result_perf_darwin.pdf b/ref_result/__bar_acc_PT-1_ref_result_perf_darwin.pdf deleted file mode 100644 index 0221b070f28cf692da4260677cad4dc044e41f83..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14379 zcmb_@2RxPE8+bx-vlW%(N@lqGx@5a%_Lh-r@4YEe_DD34O<5r;GAgskYACWt*-i&VgnHt zhQc(xoy?&yDS{ip&e0MIQzclMyF!t`0ClLSD8$^s3>=92ZGw!WgBujJJpiVwVq#-% z>IN0~@l)E%O;*E=;ARd*{{W~G+}zAv9H1ERCj_QpMKH5=u!LfFdR-h%HO$?h`T$wH z48V%Hmm3r&Zx65_xl>E+)bdcn-`GL?Edaz}*k_Ve00tj_xj|fIJ}o0DUEM zGi!pRqZdFC4*WyHar^>EI2MT(6vROhf@pp$7KcS*09r69KuQ3WD^&0YuM!Rpj=(Pj zI8Pn&OEX~it|}GH9W33fpoksvGS+s0@KBhH9iS1sxv8U>IVgjxn~OQY9^#eMt*_+R zkEN+y)|sSB6S?v6)17C2Wd~fy65m=;OH&7zIw$dPk7lh7EvUXXYQOL&_K2VE&Apue+Ochz4Xg3_SAILsR>xsx&qXAVE5*SGCm+ z6x4Wl^}aqQr|rYCTqHteahv6;7fd-NGoi?0mQyFDi%4WPH#tU; zM`EO}1-XAh;zQBR-N+A3ub+()%<{ z2$)gP5Wqj8AT#(@p2^+JN<}rI`+WM4aJt5dt}YYoW!O=#7)LdRv0kNokxypIFP|AF z*PgJ3&V-7KL$KU(Nwes}{78CA`B-b4uosMO+*)MH82NMcykWLlfl#d|Exp`2cOIpq z%6K`c65cRtE!Tei@Ulk1JEYPeD`?N94($%m2O%Sca$-)bzqM_Xb7(LEJOWgtWopLew?LznJFB3EU$ zrqxqwOL93rya9M@ya#v-fj3DT}-p6Z}fQ~oj*?&BofH=WeaOkAS`9~Z* zkKu*n58y7^`6lJMqAhnU-J>d{X5N#O zR%ydLRgib?BbDYKw4&<{PV0JpjO}I^r7GZ~Z?ipDJ}w+~`@Rs3r)$bbX3L+aJM&(4qyMAQx*y+TGG5yO6^Mk~qfKdg zoS^gbMgEB|R`*c@O`MCPib)%R4Vs>-mvF*PtI8YQo5q)Bzt49VMWl8%_KAEfyCuYF zvf%o(qTPMDjXUW@5-b@>T@ftLXRO|EgqFafCHCSt2TMfDvkcK?>a}*xT!j2(b)r{S z3MhvL#!R9o2So3sStc}SF)rmT*Uf}zG<_y4e{)K!x6AvgpV-&DNME8CRYEAe+IYxq z_*NAmm=rZXr|{G>B%*xN>$Bb3*Z41dUygoR=5*Uy7KzsARl8%5G#2P_gk@jkn;hjA zcx2sr?Q?~FH~FK>$Tv<17})ddqpw(MIqWJie766AH*`ELRM*X-k~Xi@+46jCLzFQ) zmsn+hyGbxpB|a51D(vqYK`zLcY<8W1PZ(b7^(puJSQOT73g32>u zS<02^jQfoq99cpJ`QR4=bRAPAjam7$k1!W z9LJGdE4eU)Ggz6it;8M2CgudbdCKZ_n0fR4P)6p3%qF#>m*PiGuQ|%!H+(fPL(uS^ zef&s&O-SsTg-Xhu;=BGL%SnA-3a|`o&gvFJr%Cd(4wE)C%5Jbt@?HsvE2?(t9ER7u zAqx;Q`yjF~Lr#5jqP)DN-P`|jba-5QhjSf2(s#l zBj2CO5jYoe)oiI=Z#{V^&!RvBZT861ET&}=JM|oU1AEC_S^!P5fRXO5-)cpNvYb}A zNONR(N-sT_xDyuYKKO;VZ^B}lJR~WY^abN7){G4j!!FWFwB;=2 zoe$HBa8+HG>^Z$4$F)3LrxSSlsN!ZGLLuP`?={y_8kyJ#JG1uK!w&I$H%=Umk=CdT zn90v6;TGe9Kj+{T)X1VYgSRCgmA7{oA9m-9mpf!PqS|^ zMU6CRhNn!8D0rvv?GaJOhPHzb$uxQE-Trr?7I@;jSFl&o?mWqhcw2v4bp1PXeDk_vNsaf!MS0M89*7<=p&L=p~yj^Oc@j4H=CCRGx|a2mC8tRlT!FUk_5vSdXX&FK2b2lNC>tRwv*%E@(uI_J0$6)P=j zOp=uXLUBE`b%$m}0`r>slwy~{Y$A#%5#8(bma!MNh`RBHggw-_M_h+OBmcJ=rDM;9 zpGHvN*SsNyX?5TFCUVmgq}w+uHpc@`KF61c(z5mr=-3PdYu}+?@3ThkKbKZ{=^Yg15@MIY$dC>O1aVL^jifxA1nkOm`$D$4L-MxCflR(glg=^xHSo z73JPGz3<9##WIb{(*{v%=UfmWb3?FA;N6Y>gPV-)?PjI#d93H9O(`12-2?fSqGOwq zU*4f)@v$Y?-@E*w12-uAXl-H4SIj?kfyIsEV(s6{g9O9yKdg!X_J6MqCbF9HDV&Vd z_nt9{>7^x6?Hl!jPonCwMwmmq2G3&M&NhtiJEk1i?}-m39XH5A#%P8YuCu_(Yn(X} zGLJI`wR|Ln3A02+GhW<-r{51HyH6hcAD%oBRl(Q>IK|9oh^WLUk-~yJM`_T~mWPB7 zM-XK^<33_;jS?f(+XgXf$2T|ma_X5sig~243OMl5Q_FR*6^&S4Pq?~t*t_x&vWMJG z;zU^T{wu~jEDwl{!tv+U{0JA$eAH}mD{kPvs_SK_5z1$8em7bg{kU1d-s6sv$wcK? z(IFYs5aD|J{HO9ef?j4Qx;gI%SyI%^SQjtTvJs0&ntsZ8Y0^RM#WmBnU#02_s}agg zp61fjrkWeioJLG5dp5tGDN6O{?@~0`Ln+`{=#Pa$z~TR}Qc=Z}4dN8UzDjkpDV?_u zw~s-cMrMa(F5!aqjgp*Xqa)6T)x=QLSTAFk{Os+Wj>(Ez-+EdVm-vzqeNZk0Z#Dww z!^-8Pu3XOW_yDICN!-%5twK30rqs-qby9zM$F#Wzv*4-r4`$J5#6K__D<8{@Aj7Y@ zL-@OBO}gAt7M#3ZKHho@%U#WYoF*~jXnSO*%fIeYF=dVl*<8w&>E&5pANL3<4Hj$; zQCb&v4!)=q@jm%zioMvi&Wanpx;^BX{SygqM0s2_{5#w4CLPfh7L9w%VCr1U#nfj- zqvcW5^u~70zL@{fGm|0Zs7#WH!?!1{wCQJ!#K|7KJl<25#rM_wYX;lWPh4cbUN%IXy+1RY z_NpwO{Kqlf^i<3DC+Pf9EUb|MiHl2b$}+k&qAOd@q2EN{9okox$r{YENGGcN=>2VD zr3dJODbMh{Wu2}!3K1~5+FB53=Ygpa?_5|F=0tF48~Tu&)@gk@Aay*7fDV5eOG+Iw z7{j$%YLhyl62a1CIrp~O@VGwI+)7L`BCVNKzG#(7p6KM*!=C;~^gFV&`McQKn@?BW zFDsBt+n2hI=R$1VN9w<&-0~OTQgVYx=s6Np@e{s>mC%E0mYANJex6_vn?d?V;+JDW z1(~`@hU~wkf1{im+S=DVlP$2v4gl{h{@4LXH0~dEfZ90+QUqDw#No%{cQ4D!b)dR6 z80R)CFf|?sBzYZ+MSB>zkzYJ(A;Ed{wM^UPfv{Ay+qW2p*rIu=LvPB-_T<)Cst%C8 zN^Eq+h&FImm-N6xrYfW7*EE?iw!QE9C0`nJh0@G;%;LShn)O0Yhr^O~u&%x}VyaQ|8D_$9Wq?_$eAI_f9$Db%QBoaKeP!;v)nPd!r6XByVKgnD zt3+q#RNt3RZ(Uf6EL!|VdO1!{Y!3y3H$8tU5c>~)6Qha%!@Zehh^UkLQ}NifS!?&E z*{wblUcXnW+4$@!bBMtu&8CVMuWnpm7L8g~F%1?b_Qo3;zkhQ6e)BtcltmT8k!c#@ zXF?TT&$UN{l{5vi{knVHJ1x+SBk?3cO-fH|<)yNlHLF~guYPe+`?PMxcY&uaNJh=r z<^F*HxXX3GlzD?V(&+e;`!%K1pi?_s}x(8y5q%DTTBXr#=qgqDQ`95TX#r1}#?^LIvCq1H~ zZji@y_w@=H3aY{Iti8;%Ngp;Ew32z>Mhjc$+`+u&x*|X{W@5;##H=4RDAkgysivH_ ze4*u{g+7G0kS}^r$muW&%AdVyc9cfQpg$ka`A~t?vwBPn%^6y?4irO zP9o@k(51ARyh9_LqI&(vhB%RJ`DP0D0TIP5f4&5gNmg3ZbElg$F&wWJ&4Wvt6QdlN zgo|qz&+^iFKvD^UguB^<*)n@v*gL0a-Yc@1oJlbsF!hqqC1Z>Ub5Fmp%HyV+C&xeK z``mE2>LI~A64rB~h(<^7$roMLj*RCuJWW$8Lg|?}H_yQHRfx~%0b8tNyOUr}P-wx; z4Wlop*zv8L0rzw74|dBmpI)+-vrmXJAk9w7<9!=sCTcAdk$evt%PqZj#8DXGYs1H? zKbsw^#vP$&C=WlIdCq0>8J9-)SzFf3{g#oaQ=@rp0!<~KHmcu!Ikq6y^-#}h4-M{d zwh;IS_lQCw)#0je6fs@!Jr2mnU<(L$uz~pFg!EiV{!k*i){#JYNnL2miTJz|g%tb` zh>Snn3%;c)O?Hb2UreNHeSF^n2LpT$cJ_$T1^*$mP>QSrgFxb~gh1NMBq6gJS{Jj{ z{SR65e7-{cz`m`Kq|u7_4XK-)-gSO z$U~+ZPYO3i%8fqw?F)I$r2RLCs6E0|%s+6rt1u!!VKTc4Grq(hx}SX9sV)o3OAMvd zzTR-N?&g7g@jVo`_n^E-Fp9wa!?9aQ6%k7e*x^k`iZPAXh)9cB;q%QWQP4QW@NyDl zNlz%7zA{V7ZIXxm?n;|OH0PHLrWYHMSo9=K@{hVSr}=(*IQ(tiU^f0x-bL}IcATl( z1MiM24%z1gTNIjL9tAl|&8yl-k_1;4HODsztbK}sc`(VbhRNVjUgqnVq~70zB?j>qowcTk?{(Ec(9cow~dAgdu5#B<$)wopPlYWiZb(L)wCK zeFwMj_UlJlc-Umi9%U2m4m8+05Ag{{b7-^F$eJ(*%lFzzJxa&aePo0^=Pag{fhkkn ziCgbW@-e`>*!c8~2AJ)u&l*EaY0iJ5H~3iZ+vt~vGgkaq<=v9@JUyes*0I`mgMzw7 zsmafk71kQ2_>mtqWNxcxrYmN_S7b&PJ5KMCs8c)RltfH+s|RXfwnqqzK>W+eN?DMK zSb}02*f+y{V{3MY?Ga!7nHIRLDe5nW%E7Pt7T!$HTBP&g8k4Frabk%S=A|5=C|7H~ z&-=;2;I!T+n^1P)%^N;nmX9&Kjk~4$I+OFrW*KkRd#|z+$B0a23IpX+5R99ab3=Jb z0<}+EUY+XNgZVw8H#F)WEWAjZM64--OlocGut8cr1-3_2|r{EqxuM<`fLxOft)Ex9Gk-SCZIZbWOHr#Uf?B3}0PP#91;FHz0oX zbDBGb=b~MH@%?uXY%ZCs7C?IAHN_*9n<_pRD5MqXBSwfbw>~e=D4pH#&Kn@D zVJVolC~nSuyfrqsVpi$RXP>~^bsH`l#BLjfQfyE2WN5DUMc(mdGhG$xePiR(YtVR4 z9u`8HG^pV>cvDA5S}$||spYb(9Zic(=K|G7-NHs^aw}C($7nB)rzagKAPF4X_vDE( zn*0&fu;PCAR)s=_MLY4i?pJ|2*Y#CAQ**K@iji-~i#K@F#vjJl4Ifodx^lMaKEoDS z>{RY_mKXBNAQV2 z|BKPPR!vI`CzJA3sUw3vP4$j_ao1_pk!|)cw_VFy5$DS4oR#qph|`82tmkUP*safb zl!l5irhgC`m^7h}&D)=Xex6yUdzHpZEa-W@;zxG}%?%kF#Ao^(Bi>aZl7`#}bad?M zjIsP@6hgI333<=NEUD%*t7t}S)}c^#`ea$e?CJ8akE`cb9g1KaBS@k>B-3yZ^p+gq?0xZGXY7zBa;z@2ZP} za}&;DA7^^Dp0Tx|zVa>CIi5WGZ17qt!B^MoD5Z&qmzeWe`N9HQJ%XjWvW;ktqsc~- zLNb@W^`m|7L(aV3ph_lNRX=8ppgQ{cl-npHo;FX~-R;@%_w`4IJ$eTjR&;~piW8w0 zM+@ROAhPDM$Iabx;xi1{OZ4w4o_#SL>Y+D4$6Kg4INpS>D3YkLKVZ0j*17N8b&bYr zn=P>mmQN49tU5Cy`ElIA$yieBL03jx{9!{#TYb|;503FIyQ@4!=a~@cOJyTBHz_@i z#O|jLx0W`St&E&(tKdn*Y?g+!luEWv1|Mju8RRruU!^)t|GB=kTZ&7DZF$08>v1v0 zv%0P#VNRy?h_kz3j-^%BXpp6M#zc^X#~EE>>uZ66Ri`|t!_JznrF;uhNpg9Bn)*1` zaq9^>YLx=Q2r*unCKoQOy~;^vt}=+uA~&d|M#N{6GO|JhbpoX z4(+kXdjzP6f4Y9S_JR_)K$2QW31pkJZCTFZa*5U~=QoYl<0 z@aTXsTRSPcn!a&K7>8f~>8jWbng-VelhhleoPtx|rmHs=zmFcD`0|~ocI`~)9#ZWQ zG@>y7aHJMd-#tpg=7c|71F(n^HBIrJ~d(4Or3`_4%dV1USfNw=oE4J%tsYlmq3Vj}& zz5d$C!$QyK^C!ynp|iZIVpmJV zsW@|;UPemw$0y2tQ4K>uURk99+J;vvH{Ctj4}G6_@IvvJl-|n29*-xNh$1G#kL
pe^@OPvO*Ppx#6%>A$s3QA!41sF{Aegh7kh)V8f(U<*A5SDXNzt{>GkHTll^cuh+1KI;!T-`qP)<4@LBR>-E|a8mq>- zjPrGKkGtNr)pXcE^3lmedlP9#J^R#x8JP2hZN?K(7V!k>CE`| zgaOZ}=}8|4=KAO!^8wmdH#34CYIdkclMCJtB;t{9VsN4NsQ*m;P9XV%R?3CKqay2j z2)#!fj>7(fJ~6T9qE8~ouC85#@YA#i+ow)klhHGqf09fsq*s>Ko}ULTHIt9uP><8R zS)g-irXeEb%+R%OboxC0EK%%6dSu@-D)rKAQ?2Q!9~5QqjqN zmQj&^^;Pz&u6_mET+>ZxXS7CX6X$4~`?s^ll!NEqA^l`NjWXUbfx4!KvezCO$G&!< zdAu6UZu!oukZ(WN$WdAm+T&_9Cv4_zUsqcQI|*ds_o>9a%9Y5sxdG?3(oZgKTlBgR zlZRT_D0*4rCUJeTkc;Bs8!3|3DXNWEEjZ~ZT2hNS2Hh8CsJy0`a=mWd%WX2}V~i$` z=eS>3oO+dYJ?j;#@7?>>q&Jr6SQBp+ea^MMMn4(Z^X4*d2J3;c`pBTquVxK<6692d zEL}vZsr2kT#{+K&UbK@jXJCvPC zr*0oBcyrU=bHn?)@C7k)w+Qw<7H+TO1NI+cNsQVH`zGMFz!ws2ygBD>q_z?>@6UHP zP5cDU%WGL|bWA>9!|k~qeTnTJao$jVu*lm-d|xBrn87#sI?JGu5;fN8%7gV`sog9t zkk|uSCr4V0Wni3>0gvj!bw%=hBL_~37>rC%1oNQioeKvdaE&hfK ze6iP-9UZ%QE`I)Xf1n99xjt)+QGaO&k&Xx-&vb@9m!TZY)$BUnIW0U==e=|zhdRCB zL3ZmZ`e)FIye!h%Q#fbUJASgtXE!*KWCTS#BUtO(3X3z9T<%(lnZ9B!X6Tp;c&PF0 zG$yh0{6JYsxeNbYcNr@k+nB6LC6V-twW(Xam)>@M`%XNkL{Qj6t9ub`)d-seT!5w%;Z32nl;hs?}Gv6TloOh(R;oMAM8h#}wFeP0-+GH9r0C6Dty z@7_|kkWS4eA1pFY_9}z4H%1RGP3jk`Bz1aKk|B_C^t3O^*GjU6>W#xUtShDS=Iz{; zIQfT32%p&VxA;zYtU|OZWfkg59a4Q=Hn?&viLDtDj$SOWvGA-|ge5&JuuNconRlp3 zLy90Qe>7?{var!~dHLDdJ5dwK@4he>3HV3hH}=rP9%mH<{2wk;Fi{99;G&e+*Jyk>+Al$(KpWQd+`)EFcNz2ZGH^GIRx==9rLwMk0C`C!QW+jM!B)Oj{kCa z?i4eN>8%(dDnklG&DYfB)}>SvsT>_y^>4`ySKlua3sIfW+JoD@B5UNoa2tJ&3b;;? zT1yIKYut!U%GT;`TKV2b#9;_cv5ORz_ImJ9@#tCX6Nykd|13U(vblX056~^~k;RD& z#ECvU5B1)}CSS-3jVrAYEo&`EGBqniS2#Zoo^@Swj4RYhP(0u>V{l*mI_I3j*!w|^ zjSSdD2j~0TzI#ZpM+A)&_=mFrX6I(VliM#qVMD%SIWOD6XUksc6=ypc-2;TOg#=A7 zOe>4G%)xJ_fl1}r!s(HAx+i!(Gptdk2|ts9CaLMCevPg{E04slBPAX57p~3JE?UV1 zd8T`m(9z)?TXnf!9XuN@ct-1aduv7&q!F8S>=I*X3TKX1{>;Z0 zd+6@0Jid-syZeI7mh@e@C`aUQSkbd_i^PrO>f>6ADZDm9_YyijZ9NjZ_U&1DOWX37 zjD9)aPct!)$~-|G2CB1!Ho>rGiPO5Sea{kE$x?!yMwtclRKL2J87LOkQON6!ILQpK zMxL0zJtEC$Ml0@68=aNvz2@YWFvCU%18l?ZMw z*4tU2{BYoz0|OKI&z2zs1~R4!B&P!Tp4+La5SX+BkiH3~uKr4y-338lcxwv_b0F~) z%rw=9qJWUb)rnwg4h3(UoXmj)R3If4e2L^^?&1iAIXVD*yLkdFH!Bx&b0`K7#=_bI z0J;J^!CcKf%pIUu0e}lA3=akGS7o6Bz-(nG4$ze<6bIbKXhQ`7+ytl~;E+tAFf%C3 z91sxL&DKB#^w>gSc2ES6j{|c62tp8m5@9X?Zh$5f<_?8W2y_Nw`vK_(F70|@LEQo~bwt@tK<$;0NL2dCN!NaW- z!AZ{C+R_ThFGgS?Fx{OW7z7}-Jdm$!Z7ShlX=e^@Aq_Wkdu_0xMDW^{2+#wtEq@bd z*YoH@1%NBq{|^Q7|B3?%tR`4c7z`AH!GZ;X!wEvMSOFjj839G1P(KO)!{E?A#saY2 zc{_7<=l)qkcKd&h2XKIRwhKUSTQeXrum~WL+hzMVkXJ1LFb2Ya$q)?i*nn^lAr4?* zdo0kuJ#S~+&w0Qcuq}v2Z`Zr<9VQ@PjRxW;5Em%mE|VaQ-8MMpXB*@JC}2NWgY`e-90$popE18Sqfx+zTZ8gZ3A&INL2{z?b}JsX~G9XB(ysY=R#x z0u;GzMInGqM((&RGbrGIw#S(R`|w9=+u9>{$XG*>J7jF3$Q?3v(Cs%zz!wV+P}B|? zN6@isBLW==u)y2xUH1Yk?)GmNz^eRc0p@qdcm41vz_!>P0XiAf4#gcewL=wfQai%^ za7@6})AkI|R{^PSKpO)14)hLX4=C`YZ})pbfjidi*0zH~|B&*R>D{%wpzD-DK)0iX z-#;z^(d;@J3B>On_UGp$FewRO5X_VSMSwOPCV~9F4g>to7P!_%>{3wW=2HSNms!)gBbpayo%+6)X1fPD{YYWo8dcfiqYi?fqci~bEG zObYl&1Pp(;fLO)J&e6@z+63x>;uqjY@|o&jO414t&)9GZ%rtfY0$CFhJlxVQ}1EbP-650I;9`GZ)|mh~oZ)VSpD?f5LFs zKWV^GNZem@fkf25<^pcw@3}aEzs3RY!~T>Fjurrez5nuzfdeM=PZ;WNUN8dizwiOp z|1Vy!@V|ABMf|0EI2MKbYmKm2;Pb3Mc){Xuf33Ly>aTqT7r=t{{J*l{5b!_uE*$sw z`oeJnf2}VZhXW1ifAI;zaev_x{Cf|<1#y4vT_6?%4g7!bA>b$=l>ZY3e53X!3`i6B zjjo#u!P?H;W&5K(4Qn6Z8$969pzi1h8kueDEbm|e7;=F7AD{NQx)EI5wjX~45HAQq L*x993WFY?sj9C&Z diff --git a/ref_result/__bar_acc_PT-1_ref_result_perf_debian.pdf b/ref_result/__bar_acc_PT-1_ref_result_perf_debian.pdf deleted file mode 100644 index 4a8001f3582adce375870699ef1415a6a078d81a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14369 zcmb_@2{cvT7jQ!HLZ&oGo@5MfemrA%rZUfS9-evj$e1D$QBguj#tp?#>ryPQS!lBOQ zb`WuKC`{Yig$RYo5=aCGXKN@-onTFLhav$4O{k`mB8vRcLEhPk1VwEi!1UG3 z?T8j6sPNC9a$Y0_EfRr5gra`})CnXK(ai~p0e?bZS~dhrTPJHMcC*#Z*+Pp*f*JwC z;^hHWh+ZTpOvw>oL1yzVyLnfFn*6~I;%@;!zCrHYhyZs#*@tNo-JLz$EC6{x{(+oScDQ z2(X_fI{Hc-Uo@bb0}fbdY5yaS*SJki40k_gJ+PI4m>93fr_okl9o z{aE@Zi+Yoc$r3lmKi+BdE8gQq72lw^Uv7U$k!wOT-$>fh(7gIvlUA3e5{Jfpz0Y=c z6`_5u9k(8QFqx{CGZS9a$=(Pnzdc{~l0;fkIkkLPY9eu3a&#giXzq~Igv{{E^bY>! zM3wZ=~enL(lM=|*7^4Cl-md#-7@o@ z%=87%x=bj|F{h=HoPpgA*X4UJNxh7vBSoF=o3ra&>-C?COq#iv{N^2U(Jb*<5PdEg zQ=xgKh#WC$&ydM5ebbl=X_C@paks07sz(q;Q_iutsU|VHDQF+f{9&gAirHJG#<<|U z@0}~&+c)|u&?3m})Sx%wscGg8(;$5&BF3BfRxNJ^-EIr|^6lPUKhm!FYKb6M5AXNI zyYmui)Z`z_=t6Er8kCnX%3OH%F=?df9u=#f`76Hf+BbQ)Zs`cUw?1|%gL${CZ=ifc z_|OriSi3OE1D(|`=nr@`T7Rb~iksui^9#~L4lw)kmp{2drwNxeELHW%LM9C>l7%`D2_*M)3 zy7(=*+BowXLpJJ##-8j*s0ocwB!reS%L^Ot3*zS2m?|){&e- z&Aei68XoR6fSh{Y|vo|_# z*q%pe>q^v_;YUJUx`CgL-y`2Sv@y(f(`d1q$D=uo(B>AYC);#o4{aE{E@rrHkC%23T z$F&l4$K=z`AI&JvV3YyMS9O!+xgu+w8^{zL*BWn~2P`iX4y);QQ?VI#bH{d;_&oMy zj(LDoA?Xw2-;sMf5Nl89WRbbt6+=B=PSt$eH*d9xS>l z`rN?cV#WO`&Z@h@3c+*;1QI(A*dM#zhX_!_(Ow7Shyc{;6QHeYN)Sw-Xp{YIXioJ}$+>6~4wT z485~j5FcF;V-JpeQTEF7W!lE<*!=x_`CDS*tI{9aI!JNAa$aH`Vy=5#q;TS;MG3?% z4CGPxd6wz-mFl&7&R@GS6nO38P=L(x(9oziKT}ClJ&w6J$7L5>GgVBGsfl9V`bOd| zQ4{gk+#lU-%HFE);&yr;&DRk4thD3dH~jPHBv0zCpI^;p zIH@QhrBKy>kf+DBp}nPJzOUfX_wn(GwNK;Y?{U-PR^Y$!v9;;(j1i#5J)$`vTb0W0 zuTG(M%|pPktQ^X$n@eWP)NT;+9z!a*(1GS*kb%N6zCG9Ihk6g_Qz z6tDb}!dc{?%EvuIp4GV)Z!{Q3pr@O$-Y*{8M0a}?<6xOZ`yNed&d%FYQ6r_6B;L>w82lMHaFS(F>I?V2mA3u*zynA9+VF+I-$$ zwrsMy+ob^GK%+rvdqH-RSM)n(C?)l5YQ1hg;?#i?-Oh3k_@?H*eAM**@D!ITHhZ`6 zp+b9W&#S7W$L)@*x~$oKegu?{bMNo^G#MMd#s zsK>#%>sWl}m#Nb586Bd_Qq?@~<&e9Ao(6+{{_K2aP z)M5Y2souj_v!NLa?Zcrmf^mDbnd7LKx!V)=JRZ_cK5g9+91g=|IoGcjA5XIYo&Tn@ z+6)6WJR9ik5LleZR&2205<#F5)Z`Zo3<$QlMI;Ub_`?6;6S*{%A}J756%z-3JX0j_ z%x7uX)4g7=dZfgDnjYHVnXO&Gz$$h8Irb*@JW);#SLCMu)PWQ1hoh@NUZB zUI=+REYxH0t3cm`)im|xgb>OX2aj{4tWuaQcSyq1kP9rsn>ntKtwd{nrDWos0(Q>@_4qs-Ukw8}2dB;TM`c`IXaEK^ALkZA5QCiv__nIX`8>QTe~az$x*w#x)K*}PNA9d>*$kB zj$4@02u$Hy!y^t4t-mt}nE66Br)#~q!^Hdl>rq zI?}l}F1Y#K3FqYdsI$l9f{zb~WL1V*oMh7tHx-LJ{k-nBYmTPbThu~`XOzl4={^~4 zb~d-WY4GkKh|-*GPN1FZQEs$AheoIpS*F%ScfFpz{`rZCm)L}N)ctN@)K3$yhr838 zmW^^xs>Ml(XgyKnkm%)oQ5+wUrXW#g$=F9dV1O&Mz3LZULaiU%KFfnQ(qvQE4SVFxwt5Y3^`$p5WIxu`Q?ClECF zWp9W{a`pGViL85Za;a;W2)x%14-RRe$*$=U*gz`&m&pm7Q=q(nP+A950r6bUX1g? zVDyI2X5tp_OloAFH%MwFy6y>HGQ2z}D0eUJ%&hH0r;JpIu0)rRBP%|;lNL0aps47U zBXi<#2l~CYvYUe&uJI$^yg@WUrlWtvRh*(q5wN<5E4yr*^00c2p#1RDb?GyWP(=kTGFW5#L7>)K&?E9|h6N>}c<)FTIj zo5m?&;_MNT2mN>8>Cdgm){_VShbK=$9T!y#r7!Q? zcL$mAvp*y^4a1*V_9L7-6)NaROpla9KW#Ye=y^xQe4=bLf1f;R zh;XBI?qkUvoR=kvan?Irff98q%FWB7c-ZPHeLwA-9OarzH`(gE3WRF?IilQt z3+>fLmtl*tp0#f$^X~=-bts$fpcHTiv~8jgaQHvWR73%7?I{{^UzKWxr1sm)t)ozv z;SWRd=S6~djZhrpWF*glRbHp5v|Yro`Z+qf99EFDjd@lc9slwmdavSTyyY-l5UY6q z?&k|Bp6}uNCF0k0?aNV43rUq9ire>p?Pguufmv`byNy{i8u1UzMkz(HA*k@n9uT1p z2J;S5(!7h;%ctvIu&kvN$O#Hd?v}?6`a&yirBg)I<+V?l^1b{kD`TF)MIpF`%PK44 zt|9&^;cpWUB{@o6YcIX&tKUPN+CLHZMv~uME1h_^xMcKG77N!WysUjT^g5pT z^>6H#9SejWH<}NrMx;_qFyEdCYB5S1j#k)vVXUV(P4Judw-nA#ldEqR-yUA)ynM&~ zqWqw0j!0MwA-!}`U5htG(C$>BbBKTp+0()T!O9DSdjpBB;uHQ{H;RY$XYNi7W4NRs zs5CyRfA6mK+oOyDD0Ysk7vmQ`y(v!V)QT)?I)i=_j(2MPyhv4RnMOHL?#C2hA0;=y z7(#oJzl&qK#`LnV`K9LEKnG7u<*D}hC2<}Ex2}l~wMDhg$33z~A_(ZPXHk^JE0cN-eEo4Rbg_(2uie(k_k_4;84n6qEjL}Hu|Wi|0=qhDtRs)`{dTMC65cI zDW)BZ+{dyY_8!AE-;-hjgn3m+5NQKvf;xV}mstfpxNME-sqE(uk+2(NdVK1_^-vsZ zC&iHC_j})IXNT5zHOyoR@2~>E(~4~?0Erg)hZUf4#)%R^)i=TX^i=uhi3QdH>4s*rY1|P?KjZlU@9ouK5PJN+F1N$2_LKrngI-sh zP&bz`FZWY-_4(VjPi@a|rhJ%LS!8cae`F+4EmxoBmDSxcsU*aa>s`;eM^%MCY{F4n zTV9XbnqjcIra647PP-8^;rw}kO{ZkUhfs1g*V(hM;(A5Utf39FqN&P(v@RHZ)0c9| z53}lTOQzS)E?>=G_)d8t+CXXt1%l^0+ZBlY2fv9?M}Xnp%pye6Mf2IIsO1l~9`zsA z`%rkJUfBjSvExLD@ph)ZASX@P0IMc7Q$D`c}T{j#{AzH8U?1_?WW`lOQ`{JdqZWSo`fOb71k@-~N@=?xmAd>t{G4c99Wv zINxb)LwXcsYwqZ|I8&KWl^zt3P;T?-T#j%0n}2`^n^2M&s(T3!u+z1V{Qn2?PAF2c>GaJleRf;IkYI* zJd`Uo%!RxiwJP#e4t-_T=q{m<)z&5Q#K`m374bTalybv9WVqFh+GpMBQ<0ON5fL}3 zqdWV0MNM!TFg!;u8$-hT)moiIfv!k#E4@3I*StZ(WTWOLd@5{45reW#S=t(^*^6hJ z{H=^20(p#yrwkI`zwuzXPc?oKRcbQuw4GL&f7jd+^_Rm^Z_Iq$4!QKm(dY|{KUwj2 zcW*n$t@WNw`{A81$8TvC=jUiqRMRtWNrUI3hL4H5B3~|4nP*k~*wygXBxwg-?z9s@ z|AQ{&G?bj`;4~F02Ubs!*_W&(@$HdNUJnqAqnPAiusCy~UK_*xYJnJ1*bpD#%qm{+ zWI;@T(Gzl)fFq=55)h&syB{2ydu!G7 zD=KPi{r-T*nYVj819i!i3lOvo1K3bvHA6%9|!fJX7jEgx_eNBG(aau|Kc zjMCr>H!x9xi>01%n{4FO>J+o*NZoCH6?J?hyG6LZ@Z)Ml_t(SoQXP*BYv4e$7bq@&(9-cwTM5`_ z%l{>4|3k-?I*K|Q@;8(uMT3>ZsLKX*Z?EzDss~3kQQ5KGDHhXFIUD0z-hbOJ?Z6|} zn^k$M!zHHg{dQe`&8qu1ho~LGRLnncxTP>MKwj zfmL1FIcD_TusJcPlqjBQA=*@;Z{I$7kNjjhBwzZ)V6lO9ywo@2fSHBe-1>OI9W<~* zYYkouVP8~OmXGXF1-?!7sar1Cx>)#vweHl z@s2A8n)o>tiXUeZ(g$koU55n4Be`|iD;3PyLX>(PWFOzdRF5BoJ?AOdFAr0tyA!?A zm*8WJceC^98@Xt?t0rv}F{M5Ck;!lYxXj~*tokk1QLN&(Ww8>@Zls%m}dm)z6I`Fe5szSwpIrNUN;)Ormt65G~3nd$zVl_GVoy4lhGUIaP zGt-Ts6^W*Bs!M}OW+%u)^q=>;n%6uC!=p~oYo;XSs0e7D)XiG|vN)q6w(6ZdKv~J2 zJ8f0ako9zZbnvrfnYW;0oIuBIxI!?OeFRFmHTfJ%Lya%;jyI>pl4$Q6JD*Ex*BAdU|pOsk@Ia7GG+sU#LG5s5wHqGBT4@riMDqaAE9T!k%1;z|mb*RjO#} z$8_(McY8FS&SP0{I5pe(Dp2o+k=nVt_cLhrF_J9JtlNUZz;%Q{um zRMzxW|CiD1BsG?CFVP#A=KFhEJ9a5v5Y0=Q8V$R&5c;F%dis0-Ei9j7hxy+j_(Y)p z#pqqDU?7K6$@;2QQ$e5I^^SUx?y}^}`GJ|wp{Yy4wXEX)=dt&Q6DIF%XDg++Y|nTW zg-RW~_g-{h(wr$Odv_B0d1|%(C3-KZ;O9BY;~q}htMYb;FHHAM1(rl9YO})8kx@%C zW=daB2=!tWWQMtALgg0@$&{$HeW6@Ti3*4hCrZ9Ot(aSK%7<|eBguAna*>~cj)zT(#SzPL$;g;sqx*9mB-AUy@M>D^@9});-OZD za-+E+3dE=*MAH4(6cesOqYP!S7t^7h1_O)&dCG%h_4v|!=}N~vCc8hl_MN$*Rd;Qz zDQe#O+1{7sCx>Om$DCZuWON>Oq(sLun?PD>>Q{S24t8<5E77!{yexOVc=*;Dt>=NL z-ArM&azur)tCKCI{PCEzqRUN1GR>1Ad+I9(c`R3!=uR+wscG(%<(21LoN&~6T7WrM zQC%80E8l#;)dP3m+9quz*xEZ~BG}6Fq&~UrwLo0?anJo%#4MJRzF$#GaC?ZF8lP>8 zsX|9A(LfGD%sx+3i|0ML#QtR2{LA!g$LE_$oQcJ*+PysbGO|JeKMu`L3@xthD@}{r zx5Fgw5TGLd>Gwy@M)i8Q%8dr5L>x!x$c=1XK0lI6 zlGRtof?WH`;nWOaofiC(%%qIrvy%)AA|Vyete@%D7!uMubd&|yrYZN4t8gmX%nZC6 z88G8)rR35uGAq2o?bm;zJZhD`)_vam?oCP_+|>8!iq(Z5BS$8_{vdm@d@^(gsdfk& zQJ8<&QcGxVZK;iuHNQqLD(iT>Jh$#hJ2lfBdIf~IPpWu9L4So}cwPCrCL;8Seula8Wt3$CzK<~; zUnbos3VG4?Nxmkekl4FVoMt?JUsFN#qjSy~JbYqw6bwy8{E;1Jj_J!L%tuSkurk~& z_+Ea1t`FryfZd_|_P-#MPCL zc9`YpXB}}*x79=8Wz{@rTO#G~W41wjjQ-k`Hb1y)W$;}p8M`f8dnjR^alN10&z~Io z5jSuyVtUfYiLEBGhd99S>Q+j~BkeZLNNU`JKr()57Zx`r&zdj$yM+_q>m;4cJ0!8P zgU~y~;VA4s=o1rlCh{19>e8}5M2Nmg-0|+jHF*QexvIqdq6WputvT7yB1@&%Rn2Ja zTe*7YXKKTfP7YoB&S=CRz#hRB-&`t(xsqdiyWQd>AJgK^32J(o zjVYx$mtJKq=^K@D&eq?8wnu6e)$@$BczhQ-tQs=gjr5cMICAigIn@1bDA$vHW7yYj z^iP)}xvaar@&tGD4j*EWU^t>tdDL#s{&j_wxQlQqewSMGt1RgpyPI$U8>7U6mIbe~ z*RxTdSMy(1lB91;=JC=zdLv8GJVm$qs!2p{ih4{TR z8?ttjsGu`&I5!q}6X)+BZz*t=X}rv54amOfH(TXY%6X>M(tY9N()0$0Ur0Cul!rx@{?D8k$=Z{S^%!>;PRm8;{{ zWWw#`vMpz7gid6qQ9d~?;;MefPeD~|l{-NmCvh&EqoyUVAXUXJ-A2md6vza=^eDX)Aj;~`rmPk ziu#r_0M1u;U3}(kDB*#4vc448BA&I+(ozyzwwK#{_zcC#gT@dkWS7H_`FC{2wKW{n z(cXq0P1W=F_H!zQNX${aN}=q%ZUC2~^ovrFJ+UOi5=c98!WZRhBU4HD#_2oOoz`{j zc2-lg(xU`~Pn2Pd;8D*dh)$Wp>FOeLo>Q^t|MG8BOa|_zV#WfBq<<-I_yuX)^%13 zT?osXVq>?6xlTrBLSv%+dVh&+5#7XH?zXg=E^3pdw~OSWbVqe|;C83T8u>5WMxLPq zjuT{;69PHwR-+O!bvo-m|L7y*Hi0HNTosq|dN{6pNDNyg9m*JxCTLtdyUXe!x+(T* zK|Bk2ybu2)gEvu$XVXHXiz+3Hn{yK^Ec4K%u1`ZgxGy_L=jp{M@9~*2esJmr&#cqv z+d-|>6qvu0>jOUD9VFNxf<_Ae!`=Y1d9vTm=Xde+s#4oxcBYfhx}(Y~o>nTxhX_?G zY5EYDPMScmliy4&t6JmyiQ!hpD!ia2*0kMRi1oBKrNzg7t&Tw(&-ib{g>6h{ugyGJ zu#pcwch9qskrD6Qtk3&uuUHuFq|Wo!=9F?s9X9RoNj~OBV>Wbc@giZFg=TUzuH4N+ zsZafT81H<3dIPVK{({P$vb#i*`|7(Z`Hf>%@vDgyM|2jF1nfjJ;@Up0KbE@oy|JXJ zW$|lDzoPHQnd^|UY@8koofx47ccn3YTHn2|F`k1eDa2)jP1r#F8_CjGIj@>V$za$; zet_fZ(Yf=^HfIjsjZll3puQP1(Hs5f(YvDN2VZ`W$IDvnaMXhMb9Ma8BHc(0g~<@y zi9c%!YVro^1}CN6YzYo3+Tio<=1!8eA-DnO+c0IqW*v#aKww}8Jkj04&DMqF>;^?_ zCXi}*n3FaVRKWoNi=j$z1kybJOZR7InV)%`f>h1OfvYQwLH~fh^CB1XT!3&Iw4`1QS+&C(3StATYeGl@$?4 zHwE)cji4wXq;Yp4SP-G$X_E^PNIL}*QNfo;K14TXD9qUj;G1*~sF7^kh(ss`5XQ>Z z69BpcJi*+FoJ3&O}t0C3VkI^Z3kyFm-21DJ>3jQj$CQvCV> zcEN%w+FU;%TVMeZfC52nlHMTotJhB^P#cQb7#tV}BnvVPB5eu0bqARJ#gs5$K~Tb* zd;!dZ_#g~}K|;|WzrgD&5Vln?Kn@HB?DHE2K>=0}q%8s{Vyl2sY)lZq$M*XM_n;QH zHEhfv$n;j(SPcN)#)<*pf8IAV4Z^_oKguRe2*|*ez}xTOqp&f90MmcAZH>Ei-)#G} z8veuu6|>_QTaygjf2Sz^85+n!5u2Yd;Gw|27qp3h{RJ$}Moks)B|mHGP$2x-fawB@ z;Af2hMQ&J82w;`?oeXMo#7#G~IV#|! zHii4?n1G|FjUJ${0#e_AIt1_?=*^Kmp}>>A(S8mJoUv}yHXI!KrA#p(Dvkeag2Ojb5FfdyA3kEKJ@cK0_PzUVUb{Gm&9eDl5LxLx#+hB0u zGoS4+Akg@0Tm%M;7XCvIfj|NA$u<~V1bnFei-*Kuf%W_!7#xYf{>2Lt13v8A@I(NU zw+)6u0ukVL7#g%k|Lu#y;{NU{0yx}lcxaT!cG=))VIWO$8y*G@yg1tqL;cMQ1`BN8 zx8sR`rU$%!&j$wluxc9&`?v1lSQOwkw&P(%{)UMNZ`VCs81>h>f(v7Tcz7E<5jbx9 z+6AWaula?GfWCHnUlGu7gV%4_aB$!lb35$sH3Y|r01@AR@qkzi^dsQ)iyi_FCdhAx zq5qN%fdZrCUw9-pf~^D5ZR1luEn6Sp%R6Axpy})k8kr62tmI?`7;=F7pK-oBiQq=s Sc>ED46cPvF;*wL7hx{Lp?+&E^ diff --git a/ref_result/__bar_acc_PT-1_ref_result_perf_windows.pdf b/ref_result/__bar_acc_PT-1_ref_result_perf_windows.pdf deleted file mode 100644 index 6fc3016ca1bd22c74bc00d595bcff5946ed768a5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14349 zcmb_@2|QHa7kEN3*;U%qs0d;9F=OAC?E5am4931^Nh14}&_cGPgd$5+mO?7BrIdY% zvhPA6|Mx`omH+40=l}VS&voxR@7#0GIrqME&b{wEPIVbMeuMxT!g;?7n)4KbfWo1U z<~9%!5hzUC%ZUVqNfX=%c8*q1m^#6V(ZcxFWm$II2@>*^LHxd;66QE9Tb0fJpKr!GY1g2$8Ad(%dpxBMOE{+yjBsZuL zFszImzzWIJ4GL4V2Uw8WsHHb*MX1Ri>>&OY0OT9w-h~8k_mh2?Cdt*&-Ngcs2jm|Z zUxh>@6QmqHff3=r9}mNe1$FVi<);mGuZOz5a9N5hZOSq-$AWQN9d_&t**pTksh3eJ9%vNfX^ zGdNT(e+A{JJO6g9X5#a=PnY8cCR=@+(MOa0Ses4yhtjpW_g$D28djZl<=A`FWY2?i zIzz9CX(vTS4ucWj`ey<&M2{tdP_<9PoWb9MuP4(GV%@TyDIX@=t#v-SI80KGu2zCv zde`D9+-JeGYf*5yqt$@|t8m+{D%Dzfw~*o;`*VfEKgKKENZXZUJq6KBtt)3 zq%}a8CZ!`4FHJc(g%VwGLh}K+UjC1tq&=-Xl$DO{AoepbK5u_M%~>54msUP|?R2J7 z-inE!--PGXJq|9337ncXvit?JDe?euzVUFwRjnnn_)FItFMa)XYxjA#9f_4ny9P#Q z>&Vo`_#j5PCsMkQ_-F&6*_}xP{{2efr$Ua~O@}FF4&UNdJHwk}T-PLWfP4w!pRN=j z7ZuTai1mg|_|fpuK2j*8YyjmO*si?DtXENaI>7+|J2+-s+FKv6DW&g8n?ukaUee+ zY>|2)+visBoWJQFV(pgiHR~n+-AjSH#11Fxpq{%U<0PCv?K1pAzzSJSGNqg$HBdVx zbg|nxePI~AOThBl9=#Yw{9qQCMQe=azgi!QPU|xANa4` zFJ=-dDvpHb@wdj#@!2Np?3N1A4Pmxyy=96{A@Pwf9BXf0QSEV8Ynn>`5xvrr*J!jf zD!ZDNUHUxSBPg|WrA6#mMBdIfhKmtWOqvdU2r35Eao4gpc(Frc%I7ZdN2}Gmb@WUe zRTckwbGNE^__L>}^-bNq-6Mx*#>MnAEyh08sW!4YeCTQ-t z@!m;Rrj4^zEj+mPmHBM`aMQ}blW!+6T(s)E65eiT<9Z(0m(Xz!9%@k@%EO^D19Ue0JxFt(`arEaxru{`DA~+|G0H!6R(-(++w&#e|i)CpvT3Iu~EP zaIAdwd`djMbZTlw*5pO7pT7DMrub``iVDYSXJi zv6WNdUp;1e8`=weN|!DyPBn->Xti+muDNj?_0)-jJAp!g`RMZgo%Y#i{71A z7u=)RgvWI?M!BQ>?~be%M(_xCVMYq7yZg?+>k>M{&WTt!e%CtSV^Rc{Cp}wfpDtpQ zn>L?~lXSDJ-?*subXi*~mMSZNuSzTJg^trzq`dNg zh$t)RP+kSfacTMXs61M>?j?u-HkO_v8%qC>k)8Ccg3EY*?DS*La^b*x0qEyqZhP5R z!yVtg&1*K#`N)(8i5(L>t{d;Rc-JpW#H}~!L-|6JMceJ`mewP^%3VX)^Epv9z6auK z*=YS8_NhpUPcyI42zB<^_DfNlB5{#$da1sqb}zMM?<(of7wjEfe2U9f6dG#1UtE|D z_gfMSYml%{h7}i6F_GI|OI~#G#Bv`_Hp6mH zda2!fd4fi_j46a6sPV`SlgjHyPQ?Y*c(o1IV$GMYO>hVai@(}6?&BNDOQ~^>bM9jJ zaaRAT!Gw2E#+wlBk-#TT-|ldIOM9jRf_QMqgEzCTDc{5u4q*qWvt#Snp(RS`PMTt=v z@}e(@ES)Azji|oJMR=XEwf7S2f+!-(NxF1s^hMqd|bxYbJCyiv4Ox0X| zAZZ&Zk9g<$iXL6|E|xZ~H|(Mvd;a*|zQ>%I{t2EKj9xF=OvIwkq)O`bOEIlv=UqVy zh9N!tvbPeHCddyR?%o#tkn9wEXxV#b%8X_$6cty0Xi@}j$F%!uc3sfR%e=_Qin>RpWhM zK|_u}V9T#v|;#i;=walh25 zV{Tl&<$q5P5)8DrSQSC+|DGL8WGVe~I1RPWT{9|+3$tRn@l`#~qAD}`*+V>ggt2bI z)q^_@ss?nOlL@6BG|oWAXou%6AA>zAbLL98eTXfn_6s#k^7^f=Bm~ax^?JMq*+y$8c_b`(=Or`VV-G1!!)25f&lAp`{-Ryu zmS4>isPAc_70Pc~Qo@t#B!%Y;{iJT>ZS ztc#~bQNLv*Qy2ZDEOn3W%%a7IZ_<^yr3lrUb0k?t3+D~q8k zBhO(DpSQPnIw&tjzV`fST;f|c^lpU^8Dc-2AFFULb^cs`zqOvUF9PFzS(UE*0#a{e!Q%Ss!ejDxW zuAzkYV!Wl@ga4b?xAYUU!z&$g9S7G=gxqxXm+Mi@!G+fo?vxCxYjLOZ+ej2R z2J=Z#R21a%mt7>>>P~JD8S>@4Qq;?sx$|~7bD%uG;+FyaTd7tZM_Bw&$2cPW6K7`M z7p1pqML(`pLcfoYacG#Iqp2olP!Byl&+2CzE8EQyOn;j91IK8UX^5bCVBLcNI}c2m zMAOuQ$SDMuu8B9TMWxR4F6l#21a$cGSZc3yIy}#;v$~8YhZWT9( zq=6$rU1rEW`Ti@v?de?Tm9Lp5kW(9yJ z6I)gQ5{>(Z6`-NyK#id39NJqUap$6788t#qO|FjqBIgdQU|2mWEzckZ;SXv{LV;Zb^LMiV>?mRa(#n4;gtJJ-MjOjG>xN+MV$NA32S=xhh=Mr>>JL$Vw;D$H$iE zj=j25Y$RGKTa)3L^|5|fQGnxtSIvoCsw%wUL-yL*a(Y}=%srJ=brB;kv|nL{9Ot{) zbsl~4COnFK;OJ3M8dG{{!q9rJf~m^Bj1MrT+OJQ=#wXM}9*wS@U5w0|`A&T?&Om${ z1%hWeTNQ}?2fv9?M}Xnp*c?R6N%Ofx?BY1ty=Hu^6D4ERF8$I>_&5n-d_lXW!55lSx^6SIzIhWB57ghyFE-M?>?iSm_DiRWwGei0RI!OZimZSGB$=okGr zsDx@%o|h|1XTH>a>N*#=u&`2&XIrQK(UuABZ z`*HHayf~e4tMhvv+wJ#YY+5>O^VUJUVIIzcont{A%ySr?3AcU8G#*+*(xKV~rSLpO zUCZ)ivYL-h{yehFG;04H?p+q_E0mF@hn^jhxxVv|t1j8=%G(L#otK^0BF-A2!)w?M z?JoDdU3Oq>mC?cGEO8rkf@fh{)rrCX!+KP&wWmVREtf*DOcI%wy}9%U^GDVWpXo6Z zVInAHysoG9hn>$lj=3VR^nv+q^OI>|P5Qdr#n8eu^H9zk;ZBrIs1@9jEIMq$D3wrf znCujN^3&^vWsw&e=}!$ikr9?xs-J&UABi6Jh>D7*jce^}7c#+Xz+^bu*_o3*uT<+K z^L>aGvDCYXdB=T8kYd2xghz$lD5^)gHcMMWHGA%Ct*@mKgfEvRS;8Rs^LzLG_h`QO zqe@J=E1Kw)d3Q`M(0)BA{@%>n<$zP0ESNOj5Sd<#sy}^grlQRzuO@1)Q#QdEbfzh3%u& z6rNq8%4>f72~@)z%oa)~YqT+3EidPp4uKw$?EPVFNAj5T@Xwa?IU3Vn zm+{t&%nRMRjdME}VE7dA72R!%b!>3L-wO(TaCOCW2^BlIcCXuAsbhDm+{=@*WCi<# zC}Zl(q-?$qK}0dKP(<=wXe^KH;yy8y^RL9*;@$ygL_0Fk(zLIIEs=b_%0XZORyz` zC)ik`BH>n+lt3s2OI?3}qLe2^&j?270>Orx!G&bxvi-dJl&R%nV+I8I~V_)&%_-DB*{f|sPpWhMk?y&CP9HO=f zQ!)R*;ikeU0ENkID$MMHKo+G|Dj|tvS_suX z(zk6IzD0TZ4kSF(Q~vVQt2XWRt@K7)$5NjaI>s`Un)9w4u(~kiF9N7ulC1$zU)+*E zkH```$m)`)S3c&HL`ie44R?S&F@`kEe1Oz%4Zo`9NVi{EjOvCy7q0d*?_H)2_51R- z)1D01^&MpV@$vf#MP-QI-EFkCO$dxY{L9Wt70*B^NjD0to4(G0MZ3NBh;M#}r?|;< zRTo1Q;DJ85S8rv^uy}J1NIx}qIuA+!UwM= z5i=VsTdz+PBvzYVmT#N4Oj$0HDSeQ4s-QQnTO#mlnmdNq*Dfdj{>O(l7t9wPK-zC; zOGK*Hlze?~CM{1dLEi_vXHr{+BD1cnoinW2BdE#R)e)|Nz z=Ie0zAWqvTlyXDbx&1Gze2_Q2PFO4mwZFIVZa02$R}mIMoz$aszUQi*o~*&`oyX^j z0vl^)YLo&rKe>f{8q0dDhC0Z6aqw2st_M^B13R8QQ$^E0Vdzud>0WmxcmIr?#6)XL zfZi1&wR5TWGU@V>?`iW__|gW8Z&dajIHPh&`04%qYc#PVS)-A@Z{v=+sqO#bDRc!> zcW+li^A3fJLb(|u1L1))p+DMU?o9d7!}2(`ng4BqPXzj3jNau^W=c4Xw2xXP4fJ`c zSL~ZRP797F#`p5r)qW6leq4HQe(*Emq{(OUM431zS;?a?RGjVBXQA$4bJp1Gohj(o zw=4AnnLNdVUgs!(ad*&Ok+VU3WxZ$0w;)7SofUzOj$IftQ~ZiTs28aq@0t^n%D!@l zrN?IM3FTx>mPd@AeDtlNbaKHV560Dxq}WEXZQ?*Qa0dH7Pf_{f6kSe8(#V24)3i|H z>k$zboHw{LojL*2Rwr`$?Px&Q=|3&+dT_0?o^bYu`V4+z$XWc$SlimG6ZNQX{BxC# zM}=SYTuvqU=zAWZH}~)qcNSL6ePC-qu+mhu5xeJTzEX20ncIl`WJgEH>31s($utX^ z2gwMA1MiNzePWYg&X#p|d)4=2`N>|7_MZLo`auf$iBQV}58}8W@}$^9B)5Av(oHxE zjP5E6zZnhnFz9CC%T?|ftdS|nlPt5}WwLYJxl`$i){D!lwXsuH&v(Cldb(fg%bsKhk`6UPU};WF9+bC9`|4j6Si1P`5va0G_LL<2RahLKr87}xo( zlTTw;n5tc;%v0m3PvJ+tkCv{?{P=WeXz2$<`Qqu&ZKT>JXhdQDVM{Hlxw)k_PSae9 zn^V?te|v7to_=JkE;I~*9XMK~7$J6})=O*lmYS(Yc-IZnH_dLC?*_#|BH{CWBN{Er z;l9HLqi1dn(=-QUH1-vqT=GCqrz^DaOph8EH|Tz%nWgd`^YrsiOor}1Rm~7}=t1=j z;^E}INCeAtgU6Uecc1Ler03UNclneg)nS{T7kV_mqch^g*&8h$-MjDnNcN(uTRS{I zjaCddk@@=Jy_C{7R7_!1{cFlGnuySH{k!IlAt+*!OeagC%wx${LXbC&vvO6z1*G;p zB6MF8_tfTB7N2vxdx}Sxfr`1dkT<$n>8QSR(o~$-*kR_>{O?cqF?3?R8r8L=QG^s$ z$-VwPs%N-tlHiI*2AK7|DEBi@ruzlS;HYOO5yj4m>dMWv5y#_5dhKlVniWSL`Jk$M zgFG_|{dG+OAFsN5H0=2?^ze=HL1}~e;x>mLmM^BBcXl1>B8y39|_&(qDF_k9oVa)+eqEa zEmzZni?tgyqiON?11NYUo%Xx1dQ^R7{3w|GStsRe?g7!|ZG_$?4o6}CL7$jdrRbvw zn!rV0hyYWqh<)nNWjO=lZ_690gVFbBsIOvSLr1=fi0N}`bH%uCTgxi zo1(P}YfgQtcmFPYP&IhsBl5i5^e48P=1|wvP|or_gV=X2Oce{!oK_z_bNP32_a9&u zWj>@)cEo1V_Fbu^h?C%LnH_3zEm@K|Ht}#iYop}+`Wer&G1;j3mAtoQZjx7qbGhk? z-%C@~jWDdV)Z%1En5iu%_UpePqOxno9vQUiU#yoij4>TPbkF_l%t%Ydm5de+pF8)- z)bX<{9En%+zGjgxvkphLy}!tp&atbgDl+J6%eYBff`VGFm5XR8gMr<-!GL(Yubmu` z?=0(=$KITHER=j?QoE4{a zi{RX5;<8d zv_>3drd@?06ndikyrbzx+$IVzSK<{J=d|!hy$;zIT$-%--JIm7tgoO$*%{R3$8pZ; zH_ywf3a@Y_$>BxMMQ~Kr=jPv5ak*nHZqdS?zrS(9zgX+lNlapsVRuo=BNu@??sC?8 zwlNvQDx$Z1%Tw2UE_`VE{)2Ktg>YsYt!}ryz@z?m9HXMX=?sAL)gAuNy$nU&5#?(O z3H2gbdx%6a?Bm^B=KV@kr`e1l;>ZtnKc@N^jH|0SXyd#L-D@kSZZV!v3>KZFX-TJU zk1>GDQlF1ikv_Q~wLgIV(@7tckF``8!+VGCSXX-I)$3Wcaf-!B2=7?KYy3w%79cv0 z<8Y14}A?V3ol6HhWKvmNY>`@j%pSWbO-#xw%)u zH=~Aw8Mq9um*SC;*JqT^5R zc=H_B-5+}QLrx6BIRsI>jQQ4Sus=l}A<%4lTJ=S1rNG7TtP%EO7T01Z7)%r%d$bEjD-`n|wASG_J5r ztf=lmk_9mrUE*93Jnp*a7?-P;puEd_%=o^KnpmVo83RqZV9P9MCTXqYF z<4^0nZm3It3VDIeICz?8Z}FftLt`Q?JhQ+|md=^0PT+QhZyU?a`HCwt8h75%*iwId zB*qol7nb*G&@yo)x%80EObVZk(A|W_>9r@~m%qPyR9io{l-{M_Gd&gqd7O>c+s`0O zsK2K>eE=(O5aRl1|Z}-$|~UBl5`P1xIV8gQ-z!*M?~0 zuMM@w6&Lpv*4_X5gECRta+{+T#Gk9ZjLTc#745Hmb@4*&hm#+R#X$K{K*A;x4!m&$-a2Bz)J>4Q3c<~Vyq@PN00*8qFff7tYMDS_ zAYhGl3O%MboL$ASYfYTUVC?23qfZ_p%WC4W{p)e94Ah4RrKn2{fg~IHh2p}H^ z<^T)`K>$jGxd60*F`+PbD9i&2I|t zg2G^+7z`FH5F8E<#bO14G-Cu5g+l!-01Sge|HK7goBcNWZ1&w+LpJaK!Ut%8bk++n z-nwSM!N4MjM6Q?hYanx45MT_10i7Wj;IRSWAVD0!z&b8)f4$!Z-miW@AFz!_qu1+A z_y!XYuto#%6G#gbaFa=JjLkNP^Q#T=02J^(ScBtl-1{ZqIvw!-WSXALR?K=VHgY&iU#=wUSEN*&4K}P zU@%~x-!KRYu!10M9H5BJ0!pzyK>#0H>viryEpEB7K7%0Bn`M190C?*w27v!ruWK5F zf%pF?8#Eyx1DgVGt-(iOeFg!h|Gc+3?qwqtBgGEo0cQT#JBkcJ{Q zK39-|0{dPdogcKnfW=vFsRF*_XG16s!; z0qgK*Yu(x-H-;fYksHI5 zT>z`{vjv#n4d3*8$W)S~~O2$Kdr0|CPyZXi~1vU7B^Bb!4# zPy&JiNIs~wo12rX5DaGj>z06{ixm$92E;5xcMIUFl0SPn5iOzS1PfbWlnn|zf5_|U zLb8Owf$#(Z7{HAm6pO(C%g7S?3kHTk;1BAs0Yjh=;Bx#g42i@4g@?q0%lp50C^+B< z{(=FoZnwe^pt}Ck7q|fI*;W_|RNa5^fB@p}zQ89vf8zmC{kJa=Edg87tuQ1IcWr^e zaiFaLf4}ttHvK-nL`fFXm1u?k4@NjV8*m27kxWDHYjuZTAe&INv^VYt2IN*=B z(8K?|hTwP{Xq>^{FP#Fh7`Or10s}XuTVY`Ev;~Gj;Qtuc&4oa=Be|@9wx>n*2EMWb tHVvANj-ZiQx6Xn-T3SpxEtJCwo&3f(uk1 z7*umy+A1Jk;SPMY1M&}y zuS75-;w9|eff3=r9}+IW$B%?#k!T!F0E)n&`LI|4ED{5Z1(O7%1V}kUaX)w!x3jYc z?hs%l?#Jy*Bi5(b*u(n* z8JeEeM?(|8qRP%)bNyQ%d$J;p7-A_ZrS=+p^i7F#iy{M`YdrB(_>6Bgfy(Z=do*1*oJybDjh+}T;6*9&tl&W`!1T=wMU(8CbbM_v_@c3FQT4rQt2)ff} zXx2TOCW($!PkN+4wMxM~`Y29{Uq(f`lD{$cpkh?7Y(l%!xbg6@W0vXwnF%+C42ApA ztvsx1_qrs?Z$8rJ@6k}Rt=YqX=$5aOX+7SBE?5eUB$rvfEm-=AaAk_(+#X7-d|Xcd zys#1OI)OgrN2-y?MIo3t{`b4FtJU@jBc+SFX!j4C6Ri$mF;SNd3p>Y4#_S ztCT~iSH5u_k~=`pX{IB3cA0kr!d^G@-jzA_g!h#RiBQ5(EZSpA($h0o4w4DW0vR><*@!$o2*o0GbP~$ z=G11!Px7V(Ul=1E(}lzN)~VAR(9GBSsTUb^@7TrE9=*^-uRQHh+sPVuHuq^TeeTmE zPYyj$%Uy0QGt!`EGti(H*6FLoWn^-h$W(Df3KWFCGV!2G%`9XStSXFv=URVtxfQ z<)Ck9YP)_a60i zeP(2Mzx3i8bSq5Mjg{8G+xYT4Yy0&zd*At+>)s6`_LSM>-8g!$v@_m9$J3aU8+AW= zPiA`fqUEeeqM7dn=TyjljmZ%IQhdYv9^|!H+AS3c$!JyeeQ~KQ)J)E8VYt23w&&wg z57KBGKF#mkUz7$r)Em?>-E{USheC%v@dR^PTKe*`@iA?M;M%==X%hk`nG`o$v^d?} zRHFtf<6+o2+K^E$R?^zdV7vPIpanAytC%niPD`D!^jLc4A!4GOjJr_SC$dMAjfvQ^ zVgZ5q)B*mFd3})0y3iGRW^7u{m&{;Q7mjXvOJsCJ5>y?ef!zhR=sfJ>zdLjZ8}fS zNZ(ps<@@X5X$iMCEuUPNzLk_#`RrCwe0I*Qq&V|U4#gfrK{@^c zkB$qr8F@hsmR^$@c)R!Rfb6`bt0r3ZSFPN*N2^Becm?MYyZ$}Yp{EGwSUb3l6&mUL zM&D(T^n&-)tNl6t6W_~&{h~kGS1Wy}TfJN6_TBeHV`?qhM`6?L?IpW)eOu1i^>oF$ zV1z@yG324&_0O_drPOrPFRQ zs6}qs&PJYVK2SWT`ng^0yv}F#=AP4Rj;UCWl8F8qf>$tKQDHjCsuAm*wsQxqdW0Wk zbW1&gdwvvn`r__hZ9zQWkaw=UPuIr&)A@JeK^MO5a<-lMmVW|$76OZg?|Aqv#|8rR zp@u(Upj-dLxgl{Fz@z;S&&IAUA4!Iws2)GG@kkDTW-eFFn&!o3eT6*hc{*r|Tb^by z{V}nVPqDFBUxGA0nrsdu{i<=Z4IRvSTE&O%(3?rU+kvEAuwd7b6`sLy^C`-pv@Rt zi7PzUolEItqQh*=I-(ESCGf@`KX^l0qsnhOKd0oZ7$^KG2M27WRh5Gfzgx5Jy&t}qrfl}@?GrVZBF(yry_$Zj zJ}<1dF$B%u<|(G<%&+z*2DH67ZJ(Zlx_Cl5;N-AC zZcVT$H;Z!uzg5K*B#H@W)XK4m3~)XxONz*p z6?tsNFi1J9hbtk5dxw=%>IQVpa3U%fTh)$9RPhVO^wT#S_$1<=*F2~c{UyXItcV)% zYL(F<+GmsGm6RcV7d7q@lcCVa|E)&pXob+z2r8*%4~SuU!}r1Q+}nxL9qX0r@BL3a zl`0XXXB!ySznS?gKCkRPGeE~}s zf<}0yZzn3u5bxXFy(7|_;tbW;{UcD8HifLz_6^7f;?T^cA3uw8{g?#xE(F&Q0 z*+{HDN0Gr}?uV-iGRU~!FpF2b|K*0b0@3K>cgq2HsjKW-i(0sw<*&}XEv#(pyyt^# zVF+#I>2{jxOiqar$CsQPcKF5+Afhv5+tg5$8*h5gnd7QOI_D)TM6HcuL6}S|uARRx zc4+_l;f@Zo($`$XS!q+MruVM?yk8=tn^SsjQL}nl<8AL=`OqmaBK%-^?wyy|<+M3g z7Y?7=zo!QYhR}al6@Kjho*hhhHT5Gn1-aKBRNc%H6rq`&n`Uux}n&4^5FmQ zyByMwX9a zZmDeicD#(Va-FA&#w?-|uYEb_QFQ>>PiZ55JS1iBRbwvJ`=mypQVPr7_>0^hHJe?E zo6cU-bvM)q=CvhcMM|UVTF%?L-BL0cuXKn696kLsBHTz(7ZZiZr*@d%S8N5w@u zxto@anTOL2QO`<~k7&;?oA!Q_Y$&WoC^ugsNYk2Xu03%WGp*`h|HfUEcA2kR(PS5; zfcuy~CJF(E|HDj06jL{yqayWEYM@W;ia*%#4(c%WX;j8nAYjj1vJhfe z3z%cxwzdvOWkrcMA3cmo>N$+wFBc?bHU{U#%H^ajUdeF#0H+m6+SImwh_ahct@%{e zMZ3~>Y<(AI!5!%z%%ahVe_%FRKAHtVA+_uZ;p?V1>2^t-b8zpe+w6tqE@ePYlbLa} zKd{l|TXm|OB%p%Uzhuh{aILPscMB*D#I*z|tqMB^`Y46HPC1flD|WrBGS*ACpYqPo zc;ZV@E@zF)T^(7;hqQ%7W9pbp9cwv{4O-G^xfM0Pv|hF?=6mqOWK=og4%zs@`0=al z`k7-fviq;R?=Q>b{pRs4^O;LPB)I#*&J z$Rj~gS5nMda|M5UIHf~)+=o4?Y?L;8@10QkYqGrZAK&TTPP2G@oZ&Kxl`Y&aY5vR0 zvW!<6kyWh<=$Bzqb{&fg6isHCzZ2Pm(zTj}CnlO->s$a)WcJ)GBRUC5*M(Vy3s+kXau}uoROB zOK)M5FIu9JCpq!1qJJnH-A9o=n}w~7d$i1_U8RseYB z@W%>3q6Pk81*j?5ks~Mu#}C$>%eo>j*NJ+iad>9E5>w-bKvFiansfksz+5X%H3)Nxr=Sh#9F``Xps!RIeL6cRHv&)(+80&%8 zd=fnd-NAIzZl9z)+*|a5Pv&TI*u-^Z6ra)?aK!QTu^Vu5)H$op#S`m@3a2tYO|LGn zKFO-k7io}g&UDZ1YoCzkV=M4zKDAF-i7Ry6R#Q_(hr@z?q@l4bZ1S<@6U?~%;xLO= z`CCtXd3b@nTS@he>Z>ysEDy>VDILn}h0(QseJJ{AM)h_1)aJ$I@S^$e4o%UKJ`t>Rf2ututfa}G?ft6XwaXm+cr1ZT zuvzI*t-NG*i{?Y;g=;HLYM)olcrS7_1jwivI~`!245SYgSMrxRr({_RJ^Goap)lR` zB=LSxjMgU$?*mmfOm4JYE61#Q?8G`2;SAUX2Gr3ayNUTw>lV6C!Oa9MvVBkz_cP?J z3?FBkc&KE(kwZoiOj(@!Oj*`Q!;P5hnWf_QGCOM%J)(MMkXbE*H^VOKqeGhyGw-kU zxl?mwdY#tJ>Y~{$>I6@@{!}Lh_Ydn)wbhmkLA6>9!P1?}zV6AP`@VQ`^Vs9q})6{pC8bvWR z{n};3H+t-Zpd+$pzTPCaeq&F|Ys1uCbh+D31pN=Xlvb0sdkm+lUOlvSj>NirJ@xE9 z5yj2RyoqEJZ1koIr<*k~9M9(ofh8?T5%$M~i)-hFco^IuX?PqyD;xi*%vKuG{X@bkbhHcZb@9xil8!D-?Nr^YKGYqmRk zEy7VJ-{!URHplp*jg2jpX*Ipl1h z!MVD`+qn{a!6XcAWB&3Iy3p3+33Q z_dO_bmlz%Q521xpcmo&&l5QsY(_bMA`lO-dlev2N0FmqKRoeTu?T^VGTavyccahUu zO^FWDd;I!3mzQclbSs4w%dIjYEv1V$9Ul(GTV)=qI2K!9xHeX9^uc>i(2HZ*e=7{N zOPGrJ2M%`>Mgk~IW=CPhzI?%ZDc?IZWI}mJq14(@O>qrz`}QRCQ|+=pN9_`fA_V?n z+pVOEh$aQ>@H!;bn9hAnq}8nO>3V$xG)6JBoD5mgAACw*nKd<@tYXMjX?>LL!WV<7 z`KDx6JqeTiBTg;pUY{%8e4jP=lyD%==Uj7#fT_!UkIt)h*%xrF=bK?}1vy_@mb8&% ziO#HQ_Ae9J1{MADU=r_|CIa8`utZ^!2VTeP3Et8dl1M(LDsF%C-J3BJ!qrNGaJH#n zYq_p<*Tij7?kq@=__L8RJ&PoX1+sA zYR-OUH2B!)_1HU4z*zC)Lyy+D=8#YrBNN2*khatdwyyq~cUlz`A)e_-@(epe^FtG;v@J7v`ZJqbgTxxySR_&fv7(XRBa#;q_S0m4%~B zy)id+U)(u!XuXUl^R;`~@uMWBGKK!~sffet7Bi!HO8m9;PR~zv@51~p(Hk1|4;Ee| zMm*XSK_R)kdC(v|p9_Rl|Dt*7YME4ZLD89#(U{?L*S@B^Vz_*4@{8~F-M8{JSt@`G zBxs%sS8lHST5vwSNGDO(3wvNzQ;HpeA8<*2nf=Nauj~O2R?s80yY1$!Vkuhp?RirhH;@U{6@oCaP z-KRs2CXM$(rBLVS)H71^m3Y*-wR1PWE=(&4t$E}Plh?2oOqmzAy^=z4`|*f?saWDU&u6Xb8hC< zbAO#EeU(dTIoVXj$d{DGYdq=iD-s&s967IaRp{Y8rcH|I$=s=MpPm?27Zs+D?t)R6 zww!$(-FxJ&2o`2ez6-rJAH30jBWvz5HLQqjm-*i%_(Y)p#pqqHrYD6{NP4L>P(UB0 zc|hkR-cEwo4p{2F_sr=P4?0jJ(q+aSCa0XPO>t z_H3J5+FUupNJKLvMy{vfy>#7=P@A~9i#ZC(7ZzCS;VsmatweL|P1c&vr*P^MAMAM@#QkE8CWT^2 z{U{MZbL7QIm$!$d=<}pqU7ozzSbcENZD53HQ8z%YI0#MQ>ZxdzFDfWNW8{&pW)t5j)MwO8jr89w?@xdJlfy$kb6wx<9j;? zV+pPM-5D_n2MrNQ0*8Cqo#m;zxPzp9%f{l?sof4m?_~@nN)u$O!YA4*xsovJ zr9rKw5^WQK`^9&yvt3j|g_vUsH3cmbc)t)_18^o>hGIJ}2W zKa5_ZYjU15NsA>vgPZ(5RlPR9@s@dfWrL)4nLBtFsdfn(QJ8<&Qj4hXY^e>>)mLH` z6t!G?E^XRUPfoW5haj-;PL#=qi6*pqXneV?V&oP&lwkC%+XeGouOdJ=bn(rk+H=KF zp9#IG^S37`y8Sad-;|zSaYKL3kn2DDc}mZqL;Eeo7c$Rj_sf1sDNv>}O*9eA1x*QN z$5IX=5e%O@+@{YBzmeXX{3zaepI2pa8@BsVsay98Dt#^iTm9t|2M>8q<}EwB48V)i zDMbZlvY+11Nv(cHMi)XhwyAhS9T8lsd)LH12xXQmHOP=8RVDsJ5b~_^i%er+31Q%X zFxAJT1Fgjk6_@Pqo;fQ-Lq^|P$`#qIa6(ryc`inD`WSs$@%M*^Xa=z!ovIp=D10iT z_(8uBW*;R@D01y|H;ADEIDRn*la# z_hBxZv4r$vHQK`W9C+N{XsDxV4y-=vIaeW~=hdj!p7{9TyKdur-CSnpEFcR)N>as@ zx;;`_>OJx5Sq2|Xa?1^lCJCutrMr90I{L|BZixenUB2A!Hxh?0MNCb2+Oaf7_7jHb zpT}hcR%mvrM^fVM`IB&oJ1{vhx;1{K?c-1Rpp|;D@QBFjE<*1Thoi9npifM+LgWbq z#kFN02p?Umux;A-bs0Ug+4>Y(LA|o{j{H1mshNDjntF_8T!D`7bW>O=_vrQS4EkJ` zStHn!+A5_nA^8UJU8dY;85d$tdK6*Po=gV5Q!9~7R?*3Rl2Ms|?RoZ+u72gIndUfX zSENR1^O?8puHS`@DhJN=A-!cjzdd}*1nQg?%wBuoJ@$nYUENY7yG5USA@5$!u_N>% z^vr5C$E{|qUsRh5JMiC;+M^QlJXbv5Di+RTsh?8ZKJR|`oXokYKu2!FY{)lpF7Ugb3OAE!!gfqp|+e4 zR-#{xIj$++pXV7Qy{F-K)Zn{(gGInti5lBf)&9niv{$T7km!9{C&pThWngC}{2nxf z>Wbuhg%6()F&G=C3gkjDIu;Ix2|RY<8y$Xkm`D>N;VCgKJzMO)X~V!y$d#IXG30MT zOR3LRV>DD6M4}_Y%Qcmu&uJ(Jb2f{Ta!d~m*Lf}dm_wZrx1XK(knst0JTH^H_N0KL z>Md_sWuY~WWEq^ur7*U}_QK*jN={jpVy4eoikUiR{3+|T^KG)EqCI}a+R^v zvA&Txp(JwKr#5ZV%eS}d`v&QZ68`)yTHS4XfkXZ8I7Ufz#~A?Ut9$$&d0Y^2MbvIC zCAJIa9xyW##a8X-Fd0)I<34Nv5kvOcY|Oo(F=%RJqm1#m;M&?ScboQ#vqCCSrE5={QoZ%=!nyeuVZXkOZV$2wCxuE*!L#>iJBBRr!o+~hs(wgl0t zl0Dy0YM18aw8oihK}uvwJmOPgWqzr09+q5DV3Ek)lXsw5LlQ46ebHB9#|xqgF3_MaFXP zdG<(Pcr5sCZ~hI0V-TWZ74vOSk116a!Pjlgt^D{^1K*X<+({Nz)0;O)XbhOXa8+$1YmtHTB3ep_c+J)QQB5UNoa2u&W0~{wvE+_k+db}2$oUQe$d2wTq zgu@V;Y7;Ij?SB8G;t?Tiy?8LgR_tC8h;l)Wzq)DD!6?!kDQ!Zu($CTEH zmbDcmo0=7(D;?_sKRGYk#}w)$D(>@~Hn?{#>dcJYyVoNcYZ)*fJI8xxy>^jcmk1ik z{||cu%=XEC*I94B^K0^*3whago}0Ex&(CyFFx*Edn~T#0!n87Z%Iv(Sn~teGnL9n! z!B8*7Ylb!IGT}RRUX$GP^N>dOh^1T7x3Q8=#*5ddYv(Ox0xsQlD`8-evTxJne7;{O z6vwUgw4*KKA>=VO^C+|#tNL3zAhlxfA z-;N7;k~F33JoqGujUqMB;Vlclp6WLjGXurK1}b^IF$bApw(#S#zV?<1N7EuyZjMvN z-W(r@si=5U+IH{j25FL{`7TE-h+kL7k6h5L1W%X*-kI>DCaWT&r>e&-?nK1fC~1Pv zyOXnvq$S=7INydT; zpy2tcER-MUtqc_abfpRv08V4Hp*Vmx9*P4Tk|`8s289s-0fE&_1PY+V8Va+4B7l4x zm>n=61OX@!<^<3N#)QILp)fZn>=KZp17woIbckjymVlr?vT^@QO5M`?&Z~$2CiRz@ z*;3_h(*P*h25;dEXyRWcMBI5xK{#M~U?6o+Tf9hcb1R8=kRuQ+EP>o$1Qr6*-M+yf z0IB7HTw|iCxSfR!0bD{FE(BX`u%d)_-;xN>1F$TA5ogEq=tKE|BiR2B7v%qy1`?P} z@IqlQPz(kOUJwBR92AS?2NH-8P!tOF;{sq90rU@C0JhU_yU$MFKTF6?`%ip;21sY? z0><0Y3^*8g@gtF2*Va9d6U`4W2Eu^O5Df6xfN+qY0KmW&F3`T!ZyWDtKcEj-$Dz?% z7~(AjtI2wY3@mysZ@jz<({b zG!4SQ_Ft}Tnh=nI9fAKW!AD_h1_7pjZQB`lr`&G)xf*`O1r@XF7(0^;l>enD{u&xc zLJ`{^B1l1jeJ_xx587YA;%wEF0blZ?rV0hZpDma+un2zC@KEHI6@>sc8M*DY%%Fe+ z+QK6M>+nZy%i1Hihap0d+rwBxx87(01KL2h-W&m68`wd&-sAus^T+7=FQQrp7)a7@6_(^e1AR{^PS zKpg`34)pfOZcyM!-)g@E1u_%>`C^A|tyODOZ71ivhh|A4=o z*8K+r)wCna52yL#V;GnM(F_a@fOQXQYU|?=SHRJ2iL;$ki~fZoOcMCO0}OvSfmp@C z#@@w-XaaRZ@$vH^d7zdqE)LFuFqrMnCO&&7i?a|I5VM%MngZX8{MyUG%p7WhH?;;v z*`{#z7kS;C2<8wt5S~B)1GxQzVlfzC8JR3*xN7#xX!|HTUu4McW-;_(BY5d8r|A;HKD{QVpQ zjzVGo>WjjH5A`2-0^oA{9fk%2;y+k7^fx`IFO1mK{#{JpQh-}4I>!2UJA zZ~=ZGi20*04h{y(zthA0y@ud8ejsZ99S_`<{5ih}IQX9WPdxNrvLR3iAhG)=o{JNn zXhU$?`XEn(=m~s32W%SD?d?G$vt^y-?aTo~4jP#)>+9@-cXHW!{1GT5KLWzeF0CR1 F`9H2e>OTMg diff --git a/ref_result/__bar_acc_PT-2_ref_result_perf_debian.pdf b/ref_result/__bar_acc_PT-2_ref_result_perf_debian.pdf deleted file mode 100644 index 478e6afc3027d5c26b028bf15faa5697fa909e41..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14295 zcmb_@2{@JC6L3OtL$)ZAEBku)%gvUZkbTRN>$;ZgYnCL+9*GpPWl0pW%TghVY%N0e zQnu_uA>aFoekI@Y@A;lj&*_{yXU@!=Ip>|3bKZHmHRTnB5h7>^_k%v@{YnS|3Wqvb zoP$V6Kw-Mx&O|6o4)2b)cd~}UH1XC%Hz*Q7(1uD$L5Pl)AfniB0~DPc-Jz&W1el?Q z#W^Cu9V+_sr-GNel8!sxod`w$1Zd*j-HEP_Pz?AJ0@JagED{6og2@511W36-aX)31addP7 zej&hd+K~S=19oq#QjO?n?QR1_Y_V6gwFf2-g(=zt8j&XwoGgjp9NgSpiFgNyS5l|3 zx)ZA?ZOwwgEPa~fjnB)^U#XlS#gVBKGDFh$*J~Zkb`2NHSzi|Kiz`7f@)rnu7b2lg zS6h;kMOE9X>BeKbBB~}|Hn3a6zTIE`Vp&nS*2cK8H&syN8}mw^TSNPhTLYI#KU_G2 z;*8JKkZTxBuc&S{-ruy&-1S>Rp~jaH#+C~qy={k4Div@0??%GQR;Pk<#uU_|>Qt!& zvxDZ!p6}-*n?AM_*>&uWJ+0zSSJS5MN%T_~Kq9Ho+^pj#rqp_7<7wX}+nf@*B8d z@y|0WYme%SGxQPsLuNsg3VHI2ZLIyl*Mc9-*Lf5DmS0EC^5k9`<_^Gnc#oF|?F}5x zz>Cxh=43{Zel1@@K}0cq4oEGcH81Fg?NWpkER2Y}FK60)wBv z?om0#uRg>b>~|&3Y3OtvYsT}sUe;k<$X(@0QnjK=w};b_)aV?5jF?_ zV$D46mAu;M-4jqRmZ@2);-wtzJ0vsh z+9aj4csjA<2k9}^L-89O$4XUssl0DaTp>wSCBu#leFq(`!1 z5-)MNmq`mB*zQ3R!_bG#h4!+9Lnn{g#idp&9jDlL z_|}i$)A7{LkK7MP*=zUKp-5D25?5y{SbE-D*AY>It{K|W8fq@;3N*us z;X^r<(+<)qde4px=ZXz6d~GpyN_mSJp~<%+Rhu|d(w6%HZ+9cqZvB z6VNG4s%EEJIwGMeq~dXMdT0XvH1ovUh{j^Cu!Yqj>b4o10^bW4t-ngnD>)^X&vq*M zwTGXTQCQuZx>{|UTF@i9j~KaeoJ5zu{E7MAK!&QLGp2oVs7SdCs#^#jdUC zoc9`9<6Mo{u)O%$e_=uNOPAY{V+bxUs4c8}RbsgJ^NN@oy+y)8R;-o3UG&?%To^+- zt*P`0C8NtD1deZphe`$qMN_W?bKS|L5}0zJW^!wezy+SKiWiVzl>ux9F9; zc|^zP!LjL4-^0Ac8Q4c+jpS1TQhdZK`#jF@#E=Y3Dd_J_bH`_@H(zt9@01PT5_ch7 zVt20p93Y?(PN86Zh4l_gf~QHx`pwHF*^)NK_XU2r0(o1y3hU++)at=_~^W ziIYh?F0WJyhAb8;M778Z1Ngme)GRd*7j>D{Hq;gm z$RB#P@9vP6=SRDc2*(TL%=?>s81!ga;{%m?3JTanzJEPrVn4IQlMBVd$*-knoc(6( zz9?8CHqps?FL3M!by%Eg;*)V~-0Jl0%CGkp7R;AM-YyES-@ZQD_uZ-VPQ|j!5kCe( zn!XI9d%(3>rL1DA$9NIlA?|?sJp@jQA4ze`6$QN2xe^6=>eC`OyVwiGjFi3v2>5v%$y6i0%v*o-u{SoM+5P+xK_UN0A3yHQ17VK1Z7B4F!@X7;~eR0 zWTw%HbGrAMoL@08TwM))Jgl#5t>VH+C&^cNUACKRMwxHnORYiRt;1^TxrkEM{Xj8@R82$;NouZUlo5B`Ej2&a?DUtjpNTX}6!`M!c)L zCAIp4ExvKpB|@WfLlu7tEl-~{>+^wSg6TC&=l+DC<{>Vpw0o$t#}tB&_lxCJhY|$Y z^dro~6HdK&cFX0yw)uP1T(D=1`W=~GSzUHE*VIgS*JX&xwC(-Cb1p}D&_W$rp(-TV zIstA5-MxJ?qoc2|Nkf!#zD04{m@VzWajLcM$d}c}C zOWALPE3&*ebsfyZ9R7fI1^ z^z<9w%zSw(x8!b#!?z5LdtnZ7{#^^FyrpA*%#5G$(oaY}Kbs&ZOl`ALZoJihGADxU&Niyf7GpLA1F9VaT*j_C=$VPO{6sprtb*1HuY- z5>%&bA3EO6kbIZy9L%!nvo~cN1p-z5h)1CCd{j(B@~hj_>^^pQhr5?PwTTT#JX)Ul=qv4?I>YYH<5%J~}LZoCqker%P;VBN9-|&?2;R&LSXWm1~uTx~EdCMevnPq)l_+A~`XKBwA4pB#8 zhJ(s&oCU+y*AuSHGkRAXKz38w%Nz|$-h0_xfc+tg|W)_QWr0!dwzl+luX>vx2r@s&ZSg;DQQ2r(#5*I zi?HAcbce8LG~yqGjZukVLr};sdq6}w=qx(iQ)ZmKUOnD;2g_MXhj5Wu^0Yj%HxyZQ zEgvVMLe}TA6?+6$S4TX9ii2^DA?m9VF2R245$}@^r#MJoYcIdyYuHVh(Knj#R!YE4 z$G^QbE9sEFgjDQfCW1>1A8W4-t)6E=!&|#$heDA@&n*T|M`nCLD_0Msgr_9>aCtQ*Qc;@~OK+WQ?5fn)0K* zjOyrtWot}#b)P`6;NrQM{)#3?*W8B5kG zG-P_^bhV7Bh;Zb2H*oGdtu7Eg>Y%HuXuxAlH&9#G95Mb(_c>YsrmaSXC5?m!6U6InGQ|Rl0H9C?)5@{SVCP_H2Xqlw@14b`q^+i*@*`Ar!^{a z*^RoDZVOjdT(xFbErrht)CMVPnY$if84solmr)N?JgII|13fY;RGXjXaoqM{L9E^v z>k9`e?3p|dwy&_Bd*vwIx(KJoF3_Wn6gZB}h1)gKehFXGe(l6diwH_?Bdt{0+G zx`6C6i)6~;+ectqBfVj|1bl8+}D!z!-);Wsn!;98c ziDz2rm1la95mwjhpLS`EM~!(#M&6)|?dOWpEqw zhVQZ{$wv!QeswnE$N{;g99^x`xeI5T{H%;2LizN`CykOnz4c(aNAWoTRc_kaZNpzU zfozO}j-blbpo`&oJx2^KYD|oDVwEVYnG+YH_k^V4armrk{Ff33v9K=ZD51+r8N5l+ zpD=Z@(0Ox=1zUHYgvMjwuskw*-{*$o6;Bzq;jr$b1+)gZCo6^=ZRszn1scW|Pu$56 zb3Y$=rV{ZL-EW6=YIVll3kuDPTQgfh#f)s+>-SK7zrRzlk!#*o*&!j)ggiSbSLj`k zrIhW7h~&G_7=DH2LrxM1-*dto#$U2ywD=>8OjY3G8LF;h&-rvZ#qBsU_F7*>9siKq zBHBw=d`9cyyTb4aDJWTLi#MvcA$NfWS zp?-M@b&V+hYl^z$e!7dz9n~8Hd;-N2{C&1{+fWV zW>8EM#W}XyCE|MOXK%Vx_T4&{d8mx_#*_TD;Zn0t7xsj_Vb%XzLewr{D&`*~+*TL~ zpfJU4g_&Oz3EfLM;#`{v6(WUF>tC;rtBu>YC%&8N)-IfP2}TiO|8VS9*F?mS0(N*E zl44HlH7wa=ng3$_NhCB@ExeQrS=1fMX?&VJnC#wKSG z$Q3hJ`&{YWl=k9IdYhe7mG2tW!D{t}3vL{+<}kI-BB(*4osp%Xw3TpyC4I~YgKMHe z&7^Y@DaFlhvBPYM(Zn&j!^B}5c$|h46_c72S-c^SSkotgyR`2$A1Gy{JsN8sI&$zw z*Y`D&+7N@gyJ&5f5Ey~@my^|L91W=q)da9_hI&6P+cP>KzWKAx@Yzz;T?$o(U-8Y4 zyOTLb@5A>|uF}Gp{i=iz^)OYbR^tPqSw|BtquFzz+!E_Id{!2YFujYtY4|3C_t1KY zQ09BDlA}jR2#WcEDk%u&b?d3YTy@c!C$6uLckCkkF3}qr^$!+aGFB#rfS{0D-e5FI zyHADfmeROeX?6YahG)u`MIri7x+1c+GrftQiVBpC3+?B>eLTpE3tmsM%x<;nyfsyn zSZ{Vsse92XWwk`UDzAXIXfU?_6tx*ndpXt9(B>511pdG&Nd`PaNt zX$1xehQ8PX)4K8`+0E7Emz@!g{P84B2=2CPv0V$xl;`g!R=C>4J-Wb+KzL@ZP~4|= zc%3LZf2VFaO(T?KV4T&I`MW!-GSj4U*doLA!Bxqo2#PBMDdt?H!GSb-oTXxNfX*KJ{KU zRUz^%W#O7o+DKV^?a<*<>X*eUA24lD#Ej=mT=jbu%kHkh^x5mgbxiZUeXSjPlrNpg z&m8|4eq}E7M|X7Aj6XH3fMb{S-zE4&p#R0_U8|xag;U7+YSdCdpQd`pyv%Z5a^n2L z$Zy~DPST~K>fYkWCj^)2C)=rNX>MCp&*D&N<~yHG^p9CE#N_TxLBGhTHM~OWB^~tQ zzS?IGN8L5WbBM1D_soQrPLS2-M4+Q$mL|GCCF;n#7M4lXUpb`GV=@nfax)|= zA--^xetTRsz2sN`;~7Sh>@u@m;y^TT2m3#7QH5hwTsdVZ6u^^dTBzlVaS2y3AMj*4 z!wJ*XwB+{R(~7V+tgPwFyV=`)?d*eB03+kKjLaoy=@#h2AQt`fq zUWchIJiVk{#8vY1?2Pc%+NaM+-E*>7YdDq6XKedu&-)O;H)}M>6ieDiY!Nhv-yC=U zz${OftKi}OeCWsOBSz1j0j5R6AmzeDsMX=TSRRNHF@}ZcelI@Vl)K3IuA2DEiBM0Y zetMyNwSkca`SJppYKMKMd%w8!s$SQ5c5S^WX2$yI{#TWP!?K@89G%T&^&WPl$Hp_7 zLR#t?*1E-*-*LOCP_+w&C|oQVj$5bpJQTB+A>3Ahs8n%vtfgEa5wl(#(o`(lJQlpK zp?ZMVa&?J@i{WcsbEh1iBIm-WgWlsp%=xO?@`NeH=0h$XxO>(%nID3zz0*g7tULt` zNo}tM;wq1O9t;yFET?=A(@1iCh#LPq)pqjA*uerirvuyEoVya{D zh6-nLiHmLzuc54*i2sknGoynG>j%m+6AtXM$h!onh=00%xb~77xImJdNeSc}vuj$& z3=A5nM+*-!wMPWP_`nW1%)@bY9W-L-~ zkn`fkzfV-H&HebmGP?4Eq-I$#beCD}5;USP|8S(1)ZRW)o1|&4#4f1mdAvHm;Xpk; z*&G^%z}7{L=DSf@kVN?6(74uXwQ#>Nqlr^@#wa=h zGuwuWxmG;Uv+2s+{Ie5ACawA(DCWt0CcXRv5|g1!y!AAZEP3_umaNH)NCf?CtLNm& z{vn0ENl$ON?ei^9YQ}avE%xkqLuD)==3u;hjPcNg@!VxM_a1m*8l{xjRQ8L9_fo1} zlF^2d4R5GLYa>Ey4DVVvg`g~xKpMbn9h2JX=(ez@y+cb6LQ1}!E8ODHt)2DdOCBapW zexx(>CS{_VPWKOz7bBmaMwGdzXsUHIMI4VM8uTzzYd=0(>Wiu$4D!k>4$wEfQnBvg z*?Qo|=);$4N92qa%epL&Kny^c!{F4Jw(j-tdp z2qY1Zab|L5@T~iKuuC-glU~Z%{KJx~yG(kQI2?uj2Yq5j(ZniQ=g9qf7B|HOVTjNex6=_|H|v^B}3zK&Z&ktXnT}SaRcv%7LV`Z zM@|P%bs;Y(&VFFNZ2@&l4dt#mFoJ#KO8a;zirc!&D_?jo-|%5NNjes->Z9kT?cP*b zNjQsU$nVjJeVrq7|J)6@kd1M2VauG?+2~x<;#$G0YIm9IWBGhkWpCxkn#XC@UN?y; zjMI@@O)(k1v_$1LOqLpT8eVEqJQHm;#&XZ&?A-Y4%p;#- zpZU^-(a|>TRpr&k@0a>Jy%HaOw^*ZVP-NQReGY-ey{h(bN9IS!pMa3cR|O$w|-Z^DW$g@6k$3=djD#>4$Sd zy`&Fx0*;t`SE;oQ8ZOe}n5fua7na(|?h1+7r*~|)$y^b}I~MS$Hr!D1zVFrkW0EGr zqg25HC?X+wlU2`rmPky5_bs0Pa`!1U&UVBk6&t*;qwU(fsdnbhK~9xm$!Ut$>Eu1pMsNl43o+_) zTuZV{fz%(kd{MqOveh(i9lvAUs9n}?g&43 zmt#$8%am}~ujriB`SLkfQdypL0{5%j0}VQIcnOumk?U9Up8+Wj&&6*?jwW}luoZ~< zN6N45qKRG3DhT*LT&7?m5j4O>DXFj7Fz+RXq7ljRGY;2R{e7cu2=00LRIGnE^zOU+ z(Fm6iMA<6lTdxsQiV{Mk!%pz@v(8$POW`@=Z0v-a(IhmcRHnLb4wl*$(~PF_v}M-4 zqcmN5zd(9|=BVB-((V>nBmYI(C{-HZIzetZDUkEoT1-;5UT4GNk6scUQ)r6)RS5;J zho99Bi({Y2gwp$G3Y(Nn?Xh}@Zi>HJn8-w$=p#^O^fo5>Y-VU|akW%Qb6yg`G9O*; z@;La5+p<$^zCnW8KA%aG2Pd!dPC0&jKcKUg4)b$#dBE?x%M5mjppl~ga5lhf-R!sX zUkEt0rqZ^Mo9*bc;h_GSx0QnaA>y=^3~ew>FH@+*@xo+1tH$#guHjbtC-TCUShIEu z5!O?>HiosIz;&%|%TMQsdcuT9p>*(e5`zvEd%PcQG(Y{>U|zj!!KQ13-+b9yD@ z88-8XAU|W-hz(6!qF8u#k+}ku3s191#$&&3`rC_-ughy?y`-=s?<$qzxjGb9@O;E7 zaV@!uMQ<)e=-i3B32n0*kEE}Ce_q(T1)q0UH+MN3yen|O z4O7E!wUHPM1O^U}C%O?_ZJphnT%m}qv`-xm3-`^mQjh^)F;3$hfaJ_S>HZun`!gF; z7>mU~g;7B2B@zz2aRlBvV!SW!R!+y9bKH}$^#>fygd{cUA7Rk_3xOGK{lFlAS*rlK#kK?) zM{9c`xP^4wi4OW;Lmlt6IU_(1z_$EtI@_Md7%B=}!Tx_JkpEX2NMJR=g2G^+7z`FH z5HT?v6pIxF5{3~_6bkjT05FUg`X?>`+a9+yW_#?;8nWI03m>2X(%CEkzfH}6%)law zL~fSN-#`wuC?FUJ0|rAdz+(f#L4sm{0Gqf#|K_+YykFyhF<={qMsL>J@GT)AV2uXi zCy*96!EGTy9@}jY=T{q)0VrTUScCkx`hHDtlMdLwJq~DtGH%iY=>i3X0u+e-S%V4y z7Y(EX)&Sj2S|A<3Jp3ofF90};Uq8ShSWrb<`v(*YEFc0%$<_dyBZY__iH!&jteSgS03Ao4AlRnDE=NA$UzZXA0Eg0 z4+ZX6H(Q$y4*heM|C!!x%L}?rMFey+TKN4#5RlBaqme=U?qPp@`T>)Z0gzw>JQM-i zbeIhC|B43q%m2_&|5r4)2>3Q^3-dQW@mnbK-wb}+ApZe>yRG{V2&!p&E-40s~?eOAi9@eaP>loGq=O7I=akz-5a9|8Mj4awS?p;6Qi+ z0Sw^Q9~6th0Ncn4`U?hzLEsGe; zf5L$8et?JQ4;aYyFIkXi!29pS6BXOZ1CBxfk=_nGz^DB+7KH_^(4Tx!qClec4j38* z-#Irp8Vj16KgXhRf9n~8{96_b7PO6jjs@J%U$S6uphf%>4-0Vq3x)zxCU@dt!C-X9 zSTXoc-NQvuf9)%{CeU`=#~A^`+VsJJ+U=wt~Z3ja1h&e_QWirPeg>8qLB z5iLBRLO*}X`gq7|c@R8^Q1nlLI>Ez(=;j2)fIlHHEgOQRt&=qryVdLFY@tQ;fEoeQ z;^hEVh&~=rn4%-Vg7j7`vsEiXO@3ns@wWgV-yrvHM1Z@W?87vP?#`ZW7Jxh;|G@kz zL`z$Ow6hN|BOLgLgbND_A>mjg8ix~xB5-IyELIqc!~k=_WB@4vQtnXPPhO>*oScDQ z2ymPxM&Ar z?#0qpFX(-yPZhiIY1OqU9dk{ltwheB#}A$qHR6$cLe*|#orZJ9E<`Dx=r3SH&2JWP zGh`v^WrFj~#m3tmGadDlWpDM?zRpcsR+g_cFm6!YQ4;)Sz0&JiuQue~$YE@P5=f^o z^qU!Ui-PGC)vm?|n$$Iqeqpd+>uOYjFHj#BAvSRgX)ecOYddGl?C+&9BXE!c#DgTJ zhMwbWQM8X)rcV`6-*ups>n4+Ub2#6FUeoA&@o?Z5*xG33>@h~eC)2(UqibZt-t#aZ zEqsJ!ZDVv(1|-AqZnUgxdz{6eISC@1iBZ=ySR9e_&|o#geL)43PD$$BO3u9QRNhSk z3m@G>3)6KGyo08}l(KpF#TJ&nkgFk2zSQ~>1D4x|I(RcFE4kng1YyZHWhLR~&`q2^ zoN@0ETFaCU9E$UphFDU1>Qm)&==Vy&$tkIl<6sk3;o7|z*n ztJ!5CIxB3@hr>&By44rCDV?tzKj7?qS&R3ujPF(W(9E$1{uZPvlZVBa`e~dzD0Ptu zA5TVnGst>RzxG~bQD2`0_ALDH`0M_(Q4?#SyrqbSz3HK5$yL*~d!{HP_du|UaXek~ z;-*ioVmKP>GS3`1OaCDRslf$db%X??*;honM_rsBrQweA%Nnkv#p`l5T*{OuHKl)B z-mczVlT~v_leA{RCaSKvHzLF*tHkEnfsfyOc;8&O^}IO$*bN1+dNaV&c0w6_K+e;Rac`C<<=BvGw?`_+o4a;qc*HHoq^_gG2cF$362vaBMQ_d z&+E*61dZ8`g%Ool&7KyXK0I&muI8ftd-NW5#f%yX3%WLSs>q?oHYP4E(FfjGK z>oJFLm+kq2#wN?X*Ew|P{4eJYhsEaT!)~??b7$MXWHtUE7ys^&-xL3vad^JB<22dI zCq=bhl#g&`3jm!)az6LYyN^Z1PG0D(=^$KI^vq(8vJU5LnM)TEewto2JMk^KWBDap zrDuO_Xw>(VLr*V9EFBT|_I)dUwzz=m)zJ;by%a^v-Dlw>HQ^@F-nbX?7Rf`*AErBw zh}O^%4|S+p*>i{QS*ER zOFk2XJlyEJWrRL-<+zYTwZ=z9y1V1bE+@6qU7Zc9UPPG%$5gwMPCh3SG7GP^X=(d!v8tt)UpxJ-uU#D;dV?QxmsC@}P^0$JYJs zB%7@skwD07yeWP3Q3xY3cV7qtQzHmrPJ#b%&shs$4uLJYThl4uSpy1`B1R%{k^329 zx*X4nEehncS37+kzT9V)8Ewu2O*|jyg(QYnxRy`5lnM96_v>e;zii;QjNtMaaeqVR zJUGnQYnC2;f}t4B+cTQiDtlXWus*xAU*Ql3)2CETuXdKJk*aZY3X=?_9Q%9mW*)r0t7Jg5*%OGI744w2C}&fFi)yq4d%tnc{M>4KL4%95U;T^YTZI`U&;) zv0jW|HXpb%Wf6zW6T~T^k)ai@mkwOtm5&Rf4bK8Z0f)`ozVL<|k`{NlCDGtlc z9b@tB)qbYT-4vV)!{p3$Z4CKmb{27{HbN50}QNsTfU$pr{x-^y8TVfqO1n!=C2t#?w+owllQQ2Cp3LLOK?S6ECqh zu;+=gLTIu%jBH2k#w&Cf8;4o|?V-U5gS#Q5t*|i9{uTb7F{??+(4-LZ*UTr_)7HpL zt|iWEXEnOKVPLqj8dg1|t6;6@%19^1Q+{2#lVeVSXJNWVFX+}`<@FrInS>SotM0|L zaxsw(md!DYPVoXajxk=7)p~q!DmSZ$SAqxrl8YawmBC;MZ%RI_=;$;$=qV7daKK?e z`8I3%^(J>U`CtVBTCdtmr(aqhOuSroDc|LNxWpTtz$6Pf2Ft~ES>Dju9-P+4Sd9er zvX%fl>rq=t?FSLY+M>8+@WqIJ+{;;uNcH zq^U^4nU~LRx#nt`b))7(ykb=DO7%!j1W}x|%?+}1J;sIRZ_@}EjWuakT^qVXZ0!cCjv=*CmNj8S-M!E$MqG z`wVbJwpY$YKBCkQZk^#llr1)Dut+}^Ivv|dS94%mEGVbGMH1dC|Q8q>?f&)Q?U-pHVq}F`z8Oy$#AltlNwmuqk{3X6foQ}P# z&*)~x>svV`_evbUrK@FyJH`cmSUBS=5&L6q!q7)Y{?&@MD2I`(lBt^84vgQFDagKM@xYzyl65MNw;iI&!8I>Z?gp+&=);ZP{p-xl z&6dU8e73W)7F2bkox~a1A2rTEUek`qUuAbRIX zIMviwh`7`+$(cDtuHxV?8(va=T*1`w%zH?AwTevDZy8@(KDxdxkX6h2Ny00IUC2p* z;h;jx$$}y4>j{^?F#0||fb67nkUAEgy!VnBAKODx(+IrM@;SoUQ=hc!Jqqi1FYEi5 zXoU$l5;LP^(bWxS9KCLgT4 zLfcC{D@)$5JHKr4{+moqeg#6c-kT_U&_a9dh0Bn|I%h2?I+PJ;P+jPtud-!YZ#(RoX6KSk5^* zx*U-gx4rqSJT~zSGkU*5DBf}iE`U|Yy0ds8&FdrlpjhIDu6;SmX+EWLx}^2s$_JM9 zU6=)rqC1#HqY?kWY>Z+ID}n;Q>GW53lMY(>}T;Yf<1^xx)x8GfSqKKa|pjQNp(@7 zK;;F(-M-{z(Xjx|>m>sR@9#~Ipt~$Dp!jJ-|Lz^@?ql?UC^q&h7Zc~dyemoT(29QC zsDyqOiFayVT%f44%pe~tKgSSgA0yjGA3}YK?>+lut!b!``Q=x6K@MJ+O3Bu_B~fk! zm#&E)rA3X-=Y29qqX_7TXEEdlL;J7sEEU_`8B>d7YqOqtUtx082uie(kd91kU{@?y zqERF{K2qA*djS{RO{gKWU@``z!cZ_%)cSTVrJapo5^z5=WE5^R7TTuFqaa$Pe zl-D%g*QdcC>_nC>mqT1@TH#59E?1o32To%iu4;Gnxm&i?wn`_{rl(dH*j{9o8j01& z)@S%+e`p$46lBlytv|U>RfR8N%u!ogPLIo)uD_=CRpi8T?H8Ca=fyr&okzodghyBM zoV|)Fu2o!`F|=V+FjYB}@g7Fo__bVodPcqb(d5S2ChxC>5J24J}-5LL{+qf?w{-E^lY_4KOc%GJ6*5xtV&Vl zeuH+o`@-cFH;vD$mI7z_YJ%l7%-jwfod}_ekWvYflT@*(f*$$IUz4BedBXN#L9EWS z^|=F&9hkfhwyv<)y>XIgUWC(Q7wAz(3Y^C0BkUV!r^D)rI%NBxBtG2ajr5;p>-eeU z&mntFqnI*z_F1s5kzO%9`t&IN*50G;y0*U8-^?I08+vX=o;5;8)H5I5UlovEd3b95 zpp)HM%U#q7UT^JGCkFQq`%%5okqkk#S^>e*O5VTf$E817II+QUrr%7ImY|gJvWfB{ z>|FK<%yq$)_jLE#o_rS3q<)pZ99Eoa9>y6T;X>MqS`%K8MTgHA-60gQ*t$e>48Lq% z6@9LeR&LmXjI_F5_w0lEMD(~+1GNJh*}cvV=9qWWbTv$Zu;a~94v23Q$E`19$LB@L24zVl?tqWE+XRc6vx-Ab*@ zw`X>V^6L?acV>QWhg~{lsq}?Jt5yTt-CLNsv_7(GKfE2`_$|ZY{46z!Vsfh6qyK!& z(DBo*$T#y(&9k5W*wfH$lCp~~cRPuo|3R0s8j4QO;Zzl?ht?!X>>sVC@a_{+-Ut*( zARA|=vryuw*T!(Q&l5w68WN+NSwst~=0*7Fy&!i8I6~%q!gPtFaQFw8X#Pv`>D)=z zK4NO6py7KZq0NB6*Ib#)#s_yy}v`Qf#Zv^k>{OttTji&F%Xz)fFm?*+U(v{rCU+`#kh}g5I@3p>yIx(En zBvfDYd9C8Z%8@yVwo(I|T{O7M*+S?a+#?E!)P$?UQKa-C_qZURLaZRXA;yx`33s!l z1;a?_UkwE*O6x-#kHzO4%cl~2NMiQ!UdT;#S&Ew^_(BqW+oOBtxR~I(u(L~yj{Aqu zLgh*g7zC1TBm~i2APb$=(h0~|4Lo4W_w~}jhmK9p$)4Mgz9aWgFj!5F2{m}$eU;B& zJvgS3!jAQJiHMHM*_*EAy|?T#4wbUpc$&X9^vLw%xjmt8S#irls3T%X0Xw`7 zNin1K84_!>%zwH5GzuE49Px+@S=1SJ(nysp5}kxdD%R`RbwXlGgl`jr0EPT@#rb zPnnVy$-*!1(R-xwx&-EZYCsn^)3bjA@3?xXk?*8@$&>qp%)UB%*8u_1Xf9p0N_lhE z5XCMBnJ0HKHJ_MaFS!d3%E44=ZpW_nB>5TR-R%5&hA&#~sm&NcOlZ%3W-$I#>;L>* zjCRFdoGIS48=#ho~NDG(6z{>}1Sg@YybmQ*`}?-^#)fruVTo_1~s*A6hTr z&*=6kId+7^LM}f@F$KZAZap)QqasxG)UEwQ+b+!S61|~O|6t+8Vx?j%5EL@Y8;r)O zxm4IrakYEpR@bXHyi%qY`RT*xipW|FdlEku6(|@LI(+#ya*!JrvYuplzuBte)=W`i zo#|Ei&PA(~)e?L~UIBN}Ky07n<*%up7`_09+`gG&T%3TUKuMCxwMY$ zi9%oQ)-I>2g^~16u$VA^_h3o;%`Ff=w&0&x5 z;i>G$YN#W07e?gY>Q&sd?YYx=&Sz zd`DTh#-BP`8ecPb_>9UWk@5#j8x%1U*^^fS-o&zbs4;!=Iei`TDr;YJ+a84rr}Hx= zMj|fHhyCchmN^$l4J%;ZW&L*vJ`w1DF?v@k=t$udGX82c6wqgPd}Cf`x-2=LoMz;8 zXnZf``nV!%ar7gC!{npwOr->;t&&%9m<03PkEi>_%^6~H_NJg;rq}3SruC5sewnNM z$PrY5ENv2rRJYtKWIsEp7$1pRV zE=ShW(@Gkncs7|D^j(d3Y9%yG8DH??R6+-FGGZ_EK&aPmGP!BzC_G=acE<)^sDiZef5?7 z+?K0LG#m_HYhQK9@W`E97<1IAF2s0O)RZO6$h|t`>WRy;w#gU{w)RaM3%2q)rB7;m zH3(OJ!s}qTh{bZs_i(i&w}+^SPctnypQ59ds36P`v&BhD(fq2*Y*owVUnghU7H=q? zOfGTN?&8*$mJtm6ad>WQU}610Sw_NvT^4zl02T31*AG`;Qv(-BGIJ?GC&%p@7czL< z6qcXfRJ|6da=n2e8OI(rd_9MU_ne1Iiq(o^VV-^EQ2IR~ohJN}^tiO)vr}|*!XXvU zte@%D8WJ+wbd>p7C&>?xs+?4?nd%!H?lU{tOwOrcWL6Z;b*`7AJZ6oy&VA1O&JA*I z+{E|EinaM4!$-$fevnixp9?l8p{GQTOGz>A5=P$up=ny90Bb@7%g$&5$@{pV({DapP; z*}X~6Zn^LCFH3rbZF^Si)%KRkh)>wjX!$tfp>q>C%kCas@WNC|ap9T!FCS*5RJY+XYL$|vP{DF&?n*_OT9P^dEN3wt~R8I*mXdZ>Qmx@#=@FXZ|8g5 zydpGYbdAM)(QQh{^<|RgV#TLe=3tdh$ZTEF;i<+AA97Fsv8LQ$tb?4YjXMVx~Et3 zfgfWJUn?JxF<30^^m=-pByv3B&@RihOHhsYht)zM%hcf)>KJA6(R+@cCe{0!GJ2O_ z7$NizHcVDsk%p9=s$u~WXvSfF^V+hq&eygd8-}6T_m~~J*s*QQd=5kLsVtScq7Pj7 zeFn8OQI!W)pS_VR6*KU!HE2qBUOv)hmaCtA)IHNy4~3Ue^Q3Nymc@_SUgD(>&?dL} z!Br=Xf3I?HuVr&5Im|1eYq9nGsnH(^ecn-%<9<%8wb7l#KDzd}w2)Hm7R_i%+=Czz zJ}DO_HwLfTuLnN}C4baOIh%i2Y;_l*cZtJM*niL`CPpdxID+ExasWh-wo%ma&e&Bs z1IyW`$p=pxl%zK2=0J-r730@5W3}V*^v+M!MW&n@xcZ&mh%b;WiZk(5nJgwe*Z5Ye z#VKBfg&QY)3$S-yOoWVR6v-s1>D_;kR+f9Y{r-}^QQ66v`Z#E7v{rFF_i&TvcabBi zAu}J4=j1*QGv79cy59-otU54?ed|VBy%f!9{lO<+U@y}KuXR#=I; z2&LoqsKvHtOXb?#fb-iJB^Nf$`<%U&gIZiGcvI;ib$vXahpP0Q4B4v*nzi;uVc7{f za;q68{nwVLocgIp1|9krn&b?xnT{XL@;o~~(VlTVqn+J9^MNh-jW6`3Tgw^ZekTQ1`gh%K{s##4sw?KXBj>{_FD&%XL`++ z&W(+=Xs#-(R(CJ;b@(Jc9O1Z4-I{db){(q-ae>}zzCT3IN>F-4a_+KlyB!~}{}4-J zG+sN_1GfeKkZ7~@8DCS4#cQ*H0-33j$N1h{%{WQV;`c4Wk>|-uOvjMxn(D)O{vOf? zS{IKPe^;!r4jwAfV4r-vzc&0%2b&utW}nXSp+++~829+aCp8iJV!8fT`i_ek4~+ncmW;D$sn&fA?&Jt`<%S0$QoCY98SzT zlD)Pmzc5|JEz?HAqMfynsb%J3sn!b)Ok%5HUrEX%H^EF#IU7CuYZ>DzVs`_o?riv< zf8YB32kDFo;mj^t-R*dRL;deKMn!$w836aIdoDinH5BtiRBbFJG>K*(u(TA%KHkq| zKBPo;irE+?P=s)*9_pYeDwZZ8{9+7m3LNuVg6KS!KT}iebjROqjVIfh z)RrmXa6plrm3P@ZEU7flI)U>|&VhO@8G@+d;i&a1`OhsD7G8+ljv7n;u)7U{SEU~7o=7KC| zw8l$3!DSXu^fRNSxGn?|G5P zPLUGgGR!Jup#IIn(pWjahDy<3$VIM?{mQY~^UgL(NA5(a-5jI5adWIIwzPDx_|=22 zKS&d0taiC-LHxcuekOr#W_H4)3GT$7EqOIL19gK_Qf{^c2NiAbd3SU7kg*}S0r%T5 zWx`e)iNQc%-~c?)-NMb*#lzVRirC8k)bcd<*vu~lCjcylD!~!R()^R|&%x3^(=r9H zSPWDE1>|5N;lLY5;H@JT%*F)Cs}MZgY&TOk1>wLm2L>kaUo8^|3}j3l$S4JpG&l20 zAuw4dAmy~bMFAm=y9>dB2nBDOT!=u{D3C`AzC`jP zx;aB(&Q1W|9^OF9!^VwBgkk_;tZcmipgX`5%$?{(bb?}q04|^~JQTcNm4^xeqgA28 zfUeY`!oY2eE))mQCO~n3L$ZLvETJ$WARw@tZGj5tv4_GOpa>uz2j&D!2tfc!gt-B< zfjOZtPbka_3iAeXbbu66n4YbrhYcX;&otcsvQannzWwUqzeW9RWj0m0+cE%3b|6^0 z1Dg1k1(9;!R1gkW9vDa+)Rq7eJlx62LkOAONWqfh1#F3n?dS z2O_wIv^8;3@3*4ywcCLmyq2I41> z7AW90li(cNZ4l>I8{`2fU_V%c^KbS25^$3a*uOmvXoEa%(gf)O1%(0>i2YfE3IG=k zqyyFf-A!5`9l$*N$H*@LD8;WI;1DdRqOJV{vIQ0p0VojE7U@k=zefFJ0=21_&B=jz zK(ZjyAkwzL+cm)KFQ$Y53xX2b;tOCN#0Oy*3=)b4`2}8Ifw1j@0din4;F$kl5ENhq zLE6HABDM=C#pVJ5eC(_@xd*klqhoUgL8iCM=57G+Hg^mF|GnPSGzbIxe=A!wAs_?W z0`IKBM`3dX0j7WN+n#s3-s=0c8-B+H6|?Ic+lvg;|D`DY9va9%5nEp%;Gw{|7s${D z?Jr<)He0HIFZtP0hXUcxCQKLD1V38@D00(^LI9hL+;Ur%P{09g;t_#;__MWX?U7s4 z*g}z8)7V3iThll|H{ToqpBXqoQCrhEgN|jB5a>XF1>S6LyBA<_H-EbUR^?|4Fuz;A z>!(Knw#6m}=wwh^Gj6%5tyuvlwI$q7#{^tGZH@qa6_EM{v>|}+KyS_L1qGh;&3{ut2+*d( zq>%qtG{9f}hlcvUqQM2hw_#hDzxj#ZLYe<0_-%vy2mI}}?mr-?rfpe%I?bP7!N8Pk zEy3Ub*!Q5OHopq-1RUL_I9oZj=-(*9WPmR_!0?9$h*ex1oIMK_NjTKh(y< z!^QnH4CeT&OVHWPnim2CVirqJ3*bYM-$%JvT0zYT7WTj_TNHSIlh?3!2AD64~_ypD*6kC0up+6;$cBU0sj43BREPJNI%>G zL!&^e_9q@13%n%T3Bw8fH5P;Vn->fgG+%$x!w3OU>rNOBG+BS*Vc~!29uA1{Hy#$W zt2@RDgDL2L!h}$N?JKwt1~grN;t2zq{tNc^`oe{=f2}XD>w#>h9rSQ;Am-i)`+E<; zaY8_P!k>76w*}nXP8bq2|KQ)RwM4+tf60bGA%5TY9&Q9%2cp~Nmv~yXe!z!wz@b6Y p*%>r4o7P#;$qF#!0H;5{$8+}}xOr?o{sQDdx diff --git a/ref_result/__cls_acc_ref_result_perf_darwin.txt b/ref_result/__cls_acc_ref_result_perf_darwin.txt deleted file mode 100644 index 9312471..0000000 --- a/ref_result/__cls_acc_ref_result_perf_darwin.txt +++ /dev/null @@ -1,7 +0,0 @@ - classifier exp-name accuracy±σ -Neural Network PT-1-03 91.5 ± 0.1 -Neural Network PT-1-06 91.4 ± 0.1 -Neural Network PT-1-12 91.6 ± 0.2 -Neural Network PT-2-03 91.5 ± 0.1 -Neural Network PT-2-06 91.7 ± 0.1 -Neural Network PT-2-12 91.7 ± 0.1 diff --git a/ref_result/__cls_acc_ref_result_perf_debian.txt b/ref_result/__cls_acc_ref_result_perf_debian.txt deleted file mode 100644 index 4750c12..0000000 --- a/ref_result/__cls_acc_ref_result_perf_debian.txt +++ /dev/null @@ -1,7 +0,0 @@ - classifier exp-name accuracy±σ -Neural Network PT-1-03 91.7 ± 0.3 -Neural Network PT-1-06 91.7 ± 0.3 -Neural Network PT-1-12 91.3 ± 0.0 -Neural Network PT-2-03 91.4 ± 0.3 -Neural Network PT-2-06 91.5 ± 0.1 -Neural Network PT-2-12 91.6 ± 0.2 diff --git a/ref_result/__cls_acc_ref_result_perf_windows.txt b/ref_result/__cls_acc_ref_result_perf_windows.txt deleted file mode 100644 index 74f3b1f..0000000 --- a/ref_result/__cls_acc_ref_result_perf_windows.txt +++ /dev/null @@ -1,7 +0,0 @@ - classifier exp-name accuracy±σ -Neural Network PT-1-03 91.5 ± 0.2 -Neural Network PT-1-06 91.7 ± 0.2 -Neural Network PT-1-12 91.6 ± 0.1 -Neural Network PT-2-03 91.6 ± 0.1 -Neural Network PT-2-06 91.6 ± 0.1 -Neural Network PT-2-12 91.5 ± 0.2 diff --git a/ref_result/__config_ref_result_perf_darwin.txt b/ref_result/__config_ref_result_perf_darwin.txt deleted file mode 100644 index 063d244..0000000 --- a/ref_result/__config_ref_result_perf_darwin.txt +++ /dev/null @@ -1,7 +0,0 @@ -exp-name records cls ft *val #C ⊥ P ft/P -PT-1-03 10000 2 32 27 3 0 3 2.0 -PT-1-06 10000 2 32 19 6 0 6 2.3 -PT-1-12 10000 2 32 15 12 0 12 2.5 -PT-2-03 10000 2 32 29 3 0 3 1.0 -PT-2-06 10000 2 32 26 6 0 6 1.0 -PT-2-12 10000 2 32 20 12 0 12 1.0 diff --git a/ref_result/__config_ref_result_perf_debian.txt b/ref_result/__config_ref_result_perf_debian.txt deleted file mode 100644 index 063d244..0000000 --- a/ref_result/__config_ref_result_perf_debian.txt +++ /dev/null @@ -1,7 +0,0 @@ -exp-name records cls ft *val #C ⊥ P ft/P -PT-1-03 10000 2 32 27 3 0 3 2.0 -PT-1-06 10000 2 32 19 6 0 6 2.3 -PT-1-12 10000 2 32 15 12 0 12 2.5 -PT-2-03 10000 2 32 29 3 0 3 1.0 -PT-2-06 10000 2 32 26 6 0 6 1.0 -PT-2-12 10000 2 32 20 12 0 12 1.0 diff --git a/ref_result/__config_ref_result_perf_windows.txt b/ref_result/__config_ref_result_perf_windows.txt deleted file mode 100644 index 063d244..0000000 --- a/ref_result/__config_ref_result_perf_windows.txt +++ /dev/null @@ -1,7 +0,0 @@ -exp-name records cls ft *val #C ⊥ P ft/P -PT-1-03 10000 2 32 27 3 0 3 2.0 -PT-1-06 10000 2 32 19 6 0 6 2.3 -PT-1-12 10000 2 32 15 12 0 12 2.5 -PT-2-03 10000 2 32 29 3 0 3 1.0 -PT-2-06 10000 2 32 26 6 0 6 1.0 -PT-2-12 10000 2 32 20 12 0 12 1.0 diff --git a/ref_result/__plot_ref_result_perf_darwin.txt b/ref_result/__plot_ref_result_perf_darwin.txt deleted file mode 100644 index 96e5c7b..0000000 --- a/ref_result/__plot_ref_result_perf_darwin.txt +++ /dev/null @@ -1,19 +0,0 @@ - classifier exp-name attack accuracy evades valid -Neural Network PT-1-03 PGD 91.6 89.8 78.6 -Neural Network PT-1-03 CPGD 91.5 59.0 26.2 -Neural Network PT-1-03 VPGD 91.3 91.3 91.3 -Neural Network PT-1-06 PGD 91.3 90.4 76.6 -Neural Network PT-1-06 VPGD 91.5 91.5 91.5 -Neural Network PT-1-06 CPGD 91.5 63.3 12.5 -Neural Network PT-1-12 PGD 91.9 90.3 68.9 -Neural Network PT-1-12 CPGD 91.6 73.1 3.8 -Neural Network PT-1-12 VPGD 91.5 91.5 91.5 -Neural Network PT-2-03 PGD 91.5 89.2 89.2 -Neural Network PT-2-03 CPGD 91.6 72.2 0.0 -Neural Network PT-2-03 VPGD 91.5 91.4 91.4 -Neural Network PT-2-06 PGD 91.6 89.2 89.2 -Neural Network PT-2-06 VPGD 91.6 91.6 91.6 -Neural Network PT-2-06 CPGD 91.8 73.7 0.0 -Neural Network PT-2-12 PGD 91.7 89.3 89.3 -Neural Network PT-2-12 CPGD 91.6 59.6 0.0 -Neural Network PT-2-12 VPGD 91.7 91.7 91.7 diff --git a/ref_result/__plot_ref_result_perf_debian.txt b/ref_result/__plot_ref_result_perf_debian.txt deleted file mode 100644 index 4879492..0000000 --- a/ref_result/__plot_ref_result_perf_debian.txt +++ /dev/null @@ -1,19 +0,0 @@ - classifier exp-name attack accuracy evades valid -Neural Network PT-1-03 PGD 91.3 90.9 79.0 -Neural Network PT-1-03 VPGD 91.8 91.8 91.8 -Neural Network PT-1-03 CPGD 91.9 59.2 34.5 -Neural Network PT-1-06 PGD 91.4 88.7 70.5 -Neural Network PT-1-06 VPGD 91.8 91.8 91.8 -Neural Network PT-1-06 CPGD 91.9 59.4 21.8 -Neural Network PT-1-12 PGD 91.3 90.2 71.7 -Neural Network PT-1-12 CPGD 91.3 68.0 3.0 -Neural Network PT-1-12 VPGD 91.3 91.3 91.3 -Neural Network PT-2-03 PGD 91.2 89.1 89.1 -Neural Network PT-2-03 VPGD 91.3 91.3 91.3 -Neural Network PT-2-03 CPGD 91.8 64.2 0.0 -Neural Network PT-2-06 PGD 91.5 91.1 91.1 -Neural Network PT-2-06 VPGD 91.5 91.5 91.5 -Neural Network PT-2-06 CPGD 91.6 60.0 0.0 -Neural Network PT-2-12 PGD 91.4 88.8 88.8 -Neural Network PT-2-12 VPGD 91.7 91.7 91.7 -Neural Network PT-2-12 CPGD 91.6 74.6 0.0 diff --git a/ref_result/__plot_ref_result_perf_windows.txt b/ref_result/__plot_ref_result_perf_windows.txt deleted file mode 100644 index eb877bf..0000000 --- a/ref_result/__plot_ref_result_perf_windows.txt +++ /dev/null @@ -1,19 +0,0 @@ - classifier exp-name attack accuracy evades valid -Neural Network PT-1-03 PGD 91.3 90.7 77.2 -Neural Network PT-1-03 VPGD 91.7 91.7 91.7 -Neural Network PT-1-03 CPGD 91.4 74.9 36.9 -Neural Network PT-1-06 PGD 91.5 89.9 73.2 -Neural Network PT-1-06 CPGD 91.8 61.3 23.1 -Neural Network PT-1-06 VPGD 91.9 91.9 91.9 -Neural Network PT-1-12 PGD 91.6 90.4 74.6 -Neural Network PT-1-12 CPGD 91.5 69.8 0.0 -Neural Network PT-1-12 VPGD 91.7 91.7 91.7 -Neural Network PT-2-03 PGD 91.4 88.8 88.8 -Neural Network PT-2-03 CPGD 91.5 66.7 0.4 -Neural Network PT-2-03 VPGD 91.7 91.7 91.7 -Neural Network PT-2-06 PGD 91.6 88.0 88.0 -Neural Network PT-2-06 CPGD 91.7 57.7 0.0 -Neural Network PT-2-06 VPGD 91.4 91.4 91.4 -Neural Network PT-2-12 PGD 91.5 90.6 90.6 -Neural Network PT-2-12 VPGD 91.3 91.3 91.3 -Neural Network PT-2-12 CPGD 91.7 69.3 0.0 diff --git a/ref_result/__time_ref_result_perf_darwin.txt b/ref_result/__time_ref_result_perf_darwin.txt deleted file mode 100644 index 5a5d148..0000000 --- a/ref_result/__time_ref_result_perf_darwin.txt +++ /dev/null @@ -1,19 +0,0 @@ - classifier exp-name attack t0 t(s) t-ratio -Neural Network PT-1-03 PGD 17 17 1.00 -Neural Network PT-1-03 CPGD 17 28 1.70 -Neural Network PT-1-03 VPGD 17 28 1.68 -Neural Network PT-1-06 PGD 18 18 1.00 -Neural Network PT-1-06 VPGD 18 33 1.82 -Neural Network PT-1-06 CPGD 18 35 1.93 -Neural Network PT-1-12 PGD 18 18 1.00 -Neural Network PT-1-12 CPGD 18 49 2.70 -Neural Network PT-1-12 VPGD 18 46 2.52 -Neural Network PT-2-03 PGD 19 19 1.00 -Neural Network PT-2-03 CPGD 19 30 1.64 -Neural Network PT-2-03 VPGD 19 28 1.52 -Neural Network PT-2-06 PGD 17 17 1.00 -Neural Network PT-2-06 VPGD 17 34 2.01 -Neural Network PT-2-06 CPGD 17 36 2.12 -Neural Network PT-2-12 PGD 17 17 1.00 -Neural Network PT-2-12 CPGD 17 48 2.73 -Neural Network PT-2-12 VPGD 17 46 2.66 diff --git a/ref_result/__time_ref_result_perf_debian.txt b/ref_result/__time_ref_result_perf_debian.txt deleted file mode 100644 index da5cf61..0000000 --- a/ref_result/__time_ref_result_perf_debian.txt +++ /dev/null @@ -1,19 +0,0 @@ - classifier exp-name attack t0 t(s) t-ratio -Neural Network PT-1-03 PGD 66 66 1.00 -Neural Network PT-1-03 VPGD 66 95 1.44 -Neural Network PT-1-03 CPGD 66 132 2.01 -Neural Network PT-1-06 PGD 66 66 1.00 -Neural Network PT-1-06 VPGD 66 112 1.71 -Neural Network PT-1-06 CPGD 66 162 2.48 -Neural Network PT-1-12 PGD 74 74 1.00 -Neural Network PT-1-12 CPGD 74 214 2.89 -Neural Network PT-1-12 VPGD 74 146 1.97 -Neural Network PT-2-03 PGD 69 69 1.00 -Neural Network PT-2-03 VPGD 69 93 1.35 -Neural Network PT-2-03 CPGD 69 138 2.01 -Neural Network PT-2-06 PGD 65 65 1.00 -Neural Network PT-2-06 VPGD 65 110 1.70 -Neural Network PT-2-06 CPGD 65 156 2.41 -Neural Network PT-2-12 PGD 65 65 1.00 -Neural Network PT-2-12 VPGD 65 144 2.23 -Neural Network PT-2-12 CPGD 65 211 3.27 diff --git a/ref_result/__time_ref_result_perf_windows.txt b/ref_result/__time_ref_result_perf_windows.txt deleted file mode 100644 index 75400f8..0000000 --- a/ref_result/__time_ref_result_perf_windows.txt +++ /dev/null @@ -1,19 +0,0 @@ - classifier exp-name attack t0 t(s) t-ratio -Neural Network PT-1-03 PGD 20 20 1.00 -Neural Network PT-1-03 VPGD 20 35 1.76 -Neural Network PT-1-03 CPGD 20 42 2.11 -Neural Network PT-1-06 PGD 21 21 1.00 -Neural Network PT-1-06 CPGD 21 54 2.61 -Neural Network PT-1-06 VPGD 21 45 2.18 -Neural Network PT-1-12 PGD 19 19 1.00 -Neural Network PT-1-12 CPGD 19 73 3.83 -Neural Network PT-1-12 VPGD 19 61 3.19 -Neural Network PT-2-03 PGD 20 20 1.00 -Neural Network PT-2-03 CPGD 20 45 2.22 -Neural Network PT-2-03 VPGD 20 38 1.88 -Neural Network PT-2-06 PGD 21 21 1.00 -Neural Network PT-2-06 CPGD 21 53 2.58 -Neural Network PT-2-06 VPGD 21 46 2.24 -Neural Network PT-2-12 PGD 21 21 1.00 -Neural Network PT-2-12 VPGD 21 61 2.95 -Neural Network PT-2-12 CPGD 21 72 3.45 diff --git a/ref_result/perf/darwin/PT-1-03_dnn_cpgd_auto_988.yaml b/ref_result/perf/darwin/PT-1-03_dnn_cpgd_auto_988.yaml deleted file mode 100644 index e4c92e7..0000000 --- a/ref_result/perf/darwin/PT-1-03_dnn_cpgd_auto_988.yaml +++ /dev/null @@ -1,113 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf1 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696859589002925000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf1_1.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 correlated constraints - duration_sec: 141.8 - end: 1696859587987899000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-03 - start: 1696859446142789000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 17.3 - cpu_usage_per_core: [19.0, 17.8, 2.9, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 5.30GB - virtual_mem_percentage: 66.8 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.76GB -folds: - accuracy: [0.9095, 0.93, 0.9095, 0.912, 0.916] - f_score: [0.907417, 0.931305, 0.907417, 0.9133, 0.913934] - n_evasions: [1819, 986, 1140, 1576, 375] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1251, 1115, 1200, 1106, 1029] - n_valid_evades: [1121, 339, 409, 725, 22] - precision: [0.910678, 0.927664, 0.933684, 0.908824, 0.926272] - recall: [0.904179, 0.934975, 0.882587, 0.917822, 0.901921] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/PT-1-03_dnn_pgd_40_853.yaml b/ref_result/perf/darwin/PT-1-03_dnn_pgd_40_853.yaml deleted file mode 100644 index 1a2a2a4..0000000 --- a/ref_result/perf/darwin/PT-1-03_dnn_pgd_40_853.yaml +++ /dev/null @@ -1,111 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696859442871337000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf1_1.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 correlated constraints - duration_sec: 139.5 - end: 1696859441853040000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-03 - start: 1696859302380299000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 17.6 - cpu_usage_per_core: [29.3, 32.3, 10.8, 6.0, 1.0, 2.0, 2.0, 0.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 5.19GB - virtual_mem_percentage: 67.6 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.92GB -folds: - accuracy: [0.9145, 0.9145, 0.903, 0.921, 0.914] - f_score: [0.9128, 0.915639, 0.900615, 0.92004, 0.916747] - n_evasions: [1829, 1829, 1806, 1842, 1828] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1829, 1829, 1806, 1842, 1828] - precision: [0.928423, 0.900097, 0.892386, 0.948852, 0.907088] - recall: [0.897693, 0.931727, 0.908997, 0.892927, 0.926614] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/PT-1-06_dnn_cpgd_auto_053.yaml b/ref_result/perf/darwin/PT-1-06_dnn_cpgd_auto_053.yaml deleted file mode 100644 index 5b58ed8..0000000 --- a/ref_result/perf/darwin/PT-1-06_dnn_cpgd_auto_053.yaml +++ /dev/null @@ -1,128 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf2 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696860026068116000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf1_2.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 correlated constraints - duration_sec: 175.8 - end: 1696860025052474000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-06 - start: 1696859849262374000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 18.5 - cpu_usage_per_core: [20.0, 18.0, 28.0, 13.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.64GB - virtual_mem_percentage: 71.0 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.56GB -folds: - accuracy: [0.916, 0.9165, 0.9025, 0.9255, 0.9145] - f_score: [0.918208, 0.917204, 0.906115, 0.927282, 0.910612] - n_evasions: [737, 1473, 1423, 1826, 867] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1061, 1096, 505, 157, 825] - n_valid_evades: [267, 648, 147, 136, 51] - precision: [0.921799, 0.887716, 0.871296, 0.910834, 0.944685] - recall: [0.914646, 0.948718, 0.943831, 0.944334, 0.87891] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - d: [3, 4, 27] - e: [8, 28] - f: [16, 17, 23] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - reset_strategy: 2 diff --git a/ref_result/perf/darwin/PT-1-06_dnn_pgd_40_831.yaml b/ref_result/perf/darwin/PT-1-06_dnn_pgd_40_831.yaml deleted file mode 100644 index 8bc8c59..0000000 --- a/ref_result/perf/darwin/PT-1-06_dnn_pgd_40_831.yaml +++ /dev/null @@ -1,126 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696859845846465000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf1_2.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 correlated constraints - duration_sec: 165.4 - end: 1696859844830778000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-06 - start: 1696859679459713000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 24.2 - cpu_usage_per_core: [18.0, 17.2, 6.0, 4.0, 5.0, 2.0, 0.0, 0.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.75GB - virtual_mem_percentage: 70.3 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.72GB -folds: - accuracy: [0.916, 0.9075, 0.919, 0.92, 0.912] - f_score: [0.914198, 0.9098, 0.917933, 0.919436, 0.906383] - n_evasions: [1832, 1815, 1838, 1840, 1824] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1832, 1815, 1838, 1840, 1824] - precision: [0.923633, 0.917404, 0.939834, 0.91483, 0.945616] - recall: [0.904954, 0.902321, 0.89703, 0.924089, 0.870276] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - d: [3, 4, 27] - e: [8, 28] - f: [16, 17, 23] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - reset_strategy: 2 diff --git a/ref_result/perf/darwin/PT-1-12_dnn_cpgd_auto_769.yaml b/ref_result/perf/darwin/PT-1-12_dnn_cpgd_auto_769.yaml deleted file mode 100644 index 7a8ba4e..0000000 --- a/ref_result/perf/darwin/PT-1-12_dnn_cpgd_auto_769.yaml +++ /dev/null @@ -1,158 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf3 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696860603786440000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf1_3.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 correlated constraints - duration_sec: 244.8 - end: 1696860602769315000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-12 - start: 1696860357995174000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 23.8 - cpu_usage_per_core: [18.0, 18.0, 5.0, 3.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.42GB - virtual_mem_percentage: 72.4 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.26GB -folds: - accuracy: [0.91, 0.9235, 0.9215, 0.911, 0.9155] - f_score: [0.909639, 0.923615, 0.924191, 0.910101, 0.91239] - n_evasions: [1500, 1324, 1173, 1792, 1517] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1, 54, 791, 58, 77] - n_valid_evades: [0, 48, 214, 47, 73] - precision: [0.921668, 0.922233, 0.891061, 0.920327, 0.940171] - recall: [0.897919, 0.925, 0.95988, 0.9001, 0.886203] -invalid_rows: [] -validation: - dependencies: - a: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - b: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - c: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - d: [3, 4, 27] - e: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - f: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - g: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - h: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - i: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - j: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - k: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - l: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - g: - attrs: {9: state_INT, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= a[2])' - text: state_INT != 1 or (smean >= dmean) - h: - attrs: {8: state_FIN, 10: state_other, 24: trans_depth} - exec: 'lambda a: a[2] != 1 or a[0] == 1 or a[1] == 1' - text: trans_depth != 1 or state_FIN == 1 or state_other == 1 - i: - attrs: {18: swin, 21: dwin, 23: dmean} - exec: 'lambda a: (a[0] == a[1] == 0 ) or a[2] >= 0' - text: (swin == dwin == 0 ) or dmean >= 0 - j: - attrs: {8: state_FIN, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or smean >= 1 - k: - attrs: {10: state_other, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= 0 and a[2] >= 0)' - text: state_other != 1 or (smean >= 0 and dmean >= 0) - l: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] >= 0 and a[1] >= 0' - text: dbytes >= 0 and dloss >= 0 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/PT-1-12_dnn_pgd_40_624.yaml b/ref_result/perf/darwin/PT-1-12_dnn_pgd_40_624.yaml deleted file mode 100644 index 3f08dc2..0000000 --- a/ref_result/perf/darwin/PT-1-12_dnn_pgd_40_624.yaml +++ /dev/null @@ -1,156 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696860354638337000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf1_3.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 correlated constraints - duration_sec: 228.7 - end: 1696860353623584000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-12 - start: 1696860124901074000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 21.6 - cpu_usage_per_core: [27.7, 27.3, 17.0, 4.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.65GB - virtual_mem_percentage: 70.9 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.26GB -folds: - accuracy: [0.9175, 0.908, 0.9185, 0.9205, 0.908] - f_score: [0.915687, 0.905834, 0.91506, 0.921559, 0.912631] - n_evasions: [1835, 1816, 1837, 1841, 1816] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1835, 1816, 1837, 1841, 1816] - precision: [0.946146, 0.93453, 0.91841, 0.925669, 0.870471] - recall: [0.887129, 0.878848, 0.911734, 0.917485, 0.959082] -invalid_rows: [] -validation: - dependencies: - a: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - b: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - c: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - d: [3, 4, 27] - e: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - f: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - g: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - h: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - i: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - j: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - k: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - l: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - g: - attrs: {9: state_INT, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= a[2])' - text: state_INT != 1 or (smean >= dmean) - h: - attrs: {8: state_FIN, 10: state_other, 24: trans_depth} - exec: 'lambda a: a[2] != 1 or a[0] == 1 or a[1] == 1' - text: trans_depth != 1 or state_FIN == 1 or state_other == 1 - i: - attrs: {18: swin, 21: dwin, 23: dmean} - exec: 'lambda a: (a[0] == a[1] == 0 ) or a[2] >= 0' - text: (swin == dwin == 0 ) or dmean >= 0 - j: - attrs: {8: state_FIN, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or smean >= 1 - k: - attrs: {10: state_other, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= 0 and a[2] >= 0)' - text: state_other != 1 or (smean >= 0 and dmean >= 0) - l: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] >= 0 and a[1] >= 0' - text: dbytes >= 0 and dloss >= 0 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/PT-2-03_dnn_cpgd_auto_195.yaml b/ref_result/perf/darwin/PT-2-03_dnn_cpgd_auto_195.yaml deleted file mode 100644 index 8d6012a..0000000 --- a/ref_result/perf/darwin/PT-2-03_dnn_cpgd_auto_195.yaml +++ /dev/null @@ -1,113 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf4 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696861000211000000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf2_4.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 unrelated constraints - duration_sec: 151.6 - end: 1696860999195115000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-03 - start: 1696860847633473000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 19.0 - cpu_usage_per_core: [16.0, 17.8, 13.9, 8.9, 8.9, 3.0, 3.0, 2.0, 1.0, 1.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.28GB - virtual_mem_percentage: 73.3 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.14GB -folds: - accuracy: [0.912, 0.9095, 0.916, 0.919, 0.9255] - f_score: [0.911111, 0.906749, 0.917404, 0.919081, 0.925088] - n_evasions: [1557, 1138, 1244, 1788, 1491] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1, 2, 0, 0, 0] - n_valid_evades: [1, 2, 0, 0, 0] - precision: [0.941545, 0.92437, 0.882687, 0.936864, 0.922768] - recall: [0.882583, 0.889788, 0.954964, 0.901961, 0.927419] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/PT-2-03_dnn_pgd_40_278.yaml b/ref_result/perf/darwin/PT-2-03_dnn_pgd_40_278.yaml deleted file mode 100644 index 15a8d99..0000000 --- a/ref_result/perf/darwin/PT-2-03_dnn_pgd_40_278.yaml +++ /dev/null @@ -1,111 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696860844294663000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf2_4.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 unrelated constraints - duration_sec: 141.1 - end: 1696860843277492000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-03 - start: 1696860702140520000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 21.1 - cpu_usage_per_core: [57.6, 56.0, 5.9, 2.0, 2.0, 1.0, 0.0, 0.0, 1.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.33GB - virtual_mem_percentage: 73.0 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.05GB -folds: - accuracy: [0.9075, 0.9185, 0.9135, 0.918, 0.915] - f_score: [0.90918, 0.916109, 0.914904, 0.915464, 0.913088] - n_evasions: [1815, 1837, 1826, 1836, 1830] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1815, 1837, 1826, 1836, 1830] - precision: [0.908734, 0.931937, 0.902913, 0.921162, 0.94899] - recall: [0.909627, 0.90081, 0.927218, 0.909836, 0.879803] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/PT-2-06_dnn_cpgd_auto_453.yaml b/ref_result/perf/darwin/PT-2-06_dnn_cpgd_auto_453.yaml deleted file mode 100644 index d0596c0..0000000 --- a/ref_result/perf/darwin/PT-2-06_dnn_cpgd_auto_453.yaml +++ /dev/null @@ -1,128 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf5 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696861457468820000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf2_5.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 unrelated constraints - duration_sec: 180.8 - end: 1696861456453057000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-06 - start: 1696861275650579000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 20.5 - cpu_usage_per_core: [26.0, 23.2, 6.0, 5.9, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.01GB - virtual_mem_percentage: 74.9 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.01GB -folds: - accuracy: [0.906, 0.918, 0.9155, 0.921, 0.9285] - f_score: [0.902692, 0.919529, 0.913643, 0.922776, 0.927668] - n_evasions: [1078, 1812, 1666, 1778, 1033] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [4, 0, 0, 0, 0] - n_valid_evades: [0, 0, 0, 0, 0] - precision: [0.923729, 0.918627, 0.90121, 0.940239, 0.926263] - recall: [0.882591, 0.920432, 0.926425, 0.90595, 0.929078] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/PT-2-06_dnn_pgd_40_213.yaml b/ref_result/perf/darwin/PT-2-06_dnn_pgd_40_213.yaml deleted file mode 100644 index 532b275..0000000 --- a/ref_result/perf/darwin/PT-2-06_dnn_pgd_40_213.yaml +++ /dev/null @@ -1,126 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696861272227895000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf2_5.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 unrelated constraints - duration_sec: 170.7 - end: 1696861271213152000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-06 - start: 1696861100503413000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 21.5 - cpu_usage_per_core: [19.0, 22.0, 9.0, 12.0, 3.0, 4.0, 5.0, 1.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.14GB - virtual_mem_percentage: 74.1 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.18GB -folds: - accuracy: [0.923, 0.909, 0.92, 0.9035, 0.927] - f_score: [0.920455, 0.905208, 0.921337, 0.903355, 0.927218] - n_evasions: [1846, 1818, 1840, 1807, 1854] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1846, 1818, 1840, 1807, 1854] - precision: [0.930063, 0.924468, 0.94551, 0.893954, 0.934673] - recall: [0.911043, 0.886735, 0.89837, 0.912955, 0.919881] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/PT-2-12_dnn_cpgd_auto_275.yaml b/ref_result/perf/darwin/PT-2-12_dnn_cpgd_auto_275.yaml deleted file mode 100644 index 974e599..0000000 --- a/ref_result/perf/darwin/PT-2-12_dnn_cpgd_auto_275.yaml +++ /dev/null @@ -1,158 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf6 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696862025289364000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf2_6.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 unrelated constraints - duration_sec: 237.7 - end: 1696862024275174000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-12 - start: 1696861786566081000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 19.1 - cpu_usage_per_core: [14.9, 13.1, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.10GB - virtual_mem_percentage: 74.4 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 5.81GB -folds: - accuracy: [0.9155, 0.914, 0.918, 0.917, 0.916] - f_score: [0.913643, 0.915438, 0.915114, 0.917495, 0.91498] - n_evasions: [819, 1244, 1801, 1252, 843] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [66, 0, 0, 0, 0] - n_valid_evades: [0, 0, 0, 0, 0] - precision: [0.922601, 0.908293, 0.934461, 0.922078, 0.931959] - recall: [0.904858, 0.922696, 0.896552, 0.912957, 0.898608] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - g: [0] - h: [25] - i: [26] - j: [27] - k: [28] - l: [29] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - g: - attrs: {0: dur} - exec: 'lambda a: a[0] >= 0 and a[0] <= 60' - text: dur >= 0 and dur <= 60 - h: - attrs: {25: ct_srv_src} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_srv_src >= 1 and ct_srv_src <= 63 - i: - attrs: {26: ct_dst_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_dst_ltm >= 1 and ct_dst_ltm <= 50 - j: - attrs: {27: ct_src_dport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_src_dport_ltm >= 1 and ct_src_dport_ltm <= 50 - k: - attrs: {28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 46' - text: ct_dst_sport_ltm >= 1 and ct_dst_sport_ltm <= 46 - l: - attrs: {29: ct_dst_src_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_dst_src_ltm >= 1 and ct_dst_src_ltm <= 63 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/PT-2-12_dnn_pgd_40_110.yaml b/ref_result/perf/darwin/PT-2-12_dnn_pgd_40_110.yaml deleted file mode 100644 index 0fcbd7f..0000000 --- a/ref_result/perf/darwin/PT-2-12_dnn_pgd_40_110.yaml +++ /dev/null @@ -1,156 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696861783125036000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf2_6.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 unrelated constraints - duration_sec: 231.7 - end: 1696861782110412000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-12 - start: 1696861550450529000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 19.0 - cpu_usage_per_core: [17.0, 15.2, 19.8, 14.9, 10.1, 10.0, 12.0, 6.9, 2.0, 2.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.05GB - virtual_mem_percentage: 74.7 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.03GB -folds: - accuracy: [0.9285, 0.922, 0.916, 0.913, 0.9075] - f_score: [0.929383, 0.922696, 0.915066, 0.908131, 0.910843] - n_evasions: [1857, 1844, 1832, 1826, 1815] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1857, 1844, 1832, 1826, 1815] - precision: [0.94006, 0.911851, 0.924413, 0.914894, 0.900858] - recall: [0.918945, 0.933801, 0.905906, 0.901468, 0.921053] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - g: [0] - h: [25] - i: [26] - j: [27] - k: [28] - l: [29] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - g: - attrs: {0: dur} - exec: 'lambda a: a[0] >= 0 and a[0] <= 60' - text: dur >= 0 and dur <= 60 - h: - attrs: {25: ct_srv_src} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_srv_src >= 1 and ct_srv_src <= 63 - i: - attrs: {26: ct_dst_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_dst_ltm >= 1 and ct_dst_ltm <= 50 - j: - attrs: {27: ct_src_dport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_src_dport_ltm >= 1 and ct_src_dport_ltm <= 50 - k: - attrs: {28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 46' - text: ct_dst_sport_ltm >= 1 and ct_dst_sport_ltm <= 46 - l: - attrs: {29: ct_dst_src_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_dst_src_ltm >= 1 and ct_dst_src_ltm <= 63 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/REG_PT-1-03_dnn_pgd_40_346.yaml b/ref_result/perf/darwin/REG_PT-1-03_dnn_pgd_40_346.yaml deleted file mode 100644 index 674bf66..0000000 --- a/ref_result/perf/darwin/REG_PT-1-03_dnn_pgd_40_346.yaml +++ /dev/null @@ -1,111 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696859676361295000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf1_1.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 correlated constraints - duration_sec: 83.2 - end: 1696859675345709000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-03 - start: 1696859592190179000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 22.3 - cpu_usage_per_core: [43.4, 41.4, 6.0, 2.0, 2.0, 1.0, 1.0, 1.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 5.28GB - virtual_mem_percentage: 67.0 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.84GB -folds: - accuracy: [0.9225, 0.9185, 0.91, 0.913, 0.915] - f_score: [0.923305, 0.917718, 0.905956, 0.910217, 0.915254] - n_evasions: [1842, 1742, 1749, 1826, 1820] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1700, 1747, 1720, 1784, 1764] - n_valid_evades: [1553, 1558, 1523, 1622, 1604] - precision: [0.918307, 0.92472, 0.950658, 0.922594, 0.924471] - recall: [0.928358, 0.910822, 0.865269, 0.898167, 0.906219] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - enabled: false - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/REG_PT-1-06_dnn_pgd_40_325.yaml b/ref_result/perf/darwin/REG_PT-1-06_dnn_pgd_40_325.yaml deleted file mode 100644 index c4f0088..0000000 --- a/ref_result/perf/darwin/REG_PT-1-06_dnn_pgd_40_325.yaml +++ /dev/null @@ -1,126 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696860121341254000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf1_2.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 correlated constraints - duration_sec: 90.9 - end: 1696860120324971000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-06 - start: 1696860029401263000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 24.5 - cpu_usage_per_core: [17.3, 17.2, 14.9, 5.9, 4.0, 1.0, 1.0, 0.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.48GB - virtual_mem_percentage: 72.0 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.52GB -folds: - accuracy: [0.906, 0.9205, 0.9075, 0.9135, 0.92] - f_score: [0.908203, 0.921713, 0.910671, 0.907139, 0.918534] - n_evasions: [1812, 1840, 1805, 1809, 1772] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1747, 1718, 1676, 1600, 1724] - n_valid_evades: [1582, 1580, 1504, 1448, 1547] - precision: [0.902913, 0.911392, 0.889623, 0.95805, 0.92229] - recall: [0.913556, 0.932271, 0.93274, 0.861366, 0.914807] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - d: [3, 4, 27] - e: [8, 28] - f: [16, 17, 23] - enabled: false - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - reset_strategy: 2 diff --git a/ref_result/perf/darwin/REG_PT-1-12_dnn_pgd_40_775.yaml b/ref_result/perf/darwin/REG_PT-1-12_dnn_pgd_40_775.yaml deleted file mode 100644 index 79a7b6b..0000000 --- a/ref_result/perf/darwin/REG_PT-1-12_dnn_pgd_40_775.yaml +++ /dev/null @@ -1,156 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696860698792071000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf1_3.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 correlated constraints - duration_sec: 90.6 - end: 1696860697775081000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-12 - start: 1696860607219241000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 25.8 - cpu_usage_per_core: [16.8, 15.2, 5.9, 4.9, 2.0, 2.0, 2.0, 1.0, 1.0, 1.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.35GB - virtual_mem_percentage: 72.8 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.13GB -folds: - accuracy: [0.927, 0.914, 0.9145, 0.9155, 0.9225] - f_score: [0.928008, 0.910973, 0.910705, 0.913907, 0.920062] - n_evasions: [1750, 1828, 1779, 1831, 1842] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1513, 1577, 1595, 1392, 1525] - n_valid_evades: [1357, 1439, 1430, 1257, 1409] - precision: [0.926181, 0.91858, 0.941685, 0.952229, 0.954011] - recall: [0.929842, 0.903491, 0.881699, 0.87855, 0.888446] -invalid_rows: [] -validation: - dependencies: - a: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - b: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - c: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - d: [3, 4, 27] - e: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - f: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - g: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - h: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - i: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - j: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - k: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - l: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - enabled: false - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - g: - attrs: {9: state_INT, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= a[2])' - text: state_INT != 1 or (smean >= dmean) - h: - attrs: {8: state_FIN, 10: state_other, 24: trans_depth} - exec: 'lambda a: a[2] != 1 or a[0] == 1 or a[1] == 1' - text: trans_depth != 1 or state_FIN == 1 or state_other == 1 - i: - attrs: {18: swin, 21: dwin, 23: dmean} - exec: 'lambda a: (a[0] == a[1] == 0 ) or a[2] >= 0' - text: (swin == dwin == 0 ) or dmean >= 0 - j: - attrs: {8: state_FIN, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or smean >= 1 - k: - attrs: {10: state_other, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= 0 and a[2] >= 0)' - text: state_other != 1 or (smean >= 0 and dmean >= 0) - l: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] >= 0 and a[1] >= 0' - text: dbytes >= 0 and dloss >= 0 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/REG_PT-2-03_dnn_pgd_40_256.yaml b/ref_result/perf/darwin/REG_PT-2-03_dnn_pgd_40_256.yaml deleted file mode 100644 index 1faf82e..0000000 --- a/ref_result/perf/darwin/REG_PT-2-03_dnn_pgd_40_256.yaml +++ /dev/null @@ -1,111 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696861097272091000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf2_4.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 unrelated constraints - duration_sec: 92.6 - end: 1696861096256190000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-03 - start: 1696861003632564000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 23.2 - cpu_usage_per_core: [25.0, 22.2, 6.9, 3.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.26GB - virtual_mem_percentage: 73.4 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 6.09GB -folds: - accuracy: [0.9175, 0.9045, 0.9135, 0.9255, 0.914] - f_score: [0.916031, 0.911039, 0.914568, 0.925537, 0.909091] - n_evasions: [1746, 1757, 1772, 1843, 1807] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1746, 1757, 1772, 1843, 1807] - precision: [0.912779, 0.870107, 0.921393, 0.935354, 0.92973] - recall: [0.919305, 0.956012, 0.907843, 0.915925, 0.889349] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - enabled: false - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/REG_PT-2-06_dnn_pgd_40_002.yaml b/ref_result/perf/darwin/REG_PT-2-06_dnn_pgd_40_002.yaml deleted file mode 100644 index 867ba21..0000000 --- a/ref_result/perf/darwin/REG_PT-2-06_dnn_pgd_40_002.yaml +++ /dev/null @@ -1,126 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696861547017585000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf2_5.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 unrelated constraints - duration_sec: 85.1 - end: 1696861546001932000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-06 - start: 1696861460889791000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 20.1 - cpu_usage_per_core: [15.8, 17.8, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.21GB - virtual_mem_percentage: 73.7 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 5.99GB -folds: - accuracy: [0.912, 0.9105, 0.9135, 0.9195, 0.9225] - f_score: [0.911111, 0.907013, 0.914736, 0.918233, 0.920716] - n_evasions: [1819, 1812, 1818, 1633, 1835] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1819, 1812, 1818, 1633, 1835] - precision: [0.902, 0.956188, 0.916996, 0.928131, 0.938478] - recall: [0.920408, 0.862648, 0.912488, 0.908543, 0.903614] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - enabled: false - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - reset_strategy: 2 diff --git a/ref_result/perf/darwin/REG_PT-2-12_dnn_pgd_40_898.yaml b/ref_result/perf/darwin/REG_PT-2-12_dnn_pgd_40_898.yaml deleted file mode 100644 index d98898f..0000000 --- a/ref_result/perf/darwin/REG_PT-2-12_dnn_pgd_40_898.yaml +++ /dev/null @@ -1,156 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696862116919013000 - class_distribution: {0: 5000, 1: 5000} - config: config//prf2_6.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 unrelated constraints - duration_sec: 87.1 - end: 1696862115897853000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-12 - start: 1696862028754498000 - system: - cpu_physical_cores: 10 - cpu_total_cores: 10 - cpu_total_usage: 23.7 - cpu_usage_per_core: [38.0, 36.0, 9.0, 4.0, 2.0, 2.0, 2.0, 2.0, 1.0, 0.0] - machine: arm64 - processor: arm - python_runtime: 3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 - (clang-1300.0.29.30)] - release: 22.2.0 - system: Darwin - version: 'Darwin Kernel Version 22.2.0: Fri Nov 11 02:03:51 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T6000' - virtual_mem_available: 4.14GB - virtual_mem_percentage: 74.1 - virtual_mem_total_size: 16.00GB - virtual_mem_used: 5.97GB -folds: - accuracy: [0.923, 0.905, 0.9165, 0.9155, 0.9235] - f_score: [0.921668, 0.901452, 0.917774, 0.916378, 0.92261] - n_evasions: [1797, 1721, 1786, 1830, 1793] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1797, 1721, 1786, 1830, 1793] - precision: [0.90329, 0.950766, 0.914622, 0.92323, 0.926829] - recall: [0.94081, 0.857002, 0.920949, 0.909627, 0.918429] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - g: [0] - h: [25] - i: [26] - j: [27] - k: [28] - l: [29] - enabled: false - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - g: - attrs: {0: dur} - exec: 'lambda a: a[0] >= 0 and a[0] <= 60' - text: dur >= 0 and dur <= 60 - h: - attrs: {25: ct_srv_src} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_srv_src >= 1 and ct_srv_src <= 63 - i: - attrs: {26: ct_dst_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_dst_ltm >= 1 and ct_dst_ltm <= 50 - j: - attrs: {27: ct_src_dport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_src_dport_ltm >= 1 and ct_src_dport_ltm <= 50 - k: - attrs: {28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 46' - text: ct_dst_sport_ltm >= 1 and ct_dst_sport_ltm <= 46 - l: - attrs: {29: ct_dst_src_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_dst_src_ltm >= 1 and ct_dst_src_ltm <= 63 - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-1-03_dnn_cpgd_auto_118.yaml b/ref_result/perf/debian/PT-1-03_dnn_cpgd_auto_118.yaml deleted file mode 100644 index 8574fbd..0000000 --- a/ref_result/perf/debian/PT-1-03_dnn_cpgd_auto_118.yaml +++ /dev/null @@ -1,115 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf1 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696822097123805332 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_1.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 correlated constraints - duration_sec: 659.2 - end: 1696822096117302513 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-03 - start: 1696821436869718319 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 13.7 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.77GB - virtual_mem_percentage: 5.1 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.14GB -folds: - accuracy: [0.9225, 0.926, 0.9125, 0.922, 0.912] - f_score: [0.921914, 0.927522, 0.911213, 0.918495, 0.910387] - n_evasions: [1391, 1651, 907, 978, 995] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1109, 1111, 1217, 1137, 1122] - n_valid_evades: [692, 1002, 280, 757, 723] - precision: [0.929878, 0.930255, 0.921026, 0.943133, 0.924509] - recall: [0.914086, 0.924805, 0.901606, 0.895112, 0.89669] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-1-03_dnn_pgd_40_926.yaml b/ref_result/perf/debian/PT-1-03_dnn_pgd_40_926.yaml deleted file mode 100644 index 056ad53..0000000 --- a/ref_result/perf/debian/PT-1-03_dnn_pgd_40_926.yaml +++ /dev/null @@ -1,113 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696821095931680537 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_1.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 correlated constraints - duration_sec: 473.5 - end: 1696821094925900863 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-03 - start: 1696820621458205708 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 15.2 - cpu_usage_per_core: [0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.77GB - virtual_mem_percentage: 5.1 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.14GB -folds: - accuracy: [0.9135, 0.9255, 0.917, 0.919, 0.917] - f_score: [0.91267, 0.926274, 0.918787, 0.913369, 0.918227] - n_evasions: [1827, 1851, 1834, 1838, 1834] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1827, 1851, 1834, 1838, 1834] - precision: [0.91498, 0.927651, 0.919687, 0.94469, 0.910156] - recall: [0.910373, 0.924901, 0.917889, 0.884058, 0.926441] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-1-06_dnn_cpgd_auto_124.yaml b/ref_result/perf/debian/PT-1-06_dnn_cpgd_auto_124.yaml deleted file mode 100644 index 825d7e7..0000000 --- a/ref_result/perf/debian/PT-1-06_dnn_cpgd_auto_124.yaml +++ /dev/null @@ -1,130 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf2 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696824366129586435 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_2.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 correlated constraints - duration_sec: 812.2 - end: 1696824365123898059 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-06 - start: 1696823552954059298 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 13.3 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.79GB - virtual_mem_percentage: 5.0 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.12GB -folds: - accuracy: [0.9105, 0.917, 0.912, 0.926, 0.931] - f_score: [0.910812, 0.916834, 0.908903, 0.923316, 0.931343] - n_evasions: [1416, 1533, 728, 1331, 929] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1097, 476, 765, 907, 898] - n_valid_evades: [716, 298, 46, 442, 682] - precision: [0.905847, 0.917753, 0.954348, 0.925234, 0.94929] - recall: [0.915832, 0.915916, 0.867589, 0.921406, 0.914062] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - d: [3, 4, 27] - e: [8, 28] - f: [16, 17, 23] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-1-06_dnn_pgd_40_650.yaml b/ref_result/perf/debian/PT-1-06_dnn_pgd_40_650.yaml deleted file mode 100644 index b462dba..0000000 --- a/ref_result/perf/debian/PT-1-06_dnn_pgd_40_650.yaml +++ /dev/null @@ -1,128 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696823211655203408 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_2.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 correlated constraints - duration_sec: 560.0 - end: 1696823210649598281 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-06 - start: 1696822650679643541 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 15.0 - cpu_usage_per_core: [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.76GB - virtual_mem_percentage: 5.1 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.15GB -folds: - accuracy: [0.918, 0.9105, 0.9205, 0.93, 0.91] - f_score: [0.919291, 0.910186, 0.920856, 0.928131, 0.906929] - n_evasions: [1836, 1821, 1841, 1860, 1820] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1836, 1821, 1841, 1860, 1820] - precision: [0.921105, 0.932169, 0.940041, 0.932921, 0.898566] - recall: [0.917485, 0.889216, 0.902439, 0.923391, 0.915449] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - d: [3, 4, 27] - e: [8, 28] - f: [16, 17, 23] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-1-12_dnn_cpgd_auto_094.yaml b/ref_result/perf/debian/PT-1-12_dnn_cpgd_auto_094.yaml deleted file mode 100644 index c48ae48..0000000 --- a/ref_result/perf/debian/PT-1-12_dnn_cpgd_auto_094.yaml +++ /dev/null @@ -1,160 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf3 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696827127101109857 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_3.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 correlated constraints - duration_sec: 1072.2 - end: 1696827126093895762 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-12 - start: 1696826053923065926 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 13.2 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.79GB - virtual_mem_percentage: 5.0 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.12GB -folds: - accuracy: [0.917, 0.9175, 0.8995, 0.9085, 0.9245] - f_score: [0.914872, 0.918357, 0.898434, 0.90649, 0.923699] - n_evasions: [1624, 1742, 1291, 1815, 333] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [118, 0, 6, 17, 491] - n_valid_evades: [110, 0, 0, 15, 175] - precision: [0.915811, 0.920635, 0.916495, 0.928796, 0.933606] - recall: [0.913934, 0.916091, 0.88107, 0.88523, 0.914] -invalid_rows: [] -validation: - dependencies: - a: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - b: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - c: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - d: [3, 4, 27] - e: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - f: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - g: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - h: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - i: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - j: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - k: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - l: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - g: - attrs: {9: state_INT, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= a[2])' - text: state_INT != 1 or (smean >= dmean) - h: - attrs: {8: state_FIN, 10: state_other, 24: trans_depth} - exec: 'lambda a: a[2] != 1 or a[0] == 1 or a[1] == 1' - text: trans_depth != 1 or state_FIN == 1 or state_other == 1 - i: - attrs: {18: swin, 21: dwin, 23: dmean} - exec: 'lambda a: (a[0] == a[1] == 0 ) or a[2] >= 0' - text: (swin == dwin == 0 ) or dmean >= 0 - j: - attrs: {8: state_FIN, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or smean >= 1 - k: - attrs: {10: state_other, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= 0 and a[2] >= 0)' - text: state_other != 1 or (smean >= 0 and dmean >= 0) - l: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] >= 0 and a[1] >= 0' - text: dbytes >= 0 and dloss >= 0 - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-1-12_dnn_pgd_40_647.yaml b/ref_result/perf/debian/PT-1-12_dnn_pgd_40_647.yaml deleted file mode 100644 index 22924c0..0000000 --- a/ref_result/perf/debian/PT-1-12_dnn_pgd_40_647.yaml +++ /dev/null @@ -1,158 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696825668653313463 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_3.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 correlated constraints - duration_sec: 731.2 - end: 1696825667647202365 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-12 - start: 1696824936467419147 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 15.1 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.79GB - virtual_mem_percentage: 5.0 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.12GB -folds: - accuracy: [0.913, 0.9205, 0.909, 0.9225, 0.902] - f_score: [0.911943, 0.91867, 0.908174, 0.921439, 0.900305] - n_evasions: [1826, 1841, 1818, 1845, 1804] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1826, 1841, 1818, 1845, 1804] - precision: [0.916582, 0.950265, 0.932642, 0.909, 0.922836] - recall: [0.907351, 0.889109, 0.884956, 0.934224, 0.878848] -invalid_rows: [] -validation: - dependencies: - a: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - b: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - c: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - d: [3, 4, 27] - e: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - f: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - g: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - h: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - i: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - j: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - k: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - l: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - g: - attrs: {9: state_INT, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= a[2])' - text: state_INT != 1 or (smean >= dmean) - h: - attrs: {8: state_FIN, 10: state_other, 24: trans_depth} - exec: 'lambda a: a[2] != 1 or a[0] == 1 or a[1] == 1' - text: trans_depth != 1 or state_FIN == 1 or state_other == 1 - i: - attrs: {18: swin, 21: dwin, 23: dmean} - exec: 'lambda a: (a[0] == a[1] == 0 ) or a[2] >= 0' - text: (swin == dwin == 0 ) or dmean >= 0 - j: - attrs: {8: state_FIN, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or smean >= 1 - k: - attrs: {10: state_other, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= 0 and a[2] >= 0)' - text: state_other != 1 or (smean >= 0 and dmean >= 0) - l: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] >= 0 and a[1] >= 0' - text: dbytes >= 0 and dloss >= 0 - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-2-03_dnn_cpgd_auto_490.yaml b/ref_result/perf/debian/PT-2-03_dnn_cpgd_auto_490.yaml deleted file mode 100644 index 15fbf80..0000000 --- a/ref_result/perf/debian/PT-2-03_dnn_cpgd_auto_490.yaml +++ /dev/null @@ -1,115 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf4 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696829188497016300 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_4.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 unrelated constraints - duration_sec: 690.5 - end: 1696829187490136674 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-03 - start: 1696828496996371717 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 13.9 - cpu_usage_per_core: [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.79GB - virtual_mem_percentage: 5.0 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.12GB -folds: - accuracy: [0.9315, 0.915, 0.9125, 0.914, 0.916] - f_score: [0.931466, 0.917874, 0.910302, 0.913219, 0.917647] - n_evasions: [1345, 998, 997, 1660, 1417] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [0, 127, 1, 0, 0] - n_valid_evades: [0, 0, 1, 0, 0] - precision: [0.919052, 0.894539, 0.938689, 0.919715, 0.902604] - recall: [0.944219, 0.94246, 0.883582, 0.906814, 0.9332] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-2-03_dnn_pgd_40_707.yaml b/ref_result/perf/debian/PT-2-03_dnn_pgd_40_707.yaml deleted file mode 100644 index c5b5415..0000000 --- a/ref_result/perf/debian/PT-2-03_dnn_pgd_40_707.yaml +++ /dev/null @@ -1,113 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696828139714385256 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_4.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 unrelated constraints - duration_sec: 465.6 - end: 1696828138706977405 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-03 - start: 1696827673099739741 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 15.0 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.78GB - virtual_mem_percentage: 5.0 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.13GB -folds: - accuracy: [0.909, 0.92, 0.9075, 0.9155, 0.914] - f_score: [0.907708, 0.917695, 0.909535, 0.918711, 0.913828] - n_evasions: [1818, 1840, 1815, 1831, 1828] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1818, 1840, 1815, 1831, 1828] - precision: [0.899497, 0.930136, 0.892514, 0.908658, 0.922144] - recall: [0.91607, 0.905584, 0.927218, 0.928988, 0.90566] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-2-06_dnn_cpgd_auto_019.yaml b/ref_result/perf/debian/PT-2-06_dnn_cpgd_auto_019.yaml deleted file mode 100644 index cc77e96..0000000 --- a/ref_result/perf/debian/PT-2-06_dnn_cpgd_auto_019.yaml +++ /dev/null @@ -1,130 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf5 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696831419024265918 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_5.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 unrelated constraints - duration_sec: 780.2 - end: 1696831418018528583 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-06 - start: 1696830637865720072 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 13.6 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.79GB - virtual_mem_percentage: 5.0 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.13GB -folds: - accuracy: [0.9105, 0.9145, 0.921, 0.9155, 0.919] - f_score: [0.910366, 0.913767, 0.923965, 0.911841, 0.915361] - n_evasions: [938, 832, 1717, 1731, 785] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [0, 322, 0, 0, 0] - n_valid_evades: [0, 1, 0, 0, 0] - precision: [0.926606, 0.922607, 0.923965, 0.921941, 0.932907] - recall: [0.894685, 0.905095, 0.923965, 0.901961, 0.898462] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-2-06_dnn_pgd_40_181.yaml b/ref_result/perf/debian/PT-2-06_dnn_pgd_40_181.yaml deleted file mode 100644 index 50f5f89..0000000 --- a/ref_result/perf/debian/PT-2-06_dnn_pgd_40_181.yaml +++ /dev/null @@ -1,128 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696830301186307458 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_5.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 unrelated constraints - duration_sec: 549.4 - end: 1696830300180475569 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-06 - start: 1696829750757519521 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 15.0 - cpu_usage_per_core: [1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.77GB - virtual_mem_percentage: 5.1 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.14GB -folds: - accuracy: [0.914, 0.9185, 0.914, 0.91, 0.9195] - f_score: [0.91134, 0.920215, 0.914257, 0.910269, 0.917983] - n_evasions: [1828, 1837, 1828, 1820, 1839] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1828, 1837, 1828, 1820, 1839] - precision: [0.924686, 0.896092, 0.918838, 0.924089, 0.931748] - recall: [0.898374, 0.945674, 0.909722, 0.896857, 0.904618] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-2-12_dnn_cpgd_auto_735.yaml b/ref_result/perf/debian/PT-2-12_dnn_cpgd_auto_735.yaml deleted file mode 100644 index 6764aee..0000000 --- a/ref_result/perf/debian/PT-2-12_dnn_cpgd_auto_735.yaml +++ /dev/null @@ -1,160 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf6 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696834077743597989 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_6.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 unrelated constraints - duration_sec: 1056.1 - end: 1696834076734785085 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-12 - start: 1696833020614912751 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 13.3 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.80GB - virtual_mem_percentage: 5.0 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.12GB -folds: - accuracy: [0.9075, 0.9075, 0.919, 0.9185, 0.93] - f_score: [0.909001, 0.905177, 0.915713, 0.919347, 0.93117] - n_evasions: [1539, 1608, 1789, 1578, 947] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [0, 0, 0, 0, 0] - n_valid_evades: [0, 0, 0, 0, 0] - precision: [0.92678, 0.91219, 0.917623, 0.907227, 0.934847] - recall: [0.891892, 0.898271, 0.913811, 0.931795, 0.927522] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - g: [0] - h: [25] - i: [26] - j: [27] - k: [28] - l: [29] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - g: - attrs: {0: dur} - exec: 'lambda a: a[0] >= 0 and a[0] <= 60' - text: dur >= 0 and dur <= 60 - h: - attrs: {25: ct_srv_src} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_srv_src >= 1 and ct_srv_src <= 63 - i: - attrs: {26: ct_dst_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_dst_ltm >= 1 and ct_dst_ltm <= 50 - j: - attrs: {27: ct_src_dport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_src_dport_ltm >= 1 and ct_src_dport_ltm <= 50 - k: - attrs: {28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 46' - text: ct_dst_sport_ltm >= 1 and ct_dst_sport_ltm <= 46 - l: - attrs: {29: ct_dst_src_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_dst_src_ltm >= 1 and ct_dst_src_ltm <= 63 - reset_strategy: 2 diff --git a/ref_result/perf/debian/PT-2-12_dnn_pgd_40_899.yaml b/ref_result/perf/debian/PT-2-12_dnn_pgd_40_899.yaml deleted file mode 100644 index eda4b8e..0000000 --- a/ref_result/perf/debian/PT-2-12_dnn_pgd_40_899.yaml +++ /dev/null @@ -1,158 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696832683904964981 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_6.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 unrelated constraints - duration_sec: 721.4 - end: 1696832682898904066 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-12 - start: 1696831961478440819 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 14.4 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.77GB - virtual_mem_percentage: 5.1 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.14GB -folds: - accuracy: [0.917, 0.9145, 0.918, 0.9145, 0.9225] - f_score: [0.918548, 0.911353, 0.914316, 0.913593, 0.923608] - n_evasions: [1834, 1829, 1836, 1829, 1845] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1834, 1829, 1836, 1829, 1845] - precision: [0.900866, 0.926238, 0.947996, 0.930041, 0.931412] - recall: [0.936937, 0.896939, 0.882947, 0.897716, 0.915934] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - g: [0] - h: [25] - i: [26] - j: [27] - k: [28] - l: [29] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - g: - attrs: {0: dur} - exec: 'lambda a: a[0] >= 0 and a[0] <= 60' - text: dur >= 0 and dur <= 60 - h: - attrs: {25: ct_srv_src} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_srv_src >= 1 and ct_srv_src <= 63 - i: - attrs: {26: ct_dst_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_dst_ltm >= 1 and ct_dst_ltm <= 50 - j: - attrs: {27: ct_src_dport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_src_dport_ltm >= 1 and ct_src_dport_ltm <= 50 - k: - attrs: {28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 46' - text: ct_dst_sport_ltm >= 1 and ct_dst_sport_ltm <= 46 - l: - attrs: {29: ct_dst_src_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_dst_src_ltm >= 1 and ct_dst_src_ltm <= 63 - reset_strategy: 2 diff --git a/ref_result/perf/debian/REG_PT-1-03_dnn_pgd_40_873.yaml b/ref_result/perf/debian/REG_PT-1-03_dnn_pgd_40_873.yaml deleted file mode 100644 index b58fe41..0000000 --- a/ref_result/perf/debian/REG_PT-1-03_dnn_pgd_40_873.yaml +++ /dev/null @@ -1,113 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696821430878480013 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_1.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 correlated constraints - duration_sec: 327.8 - end: 1696821429872681456 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-03 - start: 1696821102121190998 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 16.9 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.76GB - virtual_mem_percentage: 5.1 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.15GB -folds: - accuracy: [0.9125, 0.914, 0.909, 0.924, 0.907] - f_score: [0.906865, 0.914257, 0.907332, 0.924453, 0.908824] - n_evasions: [1822, 1814, 1813, 1835, 1809] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1705, 1829, 1782, 1711, 1646] - n_valid_evades: [1551, 1665, 1624, 1572, 1486] - precision: [0.920086, 0.925328, 0.948882, 0.9375, 0.880342] - recall: [0.894019, 0.903448, 0.869268, 0.911765, 0.93921] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - enabled: false - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - reset_strategy: 2 diff --git a/ref_result/perf/debian/REG_PT-1-06_dnn_pgd_40_683.yaml b/ref_result/perf/debian/REG_PT-1-06_dnn_pgd_40_683.yaml deleted file mode 100644 index 98818fa..0000000 --- a/ref_result/perf/debian/REG_PT-1-06_dnn_pgd_40_683.yaml +++ /dev/null @@ -1,128 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696823546689012628 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_2.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 correlated constraints - duration_sec: 327.9 - end: 1696823545683156874 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-06 - start: 1696823217733328683 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 16.8 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.77GB - virtual_mem_percentage: 5.1 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.15GB -folds: - accuracy: [0.9075, 0.9255, 0.913, 0.91, 0.9145] - f_score: [0.910238, 0.920107, 0.912032, 0.911591, 0.915639] - n_evasions: [1814, 1849, 1624, 1815, 1764] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1499, 1732, 1431, 1604, 1637] - n_valid_evades: [1345, 1603, 1196, 1446, 1461] - precision: [0.905405, 0.95122, 0.904714, 0.916996, 0.909804] - recall: [0.915122, 0.890966, 0.91947, 0.90625, 0.921549] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - d: [3, 4, 27] - e: [8, 28] - f: [16, 17, 23] - enabled: false - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - reset_strategy: 2 diff --git a/ref_result/perf/debian/REG_PT-1-12_dnn_pgd_40_458.yaml b/ref_result/perf/debian/REG_PT-1-12_dnn_pgd_40_458.yaml deleted file mode 100644 index 6c441e0..0000000 --- a/ref_result/perf/debian/REG_PT-1-12_dnn_pgd_40_458.yaml +++ /dev/null @@ -1,158 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696826047464045704 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_3.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 correlated constraints - duration_sec: 371.5 - end: 1696826046457861376 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-12 - start: 1696825674958206897 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 15.7 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.78GB - virtual_mem_percentage: 5.0 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.13GB -folds: - accuracy: [0.91, 0.9105, 0.9185, 0.9115, 0.9145] - f_score: [0.909729, 0.907969, 0.915237, 0.910741, 0.912353] - n_evasions: [1803, 1790, 1788, 1821, 1819] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1631, 1678, 1367, 1629, 1633] - n_valid_evades: [1472, 1515, 1221, 1478, 1481] - precision: [0.918946, 0.928496, 0.961749, 0.920489, 0.925156] - recall: [0.900695, 0.88833, 0.873016, 0.901198, 0.899899] -invalid_rows: [] -validation: - dependencies: - a: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - b: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - c: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - d: [3, 4, 27] - e: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - f: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - g: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - h: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - i: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - j: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - k: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - l: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - enabled: false - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - g: - attrs: {9: state_INT, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= a[2])' - text: state_INT != 1 or (smean >= dmean) - h: - attrs: {8: state_FIN, 10: state_other, 24: trans_depth} - exec: 'lambda a: a[2] != 1 or a[0] == 1 or a[1] == 1' - text: trans_depth != 1 or state_FIN == 1 or state_other == 1 - i: - attrs: {18: swin, 21: dwin, 23: dmean} - exec: 'lambda a: (a[0] == a[1] == 0 ) or a[2] >= 0' - text: (swin == dwin == 0 ) or dmean >= 0 - j: - attrs: {8: state_FIN, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or smean >= 1 - k: - attrs: {10: state_other, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= 0 and a[2] >= 0)' - text: state_other != 1 or (smean >= 0 and dmean >= 0) - l: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] >= 0 and a[1] >= 0' - text: dbytes >= 0 and dloss >= 0 - reset_strategy: 2 diff --git a/ref_result/perf/debian/REG_PT-2-03_dnn_pgd_40_818.yaml b/ref_result/perf/debian/REG_PT-2-03_dnn_pgd_40_818.yaml deleted file mode 100644 index 2759eab..0000000 --- a/ref_result/perf/debian/REG_PT-2-03_dnn_pgd_40_818.yaml +++ /dev/null @@ -1,113 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696828490823539651 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_4.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 unrelated constraints - duration_sec: 343.7 - end: 1696828489817775019 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-03 - start: 1696828146132076794 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 16.0 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.79GB - virtual_mem_percentage: 5.0 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.13GB -folds: - accuracy: [0.915, 0.9145, 0.9065, 0.9155, 0.9085] - f_score: [0.918348, 0.910612, 0.903159, 0.91679, 0.909181] - n_evasions: [1825, 1713, 1732, 1829, 1815] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1825, 1713, 1732, 1829, 1815] - precision: [0.89096, 0.943662, 0.915005, 0.932866, 0.893659] - recall: [0.947473, 0.879798, 0.891616, 0.901258, 0.925253] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - enabled: false - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - reset_strategy: 2 diff --git a/ref_result/perf/debian/REG_PT-2-06_dnn_pgd_40_576.yaml b/ref_result/perf/debian/REG_PT-2-06_dnn_pgd_40_576.yaml deleted file mode 100644 index 27eebe8..0000000 --- a/ref_result/perf/debian/REG_PT-2-06_dnn_pgd_40_576.yaml +++ /dev/null @@ -1,128 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696830631581702509 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_5.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 unrelated constraints - duration_sec: 323.2 - end: 1696830630575819882 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-06 - start: 1696830307343063177 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 17.6 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.78GB - virtual_mem_percentage: 5.0 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.13GB -folds: - accuracy: [0.9125, 0.9215, 0.9065, 0.917, 0.916] - f_score: [0.913836, 0.923377, 0.903559, 0.915822, 0.912682] - n_evasions: [1815, 1827, 1809, 1831, 1832] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1815, 1827, 1809, 1831, 1832] - precision: [0.90625, 0.922927, 0.9125, 0.936722, 0.932059] - recall: [0.921549, 0.923828, 0.894791, 0.895833, 0.894094] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - enabled: false - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - reset_strategy: 2 diff --git a/ref_result/perf/debian/REG_PT-2-12_dnn_pgd_40_163.yaml b/ref_result/perf/debian/REG_PT-2-12_dnn_pgd_40_163.yaml deleted file mode 100644 index c3b8a2a..0000000 --- a/ref_result/perf/debian/REG_PT-2-12_dnn_pgd_40_163.yaml +++ /dev/null @@ -1,158 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696833014168973533 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_6.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 unrelated constraints - duration_sec: 323.0 - end: 1696833013163162423 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-12 - start: 1696832690190822907 - system: - cpu_current_frequency: 2199.998 - cpu_max_frequency: 0.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 4 - cpu_total_cores: 8 - cpu_total_usage: 16.9 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: x86_64 - processor: '' - python_runtime: 3.10.0 (default, Oct 9 2023, 01:41:16) [GCC 12.2.0] - release: 6.1.0-12-cloud-amd64 - system: Linux - version: '#1 SMP PREEMPT_DYNAMIC Debian 6.1.52-1 (2023-09-07)' - virtual_mem_available: 29.79GB - virtual_mem_percentage: 5.0 - virtual_mem_total_size: 31.36GB - virtual_mem_used: 1.12GB -folds: - accuracy: [0.929, 0.907, 0.914, 0.914, 0.9075] - f_score: [0.928929, 0.907093, 0.914683, 0.916908, 0.904195] - n_evasions: [1854, 1814, 1744, 1813, 1653] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1854, 1814, 1744, 1813, 1653] - precision: [0.932663, 0.874759, 0.929435, 0.897824, 0.93369] - recall: [0.925224, 0.941909, 0.900391, 0.936821, 0.876506] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - g: [0] - h: [25] - i: [26] - j: [27] - k: [28] - l: [29] - enabled: false - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - g: - attrs: {0: dur} - exec: 'lambda a: a[0] >= 0 and a[0] <= 60' - text: dur >= 0 and dur <= 60 - h: - attrs: {25: ct_srv_src} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_srv_src >= 1 and ct_srv_src <= 63 - i: - attrs: {26: ct_dst_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_dst_ltm >= 1 and ct_dst_ltm <= 50 - j: - attrs: {27: ct_src_dport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_src_dport_ltm >= 1 and ct_src_dport_ltm <= 50 - k: - attrs: {28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 46' - text: ct_dst_sport_ltm >= 1 and ct_dst_sport_ltm <= 46 - l: - attrs: {29: ct_dst_src_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_dst_src_ltm >= 1 and ct_dst_src_ltm <= 63 - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-1-03_dnn_cpgd_auto_668.yaml b/ref_result/perf/windows/PT-1-03_dnn_cpgd_auto_668.yaml deleted file mode 100644 index 70d1db1..0000000 --- a/ref_result/perf/windows/PT-1-03_dnn_cpgd_auto_668.yaml +++ /dev/null @@ -1,117 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf1 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696871242679759300 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_1.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 correlated constraints - duration_sec: 209.7 - end: 1696871241666794600 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-03 - start: 1696871032000267100 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 8.1 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 1.6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 1.5, 0.0, 0.0, 0.0, 0.0, 1.5, 0.0] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.12GB - virtual_mem_percentage: 67.4 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.58GB -folds: - accuracy: [0.9125, 0.908, 0.9165, 0.918, 0.915] - f_score: [0.910851, 0.905544, 0.918337, 0.917339, 0.912821] - n_evasions: [1649, 1149, 1254, 1722, 1715] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1112, 1123, 1112, 1149, 1237] - n_valid_evades: [854, 347, 485, 972, 1027] - precision: [0.913177, 0.911157, 0.901152, 0.943005, 0.950855] - recall: [0.908537, 0.9, 0.936191, 0.893032, 0.877712] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-1-03_dnn_pgd_40_514.yaml b/ref_result/perf/windows/PT-1-03_dnn_pgd_40_514.yaml deleted file mode 100644 index 3d74ee3..0000000 --- a/ref_result/perf/windows/PT-1-03_dnn_pgd_40_514.yaml +++ /dev/null @@ -1,115 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696871028525548400 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_1.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 correlated constraints - duration_sec: 175.2 - end: 1696871027513041500 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-03 - start: 1696870852354847300 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 9.2 - cpu_usage_per_core: [12.5, 0.0, 0.0, 0.0, 12.7, 0.0, 20.6, 0.0, 0.0, 0.0, 0.0, - 0.0, 3.0, 1.5, 4.6, 3.1, 1.5, 4.6, 4.6, 3.1] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.04GB - virtual_mem_percentage: 67.9 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.66GB -folds: - accuracy: [0.9075, 0.924, 0.9085, 0.918, 0.9285] - f_score: [0.905948, 0.925197, 0.903226, 0.918164, 0.927226] - n_evasions: [1815, 1848, 1817, 1836, 1857] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1815, 1848, 1817, 1836, 1857] - precision: [0.930063, 0.946626, 0.933333, 0.912698, 0.924873] - recall: [0.883053, 0.904716, 0.875, 0.923695, 0.929592] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-1-06_dnn_cpgd_auto_074.yaml b/ref_result/perf/windows/PT-1-06_dnn_cpgd_auto_074.yaml deleted file mode 100644 index 8f7d21c..0000000 --- a/ref_result/perf/windows/PT-1-06_dnn_cpgd_auto_074.yaml +++ /dev/null @@ -1,132 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf2 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696871854089254200 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_2.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 correlated constraints - duration_sec: 271.1 - end: 1696871853072858300 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-06 - start: 1696871581941411100 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 7.2 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 1.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 4.5, 4.6, 3.1, 1.5, 1.5, 0.0, 4.6, 0.0] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.17GB - virtual_mem_percentage: 67.1 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.54GB -folds: - accuracy: [0.9225, 0.9125, 0.9085, 0.921, 0.926] - f_score: [0.921039, 0.908712, 0.909361, 0.921315, 0.926368] - n_evasions: [1013, 874, 1298, 1653, 1291] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [642, 873, 1116, 966, 650] - n_valid_evades: [390, 109, 669, 727, 416] - precision: [0.909457, 0.932548, 0.910714, 0.931521, 0.942308] - recall: [0.932921, 0.886063, 0.908012, 0.91133, 0.910959] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - d: [3, 4, 27] - e: [8, 28] - f: [16, 17, 23] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-1-06_dnn_pgd_40_307.yaml b/ref_result/perf/windows/PT-1-06_dnn_pgd_40_307.yaml deleted file mode 100644 index 055d4cd..0000000 --- a/ref_result/perf/windows/PT-1-06_dnn_pgd_40_307.yaml +++ /dev/null @@ -1,130 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696871578317922200 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_2.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 correlated constraints - duration_sec: 227.0 - end: 1696871577305956600 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-06 - start: 1696871350267767300 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 8.7 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 1.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.12GB - virtual_mem_percentage: 67.4 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.59GB -folds: - accuracy: [0.9205, 0.909, 0.916, 0.9285, 0.9205] - f_score: [0.920619, 0.908543, 0.916749, 0.929243, 0.91933] - n_evasions: [1841, 1818, 1832, 1857, 1841] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1841, 1818, 1832, 1857, 1841] - precision: [0.923848, 0.910373, 0.905975, 0.926036, 0.927329] - recall: [0.917413, 0.90672, 0.927783, 0.932473, 0.911469] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - d: [3, 4, 27] - e: [8, 28] - f: [16, 17, 23] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-1-12_dnn_cpgd_auto_972.yaml b/ref_result/perf/windows/PT-1-12_dnn_cpgd_auto_972.yaml deleted file mode 100644 index 5c67e89..0000000 --- a/ref_result/perf/windows/PT-1-12_dnn_cpgd_auto_972.yaml +++ /dev/null @@ -1,162 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf3 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696872640986910600 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_3.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 correlated constraints - duration_sec: 365.1 - end: 1696872639971186200 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-12 - start: 1696872274839311600 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 7.2 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 3.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 1.6, 3.1, 3.1, 3.1, 0.0, 0.0, 1.6, 4.7] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.33GB - virtual_mem_percentage: 66.1 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.38GB -folds: - accuracy: [0.924, 0.9185, 0.926, 0.8985, 0.9075] - f_score: [0.922607, 0.916964, 0.923711, 0.903837, 0.908006] - n_evasions: [1258, 1167, 1733, 1797, 1023] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [57, 1, 5, 0, 86] - n_valid_evades: [2, 1, 0, 0, 0] - precision: [0.942768, 0.949367, 0.939203, 0.867273, 0.889864] - recall: [0.90329, 0.8867, 0.908722, 0.94362, 0.926904] -invalid_rows: [] -validation: - dependencies: - a: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - b: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - c: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - d: [3, 4, 27] - e: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - f: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - g: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - h: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - i: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - j: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - k: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - l: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - g: - attrs: {9: state_INT, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= a[2])' - text: state_INT != 1 or (smean >= dmean) - h: - attrs: {8: state_FIN, 10: state_other, 24: trans_depth} - exec: 'lambda a: a[2] != 1 or a[0] == 1 or a[1] == 1' - text: trans_depth != 1 or state_FIN == 1 or state_other == 1 - i: - attrs: {18: swin, 21: dwin, 23: dmean} - exec: 'lambda a: (a[0] == a[1] == 0 ) or a[2] >= 0' - text: (swin == dwin == 0 ) or dmean >= 0 - j: - attrs: {8: state_FIN, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or smean >= 1 - k: - attrs: {10: state_other, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= 0 and a[2] >= 0)' - text: state_other != 1 or (smean >= 0 and dmean >= 0) - l: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] >= 0 and a[1] >= 0' - text: dbytes >= 0 and dloss >= 0 - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-1-12_dnn_pgd_40_240.yaml b/ref_result/perf/windows/PT-1-12_dnn_pgd_40_240.yaml deleted file mode 100644 index 8b5c770..0000000 --- a/ref_result/perf/windows/PT-1-12_dnn_pgd_40_240.yaml +++ /dev/null @@ -1,160 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696872271248751400 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_3.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 correlated constraints - duration_sec: 304.1 - end: 1696872270240372200 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-12 - start: 1696871966144836300 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 8.0 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 4.6, 1.5, 0.0, 0.0, 0.0] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.17GB - virtual_mem_percentage: 67.1 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.54GB -folds: - accuracy: [0.9165, 0.913, 0.924, 0.923, 0.907] - f_score: [0.918177, 0.911043, 0.924827, 0.923913, 0.904713] - n_evasions: [1833, 1826, 1848, 1846, 1814] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1833, 1826, 1848, 1846, 1814] - precision: [0.905314, 0.919505, 0.935936, 0.917566, 0.907503] - recall: [0.931412, 0.902736, 0.913978, 0.930348, 0.901941] -invalid_rows: [] -validation: - dependencies: - a: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - b: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - c: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - d: [3, 4, 27] - e: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - f: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - g: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - h: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - i: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - j: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - k: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - l: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - g: - attrs: {9: state_INT, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= a[2])' - text: state_INT != 1 or (smean >= dmean) - h: - attrs: {8: state_FIN, 10: state_other, 24: trans_depth} - exec: 'lambda a: a[2] != 1 or a[0] == 1 or a[1] == 1' - text: trans_depth != 1 or state_FIN == 1 or state_other == 1 - i: - attrs: {18: swin, 21: dwin, 23: dmean} - exec: 'lambda a: (a[0] == a[1] == 0 ) or a[2] >= 0' - text: (swin == dwin == 0 ) or dmean >= 0 - j: - attrs: {8: state_FIN, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or smean >= 1 - k: - attrs: {10: state_other, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= 0 and a[2] >= 0)' - text: state_other != 1 or (smean >= 0 and dmean >= 0) - l: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] >= 0 and a[1] >= 0' - text: dbytes >= 0 and dloss >= 0 - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-2-03_dnn_cpgd_auto_294.yaml b/ref_result/perf/windows/PT-2-03_dnn_cpgd_auto_294.yaml deleted file mode 100644 index 75e4176..0000000 --- a/ref_result/perf/windows/PT-2-03_dnn_cpgd_auto_294.yaml +++ /dev/null @@ -1,117 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf4 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696873164311358100 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_4.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 unrelated constraints - duration_sec: 224.3 - end: 1696873163293036200 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-03 - start: 1696872939033230600 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 7.4 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 3.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 4.5, 0.0, 1.5, 0.0, 1.5, 0.0, 1.5, 0.0] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.27GB - virtual_mem_percentage: 66.4 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.44GB -folds: - accuracy: [0.927, 0.906, 0.9145, 0.9135, 0.916] - f_score: [0.92551, 0.907753, 0.916545, 0.912582, 0.916251] - n_evasions: [1006, 1482, 1525, 1681, 976] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [67, 0, 0, 0, 0] - n_valid_evades: [42, 0, 0, 0, 0] - precision: [0.918016, 0.899805, 0.903754, 0.925205, 0.918082] - recall: [0.933128, 0.915842, 0.929703, 0.900299, 0.914428] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-2-03_dnn_pgd_40_556.yaml b/ref_result/perf/windows/PT-2-03_dnn_pgd_40_556.yaml deleted file mode 100644 index 0aec7b0..0000000 --- a/ref_result/perf/windows/PT-2-03_dnn_pgd_40_556.yaml +++ /dev/null @@ -1,115 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696872935575846500 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_4.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 unrelated constraints - duration_sec: 189.9 - end: 1696872934554798800 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-03 - start: 1696872744672610400 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 9.5 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 3.1, 0.0, 1.5, 1.5, 1.5, 3.1, 1.5] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.32GB - virtual_mem_percentage: 66.1 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.38GB -folds: - accuracy: [0.9075, 0.916, 0.913, 0.9295, 0.9185] - f_score: [0.902887, 0.914807, 0.911765, 0.928025, 0.917552] - n_evasions: [1815, 1832, 1826, 1859, 1837] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1815, 1832, 1826, 1859, 1837] - precision: [0.942982, 0.957537, 0.914547, 0.942946, 0.921748] - recall: [0.866062, 0.875728, 0.908999, 0.913568, 0.913394] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - enabled: true - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-2-06_dnn_cpgd_auto_404.yaml b/ref_result/perf/windows/PT-2-06_dnn_cpgd_auto_404.yaml deleted file mode 100644 index 692ee45..0000000 --- a/ref_result/perf/windows/PT-2-06_dnn_cpgd_auto_404.yaml +++ /dev/null @@ -1,132 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf5 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696873777415038800 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_5.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 unrelated constraints - duration_sec: 266.8 - end: 1696873776403336600 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-06 - start: 1696873509556980200 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 7.0 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 1.5, 0.0, 0.0] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.24GB - virtual_mem_percentage: 66.6 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.47GB -folds: - accuracy: [0.912, 0.9215, 0.917, 0.923, 0.9115] - f_score: [0.912438, 0.920264, 0.915565, 0.923307, 0.913108] - n_evasions: [967, 855, 1032, 1205, 1713] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [0, 0, 0, 0, 0] - n_valid_evades: [0, 0, 0, 0, 0] - precision: [0.914257, 0.946708, 0.906344, 0.910609, 0.912659] - recall: [0.910626, 0.895257, 0.924974, 0.936364, 0.913556] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-2-06_dnn_pgd_40_130.yaml b/ref_result/perf/windows/PT-2-06_dnn_pgd_40_130.yaml deleted file mode 100644 index 340dd5b..0000000 --- a/ref_result/perf/windows/PT-2-06_dnn_pgd_40_130.yaml +++ /dev/null @@ -1,130 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696873506146254600 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_5.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 unrelated constraints - duration_sec: 231.9 - end: 1696873505129471800 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-06 - start: 1696873273235385000 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 8.8 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 1.6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 1.5, 0.0, 0.0, 3.0, 0.0] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.25GB - virtual_mem_percentage: 66.6 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.46GB -folds: - accuracy: [0.9185, 0.914, 0.9185, 0.917, 0.903] - f_score: [0.913984, 0.914428, 0.922712, 0.917984, 0.90329] - n_evasions: [1836, 1828, 1837, 1834, 1806] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1836, 1828, 1837, 1834, 1806] - precision: [0.931183, 0.922691, 0.911049, 0.907227, 0.883041] - recall: [0.897409, 0.906312, 0.934678, 0.929, 0.92449] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - enabled: true - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-2-12_dnn_cpgd_auto_069.yaml b/ref_result/perf/windows/PT-2-12_dnn_cpgd_auto_069.yaml deleted file mode 100644 index fddfe69..0000000 --- a/ref_result/perf/windows/PT-2-12_dnn_cpgd_auto_069.yaml +++ /dev/null @@ -1,162 +0,0 @@ -attack: - can_validate: false - config: - args: {enable_constraints: true, eps: 0.2, eps_step: 0.1, norm: 2, verbose: 1} - feat_file: data/feature_unsw.csv - id: perf6 - name: CPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696874560080782900 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_6.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 unrelated constraints - duration_sec: 358.1 - end: 1696874559068748600 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-12 - start: 1696874201010563600 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 6.6 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 1.6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 1.6, 0.0, 1.6, 0.0, 0.0, 0.0, 0.0] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.18GB - virtual_mem_percentage: 67.0 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.52GB -folds: - accuracy: [0.914, 0.9215, 0.916, 0.9175, 0.914] - f_score: [0.912155, 0.921618, 0.916501, 0.915341, 0.912424] - n_evasions: [1187, 1450, 981, 1487, 1827] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [0, 0, 0, 0, 3] - n_valid_evades: [0, 0, 0, 0, 3] - precision: [0.893, 0.931382, 0.934144, 0.943915, 0.930426] - recall: [0.93215, 0.912055, 0.899512, 0.888446, 0.895105] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - g: [0] - h: [25] - i: [26] - j: [27] - k: [28] - l: [29] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - g: - attrs: {0: dur} - exec: 'lambda a: a[0] >= 0 and a[0] <= 60' - text: dur >= 0 and dur <= 60 - h: - attrs: {25: ct_srv_src} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_srv_src >= 1 and ct_srv_src <= 63 - i: - attrs: {26: ct_dst_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_dst_ltm >= 1 and ct_dst_ltm <= 50 - j: - attrs: {27: ct_src_dport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_src_dport_ltm >= 1 and ct_src_dport_ltm <= 50 - k: - attrs: {28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 46' - text: ct_dst_sport_ltm >= 1 and ct_dst_sport_ltm <= 46 - l: - attrs: {29: ct_dst_src_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_dst_src_ltm >= 1 and ct_dst_src_ltm <= 63 - reset_strategy: 2 diff --git a/ref_result/perf/windows/PT-2-12_dnn_pgd_40_338.yaml b/ref_result/perf/windows/PT-2-12_dnn_pgd_40_338.yaml deleted file mode 100644 index 8a22324..0000000 --- a/ref_result/perf/windows/PT-2-12_dnn_pgd_40_338.yaml +++ /dev/null @@ -1,160 +0,0 @@ -attack: - can_validate: true - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: VPGD -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696874197351697600 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_6.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 unrelated constraints - duration_sec: 307.0 - end: 1696874196337106200 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-12 - start: 1696873889309900100 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 8.1 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 3.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 1.5, 1.5, 0.0, 0.0, 0.0, 0.0] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.18GB - virtual_mem_percentage: 67.0 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.53GB -folds: - accuracy: [0.9225, 0.917, 0.906, 0.909, 0.9125] - f_score: [0.922693, 0.918146, 0.904858, 0.909812, 0.912104] - n_evasions: [1845, 1834, 1812, 1818, 1825] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1845, 1834, 1812, 1818, 1825] - precision: [0.922233, 0.910068, 0.911315, 0.892128, 0.924644] - recall: [0.923154, 0.926368, 0.898492, 0.92821, 0.899901] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - g: [0] - h: [25] - i: [26] - j: [27] - k: [28] - l: [29] - enabled: true - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - g: - attrs: {0: dur} - exec: 'lambda a: a[0] >= 0 and a[0] <= 60' - text: dur >= 0 and dur <= 60 - h: - attrs: {25: ct_srv_src} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_srv_src >= 1 and ct_srv_src <= 63 - i: - attrs: {26: ct_dst_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_dst_ltm >= 1 and ct_dst_ltm <= 50 - j: - attrs: {27: ct_src_dport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_src_dport_ltm >= 1 and ct_src_dport_ltm <= 50 - k: - attrs: {28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 46' - text: ct_dst_sport_ltm >= 1 and ct_dst_sport_ltm <= 46 - l: - attrs: {29: ct_dst_src_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_dst_src_ltm >= 1 and ct_dst_src_ltm <= 63 - reset_strategy: 2 diff --git a/ref_result/perf/windows/REG_PT-1-03_dnn_pgd_40_519.yaml b/ref_result/perf/windows/REG_PT-1-03_dnn_pgd_40_519.yaml deleted file mode 100644 index d920617..0000000 --- a/ref_result/perf/windows/REG_PT-1-03_dnn_pgd_40_519.yaml +++ /dev/null @@ -1,115 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696871346528639400 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_1.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 correlated constraints - duration_sec: 99.3 - end: 1696871345517562500 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-03 - start: 1696871246200269100 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 11.5 - cpu_usage_per_core: [3.1, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 1.5, 1.6, 0.0, 6.2, 1.5, 1.5, 3.1, 4.6] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.13GB - virtual_mem_percentage: 67.3 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.58GB -folds: - accuracy: [0.9255, 0.9155, 0.9105, 0.9005, 0.9155] - f_score: [0.926709, 0.914861, 0.910901, 0.905103, 0.914082] - n_evasions: [1849, 1812, 1782, 1798, 1828] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1689, 1608, 1836, 1745, 1650] - n_valid_evades: [1555, 1456, 1650, 1558, 1502] - precision: [0.919024, 0.904382, 0.922379, 0.875461, 0.911765] - recall: [0.934524, 0.925586, 0.899705, 0.936821, 0.916412] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - enabled: false - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - reset_strategy: 2 diff --git a/ref_result/perf/windows/REG_PT-1-06_dnn_pgd_40_484.yaml b/ref_result/perf/windows/REG_PT-1-06_dnn_pgd_40_484.yaml deleted file mode 100644 index 56f1f61..0000000 --- a/ref_result/perf/windows/REG_PT-1-06_dnn_pgd_40_484.yaml +++ /dev/null @@ -1,130 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696871962501276000 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_2.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 correlated constraints - duration_sec: 103.9 - end: 1696871961483375700 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-06 - start: 1696871857571399900 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 12.2 - cpu_usage_per_core: [3.1, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 4.6, 1.5, 0.0, 1.5, 1.5, 4.6] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.15GB - virtual_mem_percentage: 67.2 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.56GB -folds: - accuracy: [0.915, 0.9165, 0.911, 0.9205, 0.9125] - f_score: [0.916748, 0.915099, 0.910463, 0.91998, 0.911392] - n_evasions: [1730, 1833, 1799, 1817, 1810] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1509, 1661, 1674, 1724, 1549] - n_valid_evades: [1331, 1510, 1499, 1572, 1407] - precision: [0.917647, 0.901804, 0.896928, 0.935517, 0.942408] - recall: [0.915851, 0.928793, 0.924413, 0.90495, 0.882353] -invalid_rows: [] -validation: - dependencies: - a: [9, 12, 15] - b: [6, 22] - c: [9, 12, 15] - d: [3, 4, 27] - e: [8, 28] - f: [16, 17, 23] - enabled: false - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - reset_strategy: 2 diff --git a/ref_result/perf/windows/REG_PT-1-12_dnn_pgd_40_978.yaml b/ref_result/perf/windows/REG_PT-1-12_dnn_pgd_40_978.yaml deleted file mode 100644 index c23f0e9..0000000 --- a/ref_result/perf/windows/REG_PT-1-12_dnn_pgd_40_978.yaml +++ /dev/null @@ -1,160 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696872740989235900 - class_distribution: {0: 5000, 1: 5000} - config: config/prf1_3.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 correlated constraints - duration_sec: 95.4 - end: 1696872739978166000 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-1-12 - start: 1696872644575461800 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 12.0 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 1.6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 1.5, 1.5, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.30GB - virtual_mem_percentage: 66.3 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.41GB -folds: - accuracy: [0.9105, 0.9165, 0.911, 0.9165, 0.924] - f_score: [0.908533, 0.912976, 0.913255, 0.913066, 0.926782] - n_evasions: [1728, 1825, 1813, 1833, 1838] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [1618, 1663, 1574, 1758, 1598] - n_valid_evades: [1438, 1511, 1426, 1616, 1467] - precision: [0.909928, 0.926004, 0.912366, 0.936966, 0.92589] - recall: [0.907143, 0.900308, 0.914146, 0.890355, 0.927676] -invalid_rows: [] -validation: - dependencies: - a: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - b: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - c: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - d: [3, 4, 27] - e: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - f: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - g: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - h: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - i: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - j: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - k: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - l: [6, 8, 9, 10, 12, 15, 16, 17, 18, 21, 22, 23, 24, 28] - enabled: false - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {9: state_INT, 12: dbytes} - exec: 'lambda a: a[0] != 1 or a[1] == 0' - text: state_INT != 1 or dbytes == 0 - b: - attrs: {6: service_dns, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] <= 512' - text: service_dns != 1 or smean <= 512 - c: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] != 0 or a[1] == 0' - text: dbytes != 0 or dloss == 0 - d: - attrs: {3: proto_other, 4: service_none, 27: ct_src_dport_ltm} - exec: 'lambda a: a[1] != 1 or a[0] != 1 or a[2] >= 1' - text: service_none != 1 or proto_other != 1 or ct_src_dport_ltm >= 1 - e: - attrs: {8: state_FIN, 28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or ct_dst_sport_ltm >= 1 - f: - attrs: {16: sjit, 17: djit, 23: dmean} - exec: 'lambda a: (a[2] >= 10) or (a[0] >= a[1])' - text: (dmean >= 10) or (sjit >= djit) - g: - attrs: {9: state_INT, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= a[2])' - text: state_INT != 1 or (smean >= dmean) - h: - attrs: {8: state_FIN, 10: state_other, 24: trans_depth} - exec: 'lambda a: a[2] != 1 or a[0] == 1 or a[1] == 1' - text: trans_depth != 1 or state_FIN == 1 or state_other == 1 - i: - attrs: {18: swin, 21: dwin, 23: dmean} - exec: 'lambda a: (a[0] == a[1] == 0 ) or a[2] >= 0' - text: (swin == dwin == 0 ) or dmean >= 0 - j: - attrs: {8: state_FIN, 22: smean} - exec: 'lambda a: a[0] != 1 or a[1] >= 1' - text: state_FIN != 1 or smean >= 1 - k: - attrs: {10: state_other, 22: smean, 23: dmean} - exec: 'lambda a: a[0] != 1 or (a[1] >= 0 and a[2] >= 0)' - text: state_other != 1 or (smean >= 0 and dmean >= 0) - l: - attrs: {12: dbytes, 15: dloss} - exec: 'lambda a: a[0] >= 0 and a[1] >= 0' - text: dbytes >= 0 and dloss >= 0 - reset_strategy: 2 diff --git a/ref_result/perf/windows/REG_PT-2-03_dnn_pgd_40_714.yaml b/ref_result/perf/windows/REG_PT-2-03_dnn_pgd_40_714.yaml deleted file mode 100644 index b96e5e1..0000000 --- a/ref_result/perf/windows/REG_PT-2-03_dnn_pgd_40_714.yaml +++ /dev/null @@ -1,115 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696873269726466800 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_4.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 3 unrelated constraints - duration_sec: 100.9 - end: 1696873268713565600 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-03 - start: 1696873167857573900 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 12.2 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 4.6, 0.0, 0.0, 3.1, 1.5] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.27GB - virtual_mem_percentage: 66.4 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.44GB -folds: - accuracy: [0.9175, 0.914, 0.912, 0.917, 0.9115] - f_score: [0.919078, 0.915438, 0.909836, 0.911702, 0.911984] - n_evasions: [1766, 1673, 1811, 1817, 1813] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1766, 1673, 1811, 1817, 1813] - precision: [0.938878, 0.894332, 0.918304, 0.953281, 0.907023] - recall: [0.900096, 0.937563, 0.901523, 0.873598, 0.917] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - enabled: false - immutable: [] - n_constraints: 3 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - reset_strategy: 2 diff --git a/ref_result/perf/windows/REG_PT-2-06_dnn_pgd_40_611.yaml b/ref_result/perf/windows/REG_PT-2-06_dnn_pgd_40_611.yaml deleted file mode 100644 index 1c2fc17..0000000 --- a/ref_result/perf/windows/REG_PT-2-06_dnn_pgd_40_611.yaml +++ /dev/null @@ -1,130 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696873885627269300 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_5.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 6 unrelated constraints - duration_sec: 103.6 - end: 1696873884609681700 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-06 - start: 1696873780969448200 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 12.0 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 1.6, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 1.5, 1.5, 0.0, 3.1, 0.0, 1.6, 3.1, 1.5] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.25GB - virtual_mem_percentage: 66.6 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.45GB -folds: - accuracy: [0.9105, 0.915, 0.9245, 0.9145, 0.9155] - f_score: [0.906625, 0.919125, 0.924763, 0.915136, 0.909963] - n_evasions: [1809, 1790, 1710, 1776, 1718] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1809, 1790, 1710, 1776, 1718] - precision: [0.919577, 0.908749, 0.915187, 0.940816, 0.932314] - recall: [0.894033, 0.92974, 0.934542, 0.890821, 0.888658] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - enabled: false - immutable: [] - n_constraints: 6 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - reset_strategy: 2 diff --git a/ref_result/perf/windows/REG_PT-2-12_dnn_pgd_40_474.yaml b/ref_result/perf/windows/REG_PT-2-12_dnn_pgd_40_474.yaml deleted file mode 100644 index 4c93b23..0000000 --- a/ref_result/perf/windows/REG_PT-2-12_dnn_pgd_40_474.yaml +++ /dev/null @@ -1,160 +0,0 @@ -attack: - can_validate: false - config: {batch_size: 2000, decay: null, eps: 0.4, eps_step: 0.1, max_iter: 40, norm: inf, - num_random_init: 40, random_eps: false, targeted: false, verbose: true} - name: ProjectedGradientDescent -classifier: - config: - layers: [64, 64, 32] - model_fit: {batch_size: 32, epochs: 80, shuffle: true, verbose: 0} - name: Neural Network -experiment: - attrs: {0: dur, 1: proto_tcp, 2: proto_udp, 3: proto_other, 4: service_none, 5: service_http, - 6: service_dns, 7: service_other, 8: state_FIN, 9: state_INT, 10: state_other, - 11: sbytes, 12: dbytes, 13: rate, 14: sloss, 15: dloss, 16: sjit, 17: djit, 18: swin, - 19: stcpb, 20: dtcpb, 21: dwin, 22: smean, 23: dmean, 24: trans_depth, 25: ct_srv_src, - 26: ct_dst_ltm, 27: ct_src_dport_ltm, 28: ct_dst_sport_ltm, 29: ct_dst_src_ltm, - 30: ct_src_ltm, 31: ct_srv_dst, 32: label} - attrs_ranges: - ct_dst_ltm: [1, 50] - ct_dst_sport_ltm: [1, 46] - ct_dst_src_ltm: [1, 63] - ct_src_dport_ltm: [1, 50] - ct_src_ltm: [1, 50] - ct_srv_dst: [1, 62] - ct_srv_src: [1, 63] - dbytes: [0, 7085342] - djit: [0, 199573] - dloss: [0, 2627] - dmean: [0, 1458] - dtcpb: [0, 4294383134] - dur: [0, 60] - dwin: [0, 255] - proto_other: [0, 1] - proto_tcp: [0, 1] - proto_udp: [0, 1] - rate: [0, 1000001] - sbytes: [46, 10508068] - service_dns: [0, 1] - service_http: [0, 1] - service_none: [0, 1] - service_other: [0, 1] - sjit: [0, 1460481] - sloss: [0, 3993] - smean: [28, 1499] - state_FIN: [0, 1] - state_INT: [0, 1] - state_other: [0, 1] - stcpb: [0, 4294729542] - swin: [0, 255] - trans_depth: [0, 2] - capture_utc: 1696874668478832100 - class_distribution: {0: 5000, 1: 5000} - config: config/prf2_6.yaml - dataset: data/unsw_nb15.csv - description: Constraint performance test with 12 unrelated constraints - duration_sec: 103.9 - end: 1696874667473503200 - k_folds: 5 - n_attributes: 33 - n_classes: 2 - n_records: 10000 - name: PT-2-12 - start: 1696874563599572400 - system: - cpu_current_frequency: 2300.0 - cpu_max_frequency: 2300.0 - cpu_min_frequency: 0.0 - cpu_physical_cores: 14 - cpu_total_cores: 20 - cpu_total_usage: 12.9 - cpu_usage_per_core: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - machine: AMD64 - processor: Intel64 Family 6 Model 154 Stepping 3, GenuineIntel - python_runtime: 3.10.9 (tags/v3.10.9:1dd9be6, Dec 6 2022, 20:01:21) [MSC v.1934 - 64 bit (AMD64)] - release: '10' - system: Windows - version: 10.0.19044 - virtual_mem_available: 5.00GB - virtual_mem_percentage: 68.2 - virtual_mem_total_size: 15.71GB - virtual_mem_used: 10.71GB -folds: - accuracy: [0.9185, 0.921, 0.92, 0.908, 0.907] - f_score: [0.920989, 0.918557, 0.919598, 0.908275, 0.903727] - n_evasions: [1759, 1837, 1840, 1815, 1811] - n_records: [2000, 2000, 2000, 2000, 2000] - n_valid: [2000, 2000, 2000, 2000, 2000] - n_valid_evades: [1759, 1837, 1840, 1815, 1811] - precision: [0.890347, 0.956009, 0.932722, 0.892262, 0.93871] - recall: [0.953815, 0.883929, 0.906838, 0.924873, 0.871257] -invalid_rows: [] -validation: - dependencies: - a: [18] - b: [21] - c: [14] - d: [15] - e: [11] - f: [12] - g: [0] - h: [25] - i: [26] - j: [27] - k: [28] - l: [29] - enabled: false - immutable: [] - n_constraints: 12 - predicates: - a: - attrs: {18: swin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: swin >= 0 and swin <= 255 - b: - attrs: {21: dwin} - exec: 'lambda a: a[0] >= 0 and a[0] <= 255' - text: dwin >= 0 and dwin <= 255 - c: - attrs: {14: sloss} - exec: 'lambda a: a[0] >= 0 and a[0] <= 3993' - text: sloss >= 0 and sloss <= 3993 - d: - attrs: {15: dloss} - exec: 'lambda a: a[0] >= 0 or a[0] <= 2627' - text: dloss >= 0 or dloss <= 2627 - e: - attrs: {11: sbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 7085342' - text: sbytes >= 0 or sbytes <= 7085342 - f: - attrs: {12: dbytes} - exec: 'lambda a: a[0] >= 0 or a[0] <= 10508068' - text: dbytes >= 0 or dbytes <= 10508068 - g: - attrs: {0: dur} - exec: 'lambda a: a[0] >= 0 and a[0] <= 60' - text: dur >= 0 and dur <= 60 - h: - attrs: {25: ct_srv_src} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_srv_src >= 1 and ct_srv_src <= 63 - i: - attrs: {26: ct_dst_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_dst_ltm >= 1 and ct_dst_ltm <= 50 - j: - attrs: {27: ct_src_dport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 50' - text: ct_src_dport_ltm >= 1 and ct_src_dport_ltm <= 50 - k: - attrs: {28: ct_dst_sport_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 46' - text: ct_dst_sport_ltm >= 1 and ct_dst_sport_ltm <= 46 - l: - attrs: {29: ct_dst_src_ltm} - exec: 'lambda a: a[0] >= 1 and a[0] <= 63' - text: ct_dst_src_ltm >= 1 and ct_dst_src_ltm <= 63 - reset_strategy: 2