diff --git a/README.md b/README.md index 286f1939..e8caa63c 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ Requirements 3. Frama-c 22.0 (Titanium) 4. Python version 3.8.10 5. cmake version 3.16.3 +6. Ninja 1.10.0 +7. GCC 10 (installed and can be accessed as gcc-10, needed only for coverage build of GCC) ** see subfolders, we modified few files in each. Replace the files in these folders with ours. diff --git a/scripts/coverage/0-download-GCC.sh b/scripts/coverage/0-download-GCC.sh new file mode 100755 index 00000000..8344805a --- /dev/null +++ b/scripts/coverage/0-download-GCC.sh @@ -0,0 +1,48 @@ +#!/bin/bash +shopt -s extglob # Activate extended pattern matching in bash + +working_folder=$1 + +sudo apt-get install make +sudo apt-get install flex -y +sudo apt-get install -y bison + +# Downloading LLVM and Csmith sources, building Csmith +TMP_SOURCE_FOLDER=$(mktemp -d $working_folder/.sources.XXXXXXX.tmp) +cd $TMP_SOURCE_FOLDER +## Get gcc +git clone git://gcc.gnu.org/git/gcc.git +cd gcc +git checkout releases/gcc-10 +## Get pre-req. isl-0.18.tar.bz2;gmp-6.1.0.tar.bz2;mpc-1.0.3.tar.gz; mpfr-3.1.4.tar.bz2 => works for gcc-10 +## GMP +wget ftp://gcc.gnu.org/pub/gcc/infrastructure/gmp-6.1.0.tar.bz2 +bunzip2 gmp-6.1.0.tar.bz2 +tar xvf gmp-6.1.0.tar +cd gmp-6.1.0 +./configure --disable-shared --enable-static --prefix=/tmp/gcc +make && make check && make install +cd .. +## MPFR +wget ftp://gcc.gnu.org/pub/gcc/infrastructure/mpfr-3.1.4.tar.bz2 +bunzip2 mpfr-3.1.4.tar.bz2 +tar xvf mpfr-3.1.4.tar +cd mpfr-3.1.4 +./configure --disable-shared --enable-static --prefix=/tmp/gcc --with-gmp=/tmp/gcc +make && make check && make install +cd .. +## MPC +wget ftp://gcc.gnu.org/pub/gcc/infrastructure/mpc-1.0.3.tar.gz +tar zxvf mpc-1.0.3.tar.gz +cd mpc-1.0.3 +./configure --disable-shared --enable-static --prefix=/tmp/gcc --with-gmp=/tmp/gcc --with-mpfr=/tmp/gcc +make && make check && make install +cd .. +## ISL +wget ftp://gcc.gnu.org/pub/gcc/infrastructure/isl-0.18.tar.bz2 +bunzip2 isl-0.18.tar.bz2 +tar xvf isl-0.18.tar +cd isl-0.18 +./configure --with-gmp-prefix=/tmp/gcc --disable-shared --enable-static --prefix=/tmp/isl +make -j $THREADS && make install +echo ">> Downloading GCC (trunk) source files ($TMP_SOURCE_FOLDER)" diff --git a/scripts/coverage/0-download-LLVM.sh b/scripts/coverage/0-download-LLVM.sh new file mode 100755 index 00000000..e9bb34eb --- /dev/null +++ b/scripts/coverage/0-download-LLVM.sh @@ -0,0 +1,17 @@ +#!/bin/bash +working_folder=$1 + +# Downloading LLVM and Csmith sources, building Csmith +TMP_SOURCE_FOLDER=$(mktemp -d $working_folder/.sources.XXXXXXX.tmp) +cd $TMP_SOURCE_FOLDER +## LLVM +wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-11.0.0.src.tar.xz +tar -xvf llvm-11.0.0.src.tar.xz +rm llvm-11.0.0.src.tar.xz +mv llvm-11.0.0.src llvm-source +cd ./llvm-source/tools +wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/clang-11.0.0.src.tar.xz +tar -xvf clang-11.0.0.src.tar.xz +rm clang-11.0.0.src.tar.xz +mv clang-11.0.0.src clang +echo ">> Downloading LLVM 11 source filese ($TMP_SOURCE_FOLDER)" diff --git a/scripts/coverage/1-install-cov-gcc.sh b/scripts/coverage/1-install-cov-gcc.sh new file mode 100755 index 00000000..1f9af5d9 --- /dev/null +++ b/scripts/coverage/1-install-cov-gcc.sh @@ -0,0 +1,142 @@ +#!/bin/bash +shopt -s extglob # Activate extended pattern matching in bash +working_folder=$1 ## base folder +TMP_SOURCE_FOLDER=$2 ## Input from 0- script +nb_processes=$3 ## How many copies + +timeB=$(date +"%T") +echo ">> Start Script <$timeB>" + +## Assure we are in gcc-10! +sudo apt install gcc-10 g++-10 +sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 990 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 +sudo rm -f /usr/bin/cpp /usr/bin/gcc /usr/bin/g++ /usr/bin/gcov +sudo rm -f /usr/local/bin/cpp /usr/local/bin/gcc /usr/local/bin/g++ /usr/local/bin/gcov +sudo ln -s /usr/bin/cpp-10 /usr/bin/cpp +sudo ln -s /usr/bin/gcc-10 /usr/bin/gcc +sudo ln -s /usr/bin/g++-10 /usr/bin/g++ +sudo ln -s /usr/bin/gcov-10 /usr/bin/gcov +echo ">> End Settings" + +# Copying GCC source and Csmith exe to parallel execution folders, then install GCC with coverage data in each folder +i=0 # compile a basic one - never to use. +while (( $i <= $nb_processes )); +do + ## Start loop i + echo ">> Copy gcc folder-$i of gcc" + + ### Create a working folder with GCC source + rm -rf $working_folder/gcc-csmith-$i ## Remove the old version + mkdir $working_folder/gcc-csmith-$i ## Create a new version + cp -r $TMP_SOURCE_FOLDER/* $working_folder/gcc-csmith-$i ## Copy the data from the temp download folder + + ### Update Csmith settings + cd $working_folder/gcc-csmith-$i + echo $working_folder/gcc-csmith-$i"/gcc-install/bin/gcc -O2" > ./csmith/scripts/compiler_test.in + + ### GCC PART: with instrumentation + # Setting the env. + cov. and keeping information of the versions + cd $working_folder/gcc-csmith-$i; mkdir compilation_info; mv $working_folder/gcc-csmith-$i/gcc $working_folder/gcc-csmith-$i/gcc-source + cd gcc-source; git status > $working_folder/gcc-csmith-$i/compilation_info/git_info.txt; git log | head -10 >> $working_folder/gcc-csmith-$i/compilation_info/git_info.txt + gcc --version > $working_folder/gcc-csmith-$i/compilation_info/gcc-version.txt; g++ --version >> $working_folder/gcc-csmith-$i/compilation_info/gcc-version.txt; gcov --version >> $working_folder/gcc-csmith-$i/compilation_info/gcc-version.txt; cpp --version >> $working_folder/gcc-csmith-$i/compilation_info/gcc-version.txt + cd $working_folder/gcc-csmith-$i ## Assure we are now here. + + ## Finish settings and moving files + echo ">> gcc folder-$i is set" + + if (( 0 < $i )); then + ## Compile with coverage + ( + ## Set output folders + sudo mkdir -p /tmp/gcc/deleteme/coverage_gcda_files/application_run-init + mkdir gcc-build gcc-install + cd ./gcc-build + + # Covarage flags: + export GCOV_PREFIX=/tmp/gcc/deleteme/coverage_gcda_files/application_run-init ## Send gcda of build to temp. + export GCOV_PREFIX_STRIP=0 + export CFLAGS='-g -O0 --coverage -ftest-coverage -fprofile-arcs' + export CXXFLAGS='-g -O0 --coverage -ftest-coverage -fprofile-arcs' + export LDFLAGS='-lgcov --coverage -ftest-coverage -fprofile-arcs' + + ./../gcc-source/configure --disable-multilib --disable-bootstrap --enable-coverage=noopt --enable-targets='X86' --enable-languages='c,c++,lto,objc,obj-c++' --with-gmp=/tmp/gcc/ --with-mpfr=/tmp/gcc/ --with-mpc=/tmp/gcc/ --with-isl=/tmp/isl --prefix=$working_folder/gcc-csmith-$i/gcc-install/ CFLAGS_FOR_TARGET='-g -O0 --coverage -ftest-coverage -fprofile-arcs' CXXFLAGS_FOR_TARGET='-g -O0 --coverage -ftest-coverage -fprofile-arcs' > $working_folder/gcc-csmith-$i/compilation_info/config_output.txt 2>&1 + + make -j$(nproc) > $working_folder/gcc-csmith-$i/compilation_info/build_output.txt 2>&1 + make -j$(nproc) install > $working_folder/gcc-csmith-$i/compilation_info/install_output.txt 2>&1 + #### WE KEEP build, install and source folders sperated. #### + + # Cleaning after build + sudo rm -rf /tmp/gcc/deleteme/coverage_gcda_files/application_run-init + unset CFLAGS + unset CXXFLAGS + unset LDFLAGS + unset GCOV_PREFIX + unset GCOV_PREFIX_STRIP + ) + ## Shall be here: cd $working_folder/gcc-csmith-$i/gcc + cd $working_folder/gcc-csmith-$i/gcc-source + + ## Remove other languages: brig, fortran, go, d + rm -rf libgfortran # FORTRAN + rm -rf libgo gotools # GO + rm -rf libphobos # D (also GDC is D) + rm -rf libada # ADA + ## Remove testsuite folders (gcc does not include these into coverage) + rm -rf gcc/testsuite libatomic/testsuite libffi/testsuite libgomp/testsuite libiberty/testsuite libitm/testsuite + rm -rf libstdc++-v3/testsuite libvtv/testsuite + + # stat for core: + c1=`find $working_folder/gcc-csmith-$i/gcc-source/ -name *.c | wc -l` + c2=`find $working_folder/gcc-csmith-$i/gcc-source/ -name *.h | wc -l` + c3=`find $working_folder/gcc-csmith-$i/gcc-build/ -name *.gcno | wc -l` + c4=`find $working_folder/gcc-csmith-$i/gcc-build/ -name *.gcda | wc -l` + echo ">> Built GCC in $working_folder/gcc-csmith-$i/gcc-build/, installed in $working_folder/gcc-csmith-$i/gcc-install/." + echo ">> Compilation info. in $working_folder/gcc-csmith-$i/compilation_info/." + echo ">> Total ($c1) C-files, ($c2) header files, ($c3) gcno files, ($c4) gcda files." + find $working_folder/gcc-csmith-$i/gcc-source/ -name *.c > $working_folder/gcc-csmith-$i/compilation_info/gcc_c_files.txt + find $working_folder/gcc-csmith-$i/gcc-source/ -name *.h > $working_folder/gcc-csmith-$i/compilation_info/gcc_h_files.txt + find $working_folder/gcc-csmith-$i/gcc-build/ -name *.gcno > $working_folder/gcc-csmith-$i/compilation_info/gcc_gcno_files.txt + find . -name *.gcda -exec rm -rf {} \; + echo " --> Removed $c4 gcda files." + #### TESTS #### + else + ## Compile without coverage + ( + ## Set output folders + mkdir gcc-build gcc-install + cd ./gcc-build + + ## Config, compile and install: + ./../gcc-source/configure --disable-multilib --disable-bootstrap --enable-targets='X86' --enable-languages='c,c++,lto,objc,obj-c++' --with-gmp=/tmp/gcc --with-mpfr=/tmp/gcc --with-mpc=/tmp/gcc --with-isl=/tmp/isl --prefix=$working_folder/gcc-csmith-$i/gcc-install/ > $working_folder/gcc-csmith-$i/compilation_info/config_output.txt 2>&1 + + make -j$(nproc) > $working_folder/gcc-csmith-$i/compilation_info/build_output.txt 2>&1 + make -j$(nproc) install > $working_folder/gcc-csmith-$i/compilation_info/install_output.txt 2>&1 + #### WE KEEP build, install and source folders sperated. #### + ) + fi + + ## Zip it with soft links - unzip with h option: tar -xhzvf gcc-csmith-0.tar.gz + cd $working_folder + tar -czvf gcc-csmith-$i.tar.gz gcc-csmith-$i/ + # Next core + i=$(($i+1)) + + ## Assure we are using the built gcc: + usrLib=$working_folder/gcc-csmith-0/gcc-install + sudo rm -f /usr/bin/cpp /usr/bin/gcc /usr/bin/g++ /usr/bin/gcov + sudo rm -f /usr/local/bin/cpp /usr/local/bin/gcc /usr/local/bin/g++ /usr/local/bin/gcov + sudo ln -s $usrLib/bin/cpp /usr/bin/cpp + sudo ln -s $usrLib/bin/gcc /usr/bin/gcc + sudo ln -s $usrLib/bin/g++ /usr/bin/g++ + sudo ln -s $usrLib/bin/gcov /usr/bin/gcov +done + +## Revert back all gcc changes: +sudo rm -f /usr/bin/cpp /usr/bin/gcc /usr/bin/g++ /usr/bin/gcov +sudo rm -f /usr/local/bin/cpp /usr/local/bin/gcc /usr/local/bin/g++ /usr/local/bin/gcov +sudo ln -s /usr/bin/cpp-10 /usr/bin/cpp +sudo ln -s /usr/bin/gcc-10 /usr/bin/gcc +sudo ln -s /usr/bin/g++-10 /usr/bin/g++ +sudo ln -s /usr/bin/gcov-10 /usr/bin/gcov + +#rm -rf $TMP_SOURCE_FOLDER diff --git a/scripts/coverage/1-install-cov-llvm.sh b/scripts/coverage/1-install-cov-llvm.sh new file mode 100755 index 00000000..93273ffa --- /dev/null +++ b/scripts/coverage/1-install-cov-llvm.sh @@ -0,0 +1,75 @@ +#!/bin/bash +shopt -s extglob # Activate extended pattern matching in bash +working_folder=$1 ## base folder +TMP_SOURCE_FOLDER=$2 ## Input from 0- script +nb_processes=$3 ## How many copies + +timeB=$(date +"%T") +echo ">> Start Script <$timeB>" + +# Copying LLVM source and Csmith exe to parallel execution folders, then install LLVM with coverage data in each folder +i=1 +while (( $i <= $nb_processes )); +do + ### Create a working folder with LLVM source + rm -rf $working_folder/llvm-csmith-$i ## Remove the old version + mkdir $working_folder/llvm-csmith-$i ## Create a new version + cp -rf $TMP_SOURCE_FOLDER/* $working_folder/llvm-csmith-$i ## Copy the data from the temp download folder + + ### Update Csmith settings + mkdir $working_folder/llvm-csmith-$i/csmith/scripts/ ## Just incase it isn't there + cd $working_folder/llvm-csmith-$i + echo $working_folder/llvm-csmith-$i"/llvm-install/usr/local/bin/clang -O3" > ./csmith/scripts/compiler_test.in + + ## Save information regarding the version and how we compile it + mkdir compilation_info + echo ">> Adding compilation_info" + echo " - date: $(date '+%Y-%m-%d at %H:%M.%S')" > $working_folder/llvm-csmith-$i/compilation_info/llvm-version.txt + echo " - host name $(hostname -f)" >> $working_folder/llvm-csmith-$i/compilation_info/llvm-version.txt + echo " - current path: $(pwd)" >> $working_folder/llvm-csmith-$i/compilation_info/llvm-version.txt + gcc-10 --version >> $working_folder/llvm-csmith-$i/compilation_info/llvm-version.txt; g++-10 --version >> $working_folder/llvm-csmith-$i/compilation_info/llvm-version.txt; gcov-10 --version >> $working_folder/llvm-csmith-$i/compilation_info/llvm-version.txt; cpp-10 --version >> $working_folder/llvm-csmith-$i/compilation_info/llvm-version.txt; /usr/bin/cc --version >> $working_folder/llvm-csmith-$i/compilation_info/llvm-version.txt; /usr/bin/c++ --version >> $working_folder/llvm-csmith-$i/compilation_info/llvm-version.txt + + ### LLVM PART: with instrumentation + # Setting the env. + cov. + { + mkdir llvm-build llvm-install + cd ./llvm-build + set CFLAGS='--coverage -ftest-coverage -fprofile-arcs -fno-inline' + set CXXFLAGS='--coverage -ftest-coverage -fprofile-arcs -fno-inline' + set LDFLAGS='-lgcov --coverage -ftest-coverage -fprofile-arcs' + set CXX=g++-10 + set CC=gcc-10 + + # Cmake and build of LLVM + timeS=$(date +"%T") + echo "Configuration: cmake -GNinja ../llvm-source -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_BUILD_EXAMPLES=OFF -DLLVM_BUILD_TESTS=OFF -DLLVM_BUILD_DOCS=OFF -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_EXE_LINKER_FLAGS=--coverage <$timeS>" >> $working_folder/llvm-csmith-$i/compilation_info/llvm-version.txt + cmake -GNinja ../llvm-source -DCMAKE_BUILD_TYPE="Release" -DLLVM_TARGETS_TO_BUILD="X86" -DLLVM_BUILD_EXAMPLES="OFF" -DLLVM_BUILD_TESTS="OFF" -DLLVM_BUILD_DOCS="OFF" -DCMAKE_C_FLAGS="--coverage" -DCMAKE_CXX_FLAGS="--coverage" -DCMAKE_EXE_LINKER_FLAGS="--coverage" > $working_folder/llvm-csmith-$i/compilation_info/config_output.txt 2>&1 + timeB=$(date +"%T") + echo ">> Build LLVM with ninja to ./llvm-build <$timeB>" + ninja > $working_folder/llvm-csmith-$i/compilation_info/build_output.txt 2>&1 + ninja check-clang >> $working_folder/llvm-csmith-$i/compilation_info/build_output.txt 2>&1 + grep "FAILED:" $working_folder/llvm-csmith-$i/compilation_info/build_output.txt + timeI=$(date +"%T") + echo ">> Install LLVM locallly with ninja to ./llvm-install <$timeI>" + DESTDIR=../llvm-install ninja install -k 10 > $working_folder/llvm-csmith-$i/compilation_info/install_output.txt 2>&1 + grep "FAILED:" $working_folder/llvm-csmith-$i/compilation_info/install_output.txt + + # Cleaning after build + unset CFLAGS + unset CXXFLAGS + unset LDFLAGS + unset CXX + unset CC + } + + # Next core + cd .. + cd .. + timeT=$(date +"%T") + echo ">> Create .tar of LLVM local installation <$timeT>" + tar -czvf llvm-csmith-$i.tar.gz llvm-csmith-$i/ >> $working_folder/llvm-csmith-$i/compilation_info/install_output.txt 2>&1 + i=$(($i+1)) +done +timeE=$(date +"%T") +echo "Done. <$timeE>" +rm -rf $TMP_SOURCE_FOLDER diff --git a/scripts/coverage/7-gen-statistic-gcov-diff-tab_gfauto.sh b/scripts/coverage/7-gen-statistic-gcov-diff-tab_gfauto.sh new file mode 100755 index 00000000..3748ae0a --- /dev/null +++ b/scripts/coverage/7-gen-statistic-gcov-diff-tab_gfauto.sh @@ -0,0 +1,116 @@ +#!/bin/bash +working_folder1=$1 # Add where gcov files are for compiler/test 1 +working_folder2=$2 # Add where gcov files are for compiler/test 2 +output_table_file=$3 # Where to put the table +unitest_skip=utils/unittest # unittest folder should not be part of cov measurements +## E.g., ./7-gen-statistic-gcov-diff-tab_gfauto.sh /home/user42/llvm-csmith-1/coverage_processed-1/x-10/cov.out/ /home/user42/llvm-csmith-1/coverage_processed-0/x-10/cov.out/ table__all_1_0.csv +## E.g., ./7-gen-statistic-gcov-diff-tab_gfauto.sh /home/user42/llvm-csmith-1/coverage_processed-0/x-10/cov.out/ /home/user42/llvm-csmith-1/coverage_processed-1/x-10/cov.out/ table__all_0_1.csv + +## INIT: +rm $output_table_file +rm list_gcov_1.txt +rm list_gcov_2.txt + +wf1_size=${#working_folder1} +wf2_size=${#working_folder2} + +## START ## + +## Constract the lines on the table +find $working_folder1 -type f | sort >> list_gcov_1.txt +find $working_folder2 -type f | sort >> list_gcov_2.txt +linecount1=`cat list_gcov_1.txt | wc -l` +linecount2=`cat list_gcov_2.txt | wc -l` +if [ ! "$linecount1" == "$linecount2" ]; then + echo ">> ERROR: not all files exists in both set-1 and set-2" + + # cleaning + rm list_gcov_1.txt + rm list_gcov_2.txt + + exit 1 +fi + +## If same population, continue: + +## file name: the name of the file we comapare +## #line 1: number of lines in file 1 +## #line 2: number of lines in file 2 +## #line 1: number of lines hit in file 1 +## #line 2: number of lines hit in file 2 +## LH1 U LH2: number of lines with hit in any of the files +## LH1 U LH2 (with c1!=c2): same but with a counter difference +## LH1 /\ LH2 (with c1==c2): same but where the counters are exactly the same +## LH1\LH2 : hit in file 1 and miss in file 2 +## LH2\LH1 : hit in file 2 and miss in file 1 +## LH1 > LH2: hit of LH1 is larger +## LH1 < LH2: hit of LH2 is larger +## LH1 > LH2(x1): hit of LH1 is larger in order (e.g., 100k and 10k) +## LH1 < LH2(x1): hit of LH2 is larger in order (e.g., 10k and 100k) +## LH1 >> LH2(x4): hit of LH1 is larger in magnitude order +## LH1 << LH2(x4): hit of LH2 is larger in magnitude order +## Flag(LH1/LH2 != LH2/LH1): flag if the unique hits are not the same +echo ">> file name,#line 1,#line 2,#line 1 hit,#line 2 hit,LH1 U LH2,LH1 U LH2 (with c1!=c2),LH1 /\ LH2 (with c1==c2),LH1/LH2,LH2/LH1,LH1 > LH2,LH1 < LH2,LH1 > LH2(x1),LH1 < LH2(x1),LH1 >> LH2(x4),LH1 << LH2(x4),Flag(LH1/LH2 != LH2/LH1)" >> $output_table_file +while IFS= read -r -u 4 file_name_1 && IFS= read -r -u 5 file_name_2; do + ## check that the files matches. (without the working folders) + fn1=${file_name_1:$wf1_size} + fn2=${file_name_2:$wf2_size} + if [[ ! "$fn1" == *"$unitest_skip"* ]]; then + if [ ! "$fn1" == "$fn2" ]; then + echo ">> ERROR: not all files exists in both set-1 and set-2" + echo $file_name_1 " <-> " $file_name_2 # DEBUG + exit 1 + fi + + ## Add a row to the csv file + tmp1=__tmp_1.txt + tmp2=__tmp_2.txt + test1=__test_1.txt + test2=__test_2.txt + cat $file_name_1 | sed 's:^ : 0 :1' | sed -n 's/\(^......[0-9]*[0-9] \).*$/\1/p' | cat -n > $tmp1 + cat $file_name_2 | sed 's:^ : 0 :1' | sed -n 's/\(^......[0-9]*[0-9] \).*$/\1/p' | cat -n > $tmp2 + cat -n $file_name_1 > $test1 + cat -n $file_name_2 > $test2 + diff_lines=`diff -y --suppress-common-lines $test1 $test2 | wc -l` + ./RRS7-2_LH_file.sh $fn1 $tmp1 $tmp2 $diff_lines >> $output_table_file + rm -f $tmp1 $tmp2 $test1 $test2 + fi +done 4> End extracting data for $linecount1:$linecount2 files." + +resT=`sed '1d' $output_table_file | cut -d',' -f2 | awk '{ sum += $1 } END { print sum }'` +echo "#line 1 ................. ==> "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f3 | awk '{ sum += $1 } END { print sum }'` +echo "#line 2 ................. ==> "$resT + +resT=`sed '1d' $output_table_file | cut -d',' -f4 | awk '{ sum += $1 } END { print sum }'` +echo "#line 1 HIT ............. ==> "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f5 | awk '{ sum += $1 } END { print sum }'` +echo "#line 2 HIT ............. ==> "$resT + +resT=`sed '1d' $output_table_file | cut -d',' -f6 | awk '{ sum += $1 } END { print sum }'` +echo "LH1 U LH2 ............... ==> "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f7 | awk '{ sum += $1 } END { print sum }'` +echo "LH1 U LH2 (with c1!=c2) . ==> "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f8 | awk '{ sum += $1 } END { print sum }'` +echo "LH1 /\ LH2 (with c1==c2) ==> "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f9 | awk '{ sum += $1 } END { print sum }'` +echo "LH1\LH2 ................. ==> "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f10 | awk '{ sum += $1 } END { print sum }'` +echo "LH2\LH1 ................. ==> "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f11 | awk '{ sum += $1 } END { print sum }'` +echo "LH1 > LH2 ............... ==> "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f12 | awk '{ sum += $1 } END { print sum }'` +echo "LH1 < LH2 ............... ==> "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f13 | awk '{ sum += $1 } END { print sum }'` +echo "LH1 >>x1 LH2 .............. ==> "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f14 | awk '{ sum += $1 } END { print sum }'` +echo "LH1 < "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f15 | awk '{ sum += $1 } END { print sum }'` +echo "LH1 >>x4 LH2 .............. ==> "$resT +resT=`sed '1d' $output_table_file | cut -d',' -f16 | awk '{ sum += $1 } END { print sum }'` +echo "LH1 < "$resT + +# cleaning +rm list_gcov_1.txt +rm list_gcov_2.txt diff --git a/scripts/coverage/RRS7-2_LH_file.sh b/scripts/coverage/RRS7-2_LH_file.sh new file mode 100755 index 00000000..470ee191 --- /dev/null +++ b/scripts/coverage/RRS7-2_LH_file.sh @@ -0,0 +1,65 @@ +#!/bin/bash +function get_line_stat { + # Lines cov one side: + # ================== + + # Line left and right that has the same counter (exactly the same) - can quit here if so. + file12_hl_same=$(($file12_hl_all-$file12_hl_not_same)) + if [ "$file12_hl_all" == "$file12_hl_same" ]; then + echo ">> $fileO,$file1_ln,$file2_ln,$file1_cov_any,$file2_cov_any,$file12_hl_all,$file12_hl_not_same,$file12_hl_same,0,0,0,0,0,0,0,0,X " + else + # Exists on left: + file1_hl=`diff -y --suppress-common-lines $file1 $file2 | grep -ve '.* 0 .* 0' | grep ' 0 ' | grep ".*|.* 0 " | wc -l` + + # Exists on right: + file2_hl=`diff -y --suppress-common-lines $file1 $file2 | grep -ve '.* 0 .* 0' | grep ' 0 ' | grep " 0 .*|" | wc -l` + # Total not the same, miss + hits: + totDiff=$(($file12_hl_not_same+$file1_hl+$file2_hl)) + if [ ! "$diffline" == "$totDiff" ]; then + echo ">> $fileO,$file1_ln,$file2_ln,$file1_cov_any,$file2_cov_any,$file12_hl_all,$file12_hl_not_same,0,0,0,0,0,0,0,0,0,E2 ($diffline:$totDiff) " + else + ## Major gap in cov.--> Fix stranger cases after replacing gcov with cov + counter1=`diff -y --suppress-common-lines $file1 $file2 | awk '{print length($5)-length($2)+4}' | grep -e "-" | wc -l` + counter2=`diff -y --suppress-common-lines $file1 $file2 | awk '{print length($2)-length($5)+4}' | grep -e "-" | wc -l` + counter1L=`diff -y --suppress-common-lines $file1 $file2 | awk '{print length($5)-length($2)}' | grep -e "-" | wc -l` + counter2L=`diff -y --suppress-common-lines $file1 $file2 | awk '{print length($2)-length($5)}' | grep -e "-" | wc -l` + counter1Bigger=`diff -y --suppress-common-lines $file1 $file2 | awk '{print $5-$2}' | grep -e "-" | wc -l` + counter2Bigger=`diff -y --suppress-common-lines $file1 $file2 | awk '{print $2-$5}' | grep -e "-" | wc -l` + echo $fileO >> out_test.txt + diff -y --suppress-common-lines $file1 $file2 >> out_test.txt + + ## Short report for debug: + if [ "$file1_hl" == "$file2_hl" ]; then + echo ">> $fileO,$file1_ln,$file2_ln,$file1_cov_any,$file2_cov_any,$file12_hl_all,$file12_hl_not_same,$file12_hl_same,$file1_hl,$file2_hl,$counter1Bigger,$counter2Bigger,$counter1L,$counter2L,$counter1,$counter2,F " + else + echo ">> $fileO,$file1_ln,$file2_ln,$file1_cov_any,$file2_cov_any,$file12_hl_all,$file12_hl_not_same,$file12_hl_same,$file1_hl,$file2_hl,$counter1Bigger,$counter2Bigger,$counter1L,$counter2L,$counter1,$counter2,T " + fi + fi + fi +} + +############################ +## MAIN: +############################ + +fileO=$1 # file name originally +file1=$2 # file 1 +file2=$3 # file 2 +diffline=$4 # Number of lines different between the files + +file1_ln=`cat $file1 | wc -l` # size file 1 +file2_ln=`cat $file2 | wc -l` # size file 2 +if [ ! "$file1_ln" == "$file2_ln" ]; then + echo ">> $fileO,$file1_ln,$file2_ln,$file1_cov_any,$file2_cov_any,0,0,0,0,0,0,0,0,0,0,0,E1 " +else + file12_hl_not_same=`diff -y --suppress-common-lines $file1 $file2 | grep -ve ' 0 ' | wc -l` # Any lines covered + file12_hl_all=`diff -y $file1 $file2 | grep -ve '.* 0 .* 0' | wc -l` # Any lines covered + file1_cov_any=`cat $file1 | awk '{if ($2 >0) print($2)}' | wc -l` + file2_cov_any=`cat $file2 | awk '{if ($2 >0) print($2)}' | wc -l` + if [ "$file12_hl_all" == "0" ]; then + echo ">> $fileO,$file1_ln,$file2_ln,$file1_cov_any,$file2_cov_any,$file12_hl_all,$file12_hl_not_same,0,0,0,0,0,0,0,0,0,X " + else + ## ONLY IF THERE ARE LINES COVERED! + get_line_stat + fi +fi diff --git a/scripts/coverage/gcc-coverage/5-test-dest-machine.sh b/scripts/coverage/gcc-coverage/5-test-dest-machine.sh new file mode 100755 index 00000000..ef77e77a --- /dev/null +++ b/scripts/coverage/gcc-coverage/5-test-dest-machine.sh @@ -0,0 +1,98 @@ +#!/bin/bash +process_number=$1 ## from 0 +baseD=/home/user42 +working_folder=$baseD/gcc-csmith-1 ##$process_number +nb_progs_to_gen=4 +nb_progs_to_gen_per_step=2 + +base=$baseD/RRS_EXPR # $baseD/git/RRS_EXPR +seed_location=$base/scripts/WA-v2-standalone/seeds/seeds-$process_number.txt # seed location +csmith_location=$base/csmith # csmith location +csmith_exec=$base/csmith/build/src/csmith # csmith exec +configuration_location=$working_folder/csmith/scripts/compiler_test.in # config file locatoin +outputs_location=../../ # where we will put all outputs +#testcaseConfg=$base/scripts/WA-v2-standalone/seedsProbs # Test-case configuration (WA and RRS) +##testcaseConfg=$base/scripts/WA-v2-standalone/CsmithSeedsProbs # Csmith mix +#testcaseConfg=$base/scripts/WA-v2-standalone/CsmithEdgeSeedsProbs_macros # CsmithEdge func or macro +testcaseConfg=$base/scripts/WA-v2-standalone/CsmithEdgeSeedsProbs_func # CsmithEdge func or macro +#DA_folder=$testcaseConfg/seedsSafeLists # DA (RRS lists) folder +##DA_folder=$base/scripts/RSS-v2-general/seedsSafeLists_small # Csmith mix +DA_folder=$base/scripts/WA-v2-standalone/seedsProbs/seedsSafeLists # CsmithEdge func or macro +runtime=RRS_runtime_test # Runtime for WA and RRS files +compile_line="-B$working_folder/gcc-build/gcc/. -lgcov -w" #how we compile it +compile_line_lib_default="-I$csmith_location/runtime -I$csmith_location/build/runtime" +#compile_line_lib_default="-I$csmith_location/runtime -I$csmith_location/build/runtime -DUSE_MATH_MACROS" +modify=1 #1 #0 #2 + +## Check we have process number +if [ -z "$1" ] + then + echo "No process number supplied." + exit 1 +fi + +time1=$(date +"%T") +echo "EVOLUTION OF GCC COVERAGE WHEN COMPILING "$nb_progs_to_gen" CSMITH PROGRAMS BY STEPS OF "$nb_progs_to_gen_per_step + +rm -f $working_folder/seeds-$process_number.txt +rm -rf $working_folder/coverage_processed-$modify +rm -rf $working_folder/coverage_gcda_files/application_run-$modify + +nb_gen_progs=0 # Start from 0 +i=0 # Init counters +echo "--> RESETING COVERAGE DATA...("$time1")" + +# Keep current folder: +current_folder=`pwd` + +######################################## Prepare the env. for using GCC compiler ######################################## +# Restore softlink linker lto +#cd $working_folder/gcc-build/gcc/ +#ln -sf liblto_plugin.so.0.0.0 liblto_plugin.so +#ln -sf liblto_plugin.so.0.0.0 liblto_plugin.so.0 + +## Assure we are using the gcov in the gcc built: +#usrLib=$working_folder/gcc-install +#sudo rm -f /usr/local/bin/gcov /usr/bin/gcov +#sudo ln -s $usrLib/bin/gcov /usr/bin/gcov + +# Back to original folder +#cd $current_folder + +######################################## Start Cov. ######################################## +# Loop over compilation and coverage measurement +while (( $nb_gen_progs < $nb_progs_to_gen )); +do + i=$(($i+1)) + nb_gen_progs=$(($nb_gen_progs + $nb_progs_to_gen_per_step)) + time2=$(date +"%T") + echo "--> COMPILING "$nb_progs_to_gen_per_step" PROGRAMS ("$(($nb_progs_to_gen-$nb_gen_progs))" REMAINING)... ("$time2")" + + # Set location to record the data + mkdir -p $working_folder/coverage_gcda_files/application_run-$modify + + # Run compiler and save coverage data + export GCOV_PREFIX=$working_folder/coverage_gcda_files/application_run-$modify + export GCOV_PREFIX_STRIP=0 + ./RSS5_1-compiler_test.sh $process_number $nb_gen_progs $nb_progs_to_gen_per_step $seed_location $csmith_location $configuration_location $outputs_location "$compile_line" "$DA_folder" "$testcaseConfg" "$csmith_exec" "$runtime" "$compile_line_lib_default" $modify + unset GCOV_PREFIX + unset GCOV_PREFIX_STRIP + + time3=$(date +"%T") + echo "--> MEASURING COVERAGE to $working_folder/coverage_processed-$modify/x-$i... ("$time3")" + mkdir -p $working_folder/coverage_processed-$modify/x-$i + ( + cd $working_folder/coverage_processed-$modify/x-$i + source $baseD/graphicsfuzz/gfauto/.venv/bin/activate + gfauto_cov_from_gcov --out run_gcov2cov.cov $working_folder/gcc-build/ $working_folder/coverage_gcda_files/application_run-$modify/ --num_threads 32 --gcov_uses_json >> gfauto.log 2>&1 + gfauto_cov_to_source --coverage_out cov.out --cov run_gcov2cov.cov $working_folder/gcc-build/ >> gfauto.log 2>&1 + ) + cd $current_folder +done +time2=$(date +"%T") +echo "DONE. RESULTS AVAILABLE IN $working_folder/coverage_processed-$modify/x-$i ($time2)" +ls -l $working_folder/coverage_processed-$modify/x-$i + +## Revert back all gcc changes: restore gcov +#sudo rm -f /usr/local/bin/gcov /usr/bin/gcov +#sudo ln -s /usr/bin/gcov-9 /usr/bin/gcov diff --git a/scripts/coverage/gcc-coverage/DA-only-gcc/5-test-dest-machine.sh b/scripts/coverage/gcc-coverage/DA-only-gcc/5-test-dest-machine.sh new file mode 100755 index 00000000..33f73b57 --- /dev/null +++ b/scripts/coverage/gcc-coverage/DA-only-gcc/5-test-dest-machine.sh @@ -0,0 +1,107 @@ +#!/bin/bash +process_number=$1 +baseD=/home/user42 +working_folder=$baseD/gcc-csmith-1 ##$process_number +nb_progs_to_gen=4 #100000 +nb_progs_to_gen_per_step=2 #20000 +#seed_location=../../Data/seeds_all/seeds_out$process_number.txt # seed location + +base=$baseD/RRS_EXPR # $baseD/git/RRS_EXPR +seed_location=$base/scripts/WA-v2-standalone/seeds/seeds-$process_number.txt # seed location +csmith_location=$base/csmith # csmith location +# csmith_exec=$base/csmith/build/src/csmith # csmith exec +configuration_location=$working_folder/csmith/scripts/compiler_test.in # config file locatoin +outputs_location=../../ # where we will put all outputs + +## Change per test: +#csmith_flags=" --bitfields --packed-struct " #" --bitfields --packed-struct --math-notmp --annotated-arith-wrappers" + +## 0 +#modify=0 # 0-original, 1-modify +#compile_line="-B$working_folder/gcc-build/gcc/. -I$csmith_location/runtime -I$csmith_location/build/runtime -lgcov -w" #how we compile it,modify=0 +##compile_line="-B$working_folder/gcc-build/gcc/. -I$csmith_location/runtime -I$csmith_location/build/runtime -lgcov -w -DUSE_MATH_MACROS" #how we compile it,modify=0 +#csmith_flags=" --bitfields --packed-struct " #" --bitfields --packed-struct --math-notmp" + +## 1 +modify=1 # 0-original, 1-modify +compile_line="-B$working_folder/gcc-build/gcc/. -I$csmith_location/RRS_runtime_test -I$csmith_location/build/runtime -lgcov -w" #how we compile it,modify=1 +#compile_line="-B$working_folder/gcc-build/gcc/. -I$csmith_location/RRS_runtime_test -I$csmith_location/build/runtime -DUSE_MATH_MACROS -lgcov -w" #how we compile it,modify=1 +csmith_flags=" --bitfields --packed-struct --annotated-arith-wrappers" #" --bitfields --packed-struct --math-notmp" + +## Check we have process number +if [ -z "$1" ] + then + echo "No process number supplied." + exit 1 +fi + +time1=$(date +"%T") +echo "EVOLUTION OF GCC COVERAGE WHEN COMPILING "$nb_progs_to_gen" CSMITH PROGRAMS BY STEPS OF "$nb_progs_to_gen_per_step +if [ "$#" -ne 3 ]; then + rm -f $working_folder/seeds-$process_number.txt + rm -rf $working_folder/coverage_processed-$modify + rm -rf $working_folder/coverage_gcda_files/application_run-$modify + + nb_gen_progs=0 # Start from 0 + i=0 # Init counters + echo "--> RESETING COVERAGE DATA...("$time1")" +else + nb_gen_progs=$2 # Restore + i=$3 # Restore counters + echo "RESTORE COVERAGE DATA...("$nb_gen_progs","$i","$time1")" +fi + +# Keep current folder: +current_folder=`pwd` + +######################################## Prepare the env. for using GCC compiler ######################################## +# Restore softlink linker lto +#cd $working_folder/gcc-build/gcc/ +#ln -sf liblto_plugin.so.0.0.0 liblto_plugin.so +#ln -sf liblto_plugin.so.0.0.0 liblto_plugin.so.0 + +## Assure we are using the gcov in the gcc built: +#usrLib=$working_folder/gcc-install +#sudo rm -f /usr/local/bin/gcov /usr/bin/gcov +#sudo ln -s $usrLib/bin/gcov /usr/bin/gcov + +# Back to original folder +#cd $current_folder + +######################################## Start Cov. ######################################## +# Loop over compilation and coverage measurement +while (( $nb_gen_progs < $nb_progs_to_gen )); +do + i=$(($i+1)) + nb_gen_progs=$(($nb_gen_progs + $nb_progs_to_gen_per_step)) + time2=$(date +"%T") + echo "--> COMPILING "$nb_progs_to_gen_per_step" PROGRAMS ("$(($nb_progs_to_gen-$nb_gen_progs))" REMAINING)... ("$time2")" + + # Set location to record the data + mkdir -p $working_folder/coverage_gcda_files/application_run-$modify + + # Run compiler and save coverage data + export GCOV_PREFIX=$working_folder/coverage_gcda_files/application_run-$modify + export GCOV_PREFIX_STRIP=0 + ./RRS5_1-compiler_test.sh $process_number $nb_gen_progs $nb_progs_to_gen_per_step $modify $seed_location $csmith_location $configuration_location $outputs_location "$compile_line" "$csmith_flags" + unset GCOV_PREFIX + unset GCOV_PREFIX_STRIP + + time3=$(date +"%T") + echo "--> MEASURING COVERAGE... ("$time3")" + mkdir -p $working_folder/coverage_processed-$modify/x-$i + ( + cd $working_folder/coverage_processed-$modify/x-$i + source $baseD/graphicsfuzz/gfauto/.venv/bin/activate + gfauto_cov_from_gcov --out run_gcov2cov.cov $working_folder/gcc-build/ $working_folder/coverage_gcda_files/application_run-$modify/ --num_threads 32 --gcov_uses_json >> gfauto.log 2>&1 + gfauto_cov_to_source --coverage_out cov.out --cov run_gcov2cov.cov $working_folder/gcc-build/ >> gfauto.log 2>&1 + ) + cd $current_folder +done +time2=$(date +"%T") +echo "DONE. RESULTS AVAILABLE IN $working_folder/coverage_processed-$modify/x-$i ($time2)" +ls -l $working_folder/coverage_processed-$modify/x-$i + +## Revert back all gcc changes: restore gcov +##sudo rm -f /usr/local/bin/gcov /usr/bin/gcov +##sudo ln -s /usr/bin/gcov-10 /usr/bin/gcov diff --git a/scripts/coverage/gcc-coverage/DA-only-gcc/RRS5_1-compiler_test.sh b/scripts/coverage/gcc-coverage/DA-only-gcc/RRS5_1-compiler_test.sh new file mode 100755 index 00000000..16b9213d --- /dev/null +++ b/scripts/coverage/gcc-coverage/DA-only-gcc/RRS5_1-compiler_test.sh @@ -0,0 +1,67 @@ +#!/bin/bash +process_number=$1 # Process number +nb_gen_progs=$2 # How many we will do by the end of the loop +nb_progs_to_gen_per_step=$3 # How many we shall generate now +initc=$(($nb_gen_progs + 1 - $nb_progs_to_gen_per_step)) # Calc. the position of the first seed of this batch +modify=$4 # With modifications --> use 1 +seed_location=$5 # Seeds file location, coverage takes good+bad +csmith_location=$6 # Where is the csmith we use +compconfg=$7 # Where to take the compiler configuration +outputs_location=$8 # Where to expect the results +compile_line=$9 # Include and -D params for testcase +csmith_flags=${10} # csmith flags +# Was: $working_folder/csmith/scripts/compiler_test.in +#echo "Get compilers lists: $working_folder/csmith/scripts/compiler_test.in" +### Check we have all the data we need before proceeding: +if [ "$#" -ne 10 ]; then + echo "Illegal number of parameters. Please enter compiler and number of testcases to generate." + echo "Requires: process number, size of population, size of population per step, flag (1-modify, 0-original csmith) seeds file, csmith location, configuration file location, output location, compilation line for test case, and csmith flags." + exit 1 +fi + +#echo "Get compilers lists: $working_folder/csmith/scripts/compiler_test.in" +# Sets the logger -general name +logger=$outputs_location"__logger_cov" +if [[ "$modify" == "1" ]] + then + logger=$logger"M" +fi + +# Loop over compilation for the current batch of seeds +while read str; do + arr=( $str ) + + ## Create the temp folder if required + compiler=${arr[0]} + folder=$compiler-mainRes + if [ ! -d $folder ] + then + mkdir $folder + fi + + # Logger per-compiler - add only once (at start) + loggerCurr=$logger"_"${compiler//\//\_}"_"seeds-$process_number.txt + #if (($initc < 2)); then + touch $loggerCurr + echo "## (modify=$modify) Configuration: $str" >> $loggerCurr + echo "## - Compiler version:" `$compiler --version | head -1` >> $loggerCurr + echo "## - Applying coverage for $nb_gen_progs testcases in batches of $nb_progs_to_gen_per_step" >> $loggerCurr + echo "## - CSMITH_HOME: $csmith_location" >> $loggerCurr + echo "## - CSMITH_USER_OPTIONS: $csmith_flags" >> $loggerCurr + echo "## - Configuration file: $compconfg" >> $loggerCurr + echo "## - Compilation of testcase: $compile_line" >> $loggerCurr + echo "## - Seeds: $seed_location for process $process_number" >> $loggerCurr + echo "## - date: $(date '+%Y-%m-%d at %H:%M.%S')" >> $loggerCurr + echo "## - host name $(hostname -f)" >> $loggerCurr + #fi + + ## Perform per configuration + for (( c=$initc; c<=$nb_gen_progs; c++ )) + do + cp=$c'p' + seed=`sed -n $cp $seed_location` + ./RRS5_2_constructModifyTests.sh $compiler $process_number $seed $loggerCurr 1 $modify $csmith_location "$csmith_flags" "$compile_line" "${arr[1]}" + done + ## Per configuartion the loop generates $initc to $nb_gen_progs modified testcases +done < $compconfg +## The loop is going over all the configurations in compiler_test.in file diff --git a/scripts/coverage/gcc-coverage/DA-only-gcc/RRS5_2_constructModifyTests.sh b/scripts/coverage/gcc-coverage/DA-only-gcc/RRS5_2_constructModifyTests.sh new file mode 100755 index 00000000..cdf0d569 --- /dev/null +++ b/scripts/coverage/gcc-coverage/DA-only-gcc/RRS5_2_constructModifyTests.sh @@ -0,0 +1,169 @@ +#!/bin/bash + +##### Keep all the safe ops we need +function keep_required_safe { + testcaseName=$1 + testcaseModify=$folder/'__'$testcaseName'M.c' + filename="$testcaseRes" + while read -r line; do + data="$line" + + # Get locations: + temp=${#data} + size=$((temp - 44 -1)) + var="${data:44:$size}" + + isFirst=1 + locF=0 + funcF=0 + locations=$(echo $var | tr "," " \n") + for loc in $locations + do + if (($isFirst==1)) + then + isFirst=0 + funcF=$loc + else + locF=$loc + fi + #echo "[$loc]" + done + #echo "location is: [$locF]" + #echo "Function number is: [$funcF]" + + #Replace the rest of the calls to unsafe macros + keyword_raw='/* ___REMOVE_SAFE__OP *//*'$locF'*//* ___SAFE__OP */(' + keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + replacement_raw='(' + replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify + done < "$filename" +} + +#### Remove safe calls when not required +function replace2unsafe { + testcaseName=$1 + + testcaseModify=$folder/'__'$testcaseName'M.c' + + #Replace the rest of the calls to unsafe macros + keyword_raw='/* ___REMOVE_SAFE__OP */' + keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + replacement_raw='_unsafe_macro' + replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify + + + #Add missing header unsafe macros + #echo "#include \"unsafe_math_macros.h\""|cat - $testcaseModify > /tmp/out && mv /tmp/out $testcaseModify + #keyword_raw='#include "csmith.h"' + #keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + #replacement_raw='#include "unsafe_math_macros_eCast.h"' + #replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + #sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify +} + +#################################### Modify TEST #################################### +function modify_test { + ## Single test - create a test case and its precompilation + testcaseName='test'$1 + testcaseModify=$folder/'__'$testcaseName'M.c' + testcaseEXEC='__'$testcaseName'Exec' + + # Modify the test (the preprocessed file) + ulimit -St 500; $csmith_location/build/src/csmith --seed $seed $CSMITH_USER_OPTIONS > $testcaseModify + if [[ "$modify" == "1" ]] + then + # Keep ops required to be safe + keep_required_safe $testcaseName + #Replace the rest of the calls to unsafe macros + replace2unsafe $testcaseName + fi + + # Compile and test modify program: (with input compiler flags or with defualts) + ulimit -St 500;$compiler $compile_line ${compilerflags} $testcaseModify -o $testcaseEXEC + + # Test exec + filesize=`stat --printf="%s" $testcaseModify` + ulimit -St 500;./$testcaseEXEC >> $testcaselogger + echo "seed= $seed, size= $filesize" >> $testcaselogger + echo "---" >> $testcaselogger + + #clean + rm $testcaseModify + rm $testcaseEXEC +} + +### If it is a bad testcase, no need to run it, just build it for coverage measure. +function coverage_test_only { + ## Single test - create a test case and its precompilation + testcaseName='test'$1 + testcaseOriginal=$folder/'__'$testcaseName'O.c' + testcaseEXEC='__'$testcaseName'Exec' + + # Modify the test (the preprocessed file) + ulimit -St 500; $csmith_location/build/src/csmith --seed $seed $CSMITH_USER_OPTIONS > $testcaseOriginal + + # Compile and test modify program: (with input compiler flags or with defualts) + ulimit -St 500;$compiler $compile_line ${compilerflags} $testcaseOriginal -o $testcaseEXEC + + # Test exec + filesize=`stat --printf="%s" $testcaseOriginal` + echo "Skip execution. Bad testcase." >> $testcaselogger + echo "seed= $seed, size= $filesize" >> $testcaselogger + echo "---" >> $testcaselogger + + #clean + rm $testcaseOriginal + rm $testcaseEXEC +} + +################### MAIN ############################### +# Basic parameters +compiler=$1 # Compiler to test +process=$2 # To create temp storage for exec +seed=$3 # File with all the seeds to use +testcaselogger=$4 # The logger file for the results +cov=$5 # To test for cov=1, else =0 +modify=$6 # Do we run on modified or original tests +csmith_location=$7 # Csmith location +csmith_flags=$8 # Csmith options +compile_line=$9 # Includes + Duse +compilerflags=${10} # compiler flags + +# Single iteration, requires a compiler and a seed +#CSMITH_USER_OPTIONS=" --bitfields --packed-struct" +CSMITH_USER_OPTIONS=$csmith_flags + +## For debug only +#echo "Compiler=$compiler" +#echo "process=$process" +#echo "seed=$seed" +#echo "testcaselogger=$testcaselogger" +#echo "csmith_location=$csmith_location" +#echo "csmith_flags=$csmith_flags" +#echo "compile_line=$compile_line" +#echo "compilerflags=$compilerflags" +# Check if second parameter is a number +re='^[0-9]+$' +if ! [[ $seed =~ $re ]] ; then + echo ">> error: Not a number" >&2; exit 1 +fi + +# folders for all the results +folder=$compiler-mainRes +testcaseRes='../RSS-v2-general/seedsSafeLists_small'/'__test'$seed'Results' +if [ ! -f $testcaseRes ] + then + #echo "Seed's list is missing: $testcaseRes" + ## Only generate and compile test case to measure coverage + if [[ "$cov" == "1" ]] + then + coverage_test_only "$seed" + fi +else + modify_test "$seed" # Run a single test +fi diff --git a/scripts/coverage/gcc-coverage/RSS5_1-compiler_test.sh b/scripts/coverage/gcc-coverage/RSS5_1-compiler_test.sh new file mode 100755 index 00000000..84f49c3c --- /dev/null +++ b/scripts/coverage/gcc-coverage/RSS5_1-compiler_test.sh @@ -0,0 +1,73 @@ +#!/bin/bash +process_number=$1 # Process number +nb_gen_progs=$2 # How many we will do by the end of the loop +nb_progs_to_gen_per_step=$3 # How many we shall generate now +initc=$(($nb_gen_progs + 1 - $nb_progs_to_gen_per_step)) # Calc. the position of the first seed of this batch +seed_location=$4 # Seeds file location, coverage takes good+bad +csmith_location=$5 # Where is the csmith we use +compconfg=$6 # Where to take the compiler configuration +outputs_location=$7 # Where to expect the results +compile_line=$8 # Include and -D params for testcase +DA_folder=$9 # DA (RRS lists) folder +testcaseConfg=${10} # Test-case configuration (WA and RRS) +csmith_exec=${11} # Location of Csmith exec +runtime=${12} # Runtime for WA and RRS files +runtime_default=${13} # Runtime Csmith defualt lib (with no RRS) +modify=${14} # With modifications --> use 1 + +### Check we have all the data we need before proceeding: +if [ "$#" -ne 14 ]; then + echo "Illegal number of parameters. Please enter compiler and number of testcases to generate." + echo "Requires: process number, size of population, size of population per step, seeds file, csmith location, configuration file location, output location, compilation line for test case, RRS safe lists, WA config file, csmith exec, runtime lib for RRS, runtime lib without RRS, and modify or not version." + exit 1 +fi + +#echo "Get compilers lists: $working_folder/csmith/scripts/compiler_test.in" +# Sets the logger -general name +logger=$outputs_location"__logger_covM" + +# Loop over compilation for the current batch of seeds +while read str; do + arr=( $str ) + + ## Create the temp folder if required + compiler=${arr[0]} + folder=$compiler-mainRes + if [ ! -d $folder ] + then + mkdir $folder + fi + + # Logger per-compiler - add only once (at start) + loggerCurr=$logger"_"${compiler//\//\_}"_"seeds-$process_number.txt + if (($initc < 2)); then + touch $loggerCurr + echo "## (modify=$modify) Configuration: $str" >> $loggerCurr + echo "## - Compiler version:" `$compiler --version | head -1` >> $loggerCurr + echo "## - Applying coverage for $nb_gen_progs testcases in batches of $nb_progs_to_gen_per_step" >> $loggerCurr + echo "## - CSMITH_HOME: $csmith_location" >> $loggerCurr + echo "## - CSMITH EXEC: $csmith_exec" >> $loggerCurr + echo "## - Configuration file: $compconfg" >> $loggerCurr + echo "## - WA and RRS configuration file: $testcaseConfg" >> $loggerCurr + echo "## - RRS safe lists location: $DA_folder" >> $loggerCurr + echo "## - Compilation of testcase: $compile_line" >> $loggerCurr + if [[ "$modify" == "1" ]]; then + echo "## - Runtime libs for testcase compilation: $runtime" >> $loggerCurr + else + echo "## - Runtime libs for testcase compilation: $runtime_default" >> $loggerCurr + fi + echo "## - Seeds: $seed_location for process $process_number" >> $loggerCurr + echo "## - date: $(date '+%Y-%m-%d at %H:%M.%S')" >> $loggerCurr + echo "## - host name $(hostname -f)" >> $loggerCurr + fi + + ## Perform per configuration + for (( c=$initc; c<=$nb_gen_progs; c++ )) + do + cp=$c'p' + seed=`sed -n $cp $seed_location` + ./RSS5_2_constructModifyTests.sh $compiler $process_number $seed $loggerCurr 1 $modify $csmith_location "$compile_line" "${arr[1]}" "$DA_folder" "$testcaseConfg" "$csmith_exec" "$runtime" "$runtime_default" + done + ## Per configuartion the loop generates $initc to $nb_gen_progs modified testcases +done < $compconfg +## The loop is going over all the configurations in compiler_test.in file diff --git a/scripts/coverage/gcc-coverage/RSS5_2_constructModifyTests.sh b/scripts/coverage/gcc-coverage/RSS5_2_constructModifyTests.sh new file mode 100755 index 00000000..bf75aecd --- /dev/null +++ b/scripts/coverage/gcc-coverage/RSS5_2_constructModifyTests.sh @@ -0,0 +1,273 @@ +#!/bin/bash + +##### Keep all the safe ops we need +function keep_required_safe { + testcaseName=$1 + testcaseModify=$folder/'__'$testcaseName'M.c' + filename="$testcaseRes" + while read -r line; do + data="$line" + + # Get locations: + temp=${#data} + size=$((temp - 44 -1)) + var="${data:44:$size}" + + isFirst=1 + locF=0 + funcF=0 + locations=$(echo $var | tr "," " \n") + for loc in $locations + do + if (($isFirst==1)) + then + isFirst=0 + funcF=$loc + else + locF=$loc + fi + #echo "[$loc]" + done + #echo "location is: [$locF]" + #echo "Function number is: [$funcF]" + + #Replace the rest of the calls to unsafe macros + keyword_raw='/* ___REMOVE_SAFE__OP *//*'$locF'*//* ___SAFE__OP */(' + keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + replacement_raw='(' + #Check if it is a macro or a function + if [ $headerMode -eq 2 ] && [[ " ${inocationsMacrosMix[@]} " =~ " ${locF} " ]]; then + ## in mix mode, arr contains the value locF + replacement_raw='_mixM(' + fi + replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify + done < "$filename" +} +## Code to check MIX for csmith too +function replace2AllsafeMix { + testcaseName=$1 + + testcaseModify=$folder/'__'$testcaseName'M.c' + + for locF in "${inocationsMacrosMix[@]}"; do + #Replace the rest of the calls to unsafe macros + keyword_raw='/* ___REMOVE_SAFE__OP *//*'$locF'*//* ___SAFE__OP */(' + keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + replacement_raw='_mixM(' + replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify + done +} + +#### Remove safe calls when not required (when $headerMode -eq 2) +function replace2unsafeMix { + testcaseName=$1 + + testcaseModify=$folder/'__'$testcaseName'M.c' + + for locF in "${inocationsMacrosMix[@]}"; do + #Replace the rest of the calls to unsafe macros + keyword_raw='/* ___REMOVE_SAFE__OP *//*'$locF'*/' + keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + replacement_raw='_unsafe_macro_mixM/*'$locF'*/' + replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify + done +} +function replace2unsafe { + testcaseName=$1 + + testcaseModify=$folder/'__'$testcaseName'M.c' + + #Replace the rest of the calls to unsafe macros + keyword_raw='/* ___REMOVE_SAFE__OP */' + keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + replacement_raw='_unsafe_macro' + replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify +} +#################################### Modify TEST #################################### +function modify_test { + ## Single test - create a test case and its precompilation + testcaseName='test'$1 + testcaseModify=$folder/'__'$testcaseName'M.c' + testcaseEXEC='__'$testcaseName'Exec' + + # Modify the test (the preprocessed file) + ulimit -St 150;$csmith_exec --seed $seed $CSMITH_USER_OPTIONS > $testcaseModify + if [[ "$modify" == "1" ]]; then + ## WA + RRS ## + + # Keep ops required to be safe as a macros or functions + keep_required_safe $testcaseName ## Uses: inocationsMacrosMix + # Replace the rest of the calls in mix mode to unsafe macros according to inocationsMacrosMix + if [ $headerMode -eq 2 ] && [ ${#inocationsMacrosMix[@]} -gt 0 ]; then + replace2unsafeMix $testcaseName ## Uses: inocationsMacrosMix + fi + # Replace the rest of the calls to unsafe macros or functions + replace2unsafe $testcaseName + + # Compile the modify program: (with input compiler flags or with defualts) + #echo "ulimit -St 500;$compiler $compile_line $compile_line_confg ${compilerflags} $testcaseModify -o $testcaseEXEC" + ulimit -St 300;$compiler $compile_line $compile_line_confg ${compilerflags} $testcaseModify -o $testcaseEXEC + elif [[ "$modify" == "2" ]]; then ## MIX-Csmith, WA-MIX + ## MIX from configuration file + #echo "ulimit -St 500;$compiler $compile_line $compile_line_confg ${compilerflags} $testcaseModify -o $testcaseEXEC" + if [ $headerMode -eq 2 ] && [ ${#inocationsMacrosMix[@]} -gt 0 ]; then + replace2AllsafeMix $testcaseName + fi + ulimit -St 300;$compiler $compile_line $compile_line_confg ${compilerflags} $testcaseModify -o $testcaseEXEC + else + ## WA only MACROS or FUNCTIONS ## + # Compile the modify program WA only (with input compiler flags or with defualts) + ## Override configuaration file with value from ${runtime_default} + #echo "ulimit -St 500;$compiler $compile_line ${runtime_default} ${compilerflags} $testcaseModify -o $testcaseEXEC" + ulimit -St 300;$compiler $compile_line ${runtime_default} ${compilerflags} $testcaseModify -o $testcaseEXEC + fi + + # test the modify program: (with input compiler flags or with defualts) + + # Test exec + filesize=`stat --printf="%s" $testcaseModify` + ulimit -St 50;./$testcaseEXEC >> $testcaselogger + echo "seed= $seed, size= $filesize" >> $testcaselogger + echo "---" >> $testcaselogger + + #clean + ##rm $testcaseModify + rm $testcaseEXEC +} + +### If it is a bad testcase, no need to run it, just build it for coverage measure. +function coverage_test_only { + ## Single test - create a test case and its precompilation + testcaseName='test'$1 + testcaseOriginal=$folder/'__'$testcaseName'O.c' + testcaseEXEC='__'$testcaseName'Exec' + + # Modify the test (the preprocessed file) + ulimit -St 150;$csmith_exec --seed $seed $CSMITH_USER_OPTIONS > $testcaseOriginal + + # Compile and test modify program: (with input compiler flags or with defualts) + #echo "ulimit -St 500;$compiler $compile_line ${runtime_default} ${compilerflags} $testcaseOriginal -o $testcaseEXEC" + ulimit -St 300;$compiler $compile_line ${runtime_default} ${compilerflags} $testcaseOriginal -o $testcaseEXEC + + # Test exec + filesize=`stat --printf="%s" $testcaseOriginal` + echo "Skip execution. Invalid or bad testcase." >> $testcaselogger + echo "seed= $seed, size= $filesize" >> $testcaselogger + echo "---" >> $testcaselogger + + #clean + rm $testcaseOriginal + rm $testcaseEXEC +} + +#################################### PREPARE GEN TEST #################################### +function update_csmith_cmd_options { + confg=$1 + + ## Update the csmith command line + cmd=`head -1 $confg` + CSMITH_USER_OPTIONS="$CSMITH_USER_OPTIONS $cmd" + + ## Get location of probablities file + prob_file=`echo "$cmd" | awk '{for(i=1;i<=NF;i++) if ($i=="--relax-anlayses-prob") print $(i+1)}'` + + ## Create the probablities file + if [ ! -z "$prob_file" ]; then + sed '1d;$d' $confg | sed '$d' > $prob_file + fi + + ## Update the line for compilation + getlineconfig=`sed '1d;$d' $confg | tail -1` + ## Replace vars + compile_line_confg=`eval echo "$getlineconfig"` + + ## Arrays to choose which one to set to functions or macros if mix + lastline=`tail -1 $confg` + IFS=' ' read -r -a inocationsMacrosMix <<< "$lastline" + + ## Check which version of headers we shall take + res1=`grep "USE_MATH_MACROS" $confg | wc -l` + if [[ "$res1" == "1" ]]; then + headerMode=1 + else + res2=`grep "USE_MATH_MIX" $confg | wc -l` + if [[ "$res2" == "1" ]]; then + headerMode=2 + fi + fi +} + +################### MAIN ############################### +# Single iteration, requires a compiler and a seed +CSMITH_USER_OPTIONS=" --bitfields --packed-struct" +# Basic parameters +compiler=$1 # Compiler to test +process=$2 # To create temp storage for exec +seed=$3 # File with all the seeds to use +testcaselogger=$4 # The logger file for the results +cov=$5 # To test for cov=1, else =0 +modify=$6 # Do we run on modified or original tests +csmith_location=$7 # Csmith location +compile_line=$8 # Includes + Duse +compilerflags=$9 # compiler flags +DA_folder=${10} # DA (RRS lists) folder +testcaseConfg=${11} # Test-case configuration (WA and RRS) +csmith_exec=${12} # Location of Csmith exec +runtime=${13} # Runtime for WA and RRS files +runtime_default=${14} # Runtime Csmith defualt lib (with no RRS) + +## For debug only +#echo "Compiler=$compiler" +#echo "process=$process" +#echo "seed=$seed" +#echo "testcaselogger=$testcaselogger" +#echo "csmith_location=$csmith_location" +#echo "compile_line=$compile_line" +#echo "compilerflags=$compilerflags" +#echo "DA_folder=$DA_folder" +#echo "testcaseConfg=$testcaseConfg" +#echo "csmith_exec=$csmith_exec" +#echo "runtime=$runtime" +#echo "runtime_default=$runtime_default" +# Check if second parameter is a number +re='^[0-9]+$' +if ! [[ $seed =~ $re ]] ; then + echo ">> error: Not a number" >&2; exit 1 +fi + +# folders for all the results +folder=$compiler-mainRes +testcaseRes=$DA_folder/'__test'$seed'Results' +compile_line_confg="" +inocationsMacrosMix=() ## Invocation to set as function wrapper +headerMode=0 ## Assume function version +if [ ! -f $testcaseRes ] && [[ "$modify" == "1" ]] + then + #echo "Seed's list is missing: $testcaseRes" + ## Only generate and compile test case to measure coverage + if [[ "$cov" == "1" ]] + then + coverage_test_only "$seed" + fi +else + # Check the configuration file exist + testcaseConfgFile=$testcaseConfg/probs_WeakenSafeAnalyse_test.txt$seed # name of config file + if ! test -f "$testcaseConfgFile" ; then + echo ">> error: No configuration file found for this seed $seed" >&2; exit 1 + fi + + # Get configuration data: Update $CSMITH_USER_OPTIONS + update_csmith_cmd_options "$testcaseConfgFile" + + # Run a single test + modify_test "$seed" +fi +## END ## diff --git a/scripts/coverage/llvm-coverage/5-test-dest-machine.sh b/scripts/coverage/llvm-coverage/5-test-dest-machine.sh new file mode 100755 index 00000000..aeb2f476 --- /dev/null +++ b/scripts/coverage/llvm-coverage/5-test-dest-machine.sh @@ -0,0 +1,81 @@ +#!/bin/bash +process_number=$1 ## from 0 +baseD=/home/user42 +working_folder=$baseD/llvm-csmith-1 ##$process_number +nb_progs_to_gen=4 +nb_progs_to_gen_per_step=2 + +base=$baseD/RRS_EXPR # $baseD/git/RRS_EXPR +seed_location=$base/scripts/WA-v2-standalone/seeds/seeds-$process_number.txt # seed location +csmith_location=$base/csmith # csmith location +csmith_exec=$base/csmith/build/src/csmith # csmith exec +configuration_location=$working_folder/csmith/scripts/compiler_test.in # config file locatoin +outputs_location=../../ # where we will put all outputs +#testcaseConfg=$base/scripts/WA-v2-standalone/seedsProbs # Test-case configuration (WA and RRS) +testcaseConfg=$base/scripts/WA-v2-standalone/CsmithSeedsProbs # Csmith mix +##testcaseConfg=$base/scripts/WA-v2-standalone/CsmithEdgeSeedsProbs_macros # CsmithEdge func or macro +##testcaseConfg=$base/scripts/WA-v2-standalone/CsmithEdgeSeedsProbs_func # CsmithEdge func or macro +#DA_folder=$testcaseConfg/seedsSafeLists # DA (RRS lists) folder +DA_folder=$base/scripts/RSS-v2-general/seedsSafeLists_small # Csmith mix +##DA_folder=$base/scripts/WA-v2-standalone/seedsProbs/seedsSafeLists # CsmithEdge func or macro +runtime=RRS_runtime_test # Runtime for WA and RRS files +compile_line="-lgcov -w" #how we compile it +compile_line_lib_default="-I$csmith_location/runtime -I$csmith_location/build/runtime" +##compile_line_lib_default_rrswa="-I$csmith_location/RRS_runtime_test -I$csmith_location/build/runtime" +##compile_line_lib_default="-I$csmith_location/runtime -I$csmith_location/build/runtime -DUSE_MATH_MACROS" +modify=1 #1 #0 #2 + +## Check we have process number +if [ -z "$1" ] + then + echo "No process number supplied." + exit 1 +fi + +time1=$(date +"%T") +echo "EVOLUTION OF LLVM COVERAGE WHEN COMPILING "$nb_progs_to_gen" CSMITH PROGRAMS BY STEPS OF "$nb_progs_to_gen_per_step + +rm -f $working_folder/seeds-$process_number.txt +rm -rf $working_folder/coverage_processed-$modify +rm -rf $working_folder/coverage_gcda_files/application_run-$modify + +nb_gen_progs=0 # Start from 0 +i=0 # Init counters +echo "--> RESETING COVERAGE DATA...("$time1")" + +# Keep current folder: +current_folder=`pwd` + +######################################## Start Cov. ######################################## +# Loop over compilation and coverage measurement +while (( $nb_gen_progs < $nb_progs_to_gen )); +do + i=$(($i+1)) + nb_gen_progs=$(($nb_gen_progs + $nb_progs_to_gen_per_step)) + time2=$(date +"%T") + echo "--> COMPILING "$nb_progs_to_gen_per_step" PROGRAMS ("$(($nb_progs_to_gen-$nb_gen_progs))" REMAINING)... ("$time2")" + + # Set location to record the data + mkdir -p $working_folder/coverage_gcda_files/application_run-$modify + + # Run compiler and save coverage data + export GCOV_PREFIX=$working_folder/coverage_gcda_files/application_run-$modify + export GCOV_PREFIX_STRIP=0 + ./RSS5_1-compiler_test.sh $process_number $nb_gen_progs $nb_progs_to_gen_per_step $seed_location $csmith_location $configuration_location $outputs_location "$compile_line" "$DA_folder" "$testcaseConfg" "$csmith_exec" "$runtime" "$compile_line_lib_default" $modify + unset GCOV_PREFIX + unset GCOV_PREFIX_STRIP + + time3=$(date +"%T") + echo "--> MEASURING COVERAGE to $working_folder/coverage_processed-$modify/x-$i... ("$time3")" + mkdir -p $working_folder/coverage_processed-$modify/x-$i + ( + cd $working_folder/coverage_processed-$modify/x-$i + source $baseD/graphicsfuzz/gfauto/.venv/bin/activate + gfauto_cov_from_gcov --out run_gcov2cov.cov $working_folder/llvm-build/ $working_folder/coverage_gcda_files/application_run-$modify/ --num_threads 32 --gcov_uses_json >> gfauto.log 2>&1 + gfauto_cov_to_source --coverage_out cov.out --cov run_gcov2cov.cov $working_folder/llvm-build/ >> gfauto.log 2>&1 + ) + cd $current_folder +done +time2=$(date +"%T") +echo "DONE. RESULTS AVAILABLE IN $working_folder/coverage_processed-$modify/x-$i ($time2)" +ls -l $working_folder/coverage_processed-$modify/x-$i diff --git a/scripts/coverage/llvm-coverage/DA-only-llvm/5-compute-coverage_RSS-gfauto-llvm.sh b/scripts/coverage/llvm-coverage/DA-only-llvm/5-compute-coverage_RSS-gfauto-llvm.sh new file mode 100755 index 00000000..7aca5133 --- /dev/null +++ b/scripts/coverage/llvm-coverage/DA-only-llvm/5-compute-coverage_RSS-gfauto-llvm.sh @@ -0,0 +1,143 @@ +#!/bin/bash +process_number=$1 +baseD=/home/user42 +working_folder=$baseD/llvm-csmith-1 ##$process_number +nb_progs_to_gen=15000 +nb_progs_to_gen_per_step=2500 +#seed_location=../../Data/seeds_all/seeds_out$process_number.txt # seed location + +base=$baseD/RRS_EXPR # $baseD/git/RRS_EXPR +seed_location=$base/scripts/WA-v2-standalone/seeds/seeds-$process_number.txt # seed location +csmith_location=$base/csmith # csmith location +# csmith_exec=$base/csmith/build/src/csmith # csmith exec +configuration_location=$working_folder/csmith/scripts/compiler_test.in # config file locatoin +outputs_location=../../ # where we will put all outputs + +## Change per test: +#csmith_flags=" --bitfields --packed-struct " #" --bitfields --packed-struct --math-notmp --annotated-arith-wrappers" + +## 0 +modify=0 # 0-original, 1-modify +##compile_line=" -I$csmith_location/runtime -I$csmith_location/build/runtime -lgcov -w" #how we compile it,modify=0 +compile_line=" -I$csmith_location/runtime -I$csmith_location/build/runtime -lgcov -w -DUSE_MATH_MACROS" #how we compile it,modify=0 +csmith_flags=" --bitfields --packed-struct " #" --bitfields --packed-struct --math-notmp" + +## 1 +#modify=1 # 0-original, 1-modify +##compile_line=" -I../../csmith_runtime_orig -I../../csmith_runtime_orig_build -lgcov -w" #how we compile it,modify=1 +#compile_line=" -I$csmith_location/RRS_runtime_test -I$csmith_location/build/runtime -DUSE_MATH_MACROS -lgcov -w" #how we compile it,modify=1 +#csmith_flags=" --bitfields --packed-struct --annotated-arith-wrappers" #" --bitfields --packed-struct --math-notmp" + +## Check we have process number +if [ -z "$1" ] + then + echo "--> No process number supplied. Exit." + echo " Single parameter: set process_number." + echo " 3 Parameters, resume after crash: process_number, index i, seed number before the crash." + echo " 4 Parameters, destribute execution: process_number, index i, seed from, seed to." + exit 1 +fi + +time1=$(date +"%T") +echo "EVOLUTION OF LLVM COVERAGE WHEN COMPILING "$nb_progs_to_gen" CSMITH PROGRAMS BY STEPS OF "$nb_progs_to_gen_per_step +if [ "$#" -eq 3 ]; then + i=$2 # Restore counters + nb_gen_progs=$3 # Restore from seed (one before we crushed) + echo "--> RESTORE COVERAGE DATA...("$i","$nb_gen_progs","$nb_progs_to_gen","$time1")" +else + rm -f $working_folder/seeds-$process_number.txt + rm -rf $working_folder/llvm-source/tools/clang/test + rm -rf $working_folder/llvm-source/tools/clang/unittests + rm -rf $working_folder/llvm-source/tools/clang/docs + rm -rf $working_folder/llvm-source/tools/clang/www + rm -rf $working_folder/coverage_processed-$modify + rm -rf $working_folder/coverage_gcda_files/application_run-$modify + + if [ "$#" -eq 1 ]; then + nb_gen_progs=0 # Start from 0 + i=0 # Init counters + echo "--> RESETING COVERAGE DATA...("$time1")" + else + if [ "$#" -eq 4 ]; then + i=$2 # Restore counters + nb_gen_progs=$3 # from seed + nb_progs_to_gen=$4 # to seed + echo "--> RESETING COVERAGE DATA...("$i","$nb_gen_progs","$nb_progs_to_gen","$time1")" + else + echo "--> Wrong number of parameters! Exit." + echo " Single parameter: set process_number." + echo " 3 Parameters, resume after crash: process_number, index i, seed number before the crash." + echo " 4 Parameters, destribute execution: process_number, index i, seed from, seed to." + exit 1 + fi + fi +fi + +# Keep current folder: +current_folder=`pwd` + +######################################## Start Cov. ######################################## +# Loop over compilation and coverage measurement +#>> start loop with i=0, nb_gen_progs=0, nb_progs_to_gen=100000 +#i=0 +#nb_gen_progs=0 +#>> start loop with i=2, nb_gen_progs=10000, nb_progs_to_gen=100000 +#i=2 +#nb_gen_progs=10000 +#>> start loop with i=4, nb_gen_progs=20000, nb_progs_to_gen=100000 +#i=4 +#nb_gen_progs=20000 +#>> start loop with i=6, nb_gen_progs=30000, nb_progs_to_gen=100000 +#i=6 +#nb_gen_progs=30000 +#>> start loop with i=8, nb_gen_progs=40000, nb_progs_to_gen=100000 +#i=8 +#nb_gen_progs=40000 +#>> start loop with i=10, nb_gen_progs=50000, nb_progs_to_gen=100000 +#i=10 +#nb_gen_progs=50000 +#>> start loop with i=12, nb_gen_progs=60000, nb_progs_to_gen=100000 +#i=12 +#nb_gen_progs=60000 +#>> start loop with i=14, nb_gen_progs=70000, nb_progs_to_gen=100000 +#i=14 +#nb_gen_progs=70000 +#>> start loop with i=16, nb_gen_progs=80000, nb_progs_to_gen=100000 +#i=16 +#nb_gen_progs=80000 +#>> start loop with i=18, nb_gen_progs=90000, nb_progs_to_gen=100000 +#i=18 +#nb_gen_progs=90000 +## + +# Loop over compilation and coverage measurement +while (( $nb_gen_progs < $nb_progs_to_gen )); +do + i=$(($i+1)) + nb_gen_progs=$(($nb_gen_progs + $nb_progs_to_gen_per_step)) + time2=$(date +"%T") + echo "--> COMPILING "$nb_progs_to_gen_per_step" PROGRAMS ("$(($nb_progs_to_gen-$nb_gen_progs))" REMAINING)... ("$time2")" + + # Set location to record the data + mkdir -p $working_folder/coverage_gcda_files/application_run-$modify + + # Run compiler and save coverage data + export GCOV_PREFIX=$working_folder/coverage_gcda_files/application_run-$modify + export GCOV_PREFIX_STRIP=0 + ./RRS5_1-compiler_test.sh $process_number $nb_gen_progs $nb_progs_to_gen_per_step $modify $seed_location $csmith_location $configuration_location $outputs_location "$compile_line" "$csmith_flags" + unset GCOV_PREFIX + unset GCOV_PREFIX_STRIP + + time3=$(date +"%T") + echo "--> MEASURING COVERAGE... ("$time3")" + mkdir -p $working_folder/coverage_processed-$modify/x-$i + ( + cd $working_folder/coverage_processed-$modify/x-$i + source $baseD/graphicsfuzz/gfauto/.venv/bin/activate + gfauto_cov_from_gcov --out run_gcov2cov.cov $working_folder/llvm-build/ $working_folder/coverage_gcda_files/application_run-$modify/ --num_threads 32 --gcov_uses_json >> gfauto.log 2>&1 + gfauto_cov_to_source --coverage_out cov.out --cov run_gcov2cov.cov $working_folder/llvm-build/ >> gfauto.log 2>&1 + ) + cd $current_folder +done +time2=$(date +"%T") +echo "DONE. RESULTS AVAILABLE IN $working_folder/coverage_processed-$modify/x-$i ($time2)" diff --git a/scripts/coverage/llvm-coverage/DA-only-llvm/5-test-dest-machine.sh b/scripts/coverage/llvm-coverage/DA-only-llvm/5-test-dest-machine.sh new file mode 100755 index 00000000..bf261f98 --- /dev/null +++ b/scripts/coverage/llvm-coverage/DA-only-llvm/5-test-dest-machine.sh @@ -0,0 +1,90 @@ +#!/bin/bash +process_number=$1 +baseD=/home/user42 +working_folder=$baseD/llvm-csmith-1 ##$process_number +nb_progs_to_gen=4 #100000 +nb_progs_to_gen_per_step=2 #20000 +#seed_location=../../Data/seeds_all/seeds_out$process_number.txt # seed location + +base=$baseD/RRS_EXPR # $baseD/git/RRS_EXPR +seed_location=$base/scripts/WA-v2-standalone/seeds/seeds-$process_number.txt # seed location +csmith_location=$base/csmith # csmith location +# csmith_exec=$base/csmith/build/src/csmith # csmith exec +configuration_location=$working_folder/csmith/scripts/compiler_test.in # config file locatoin +outputs_location=../../ # where we will put all outputs + +## Change per test: +#csmith_flags=" --bitfields --packed-struct " #" --bitfields --packed-struct --math-notmp --annotated-arith-wrappers" + +## 0 +modify=0 # 0-original, 1-modify +##compile_line=" -I$csmith_location/runtime -I$csmith_location/build/runtime -lgcov -w" #how we compile it,modify=0 +compile_line=" -I$csmith_location/runtime -I$csmith_location/build/runtime -lgcov -w -DUSE_MATH_MACROS" #how we compile it,modify=0 +csmith_flags=" --bitfields --packed-struct " #" --bitfields --packed-struct --math-notmp" + +## 1 +#modify=1 # 0-original, 1-modify +##compile_line=" -I../../csmith_runtime_orig -I../../csmith_runtime_orig_build -lgcov -w" #how we compile it,modify=1 +#compile_line=" -I$csmith_location/RRS_runtime_test -I$csmith_location/build/runtime -DUSE_MATH_MACROS -lgcov -w" #how we compile it,modify=1 +#csmith_flags=" --bitfields --packed-struct --annotated-arith-wrappers" #" --bitfields --packed-struct --math-notmp" + +## Check we have process number +if [ -z "$1" ] + then + echo "No process number supplied." + exit 1 +fi + +time1=$(date +"%T") +echo "EVOLUTION OF LLVM COVERAGE WHEN COMPILING "$nb_progs_to_gen" CSMITH PROGRAMS BY STEPS OF "$nb_progs_to_gen_per_step +if [ "$#" -ne 3 ]; then + rm -f $working_folder/seeds-$process_number.txt + rm -rf $working_folder/coverage_processed-$modify + rm -rf $working_folder/coverage_gcda_files/application_run-$modify + + nb_gen_progs=0 # Start from 0 + i=0 # Init counters + echo "--> RESETING COVERAGE DATA...("$time1")" +else + nb_gen_progs=$2 # Restore + i=$3 # Restore counters + echo "RESTORE COVERAGE DATA...("$nb_gen_progs","$i","$time1")" +fi + +# Keep current folder: +current_folder=`pwd` + + +######################################## Start Cov. ######################################## +# Loop over compilation and coverage measurement +while (( $nb_gen_progs < $nb_progs_to_gen )); +do + i=$(($i+1)) + nb_gen_progs=$(($nb_gen_progs + $nb_progs_to_gen_per_step)) + time2=$(date +"%T") + echo "--> COMPILING "$nb_progs_to_gen_per_step" PROGRAMS ("$(($nb_progs_to_gen-$nb_gen_progs))" REMAINING)... ("$time2")" + + # Set location to record the data + mkdir -p $working_folder/coverage_gcda_files/application_run-$modify + + # Run compiler and save coverage data + export GCOV_PREFIX=$working_folder/coverage_gcda_files/application_run-$modify + export GCOV_PREFIX_STRIP=0 + ./RRS5_1-compiler_test.sh $process_number $nb_gen_progs $nb_progs_to_gen_per_step $modify $seed_location $csmith_location $configuration_location $outputs_location "$compile_line" "$csmith_flags" + unset GCOV_PREFIX + unset GCOV_PREFIX_STRIP + + time3=$(date +"%T") + echo "--> MEASURING COVERAGE... ("$time3")" + mkdir -p $working_folder/coverage_processed-$modify/x-$i + ( + cd $working_folder/coverage_processed-$modify/x-$i + source $baseD/graphicsfuzz/gfauto/.venv/bin/activate + gfauto_cov_from_gcov --out run_gcov2cov.cov $working_folder/llvm-build/ $working_folder/coverage_gcda_files/application_run-$modify/ --num_threads 32 --gcov_uses_json >> gfauto.log 2>&1 + gfauto_cov_to_source --coverage_out cov.out --cov run_gcov2cov.cov $working_folder/llvm-build/ >> gfauto.log 2>&1 + ) + cd $current_folder +done +time2=$(date +"%T") +echo "DONE. RESULTS AVAILABLE IN $working_folder/coverage_processed-$modify/x-$i ($time2)" +ls -l $working_folder/coverage_processed-$modify/x-$i diff --git a/scripts/coverage/llvm-coverage/DA-only-llvm/RRS5_1-compiler_test.sh b/scripts/coverage/llvm-coverage/DA-only-llvm/RRS5_1-compiler_test.sh new file mode 100755 index 00000000..16b9213d --- /dev/null +++ b/scripts/coverage/llvm-coverage/DA-only-llvm/RRS5_1-compiler_test.sh @@ -0,0 +1,67 @@ +#!/bin/bash +process_number=$1 # Process number +nb_gen_progs=$2 # How many we will do by the end of the loop +nb_progs_to_gen_per_step=$3 # How many we shall generate now +initc=$(($nb_gen_progs + 1 - $nb_progs_to_gen_per_step)) # Calc. the position of the first seed of this batch +modify=$4 # With modifications --> use 1 +seed_location=$5 # Seeds file location, coverage takes good+bad +csmith_location=$6 # Where is the csmith we use +compconfg=$7 # Where to take the compiler configuration +outputs_location=$8 # Where to expect the results +compile_line=$9 # Include and -D params for testcase +csmith_flags=${10} # csmith flags +# Was: $working_folder/csmith/scripts/compiler_test.in +#echo "Get compilers lists: $working_folder/csmith/scripts/compiler_test.in" +### Check we have all the data we need before proceeding: +if [ "$#" -ne 10 ]; then + echo "Illegal number of parameters. Please enter compiler and number of testcases to generate." + echo "Requires: process number, size of population, size of population per step, flag (1-modify, 0-original csmith) seeds file, csmith location, configuration file location, output location, compilation line for test case, and csmith flags." + exit 1 +fi + +#echo "Get compilers lists: $working_folder/csmith/scripts/compiler_test.in" +# Sets the logger -general name +logger=$outputs_location"__logger_cov" +if [[ "$modify" == "1" ]] + then + logger=$logger"M" +fi + +# Loop over compilation for the current batch of seeds +while read str; do + arr=( $str ) + + ## Create the temp folder if required + compiler=${arr[0]} + folder=$compiler-mainRes + if [ ! -d $folder ] + then + mkdir $folder + fi + + # Logger per-compiler - add only once (at start) + loggerCurr=$logger"_"${compiler//\//\_}"_"seeds-$process_number.txt + #if (($initc < 2)); then + touch $loggerCurr + echo "## (modify=$modify) Configuration: $str" >> $loggerCurr + echo "## - Compiler version:" `$compiler --version | head -1` >> $loggerCurr + echo "## - Applying coverage for $nb_gen_progs testcases in batches of $nb_progs_to_gen_per_step" >> $loggerCurr + echo "## - CSMITH_HOME: $csmith_location" >> $loggerCurr + echo "## - CSMITH_USER_OPTIONS: $csmith_flags" >> $loggerCurr + echo "## - Configuration file: $compconfg" >> $loggerCurr + echo "## - Compilation of testcase: $compile_line" >> $loggerCurr + echo "## - Seeds: $seed_location for process $process_number" >> $loggerCurr + echo "## - date: $(date '+%Y-%m-%d at %H:%M.%S')" >> $loggerCurr + echo "## - host name $(hostname -f)" >> $loggerCurr + #fi + + ## Perform per configuration + for (( c=$initc; c<=$nb_gen_progs; c++ )) + do + cp=$c'p' + seed=`sed -n $cp $seed_location` + ./RRS5_2_constructModifyTests.sh $compiler $process_number $seed $loggerCurr 1 $modify $csmith_location "$csmith_flags" "$compile_line" "${arr[1]}" + done + ## Per configuartion the loop generates $initc to $nb_gen_progs modified testcases +done < $compconfg +## The loop is going over all the configurations in compiler_test.in file diff --git a/scripts/coverage/llvm-coverage/DA-only-llvm/RRS5_2_constructModifyTests.sh b/scripts/coverage/llvm-coverage/DA-only-llvm/RRS5_2_constructModifyTests.sh new file mode 100755 index 00000000..cdf0d569 --- /dev/null +++ b/scripts/coverage/llvm-coverage/DA-only-llvm/RRS5_2_constructModifyTests.sh @@ -0,0 +1,169 @@ +#!/bin/bash + +##### Keep all the safe ops we need +function keep_required_safe { + testcaseName=$1 + testcaseModify=$folder/'__'$testcaseName'M.c' + filename="$testcaseRes" + while read -r line; do + data="$line" + + # Get locations: + temp=${#data} + size=$((temp - 44 -1)) + var="${data:44:$size}" + + isFirst=1 + locF=0 + funcF=0 + locations=$(echo $var | tr "," " \n") + for loc in $locations + do + if (($isFirst==1)) + then + isFirst=0 + funcF=$loc + else + locF=$loc + fi + #echo "[$loc]" + done + #echo "location is: [$locF]" + #echo "Function number is: [$funcF]" + + #Replace the rest of the calls to unsafe macros + keyword_raw='/* ___REMOVE_SAFE__OP *//*'$locF'*//* ___SAFE__OP */(' + keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + replacement_raw='(' + replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify + done < "$filename" +} + +#### Remove safe calls when not required +function replace2unsafe { + testcaseName=$1 + + testcaseModify=$folder/'__'$testcaseName'M.c' + + #Replace the rest of the calls to unsafe macros + keyword_raw='/* ___REMOVE_SAFE__OP */' + keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + replacement_raw='_unsafe_macro' + replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify + + + #Add missing header unsafe macros + #echo "#include \"unsafe_math_macros.h\""|cat - $testcaseModify > /tmp/out && mv /tmp/out $testcaseModify + #keyword_raw='#include "csmith.h"' + #keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + #replacement_raw='#include "unsafe_math_macros_eCast.h"' + #replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + #sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify +} + +#################################### Modify TEST #################################### +function modify_test { + ## Single test - create a test case and its precompilation + testcaseName='test'$1 + testcaseModify=$folder/'__'$testcaseName'M.c' + testcaseEXEC='__'$testcaseName'Exec' + + # Modify the test (the preprocessed file) + ulimit -St 500; $csmith_location/build/src/csmith --seed $seed $CSMITH_USER_OPTIONS > $testcaseModify + if [[ "$modify" == "1" ]] + then + # Keep ops required to be safe + keep_required_safe $testcaseName + #Replace the rest of the calls to unsafe macros + replace2unsafe $testcaseName + fi + + # Compile and test modify program: (with input compiler flags or with defualts) + ulimit -St 500;$compiler $compile_line ${compilerflags} $testcaseModify -o $testcaseEXEC + + # Test exec + filesize=`stat --printf="%s" $testcaseModify` + ulimit -St 500;./$testcaseEXEC >> $testcaselogger + echo "seed= $seed, size= $filesize" >> $testcaselogger + echo "---" >> $testcaselogger + + #clean + rm $testcaseModify + rm $testcaseEXEC +} + +### If it is a bad testcase, no need to run it, just build it for coverage measure. +function coverage_test_only { + ## Single test - create a test case and its precompilation + testcaseName='test'$1 + testcaseOriginal=$folder/'__'$testcaseName'O.c' + testcaseEXEC='__'$testcaseName'Exec' + + # Modify the test (the preprocessed file) + ulimit -St 500; $csmith_location/build/src/csmith --seed $seed $CSMITH_USER_OPTIONS > $testcaseOriginal + + # Compile and test modify program: (with input compiler flags or with defualts) + ulimit -St 500;$compiler $compile_line ${compilerflags} $testcaseOriginal -o $testcaseEXEC + + # Test exec + filesize=`stat --printf="%s" $testcaseOriginal` + echo "Skip execution. Bad testcase." >> $testcaselogger + echo "seed= $seed, size= $filesize" >> $testcaselogger + echo "---" >> $testcaselogger + + #clean + rm $testcaseOriginal + rm $testcaseEXEC +} + +################### MAIN ############################### +# Basic parameters +compiler=$1 # Compiler to test +process=$2 # To create temp storage for exec +seed=$3 # File with all the seeds to use +testcaselogger=$4 # The logger file for the results +cov=$5 # To test for cov=1, else =0 +modify=$6 # Do we run on modified or original tests +csmith_location=$7 # Csmith location +csmith_flags=$8 # Csmith options +compile_line=$9 # Includes + Duse +compilerflags=${10} # compiler flags + +# Single iteration, requires a compiler and a seed +#CSMITH_USER_OPTIONS=" --bitfields --packed-struct" +CSMITH_USER_OPTIONS=$csmith_flags + +## For debug only +#echo "Compiler=$compiler" +#echo "process=$process" +#echo "seed=$seed" +#echo "testcaselogger=$testcaselogger" +#echo "csmith_location=$csmith_location" +#echo "csmith_flags=$csmith_flags" +#echo "compile_line=$compile_line" +#echo "compilerflags=$compilerflags" +# Check if second parameter is a number +re='^[0-9]+$' +if ! [[ $seed =~ $re ]] ; then + echo ">> error: Not a number" >&2; exit 1 +fi + +# folders for all the results +folder=$compiler-mainRes +testcaseRes='../RSS-v2-general/seedsSafeLists_small'/'__test'$seed'Results' +if [ ! -f $testcaseRes ] + then + #echo "Seed's list is missing: $testcaseRes" + ## Only generate and compile test case to measure coverage + if [[ "$cov" == "1" ]] + then + coverage_test_only "$seed" + fi +else + modify_test "$seed" # Run a single test +fi diff --git a/scripts/coverage/llvm-coverage/RSS5_1-compiler_test.sh b/scripts/coverage/llvm-coverage/RSS5_1-compiler_test.sh new file mode 100755 index 00000000..dee11195 --- /dev/null +++ b/scripts/coverage/llvm-coverage/RSS5_1-compiler_test.sh @@ -0,0 +1,69 @@ +#!/bin/bash +process_number=$1 # Process number +nb_gen_progs=$2 # How many we will do by the end of the loop +nb_progs_to_gen_per_step=$3 # How many we shall generate now +initc=$(($nb_gen_progs + 1 - $nb_progs_to_gen_per_step)) # Calc. the position of the first seed of this batch +seed_location=$4 # Seeds file location, coverage takes good+bad +csmith_location=$5 # Where is the csmith we use +compconfg=$6 # Where to take the compiler configuration +outputs_location=$7 # Where to expect the results +compile_line=$8 # Include and -D params for testcase +DA_folder=$9 # DA (RRS lists) folder +testcaseConfg=${10} # Test-case configuration (WA and RRS) +csmith_exec=${11} # Location of Csmith exec +runtime=${12} # Runtime for WA and RRS files +runtime_default=${13} # Runtime Csmith defualt lib (with no RRS or WA) +modify=${14} # With modifications --> use 1 + +### Check we have all the data we need before proceeding: +if [ "$#" -ne 14 ]; then + echo "Illegal number of parameters. Please enter compiler and number of testcases to generate." + echo "Requires: process number, size of population, size of population per step, seeds file, csmith location, configuration file location, output location, compilation line for test case, RRS safe lists, WA config file, csmith exec, runtime lib for RRS, runtime lib without RRS, and modify or not version." + exit 1 +fi + +#echo "Get compilers lists: $working_folder/csmith/scripts/compiler_test.in" +# Sets the logger -general name +logger=$outputs_location"__logger_covM" + +# Loop over compilation for the current batch of seeds +while read str; do + arr=( $str ) + + ## Create the temp folder if required + compiler=${arr[0]} + folder=$compiler-mainRes + if [ ! -d $folder ] + then + mkdir $folder + fi + + # Logger per-compiler - add only once (at start) + loggerCurr=$logger"_"${compiler//\//\_}"_"seeds-$process_number.txt + if (($initc < 2)); then + touch $loggerCurr + echo "## (modify=$modify) Configuration: $str" >> $loggerCurr + echo "## - Compiler version:" `$compiler --version | head -1` >> $loggerCurr + echo "## - Applying coverage for $nb_gen_progs testcases in batches of $nb_progs_to_gen_per_step" >> $loggerCurr + echo "## - CSMITH_HOME: $csmith_location" >> $loggerCurr + echo "## - CSMITH EXEC: $csmith_exec" >> $loggerCurr + echo "## - Configuration file: $compconfg" >> $loggerCurr + echo "## - WA and RRS configuration file: $testcaseConfg" >> $loggerCurr + echo "## - RRS safe lists location: $DA_folder" >> $loggerCurr + echo "## - Compilation of testcase: $compile_line" >> $loggerCurr + echo "## - Runtime libs for testcase compilation: $runtime" >> $loggerCurr + echo "## - Seeds: $seed_location for process $process_number" >> $loggerCurr + echo "## - date: $(date '+%Y-%m-%d at %H:%M.%S')" >> $loggerCurr + echo "## - host name $(hostname -f)" >> $loggerCurr + fi + + ## Perform per configuration + for (( c=$initc; c<=$nb_gen_progs; c++ )) + do + cp=$c'p' + seed=`sed -n $cp $seed_location` + ./RSS5_2_constructModifyTests.sh $compiler $process_number $seed $loggerCurr 1 $modify $csmith_location "$compile_line" "${arr[1]}" "$DA_folder" "$testcaseConfg" "$csmith_exec" "$runtime" "$runtime_default" + done + ## Per configuartion the loop generates $initc to $nb_gen_progs modified testcases +done < $compconfg +## The loop is going over all the configurations in compiler_test.in file diff --git a/scripts/coverage/llvm-coverage/RSS5_2_constructModifyTests.sh b/scripts/coverage/llvm-coverage/RSS5_2_constructModifyTests.sh new file mode 100755 index 00000000..09a04591 --- /dev/null +++ b/scripts/coverage/llvm-coverage/RSS5_2_constructModifyTests.sh @@ -0,0 +1,239 @@ +#!/bin/bash + +##### Keep all the safe ops we need +function keep_required_safe { + testcaseName=$1 + testcaseModify=$folder/'__'$testcaseName'M.c' + filename="$testcaseRes" + while read -r line; do + data="$line" + + # Get locations: + temp=${#data} + size=$((temp - 44 -1)) + var="${data:44:$size}" + + isFirst=1 + locF=0 + funcF=0 + locations=$(echo $var | tr "," " \n") + for loc in $locations + do + if (($isFirst==1)) + then + isFirst=0 + funcF=$loc + else + locF=$loc + fi + #echo "[$loc]" + done + #echo "location is: [$locF]" + #echo "Function number is: [$funcF]" + + #Replace the rest of the calls to unsafe macros + keyword_raw='/* ___REMOVE_SAFE__OP *//*'$locF'*//* ___SAFE__OP */(' + keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + replacement_raw='(' + #Check if it is a macro or a function + if [ $headerMode -eq 2 ] && [[ " ${inocationsMacrosMix[@]} " =~ " ${locF} " ]]; then + ## in mix mode, arr contains the value locF + replacement_raw='_mixM(' + fi + replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify + done < "$filename" +} + +#### Remove safe calls when not required (when $headerMode -eq 2) +function replace2unsafeMix { + testcaseName=$1 + + testcaseModify=$folder/'__'$testcaseName'M.c' + + for locF in "${inocationsMacrosMix[@]}"; do + #Replace the rest of the calls to unsafe macros + keyword_raw='/* ___REMOVE_SAFE__OP *//*'$locF'*/' + keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + replacement_raw='_unsafe_macro_mixM/*'$locF'*/' + replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify + done +} +function replace2unsafe { + testcaseName=$1 + + testcaseModify=$folder/'__'$testcaseName'M.c' + + #Replace the rest of the calls to unsafe macros + keyword_raw='/* ___REMOVE_SAFE__OP */' + keyword_regexp="$(printf '%s' "$keyword_raw" | sed -e 's/[]\/$*.^|[]/\\&/g')" + + replacement_raw='_unsafe_macro' + replacement_regexp="$(printf '%s' "$replacement_raw" | sed -e 's/[\/&]/\\&/g')" + sed -i "s/$keyword_regexp/$replacement_regexp/g" $testcaseModify +} + +#################################### Modify TEST #################################### +function modify_test { + ## Single test - create a test case and its precompilation + testcaseName='test'$1 + testcaseModify=$folder/'__'$testcaseName'M.c' + testcaseEXEC='__'$testcaseName'Exec' + + # Modify the test (the preprocessed file) + ulimit -St 500;$csmith_exec --seed $seed $CSMITH_USER_OPTIONS > $testcaseModify + if [[ "$modify" == "1" ]]; then + # Keep ops required to be safe as a macros or functions + keep_required_safe $testcaseName ## Uses: inocationsMacrosMix + # Replace the rest of the calls in mix mode to unsafe macros according to inocationsMacrosMix + if [ $headerMode -eq 2 ] && [ ${#inocationsMacrosMix[@]} -gt 0 ]; then + replace2unsafeMix $testcaseName ## Uses: inocationsMacrosMix + fi + # Replace the rest of the calls to unsafe macros or functions + replace2unsafe $testcaseName + fi + + # Compile and test modify program: (with input compiler flags or with defualts) + #echo "ulimit -St 500;$compiler $compile_line $compile_line_confg ${compilerflags} $testcaseModify -o $testcaseEXEC" + ulimit -St 500;$compiler $compile_line $compile_line_confg ${compilerflags} $testcaseModify -o $testcaseEXEC + + # Test exec + filesize=`stat --printf="%s" $testcaseModify` + ulimit -St 500;./$testcaseEXEC >> $testcaselogger + echo "seed= $seed, size= $filesize" >> $testcaselogger + echo "---" >> $testcaselogger + + #clean + ##rm $testcaseModify + rm $testcaseEXEC +} + +### If it is a bad testcase, no need to run it, just build it for coverage measure. +function coverage_test_only { + ## Single test - create a test case and its precompilation + testcaseName='test'$1 + testcaseOriginal=$folder/'__'$testcaseName'O.c' + testcaseEXEC='__'$testcaseName'Exec' + + # Modify the test (the preprocessed file) + ulimit -St 500;$csmith_exec --seed $seed $CSMITH_USER_OPTIONS > $testcaseOriginal + + # Compile and test modify program: (with input compiler flags or with defualts) + #echo "ulimit -St 500;$compiler $compile_line ${runtime_default} ${compilerflags} $testcaseOriginal -o $testcaseEXEC" + ulimit -St 500;$compiler $compile_line ${runtime_default} ${compilerflags} $testcaseOriginal -o $testcaseEXEC + + # Test exec + filesize=`stat --printf="%s" $testcaseOriginal` + echo "Skip execution. Invalid or bad testcase." >> $testcaselogger + echo "seed= $seed, size= $filesize" >> $testcaselogger + echo "---" >> $testcaselogger + + #clean + rm $testcaseOriginal + rm $testcaseEXEC +} + +#################################### PREPARE GEN TEST #################################### +function update_csmith_cmd_options { + confg=$1 + + ## Update the csmith command line + cmd=`head -1 $confg` + CSMITH_USER_OPTIONS="$CSMITH_USER_OPTIONS $cmd" + + ## Get location of probablities file + prob_file=`echo "$cmd" | awk '{for(i=1;i<=NF;i++) if ($i=="--relax-anlayses-prob") print $(i+1)}'` + + ## Create the probablities file + if [ ! -z "$prob_file" ]; then + sed '1d;$d' $confg | sed '$d' > $prob_file + fi + + ## Update the line for compilation + getlineconfig=`sed '1d;$d' $confg | tail -1` + ## Replace vars + compile_line_confg=`eval echo "$getlineconfig"` + + ## Arrays to choose which one to set to functions or macros if mix + lastline=`tail -1 $confg` + IFS=' ' read -r -a inocationsMacrosMix <<< "$lastline" + + ## Check which version of headers we shall take + res1=`grep "USE_MATH_MACROS" $confg | wc -l` + if [[ "$res1" == "1" ]]; then + headerMode=1 + else + res2=`grep "USE_MATH_MIX" $confg | wc -l` + if [[ "$res2" == "1" ]]; then + headerMode=2 + fi + fi +} + +################### MAIN ############################### +# Single iteration, requires a compiler and a seed +CSMITH_USER_OPTIONS=" --bitfields --packed-struct" +# Basic parameters +compiler=$1 # Compiler to test +process=$2 # To create temp storage for exec +seed=$3 # File with all the seeds to use +testcaselogger=$4 # The logger file for the results +cov=$5 # To test for cov=1, else =0 +modify=$6 # Do we run on modified or original tests +csmith_location=$7 # Csmith location +compile_line=$8 # Includes + Duse +compilerflags=$9 # compiler flags +DA_folder=${10} # DA (RRS lists) folder +testcaseConfg=${11} # Test-case configuration (WA and RRS) +csmith_exec=${12} # Location of Csmith exec +runtime=${13} # Runtime for WA and RRS files +runtime_default=${14} # Runtime Csmith defualt lib (with no RRS or WA) + +## For debug only +#echo "Compiler=$compiler" +#echo "process=$process" +#echo "seed=$seed" +#echo "testcaselogger=$testcaselogger" +#echo "csmith_location=$csmith_location" +#echo "compile_line=$compile_line" +#echo "compilerflags=$compilerflags" +#echo "DA_folder=$DA_folder" +#echo "testcaseConfg=$testcaseConfg" +#echo "csmith_exec=$csmith_exec" +# Check if second parameter is a number +re='^[0-9]+$' +if ! [[ $seed =~ $re ]] ; then + echo ">> error: Not a number" >&2; exit 1 +fi + +# folders for all the results +folder=$compiler-mainRes +testcaseRes=$DA_folder/'__test'$seed'Results' +compile_line_confg="" +inocationsMacrosMix=() ## Invocation to set as function wrapper +headerMode=0 ## Assume function version +if [ ! -f $testcaseRes ] + then + #echo "Seed's list is missing: $testcaseRes" + ## Only generate and compile test case to measure coverage + if [[ "$cov" == "1" ]] + then + coverage_test_only "$seed" + fi +else + # Check the configuration file exist + testcaseConfgFile=$testcaseConfg/probs_WeakenSafeAnalyse_test.txt$seed # name of config file + if ! test -f "$testcaseConfgFile" ; then + echo ">> error: No configuration file found for this seed $seed" >&2; exit 1 + fi + + # Get configuration data: Update $CSMITH_USER_OPTIONS + update_csmith_cmd_options "$testcaseConfgFile" + + # Run a single test + modify_test "$seed" +fi +## END ##