Merge 'hardfork' branch, apply minor fixes

Resolved conflicts:
- libraries/chain/db_market.cpp
- tests/tests/settle_tests.cpp

.travis.yml

@@ -1,6 +1,9 @@
 language: c++
 
-cache: ccache
+cache:
+  ccache: true
+  directories:
+  - sonar_cache
 
 git:
   depth: 1
@@ -24,19 +27,11 @@ env:
     - CCACHE_MAXSIZE=1Gi
     - CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros
 
-script:
- - programs/build_helpers/buildstep -s 3500
- - ccache -s
- - programs/build_helpers/buildstep Prepare 1 "sed -i '/tests/d' libraries/fc/CMakeLists.txt"
- - programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ."
- - programs/build_helpers/buildstep make.cli_wallet 2200 "programs/build_helpers/make_with_sonar bw-output -j 2 cli_wallet witness_node js_operation_serializer get_dev_key network_mapper"
- - programs/build_helpers/buildstep make.chain_test 1000 "make -j 2 chain_test"
- - programs/build_helpers/buildstep make.cli_test 200 "make -j 2 cli_test"
- - programs/build_helpers/buildstep make.perf_test 120 "make -j 2 performance_test"
- - set -o pipefail
- - programs/build_helpers/buildstep run.chain_test 240 "libraries/fc/tests/run-parallel-tests.sh tests/chain_test"
- - programs/build_helpers/buildstep run.cli_test 60 "libraries/fc/tests/run-parallel-tests.sh tests/cli_test"
- - 'programs/build_helpers/buildstep prepare.sonar 20 "find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d | while read d; do gcov -o \"\$d\" \"\${d/CMakeFiles*.dir//}\"/*.cpp; done >/dev/null; programs/build_helpers/set_sonar_branch sonar-project.properties" || true'
- - 'programs/build_helpers/buildstep run.sonar 1200 "which sonar-scanner && sonar-scanner" || true'
- - programs/build_helpers/buildstep end 0
- - ccache -s
+jobs:
+  include:
+    - stage: build for cache
+      script: ./programs/build_helpers/build_protocol
+    - stage: build, test and scan partly with sonar
+      script: ./programs/build_helpers/build_and_test
+    - stage: scan fully with sonar
+      script: ./programs/build_helpers/scan_with_sonar

CMakeLists.txt

@@ -1,6 +1,6 @@
 # Defines BitShares library target.
-project( BitShares )
-cmake_minimum_required( VERSION 3.1 )
+cmake_minimum_required( VERSION 3.2 FATAL_ERROR )
+project( BitShares LANGUAGES CXX C)
 
 set( BLOCKCHAIN_NAME "BitShares" )
 
@@ -11,7 +11,13 @@ set( INSTALLER_APP_ID "68ad7005-8eee-49c9-95ce-9eed97e5b347" )
 
 set( CMAKE_CXX_STANDARD 14 )
 set( CMAKE_CXX_STANDARD_REQUIRED ON )
-set( CMAKE_CXX_EXTENSIONS OFF )
+
+if( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" OR
+    "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" )
+  set( CMAKE_CXX_EXTENSIONS ON ) # for __int128 support
+else()
+  set( CMAKE_CXX_EXTENSIONS OFF )
+endif()
 
 # http://stackoverflow.com/a/18369825
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
@@ -22,25 +28,137 @@ elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
     if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.3)
         message(FATAL_ERROR "Clang version must be at least 3.3!")
     endif()
+elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
+    if ("${CMAKE_CXX_COMPILER_VERSION}" VERSION_LESS "19.0")
+      message(FATAL_ERROR "MSVC version must be at least 19.0 (Visual Studio 2015 Update 1)!")
+    endif()
+
+    # allow MSVC VS2015 with Update 1, other 2015 versions are not supported
+    if ("${CMAKE_CXX_COMPILER_VERSION}" VERSION_EQUAL "19.0")
+        if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL "19.0.23506.0")
+            message(FATAL_ERROR "Your version ${CMAKE_CXX_COMPILER_VERSION} of MSVC is not supported, use version 19.0.23506.0 (Visual Studio 2015 Update 1)!")
+        endif()
+    endif()
 endif()
 
 list( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules" )
 
-set(CMAKE_EXPORT_COMPILE_COMMANDS "ON")
-set(GRAPHENE_EGENESIS_JSON "${CMAKE_CURRENT_SOURCE_DIR}/libraries/egenesis/genesis.json" )
+include(CheckCCompilerFlag)
+include(Utils)
+
+# function to help with cUrl
+macro(FIND_CURL)
+   if (NOT WIN32 AND NOT APPLE AND CURL_STATICLIB)
+      find_package(OpenSSL REQUIRED)
+      set (OLD_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+      set (CMAKE_FIND_LIBRARY_SUFFIXES .a)
+      find_package(CURL REQUIRED)
+      list(APPEND CURL_LIBRARIES ${OPENSSL_LIBRARIES} ${BOOST_THREAD_LIBRARY} ${CMAKE_DL_LIBS})
+      set (CMAKE_FIND_LIBRARY_SUFFIXES ${OLD_SUFFIXES})
+   else (NOT WIN32 AND NOT APPLE AND CURL_STATICLIB)
+      find_package(CURL REQUIRED)
+   endif (NOT WIN32 AND NOT APPLE AND CURL_STATICLIB)
+
+   if( WIN32 )
+     if ( MSVC )
+       list( APPEND CURL_LIBRARIES Wldap32 )
+     endif( MSVC )
+
+     if( MINGW )
+       # MinGW requires a specific order of included libraries ( CURL before ZLib )
+       find_package( ZLIB REQUIRED )
+       list( APPEND CURL_LIBRARIES ${ZLIB_LIBRARY} pthread )
+     endif( MINGW )
+
+     list( APPEND CURL_LIBRARIES ${PLATFORM_SPECIFIC_LIBS} )
+   endif( WIN32 )
+endmacro()
+
+# Save the old value of CMAKE_REQUIRED_FLAGS
+set( TEMP_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} )
+
+# Fortify source
+if (CMAKE_COMPILER_IS_GNUCXX)
+	if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+		message (STATUS "Setting optimizations for clang++")
+		set(CMAKE_CXX_FLAGS_RELEASE "-D_FORTIFY_SOURCE=2 -O3 -DNDEBUG=1")
+		set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-D_FORTIFY_SOURCE=2 -O3 -DNDEBUG=1 -g")
+
+		# check and add data execution prevention
+		message (STATUS "Enabling data execution prevention")
+		add_linker_flag("-fsanitize=safe-stack")
+
+		# check and add Stack-based buffer overrun detection
+		set(CMAKE_REQUIRED_FLAGS "-fstack-protector")
+		check_c_compiler_flag("" HAVE_STACKPROTECTOR)
+		if(HAVE_STACKPROTECTOR)
+			message (STATUS "Enabling stack-based buffer overrun detection")
+			add_flag_append(CMAKE_C_FLAGS "-fstack-protector")
+			add_flag_append(CMAKE_CXX_FLAGS "-fstack-protector")
+		endif()
+	else ()
+		message (STATUS "Setting optimizations for g++")
+		set(CMAKE_CXX_FLAGS_RELEASE "-D_FORTIFY_SOURCE=2 -O3 -DNDEBUG=1")
+		set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-D_FORTIFY_SOURCE=2 -O3 -DNDEBUG=1 -g")
+
+		# check and add data execution prevention
+		set(CMAKE_REQUIRED_FLAGS "-Wl,-znoexecstack")
+		check_c_compiler_flag("" HAVE_NOEXECSTACK)
+		if(HAVE_NOEXECSTACK)
+			message (STATUS "Enabling data execution prevention")
+			add_linker_flag("-znoexecstack")
+		endif()
+
+		# check and add Stack-based buffer overrun detection
+		set(CMAKE_REQUIRED_FLAGS "-fstack-protector-strong")
+		check_c_compiler_flag("" HAVE_STACKPROTECTOR)
+		if(HAVE_STACKPROTECTOR)
+			message (STATUS "Enabling stack-based buffer overrun detection")
+			add_flag_append(CMAKE_C_FLAGS "-fstack-protector-strong")
+			add_flag_append(CMAKE_CXX_FLAGS "-fstack-protector-strong")
+		endif()
+
+	endif ()
+endif ()
+
+# check for Data relocation and Protection (RELRO)
+set(CMAKE_REQUIRED_FLAGS "-Wl,-zrelro,-znow")
+check_c_compiler_flag("" HAVE_RELROFULL)
+if(HAVE_RELROFULL)
+	message (STATUS "Enabling full data relocation and protection")
+	add_linker_flag("-zrelro")
+	add_linker_flag("-znow")
+else()
+	#if full relro is not available, try partial relro
+	set(CMAKE_REQUIRED_FLAGS "-Wl,-zrelro")
+	check_c_compiler_flag("" HAVE_RELROPARTIAL)
+	if(HAVE_RELROPARTIAL)
+		message (STATUS "Enabling partial data relocation and protection")
+		add_linker_flag("-zrelro")
+	endif()
+endif()
+
+set(CMAKE_REQUIRED_FLAGS ${TEMP_REQUIRED_FLAGS} )
+
+# position independent executetable (PIE)
+# position independent code (PIC)
+if (NOT MSVC)
+    add_definitions (-fPIC)
+endif(NOT MSVC)
 
-#set (ENABLE_INSTALLER 1)
-#set (USE_PCH 1)
+set(CMAKE_EXPORT_COMPILE_COMMANDS "ON")
+set( GRAPHENE_EGENESIS_JSON "${CMAKE_CURRENT_SOURCE_DIR}/libraries/egenesis/genesis.json"
+     CACHE STRING "Path to embedded genesis file" )
 
 if (USE_PCH)
   include (cotire)
 endif(USE_PCH)
 
 option(USE_PROFILER "Build with GPROF support(Linux)." OFF)
 
-IF( NOT WIN32 )
-  list( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/libraries/fc/CMakeModules" )
-ENDIF( NOT WIN32 )
+# Use Boost config file from fc
+set(Boost_DIR "${CMAKE_CURRENT_SOURCE_DIR}/libraries/fc/CMakeModules/Boost")
+
 list( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/libraries/fc/GitVersionGen" )
 include( GetGitRevisionDescription )
 get_git_head_revision( GIT_REFSPEC GIT_SHA2 )
@@ -54,38 +172,60 @@ LIST(APPEND BOOST_COMPONENTS thread
                              program_options
                              chrono
                              unit_test_framework
-                             context)
+                             context
+                             coroutine
+                             regex)
 # boost::endian is also required, but FindBoost can't handle header-only libs
 SET( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 
-IF( WIN32 )
-  SET(BOOST_ROOT $ENV{BOOST_ROOT})
+IF(WIN32)
+   if($ENV{BOOST_ROOT})
+       SET(BOOST_ROOT $ENV{BOOST_ROOT})
+   endif($ENV{BOOST_ROOT})
   set(Boost_USE_MULTITHREADED ON)
   set(BOOST_ALL_DYN_LINK OFF) # force dynamic linking for all libraries
+  add_definitions("-DCURL_STATICLIB")
+  list(APPEND PLATFORM_SPECIFIC_LIBS ws2_32 crypt32 mswsock userenv )
+ELSE( WIN32 )
+   IF( APPLE )
+      set( CMAKE_THREAD_LIBS_INIT "-lpthread" )
+      set( CMAKE_HAVE_THREADS_LIBRARY 1 )
+      set( CMAKE_USE_WIN32_THREADS_INIT 0 )
+      set( CMAKE_USE_PTHREADS_INIT 1 )
+      set( THREADS_PREFER_PTHREAD_FLAG ON )
+   ENDIF( APPLE )
 ENDIF(WIN32)
 
-FIND_PACKAGE(Boost 1.57 REQUIRED COMPONENTS ${BOOST_COMPONENTS})
-# For Boost 1.53 on windows, coroutine was not in BOOST_LIBRARYDIR and do not need it to build,  but if boost versin >= 1.54, find coroutine otherwise will cause link errors
-IF(NOT "${Boost_VERSION}" MATCHES "1.53(.*)")
-   SET(BOOST_LIBRARIES_TEMP ${Boost_LIBRARIES})
-   FIND_PACKAGE(Boost 1.54 REQUIRED COMPONENTS coroutine)
-   LIST(APPEND BOOST_COMPONENTS coroutine)
-   SET(Boost_LIBRARIES ${BOOST_LIBRARIES_TEMP} ${Boost_LIBRARIES})
-ENDIF()
+FIND_PACKAGE(Boost CONFIG REQUIRED COMPONENTS ${BOOST_COMPONENTS})
+
+# enforce more strict compiler warnings and errors
+add_compiler_flag_if_available("-Wall")
+add_compiler_flag_if_available("-Wclobbered")
+add_compiler_flag_if_available("-Wempty-body")
+add_compiler_flag_if_available("-Wformat-security")
+add_compiler_flag_if_available("-Wignored-qualifiers")
+add_compiler_flag_if_available("-Wimplicit-fallthrough=5")
+add_compiler_flag_if_available("-Wmissing-field-initializers")
+add_compiler_flag_if_available("-Wpointer-arith")
+add_compiler_flag_if_available("-Wshift-negative-value")
+add_compiler_flag_if_available("-Wtype-limits")
+add_compiler_flag_if_available("-Wunused-but-set-parameter")
 
 if( WIN32 )
 
     message( STATUS "Configuring BitShares on WIN32")
-    set( DB_VERSION 60 )
-    set( BDB_STATIC_LIBS 1 )
 
-    set( ZLIB_LIBRARIES "" )
-    SET( DEFAULT_EXECUTABLE_INSTALL_DIR bin/ )
+    if ( MINGW )
+        message( STATUS "Windows build using MinGW" )
+        set( FULL_STATIC_BUILD TRUE )
+    else( MINGW )
+        set( ZLIB_LIBRARIES "" )
+    endif( MINGW )
 
-    set(CRYPTO_LIB)
+    SET( DEFAULT_EXECUTABLE_INSTALL_DIR bin/ )
 
     if( MSVC )
-	add_definitions(-DWIN32_LEAN_AND_MEAN)
+        add_definitions(-DWIN32_LEAN_AND_MEAN)
         #looks like this flag can have different default on some machines.
         SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SAFESEH:NO")
         SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SAFESEH:NO")
@@ -94,44 +234,32 @@ if( WIN32 )
         set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} /DEBUG")
     endif ( MSVC )
 
-    # On windows tcl should be installed to the directory pointed by setenv.bat script
-    SET(TCL_INCLUDE_PATH $ENV{TCL_ROOT}/include)
-    MESSAGE(STATUS "tcl INCLUDE PATH: ${TCL_INCLUDE_PATH}")
-
-    FIND_PACKAGE(TCL)
-    MESSAGE(STATUS "tcl_library: ${TCL_LIBRARY}")
-
-    SET(TCL_LIBS "optimized;${TCL_LIBRARY};debug;")
-    get_filename_component(TCL_LIB_PATH "${TCL_LIBRARY}" PATH)
-    get_filename_component(TCL_LIB_NAME "${TCL_LIBRARY}" NAME_WE)
-    get_filename_component(TCL_LIB_EXT "${TCL_LIBRARY}" EXT)
-
-    SET(TCL_LIBS "${TCL_LIBS}${TCL_LIB_PATH}/${TCL_LIB_NAME}g${TCL_LIB_EXT}")
-    SET(TCL_LIBRARY ${TCL_LIBS})
-
 else( WIN32 ) # Apple AND Linux
 
     if( APPLE )
         # Apple Specific Options Here
         message( STATUS "Configuring BitShares on OS X" )
         set( CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -stdlib=libc++ -Wall" )
     else( APPLE )
-        # Linux Specific Options Here
-        message( STATUS "Configuring BitShares on Linux" )
+        if ( "${CMAKE_SYSTEM_NAME}" STREQUAL "OpenBSD" )
+            # OpenBSD Specific Options
+            message( STATUS "Configuring BitShares on OpenBSD" )
+        else()
+            # Linux Specific Options Here
+            message( STATUS "Configuring BitShares on Linux" )
+            set( rt_library rt )
+        endif()
+        # Common Linux & OpenBSD Options
         set( CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -Wall" )
         if(USE_PROFILER)
             set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg" )
         endif( USE_PROFILER )
-        set( rt_library rt )
         set( pthread_library pthread)
         if ( NOT DEFINED crypto_library )
           # I'm not sure why this is here, I guess someone has openssl and can't detect it with find_package()?
           # if you have a normal install, you can define crypto_library to the empty string to avoid a build error
           set( crypto_library crypto)
         endif ()
-        if ( FULL_STATIC_BUILD )
-          set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++ -static-libgcc")
-        endif ( FULL_STATIC_BUILD )
     endif( APPLE )
 
     if( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" )
@@ -154,6 +282,10 @@ else( WIN32 ) # Apple AND Linux
 
 endif( WIN32 )
 
+if ( NOT MSVC AND FULL_STATIC_BUILD )
+    set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -static-libstdc++ -static-libgcc" )
+endif ( NOT MSVC AND FULL_STATIC_BUILD )
+
 set(ENABLE_COVERAGE_TESTING FALSE CACHE BOOL "Build BitShares for code coverage analysis")
 
 if(ENABLE_COVERAGE_TESTING)

CMakeModules/Utils.cmake

@@ -0,0 +1,25 @@
+macro(add_flag_append _VAR_NAME _FLAG)
+	set(${_VAR_NAME} "${${_VAR_NAME}} ${_FLAG}")
+endmacro(add_flag_append _VAR_NAME _FLAG)
+
+macro(add_linker_flag _FLAG)
+	#executables
+	add_flag_append(CMAKE_C_LINK_FLAGS "-Wl,${_FLAG}")
+	add_flag_append(CMAKE_CXX_LINK_FLAGS "-Wl,${_FLAG}")
+	#libraries
+	add_flag_append(CMAKE_SHARED_LIBRARY_C_FLAGS "-Wl,${_FLAG}")
+	add_flag_append(CMAKE_SHARED_LIBRARY_CXX_FLAGS "-Wl,${_FLAG}")
+endmacro(add_linker_flag _FLAG)
+
+include(CheckCXXCompilerFlag)
+
+macro(add_compiler_flag_if_available _FLAG)
+	string(TOUPPER "${_FLAG}" _TEMPVAR1)
+	string(REPLACE "-" "_" _TEMPVAR2 "${_TEMPVAR1}")
+	string(CONCAT _TEMPVAR3 "HAVE" "${_TEMPVAR2}")
+	set(CMAKE_REQUIRED_FLAGS "${_FLAG}")
+	check_cxx_compiler_flag("${_FLAG}" ${_TEMPVAR3})
+	if(${_TEMPVAR3})
+		add_flag_append(CMAKE_CXX_FLAGS ${_FLAG})
+	endif()
+endmacro(add_compiler_flag_if_available _VAR_NAME _FLAG)

Dockerfile

@@ -1,4 +1,4 @@
-FROM phusion/baseimage:0.10.1
+FROM phusion/baseimage:0.11
 MAINTAINER The bitshares decentralized organisation
 
 ENV LANG=en_US.UTF-8
@@ -74,6 +74,7 @@ EXPOSE 1776
 
 # default exec/config files
 ADD docker/default_config.ini /etc/bitshares/config.ini
+ADD docker/default_logging.ini /etc/bitshares/logging.ini
 ADD docker/bitsharesentry.sh /usr/local/bin/bitsharesentry.sh
 RUN chmod a+x /usr/local/bin/bitsharesentry.sh