diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..ef3b9bc73 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,13 @@ +One line summary of the issue here. + +### Expected behavior + +As concisely as possible, describe the expected behavior. + +### Actual behavior + +As concisely as possible, describe the observed behavior. + +### Steps to reproduce the behavior + +Please list all relevant steps to reproduce the observed behavior. \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..bfb079c94 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,15 @@ +Problem + +Explain the context and why you're making that change. What is the +problem you're trying to solve? In some cases there is not a problem +and this can be thought of being the motivation for your change. + +Solution + +Describe the modifications you've done. + +Result + +What will change as a result of your pull request? Note that sometimes +this section is unnecessary because it is self-explanatory based on +the solution. diff --git a/.gitignore b/.gitignore index eb6da3567..781ba3860 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ tags # cscope cscope.* lcov + +CMAKE_BINARY_DIR diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..41c2b344e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,135 @@ +sudo: false +language: c + +# using anchor to import sources into linux builds +addons: + apt: &apt + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.6 + - llvm-toolchain-precise-3.7 + - llvm-toolchain-precise + +# important for allowed-to-fail matching +# see https://docs.travis-ci.com/user/customizing-the-build#Rows-that-are-Allowed-to-Fail +env: + - ALLOWED_TO_FAIL=0 + +# travis currently does not support directly setting gcc/clang with versions +# (e.g. gcc-4.8) as value for the compiler key. So we will have to manually +# request these packages and use environment varibles to create the matrix. +# +# In the case of osx, use brew to install the paritcular versions, instead of +# specifying with packages. +matrix: + include: + # gcc 4.8 on linux + - env: + - C_COMPILER=gcc-4.8 + addons: + apt: + <<: *apt + packages: + - gcc-4.8 + - libsubunit-dev + + + # gcc 4.9 on linux + - env: + - C_COMPILER=gcc-4.9 + addons: + apt: + <<: *apt + packages: + - gcc-4.9 + - libsubunit-dev + + # gcc 5 on linux + - env: + - C_COMPILER=gcc-5 + addons: + apt: + <<: *apt + packages: + - gcc-5 + - libsubunit-dev + + # gcc 5 on linux + - env: + - C_COMPILER=gcc-5 + - RUST_ENABLED=1 + addons: + apt: + <<: *apt + packages: + - gcc-5 + - libsubunit-dev + + # clang 3.6 on linux + - env: + - C_COMPILER=clang-3.6 + addons: + apt: + <<: *apt + packages: + - clang-3.6 + - libsubunit-dev + + # clang 3.7 on linux + - env: + - C_COMPILER=clang-3.7 + addons: + apt: + <<: *apt + packages: + - clang-3.7 + - libsubunit-dev + + ## gcc 4.8 on osx + #- os: osx + # env: FORMULA=gcc48 COMPILER=gcc C_COMPILER=gcc-4.8 + # + ## gcc 4.9 on osx + #- os: osx + # env: FORMULA=gcc49 COMPILER=gcc C_COMPILER=gcc-4.9 + # + ## gcc 5 on osx + #- os: osx + # env: FORMULA=gcc5 COMPILER=gcc C_COMPILER=gcc-5 + + # OSX 10.13 + # Apple LLVM version 9.1.0 (clang-902.0.39.2) + # Target: x86_64-apple-darwin17.6.0 + - os: osx + osx_image: xcode9.4 + env: + - C_COMPILER=clang + - ALLOWED_TO_FAIL=1 + + # OSX 10.12 + # Apple LLVM version 9.0.0 (clang-900.0.39.2) + # Target: x86_64-apple-darwin16.7.0 + - os: osx + osx_image: xcode9.2 + env: + - C_COMPILER=clang + - ALLOWED_TO_FAIL=1 + + allow_failures: + - os: osx + osx_image: xcode9.4 + env: + - C_COMPILER=clang + - ALLOWED_TO_FAIL=1 + + - os: osx + osx_image: xcode9.2 + env: + - C_COMPILER=clang + - ALLOWED_TO_FAIL=1 + +before_install: + - ./ci/before-install.sh + +script: + - ./ci/run.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 93067d859..2d2a2c05a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 2.6) -project(ccommon) +project(ccommon C) enable_testing() @@ -37,8 +37,8 @@ endif() # version info set(${PROJECT_NAME}_VERSION_MAJOR 1) -set(${PROJECT_NAME}_VERSION_MINOR 0) -set(${PROJECT_NAME}_VERSION_PATCH 2) +set(${PROJECT_NAME}_VERSION_MINOR 2) +set(${PROJECT_NAME}_VERSION_PATCH 0) set(${PROJECT_NAME}_VERSION ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}.${${PROJECT_NAME}_VERSION_PATCH} ) @@ -51,7 +51,35 @@ option(HAVE_ASSERT_LOG "assert_log enabled by default" ON) option(HAVE_ASSERT_PANIC "assert_panic disabled by default" OFF) option(HAVE_LOGGING "logging enabled by default" ON) option(HAVE_STATS "stats enabled by default" ON) +option(HAVE_DEBUG_MM "debugging oriented memory management disabled by default" OFF) option(COVERAGE "code coverage" OFF) +option(HAVE_RUST "rust bindings not built by default" OFF) + +if(HAVE_RUST) + option(RUST_VERBOSE_BUILD "pass -vv to cargo compilation" OFF) +endif() + +if(BUILD_AND_INSTALL_CHECK) + # (simms) What follows is a crime against build systems as we run the build/install + # for the check library up front, during the planning phase. + + set(LIBCHECK_PREFIX "${CMAKE_BINARY_DIR}/check") + + # check for a local install of check + if(NOT EXISTS "${LIBCHECK_PREFIX}") + # (simms) This is terrible and I did it this way to ensure this gets built + # before the rest of the 'check' tests run. This should be rewritten so that + # the other dependencies know that there's a target that can build check + execute_process( + COMMAND "bash" "${PROJECT_SOURCE_DIR}/ci/install-check.sh" "${LIBCHECK_PREFIX}" + TIMEOUT 300 # if this doesn't build in 5 minutes something is hosed + ) + endif() + + set(CHECK_ROOT_DIR "${LIBCHECK_PREFIX}") + set(CMAKE_REQUIRED_INCLUDES "${CHECK_ROOT_DIR}/include") # these make check link correctly in ccommon and pelikan + set(CMAKE_REQUIRED_LIBRARIES "${CHECK_ROOT_DIR}/lib") +endif() include(CheckIncludeFiles) if(OS_PLATFORM STREQUAL "OS_LINUX") @@ -99,6 +127,10 @@ set(CFLAGS_LIST string(REPLACE "" "" CFLAGS ${CFLAGS_LIST}) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CFLAGS}") +if(CMAKE_COMPILER_IS_GNUCC) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,--no-as-needed -ldl -pthread -fPIC") +endif() + if (COVERAGE) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -Wall -W -fprofile-arcs -ftest-coverage") endif(COVERAGE) @@ -106,6 +138,7 @@ endif(COVERAGE) # test dependencies include(FindPackageHandleStandardArgs) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake") + find_package(Check) if(NOT CHECK_FOUND) message(WARNING "Check is required to build and run tests") @@ -119,11 +152,21 @@ endif(CHECK_FOUND) find_package(Threads) + # where to find include files -include_directories(${include_directories} +include_directories( + ${include_directories} "${PROJECT_BINARY_DIR}" "include") +if(HAVE_RUST) + enable_language(Rust) + include(CMakeCargo) + add_subdirectory(rust) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_RUST=1") +endif() + + ################### # things to build # ################### diff --git a/ci/before-install.sh b/ci/before-install.sh new file mode 100755 index 000000000..cff9327c3 --- /dev/null +++ b/ci/before-install.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -euo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" +cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } +trap cleanup EXIT + +TOPLEVEL="$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' + +# for osx: 0. update brew; 1. install cmake if missing; 2. (gcc) unlink pre-installed gcc; 3. (gcc) install desired version of gcc + +if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + brew update &>/dev/null + brew install cmake || true # xcode 8.1 is missing cmake + + if [[ "$C_COMPILER" =~ ^gcc && -n "${FORMULA:-}" ]]; then + brew unlink gcc || true + brew unlink "$FORMULA" || true + brew install "$FORMULA" + fi +fi + +export CC="$C_COMPILER" + +if [[ -n "${RUST_ENABLED:-}" ]]; then + curl https://sh.rustup.rs -sSf | sh -s -- -y +fi diff --git a/ci/install-check.sh b/ci/install-check.sh new file mode 100755 index 000000000..628d6d229 --- /dev/null +++ b/ci/install-check.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -euo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 check-install-path" + exit 1 +fi + +CHECK_PREFIX="$1" +shift + +TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" +cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } +trap cleanup EXIT + +TOPLEVEL="$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL' + +CHECK_VERSION=0.12.0 +CHECK_TARBALL="check-${CHECK_VERSION}.tar.gz" +CHECK_DIR="check-${CHECK_VERSION}" + +( + cd "$TEMP" && + wget "https://github.com/libcheck/check/releases/download/${CHECK_VERSION}/${CHECK_TARBALL}" && + tar xvfz "${CHECK_TARBALL}" && + cd "${CHECK_DIR}" && + ./configure --prefix="$CHECK_PREFIX" && + make && + make install +) || die "check build failed" diff --git a/ci/run.sh b/ci/run.sh new file mode 100755 index 000000000..588bbc0b2 --- /dev/null +++ b/ci/run.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -uo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +export PATH=$HOME/.cargo/bin:$PATH + +cmake_cmd=( + cmake + -DBUILD_AND_INSTALL_CHECK=yes +) + +if [[ -n "${RUST_ENABLED:-}" ]]; then + cmake_cmd+=( -DHAVE_RUST=yes -DRUST_VERBOSE_BUILD=yes ) +fi + +export RUST_BACKTRACE=full + +mkdir -p _build && ( cd _build && "${cmake_cmd[@]}" .. && make -j && make check ) +RESULT=$? + +egrep -r ":F:|:E:" . |grep -v 'Binary file' || true + + +if [[ $RESULT -ne 0 ]]; then + echo "Build failure" >&2 + exit $RESULT +else + echo "success!" >&2 + exit 0 +fi diff --git a/cmake/CMakeCargo.cmake b/cmake/CMakeCargo.cmake new file mode 100644 index 000000000..81920f8e7 --- /dev/null +++ b/cmake/CMakeCargo.cmake @@ -0,0 +1,101 @@ +function(cargo_set_lib_target LIB_NAME) + if(WIN32) + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(LIB_TARGET "x86_64-pc-windows-msvc" PARENT_SCOPE) + else() + set(LIB_TARGET "i686-pc-windows-msvc" PARENT_SCOPE) + endif() + elseif(ANDROID) + if(ANDROID_SYSROOT_ABI STREQUAL "x86") + set(LIB_TARGET "i686-linux-android" PARENT_SCOPE) + elseif(ANDROID_SYSROOT_ABI STREQUAL "x86_64") + set(LIB_TARGET "x86_64-linux-android" PARENT_SCOPE) + elseif(ANDROID_SYSROOT_ABI STREQUAL "arm") + set(LIB_TARGET "arm-linux-androideabi" PARENT_SCOPE) + elseif(ANDROID_SYSROOT_ABI STREQUAL "arm64") + set(LIB_TARGET "aarch64-linux-android" PARENT_SCOPE) + endif() + elseif(IOS) + set(LIB_TARGET "universal" PARENT_SCOPE) + elseif(CMAKE_SYSTEM_NAME STREQUAL Darwin) + set(LIB_TARGET "x86_64-apple-darwin" PARENT_SCOPE) + else() + if(RUST_USE_MUSL) + set(RUST_LIBC_NAME "musl") + else() + set(RUST_LIBC_NAME "gnu") + endif() + + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(LIB_TARGET "x86_64-unknown-linux-${RUST_LIBC_NAME}" PARENT_SCOPE) + else() + set(LIB_TARGET "i686-unknown-linux-${RUST_LIBC_NAME}" PARENT_SCOPE) + endif() + endif() +endfunction() + +function(cargo_build) + cmake_parse_arguments(CARGO "" "NAME" "" ${ARGN}) + string(REPLACE "-" "_" LIB_NAME ${CARGO_NAME}) + + get_filename_component(CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}" ABSOLUTE) + + cargo_set_lib_target(LIB_NAME) + + if(NOT CMAKE_BUILD_TYPE) + set(LIB_BUILD_TYPE "debug") + elseif(${CMAKE_BUILD_TYPE} STREQUAL "Release") + set(LIB_BUILD_TYPE "release") + else() + set(LIB_BUILD_TYPE "debug") + endif() + + set(SHARED_LIB_FNAME "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}") + set(STATIC_LIB_FNAME "${CMAKE_STATIC_LIBRARY_PREFIX}${LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(LIB_BASE_DIR "${CARGO_TARGET_DIR}/${LIB_TARGET}/${LIB_BUILD_TYPE}") + + get_filename_component(SHARED_LIB_FILE "${LIB_BASE_DIR}/${SHARED_LIB_FNAME}" ABSOLUTE) + get_filename_component(STATIC_LIB_FILE "${LIB_BASE_DIR}/${STATIC_LIB_FNAME}" ABSOLUTE) + + if(IOS) + set(CARGO_ARGS "lipo") + else() + set(CARGO_ARGS "build") + list(APPEND CARGO_ARGS "--target" ${LIB_TARGET}) + endif() + + if(RUST_VERBOSE_BUILD) + list(APPEND CARGO_ARGS "-vv") + endif() + + if(${LIB_BUILD_TYPE} STREQUAL "release") + list(APPEND CARGO_ARGS "--release") + endif() + + file(GLOB_RECURSE LIB_SOURCES "*.rs") + + set(CARGO_ENV_COMMAND ${CMAKE_COMMAND} -E env "CARGO_TARGET_DIR=${CARGO_TARGET_DIR}") + + add_custom_command( + OUTPUT ${STATIC_LIB_FILE} + COMMAND ${CARGO_ENV_COMMAND} ${CARGO_EXECUTABLE} ARGS ${CARGO_ARGS} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS ${LIB_SOURCES} + COMMENT "running cargo") + add_custom_target(${CARGO_NAME}_static_target ALL DEPENDS ${STATIC_LIB_FILE}) + add_library(${CARGO_NAME}_static STATIC IMPORTED GLOBAL) + add_dependencies(${CARGO_NAME}_static ${CARGO_NAME}_static_target) + set_target_properties(${CARGO_NAME}_static PROPERTIES IMPORTED_LOCATION ${STATIC_LIB_FILE}) + + add_custom_command( + OUTPUT ${SHARED_LIB_FILE} + COMMAND ${CARGO_ENV_COMMAND} ${CARGO_EXECUTABLE} ARGS ${CARGO_ARGS} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS ${LIB_SOURCES} + COMMENT "running cargo") + add_custom_target(${CARGO_NAME}_shared_target ALL DEPENDS ${SHARED_LIB_FILE}) + add_library(${CARGO_NAME}_shared SHARED IMPORTED GLOBAL) + add_dependencies(${CARGO_NAME}_shared ${CARGO_NAME}_shared_target) + set_target_properties(${CARGO_NAME}_shared PROPERTIES IMPORTED_LOCATION ${SHARED_LIB_FILE}) +endfunction() diff --git a/cmake/CMakeDetermineRustCompiler.cmake b/cmake/CMakeDetermineRustCompiler.cmake new file mode 100644 index 000000000..33a459c0a --- /dev/null +++ b/cmake/CMakeDetermineRustCompiler.cmake @@ -0,0 +1,25 @@ + +if(NOT CMAKE_Rust_COMPILER) + find_package(Rust) + if(RUST_FOUND) + set(CMAKE_Rust_COMPILER "${RUSTC_EXECUTABLE}") + set(CMAKE_Rust_COMPILER_ID "Rust") + set(CMAKE_Rust_COMPILER_VERSION "${RUST_VERSION}") + set(CMAKE_Rust_PLATFORM_ID "Rust") + endif() +endif() + +message(STATUS "Cargo Prefix: ${CARGO_PREFIX}") +message(STATUS "Rust Compiler Version: ${RUSTC_VERSION}") + +mark_as_advanced(CMAKE_Rust_COMPILER) + +if(CMAKE_Rust_COMPILER) + set(CMAKE_Rust_COMPILER_LOADED 1) +endif(CMAKE_Rust_COMPILER) + +configure_file(${CMAKE_SOURCE_DIR}/cmake/CMakeRustCompiler.cmake.in + ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${CMAKE_VERSION}/CMakeRustCompiler.cmake IMMEDIATE @ONLY) + +set(CMAKE_Rust_COMPILER_ENV_VAR "RUSTC") + diff --git a/cmake/CMakeRustCompiler.cmake.in b/cmake/CMakeRustCompiler.cmake.in new file mode 100644 index 000000000..5916c1c3d --- /dev/null +++ b/cmake/CMakeRustCompiler.cmake.in @@ -0,0 +1,15 @@ + +set(CMAKE_Rust_COMPILER "@CMAKE_Rust_COMPILER@") +set(CMAKE_Rust_COMPILER_ID "@CMAKE_Rust_COMPILER_ID@") +set(CMAKE_Rust_COMPILER_VERSION "@CMAKE_Rust_COMPILER_VERSION@") +set(CMAKE_Rust_COMPILER_LOADED @CMAKE_Rust_COMPILER_LOADED@) +set(CMAKE_Rust_PLATFORM_ID "@CMAKE_Rust_PLATFORM_ID@") + +SET(CMAKE_Rust_SOURCE_FILE_EXTENSIONS rs) +SET(CMAKE_Rust_LINKER_PREFERENCE 40) +#SET(CMAKE_Rust_OUTPUT_EXTENSION_REPLACE 1) +SET(CMAKE_STATIC_LIBRARY_PREFIX_Rust "") +SET(CMAKE_STATIC_LIBRARY_SUFFIX_Rust .a) + +set(CMAKE_Rust_COMPILER_ENV_VAR "RUSTC") + diff --git a/cmake/CMakeRustInformation.cmake b/cmake/CMakeRustInformation.cmake new file mode 100644 index 000000000..05d96c94b --- /dev/null +++ b/cmake/CMakeRustInformation.cmake @@ -0,0 +1,106 @@ + +# +# Usage: rustc [OPTIONS] INPUT +# +# Options: +# -h --help Display this message +# --cfg SPEC Configure the compilation environment +# -L [KIND=]PATH Add a directory to the library search path. The +# optional KIND can be one of dependency, crate, native, +# framework or all (the default). +# -l [KIND=]NAME Link the generated crate(s) to the specified native +# library NAME. The optional KIND can be one of static, +# dylib, or framework. If omitted, dylib is assumed. +# --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|metadata] +# Comma separated list of types of crates for the +# compiler to emit +# --crate-name NAME Specify the name of the crate being built +# --emit [asm|llvm-bc|llvm-ir|obj|link|dep-info] +# Comma separated list of types of output for the +# compiler to emit +# --print [crate-name|file-names|sysroot|cfg|target-list|target-cpus|target-features|relocation-models|code-models] +# Comma separated list of compiler information to print +# on stdout +# -g Equivalent to -C debuginfo=2 +# -O Equivalent to -C opt-level=2 +# -o FILENAME Write output to +# --out-dir DIR Write output to compiler-chosen filename in +# --explain OPT Provide a detailed explanation of an error message +# --test Build a test harness +# --target TARGET Target triple for which the code is compiled +# -W --warn OPT Set lint warnings +# -A --allow OPT Set lint allowed +# -D --deny OPT Set lint denied +# -F --forbid OPT Set lint forbidden +# --cap-lints LEVEL Set the most restrictive lint level. More restrictive +# lints are capped at this level +# -C --codegen OPT[=VALUE] +# Set a codegen option +# -V --version Print version info and exit +# -v --verbose Use verbose output +# +# Additional help: +# -C help Print codegen options +# -W help Print 'lint' options and default settings +# -Z help Print internal options for debugging rustc +# --help -v Print the full set of options rustc accepts +# + +# + +include(CMakeLanguageInformation) + +if(UNIX) + set(CMAKE_Rust_OUTPUT_EXTENSION .o) +else() + set(CMAKE_Rust_OUTPUT_EXTENSION .obj) +endif() + +set(CMAKE_Rust_ECHO_ALL "echo \"TARGET: TARGET_BASE: ") +set(CMAKE_Rust_ECHO_ALL "${CMAKE_Rust_ECHO_ALL} OBJECT: OBJECTS: OBJECT_DIR: SOURCE: SOURCES: ") +set(CMAKE_Rust_ECHO_ALL "${CMAKE_Rust_ECHO_ALL} LINK_LIBRARIES: FLAGS: LINK_FLAGS: \"") + +if(NOT CMAKE_Rust_CREATE_SHARED_LIBRARY) + set(CMAKE_Rust_CREATE_SHARED_LIBRARY + "echo \"CMAKE_Rust_CREATE_SHARED_LIBRARY\"" + "${CMAKE_Rust_ECHO_ALL}" + ) +endif() + +if(NOT CMAKE_Rust_CREATE_SHARED_MODULE) + set(CMAKE_Rust_CREATE_SHARED_MODULE + "echo \"CMAKE_Rust_CREATE_SHARED_MODULE\"" + "${CMAKE_Rust_ECHO_ALL}" + ) +endif() + +if(NOT CMAKE_Rust_CREATE_STATIC_LIBRARY) + set(CMAKE_Rust_CREATE_STATIC_LIBRARY + "echo \"CMAKE_Rust_CREATE_STATIC_LIBRARY\"" + "${CMAKE_Rust_ECHO_ALL}" + ) +endif() + +if(NOT CMAKE_Rust_COMPILE_OBJECT) + set(CMAKE_Rust_COMPILE_OBJECT + "echo \"CMAKE_Rust_COMPILE_OBJECT\"" + "${CMAKE_Rust_ECHO_ALL}" + "${CMAKE_Rust_COMPILER} --emit obj -o ") +endif() + +if(NOT CMAKE_Rust_LINK_EXECUTABLE) + set(CMAKE_Rust_LINK_EXECUTABLE + "echo \"CMAKE_Rust_LINK_EXECUTABLE\"" + "${CMAKE_Rust_ECHO_ALL}" + ) +endif() + +mark_as_advanced( + CMAKE_Rust_FLAGS + CMAKE_Rust_FLAGS_DEBUG + CMAKE_Rust_FLAGS_MINSIZEREL + CMAKE_Rust_FLAGS_RELEASE + CMAKE_Rust_FLAGS_RELWITHDEBINFO) + +set(CMAKE_Rust_INFORMATION_LOADED 1) + diff --git a/cmake/CMakeTestRustCompiler.cmake b/cmake/CMakeTestRustCompiler.cmake new file mode 100644 index 000000000..ff75bb3e0 --- /dev/null +++ b/cmake/CMakeTestRustCompiler.cmake @@ -0,0 +1,3 @@ + +set(CMAKE_Rust_COMPILER_WORKS 1 CACHE INTERNAL "") + diff --git a/cmake/FindRust.cmake b/cmake/FindRust.cmake new file mode 100644 index 000000000..2dbf14ddf --- /dev/null +++ b/cmake/FindRust.cmake @@ -0,0 +1,75 @@ + +set(_CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ${CMAKE_FIND_ROOT_PATH_MODE_PROGRAM}) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM BOTH) +set(_CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ${CMAKE_FIND_ROOT_PATH_MODE_INCLUDE}) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH) + +if(WIN32) + set(USER_HOME "$ENV{USERPROFILE}") +else() + set(USER_HOME "$ENV{HOME}") +endif() + +# Find cargo prefix +find_path(CARGO_PREFIX ".cargo" + PATHS "${USER_HOME}") + +if(CARGO_PREFIX MATCHES "NOTFOUND") + message(FATAL_ERROR "Could not find Rust!") +else() + set(CARGO_PREFIX "${CARGO_PREFIX}/.cargo") +endif() + +# Find cargo executable +find_program(CARGO_EXECUTABLE cargo + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(CARGO_EXECUTABLE) + +# Find rustc executable +find_program(RUSTC_EXECUTABLE rustc + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(RUSTC_EXECUTABLE) + +# Find rustdoc executable +find_program(RUSTDOC_EXECUTABLE rustdoc + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(RUSTDOC_EXECUTABLE) + +# Find rust-gdb executable +find_program(RUST_GDB_EXECUTABLE rust-gdb + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(RUST_GDB_EXECUTABLE) + +# Find rust-lldb executable +find_program(RUST_LLDB_EXECUTABLE rust-lldb + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(RUST_LLDB_EXECUTABLE) + +# Find rustup executable +find_program(RUSTUP_EXECUTABLE rustup + HINTS "${CARGO_PREFIX}" + PATH_SUFFIXES "bin") +mark_as_advanced(RUSTUP_EXECUTABLE) + +set(RUST_FOUND FALSE CACHE INTERNAL "") + +if(CARGO_EXECUTABLE AND RUSTC_EXECUTABLE AND RUSTDOC_EXECUTABLE) + set(RUST_FOUND TRUE CACHE INTERNAL "") + + set(CARGO_PREFIX "${CARGO_PREFIX}" CACHE PATH "Rust Cargo prefix") + + execute_process(COMMAND ${RUSTC_EXECUTABLE} --version OUTPUT_VARIABLE RUSTC_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REGEX REPLACE "rustc ([^ ]+) .*" "\\1" RUSTC_VERSION "${RUSTC_VERSION}") +endif() + +if(NOT RUST_FOUND) + message(FATAL_ERROR "Could not find Rust!") +endif() + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ${_CMAKE_FIND_ROOT_PATH_MODE_PROGRAM}) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ${_CMAKE_FIND_ROOT_PATH_MODE_INCLUDE}) diff --git a/config.h.in b/config.h.in index e5fa50e30..f1dc18bd9 100644 --- a/config.h.in +++ b/config.h.in @@ -17,3 +17,5 @@ #cmakedefine HAVE_LOGGING #cmakedefine HAVE_STATS + +#cmakedefine HAVE_DEBUG_MM diff --git a/docs/c-styleguide.txt b/docs/c-styleguide.txt index 30aa62f20..69661c0d8 100644 --- a/docs/c-styleguide.txt +++ b/docs/c-styleguide.txt @@ -16,11 +16,13 @@ . However, when interfacing with system calls and libraries you cannot get away from using int and char. - Use bool for boolean variables. You have to include -- Avoid using a bool as type for struct member names. Instead use unsigned - 1-bit bit field. Eg: +- If memory usage or alignment is a concern, avoid using a bool as type for + struct member names. Instead use unsigned 1-bit bit field. e.g. struct foo { unsigned is_bar:1; }; + However, if neither memory usage or alignment will be significantly impacted + by the struct, opt for using bool for the sake of readability. - Always use size_t type when dealing with sizes of objects or memory ranges. - Your code should be 64-bit and 32-bit friendly. Bear in mind problems of printing, comparisons, and structure alignment. You have to include diff --git a/docs/modules/cc_ring_array.rst b/docs/modules/cc_ring_array.rst index 49791ec8c..1b8b36654 100644 --- a/docs/modules/cc_ring_array.rst +++ b/docs/modules/cc_ring_array.rst @@ -15,7 +15,7 @@ Synopsis ring_array_create(size_t elem_size, uint32_t cap); void - ring_array_destroy(struct ring_array *arr); + ring_array_destroy(struct ring_array **arr); rstatus_i ring_array_push(const void *elem, struct ring_array *arr); @@ -33,7 +33,7 @@ Creation/Destruction .. code-block:: C struct ring_array *ring_array_create(size_t elem_size, uint32_t cap); - void ring_array_destroy(struct ring_array *arr); + void ring_array_destroy(struct ring_array **arr); In order to create a ccommon ``ring_array`` data structure, call ``ring_array_create()`` with ``elem_size`` as the ``sizeof`` the elements the ``ring_array`` contains and with ``cap`` as the maximum number of elements the ``ring_array`` should be able to hold. This function returns a pointer to the ``ring_array`` that it creates. @@ -50,56 +50,91 @@ These functions are used to push/pop elements in the ``ring_array``. To push an To pop an element from the ``ring_array``, call ``ring_array_pop()`` with ``elem`` being a pointer to the memory location for where the element should be popped to, and ``arr`` being the ``ring_array`` being popped from. ``ring_array_pop()`` returns ``CC_OK`` if the element was successfully popped, and ``CC_ERROR`` if not successful (i.e. the ``ring_array`` is empty). +State +^^^^^ +..code-block:: C + bool ring_array_full(const struct ring_array *arr); + bool ring_array_empty(const struct ring_array *arr); + +These functions tell the caller about the state of the ``ring_array``, specifically whether it is full or empty. In a producer/consumer model, ``ring_array_full()`` is a producer facing API, and ``ring_array_empty()`` is a consumer facing API. This is so that the producer can check whether or not the ``ring_array`` is full before pushing more elements into the array; likewise, the consumer can check whether or not the ``ring_array`` is empty before attempting to pop. + +Flush +^^^^^ +..code-block:: C + void ring_array_flush(struct ring_array *arr); + +This function is a consumer facing API that discards everything in the ``ring_array``. + Examples -------- -Hello World! with ccommon ``ring_array``: +Multi threaded Hello World! with ccommon ``ring_array``: .. code-block:: c + #include #include #include #include #include #include + #include - int - main(int argc, char **argv) - { - struct ring_array *arr; - char c, *msg = "Hello world!\n"; - int i, msg_len = strlen(msg); - rstatus_i status; - - /* Create ring_array */ - arr = ring_array_create(sizeof(char), 100); + #define MESSAGE "Hello world!\n" - /* Push message into ring_array */ - for (i = 0; i < msg_len; ++i) { - status = ring_array_push(msg + i, arr); + struct msg_arg { + struct ring_array *arr; + struct bstring *msg; + }; - if (status != CC_OK) { - printf("Could not push message!\n"); - exit(1); + static void * + push_message(void *arg) + { + /* producer thread */ + struct ring_array *arr = ((struct msg_arg *)arg)->arr; + struct bstring *msg = ((struct msg_arg *)arg)->msg; + + for (i = 0; i < msg->len;) { + /* if there is space in the ring array, push next char in msg */ + if (!ring_array_full(arr)) { + ring_array_push(&(msg->data[i++]), arr); } } - /* Pop chars stored in arr and print them */ - for (i = 0; i < msg_len; ++i) { - status = ring_array_pop(&c, arr); + return NULL; + } - if (status != CC_OK) { - printf("Could not pop entire message!"); - exit(1) + int + main(int argc, char **argv) + { + struct ring_array *arr; + pthread_t producer = NULL; + struct bstring msg = { sizeof(MESSAGE), MESSAGE }; + struct msg_arg args; + + arr = ring_array_create(sizeof(char), 5); + + /* share array with producer thread */ + args.arr = arr; + args.msg = &msg; + + /* create producer thread */ + pthread_create(&producer, NULL, &push_message, &args); + + /* consume from arr */ + for (i = 0; i < msg.len;) { + if (!ring_array_empty(arr)) { + char c; + ring_array_pop(&c, arr); + printf("%c", c); + ++i } - - printf("%c", c); } /* Destroy ring_array */ - ring_array_destroy(arr); + ring_array_destroy(&arr); return 0; } diff --git a/include/buffer/cc_buf.h b/include/buffer/cc_buf.h index f0d7924db..4fe111dd3 100644 --- a/include/buffer/cc_buf.h +++ b/include/buffer/cc_buf.h @@ -169,6 +169,10 @@ buf_read(char *dst, struct buf *src, uint32_t count) static inline uint32_t buf_write(struct buf *dst, char *src, uint32_t count) { + if (count == 0) { + return 0; + } + ASSERT(dst != NULL && src != NULL); uint32_t len = MIN(buf_wsize(dst), count); diff --git a/include/cc_array.h b/include/cc_array.h index 4e9dcdbe7..b4aa1feb8 100644 --- a/include/cc_array.h +++ b/include/cc_array.h @@ -46,7 +46,7 @@ struct array { uint32_t nalloc; /* # allocated element */ size_t size; /* element size */ uint32_t nelem; /* # element */ - void *data; /* elements */ + uint8_t *data; /* elements */ }; @@ -93,7 +93,7 @@ array_data_assign(struct array *arr, uint32_t nalloc, size_t size, void *data) * element is out of bounds, return -1. */ static inline int -array_locate(struct array *arr, void *elem) { +array_locate(struct array *arr, uint8_t *elem) { int idx; idx = (elem - arr->data) / arr->size; diff --git a/include/cc_bstring.h b/include/cc_bstring.h index 65416dc6b..b82c8b3f5 100644 --- a/include/cc_bstring.h +++ b/include/cc_bstring.h @@ -56,6 +56,9 @@ rstatus_i bstring_duplicate(struct bstring *dst, const struct bstring *src); rstatus_i bstring_copy(struct bstring *dst, const char *src, uint32_t srclen); int bstring_compare(const struct bstring *s1, const struct bstring *s2); +struct bstring *bstring_alloc(uint32_t size); +void bstring_free(struct bstring **bstring); + /* TODO(yao): is this endian thing really useful? */ /* efficient implementation of string comparion of short strings */ #define str2cmp(m, c0, c1) \ diff --git a/include/cc_define.h b/include/cc_define.h index b542d0201..b521b133e 100644 --- a/include/cc_define.h +++ b/include/cc_define.h @@ -54,10 +54,9 @@ extern "C" { # define CC_BACKTRACE 1 #endif -/* TODO: add compile time option to turn chaining on/off */ -/*#ifdef HAVE_CHAINED*/ -# define CC_HAVE_CHAINED 1 -/*#endif*/ +#ifdef HAVE_DEBUG_MM +#define CC_DEBUG_MM 1 +#endif #define CC_OK 0 #define CC_ERROR -1 diff --git a/include/cc_lookup3.h b/include/cc_lookup3.h deleted file mode 100644 index 204d74f97..000000000 --- a/include/cc_lookup3.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * ccommon - a cache common library. - * Copyright (C) 2013 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#ifdef __cplusplus -extern "C" { -#endif - -/* -Excerpt and modified from lookup3.c (http://burtleburtle.net/bob/c/lookup3.c), -originally by Bob Jenkins, May 2006, Public Domain. -*/ - -#include - -#include /* defines uint32_t etc */ -#include - -uint32_t hashlittle( const void *key, size_t length, uint32_t initval); - -#ifdef __cplusplus -} -#endif diff --git a/include/cc_mm.h b/include/cc_mm.h index a1cc574b3..2d982196e 100644 --- a/include/cc_mm.h +++ b/include/cc_mm.h @@ -17,6 +17,12 @@ #pragma once +#ifdef __cplusplus +extern "C" { +#endif + +#include + #include /* @@ -41,8 +47,13 @@ #define cc_calloc(_n, _s) \ _cc_calloc((size_t)(_n), (size_t)(_s), __FILE__, __LINE__) +#if defined CC_DEBUG_MM && CC_DEBUG_MM == 1 +#define cc_realloc(_p, _s) \ + _cc_realloc_move(_p, (size_t)(_s), __FILE__, __LINE__) +#else #define cc_realloc(_p, _s) \ _cc_realloc(_p, (size_t)(_s), __FILE__, __LINE__) +#endif #define cc_free(_p) do { \ _cc_free(_p, __FILE__, __LINE__); \ @@ -59,6 +70,11 @@ void * _cc_alloc(size_t size, const char *name, int line); void * _cc_zalloc(size_t size, const char *name, int line); void * _cc_calloc(size_t nmemb, size_t size, const char *name, int line); void * _cc_realloc(void *ptr, size_t size, const char *name, int line); +void * _cc_realloc_move(void *ptr, size_t size, const char *name, int line); void _cc_free(void *ptr, const char *name, int line); void * _cc_mmap(size_t size, const char *name, int line); int _cc_munmap(void *p, size_t size, const char *name, int line); + +#ifdef __cplusplus +} +#endif diff --git a/include/cc_print.h b/include/cc_print.h index f26da912a..55d87edf7 100644 --- a/include/cc_print.h +++ b/include/cc_print.h @@ -44,7 +44,10 @@ extern "C" { /* behavior undefined if there isn't enough space in buf */ size_t cc_print_uint64_unsafe(char *buf, uint64_t n); +size_t cc_print_int64_unsafe(char *buf, int64_t n); + size_t cc_print_uint64(char *buf, size_t size, uint64_t n); +size_t cc_print_int64(char *buf, size_t size, int64_t n); size_t _scnprintf(char *buf, size_t size, const char *fmt, ...); size_t _vscnprintf(char *buf, size_t size, const char *fmt, va_list args); diff --git a/include/cc_rbuf.h b/include/cc_rbuf.h index 741ce8edb..7db08335b 100644 --- a/include/cc_rbuf.h +++ b/include/cc_rbuf.h @@ -21,6 +21,10 @@ #pragma once +#ifdef __cplusplus +extern "C" { +#endif + #include #include @@ -86,3 +90,7 @@ size_t rbuf_wcap(struct rbuf *buf); size_t rbuf_read(void *dst, struct rbuf *src, size_t n); /* write from a buffer in memory to the rbuf */ size_t rbuf_write(struct rbuf *dst, void *src, size_t n); + +#ifdef __cplusplus +} +#endif diff --git a/include/cc_ring_array.h b/include/cc_ring_array.h index e9efbb745..9caf1cd26 100644 --- a/include/cc_ring_array.h +++ b/include/cc_ring_array.h @@ -26,8 +26,13 @@ #pragma once +#ifdef __cplusplus +extern "C" { +#endif + #include +#include #include #include @@ -35,7 +40,7 @@ struct ring_array { size_t elem_size; /* element size */ - uint32_t cap; /* total capacity (# items stored + 1) */ + uint32_t cap; /* total capacity */ uint32_t rpos; /* read offset */ uint32_t wpos; /* write offset */ union { @@ -45,12 +50,37 @@ struct ring_array { }; }; +/*********************** + * Producer thread API * + ***********************/ + /* push an element into the array */ rstatus_i ring_array_push(const void *elem, struct ring_array *arr); +/* check if array is full */ +bool ring_array_full(const struct ring_array *arr); + + +/*********************** + * Consumer thread API * + ***********************/ + /* pop an element from the array */ rstatus_i ring_array_pop(void *elem, struct ring_array *arr); -/* creation/destruction */ +/* check if array is empty */ +bool ring_array_empty(const struct ring_array *arr); + +/* flush contents of ring array */ +void ring_array_flush(struct ring_array *arr); + + +/***************** + * Create/Delete * + *****************/ struct ring_array *ring_array_create(size_t elem_size, uint32_t cap); -void ring_array_destroy(struct ring_array *arr); +void ring_array_destroy(struct ring_array **arr); + +#ifdef __cplusplus +} +#endif diff --git a/include/cc_signal.h b/include/cc_signal.h index 61e151d9a..1fe67c8ff 100644 --- a/include/cc_signal.h +++ b/include/cc_signal.h @@ -48,7 +48,7 @@ struct signal { * - SIGSEGV(debug): print stacktrace before reraise segfault again * - SIGPIPE(channel): ignored, this prevents service from exiting when pipe closes */ -struct signal signals[SIGNAL_MAX]; /* there are only 31 signals from 1 to 31 */ +extern struct signal signals[SIGNAL_MAX]; /* there are only 31 signals from 1 to 31 */ int signal_override(int signo, char *info, int flags, uint32_t mask, sig_fn handler); diff --git a/include/cc_util.h b/include/cc_util.h index e0967a852..d9c13955b 100644 --- a/include/cc_util.h +++ b/include/cc_util.h @@ -57,6 +57,8 @@ extern "C" { * # define UINT16_MAX (65535) * # define UINT32_MAX (4294967295U) * # define UINT64_MAX (__UINT64_C(18446744073709551615)) + * + * # define INT64_MIN -9223372036854775808LL */ #define CC_UINT8_MAXLEN (3 + 1) #define CC_UINT16_MAXLEN (5 + 1) @@ -64,6 +66,8 @@ extern "C" { #define CC_UINT64_MAXLEN (20 + 1) #define CC_UINTMAX_MAXLEN CC_UINT64_MAXLEN +#define CC_INT64_MAXLEN (1 + 19 + 1) + /* alignment */ /* Make data 'd' or pointer 'p', n-byte aligned, where n is a power of 2 */ #define CC_ALIGNMENT sizeof(unsigned long) /* platform word */ diff --git a/include/channel/cc_tcp.h b/include/channel/cc_tcp.h index d1dad2bf3..c4babcae8 100644 --- a/include/channel/cc_tcp.h +++ b/include/channel/cc_tcp.h @@ -128,7 +128,8 @@ ssize_t tcp_recvv(struct tcp_conn *c, struct array *bufv, size_t nbyte); ssize_t tcp_sendv(struct tcp_conn *c, struct array *bufv, size_t nbyte); bool tcp_accept(struct tcp_conn *sc, struct tcp_conn *c); /* channel_accept_fn */ -void tcp_reject(struct tcp_conn *sc); /* channel_reject_fn */ +void tcp_reject(struct tcp_conn *sc); /* channel_reject_fn */ +void tcp_reject_all(struct tcp_conn *sc); /* channel_reject_fn */ /* functions getting/setting connection attribute */ int tcp_set_blocking(int sd); diff --git a/include/cc_hash.h b/include/hash/cc_lookup3.h similarity index 90% rename from include/cc_hash.h rename to include/hash/cc_lookup3.h index ec35edc1c..af6f8318c 100644 --- a/include/cc_hash.h +++ b/include/hash/cc_lookup3.h @@ -26,7 +26,7 @@ extern "C" { #include #include -uint32_t hash(const void *key, size_t length, const uint32_t initval); +uint32_t hash_lookup3(const void *key, size_t length, const uint32_t initval); #ifdef __cplusplus } diff --git a/include/hash/cc_murmur3.h b/include/hash/cc_murmur3.h new file mode 100644 index 000000000..8c736481d --- /dev/null +++ b/include/hash/cc_murmur3.h @@ -0,0 +1,47 @@ +/* + * ccommon - a cache common library. + * Copyright (C) 2013 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The cc_murmur3.[ch] are adapted from the canonical implementation of + * MurmurHash3 by Austin Appleby, released as part of SMHasher: + * https://github.com/aappleby/smhasher + * + * Changes include renaming functions, removing MSVC-related code, adding "static" + * keyword to local-scope functions according to C language spec (original code is + * in C++), to better fit them into the scope and style of ccommon + * + * The actual implementation is untouched. + */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include + + +void hash_murmur3_32(const void *key, int len, uint32_t seed, void *out); + +void hash_murmur3_128_x86(const void *key, int len, uint32_t seed, void *out); + +void hash_murmur3_128_x64(const void *key, int len, uint32_t seed, void *out); + +#ifdef __cplusplus +} +#endif diff --git a/include/stream/cc_sockio.h b/include/stream/cc_sockio.h index 89fe60471..9d2f9f570 100644 --- a/include/stream/cc_sockio.h +++ b/include/stream/cc_sockio.h @@ -48,6 +48,7 @@ extern "C" { #include #include +#include #include #include @@ -62,6 +63,21 @@ typedef struct { SOCKIO_OPTION(OPTION_DECLARE) } sockio_options_st; +/* name type description */ +#define SOCKIO_METRIC(ACTION) \ + ACTION( buf_sock_create, METRIC_COUNTER, "# buf sock created" )\ + ACTION( buf_sock_create_ex, METRIC_COUNTER, "# buf sock create exceptions" )\ + ACTION( buf_sock_destroy, METRIC_COUNTER, "# buf sock destroyed" )\ + ACTION( buf_sock_curr, METRIC_GAUGE, "# buf sock allocated" )\ + ACTION( buf_sock_borrow, METRIC_COUNTER, "# buf sock borrowed" )\ + ACTION( buf_sock_borrow_ex, METRIC_COUNTER, "# buf sock borrow exceptions" )\ + ACTION( buf_sock_return, METRIC_COUNTER, "# buf sock returned" )\ + ACTION( buf_sock_active, METRIC_GAUGE, "# buf sock being borrowed" ) + +typedef struct { + SOCKIO_METRIC(METRIC_DECLARE) +} sockio_metrics_st; + struct buf_sock { /* these fields are useful for resource managmenet */ STAILQ_ENTRY(buf_sock) next; @@ -79,7 +95,7 @@ struct buf_sock { STAILQ_HEAD(buf_sock_sqh, buf_sock); /* corresponding header type for the STAILQ */ -void sockio_setup(sockio_options_st *options); +void sockio_setup(sockio_options_st *options, sockio_metrics_st *metrics); void sockio_teardown(void); struct buf_sock *buf_sock_create(void); /* stream_get_fn */ diff --git a/include/time/cc_timer.h b/include/time/cc_timer.h index 13112c682..6a8c382ee 100644 --- a/include/time/cc_timer.h +++ b/include/time/cc_timer.h @@ -89,6 +89,8 @@ struct timeout { /* update duration */ void duration_reset(struct duration *d); +/* get a reading of duration and copy it without stopping the original timer */ +void duration_snapshot(struct duration *s, const struct duration *d); void duration_start(struct duration *d); void duration_stop(struct duration *d); /* read duration */ diff --git a/rust/CMakeLists.txt b/rust/CMakeLists.txt new file mode 100644 index 000000000..808fb4391 --- /dev/null +++ b/rust/CMakeLists.txt @@ -0,0 +1,5 @@ +file(WRITE CMAKE_BINARY_DIR "${CMAKE_BINARY_DIR}\n") + +if(HAVE_RUST) + add_subdirectory(ccommon_rs) +endif() diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 000000000..5ae870301 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,496 @@ +[[package]] +name = "aho-corasick" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "atty" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "backtrace" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace-sys 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-demangle 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bindgen" +version = "0.37.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cexpr 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "clang-sys 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "which 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bitflags" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cc" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cc_binding" +version = "0.1.0" +dependencies = [ + "bindgen 0.37.4 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ccommon_rs" +version = "0.1.0" +dependencies = [ + "cc_binding 0.1.0", + "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cexpr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cfg-if" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "clang-sys" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "clap" +version = "2.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "env_logger" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "failure" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "failure_derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", + "synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "glob" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "humantime" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lazy_static" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.42" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libloading" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memchr" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memchr" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nom" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quick-error" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "quote" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "redox_termios" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "strsim" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "syn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "synstructure" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termcolor" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termion" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "textwrap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thread_local" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ucd-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-width" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "utf8-ranges" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "which" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "wincolor" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum aho-corasick 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f0ba20154ea1f47ce2793322f049c5646cc6d0fa9759d5f333f286e507bf8080" +"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1" +"checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" +"checksum backtrace-sys 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)" = "bff67d0c06556c0b8e6b5f090f0eac52d950d9dfd1d35ba04e4ca3543eaf6a7e" +"checksum bindgen 0.37.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1b25ab82877ea8fe6ce1ce1f8ac54361f0218bad900af9eb11803994bf67c221" +"checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789" +"checksum cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "49ec142f5768efb5b7622aebc3fdbdbb8950a4b9ba996393cb76ef7466e8747d" +"checksum cexpr 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42aac45e9567d97474a834efdee3081b3c942b2205be932092f53354ce503d6c" +"checksum cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "efe5c877e17a9c717a0bf3613b2709f723202c4e4675cc8f12926ded29bcb17e" +"checksum clang-sys 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d7f7c04e52c35222fffcc3a115b5daf5f7e2bfb71c13c4e2321afe1fc71859c2" +"checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" +"checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a" +"checksum failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "934799b6c1de475a012a02dab0ace1ace43789ee4b99bcfbf1a2e3e8ced5de82" +"checksum failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c7cdda555bb90c9bb67a3b670a0f42de8e73f5981524123ad8578aafec8ddb8b" +"checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" +"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" +"checksum lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e6412c5e2ad9584b0b8e979393122026cdd6d2a80b933f890dcd694ddbe73739" +"checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1" +"checksum libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3ad660d7cb8c5822cd83d10897b0f1f1526792737a179e73896152f85b88c2" +"checksum log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "61bd98ae7f7b754bc53dca7d44b604f733c6bba044ea6f41bc8d89272d8161d2" +"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" +"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" +"checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" +"checksum peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +"checksum proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "77997c53ae6edd6d187fec07ec41b207063b5ee6f33680e9fa86d405cdd313d4" +"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" +"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" +"checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" +"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" +"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +"checksum regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13c93d55961981ba9226a213b385216f83ab43bd6ac53ab16b2eeb47e337cf4e" +"checksum regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05b06a75f5217880fc5e905952a42750bf44787e56a6c6d6852ed0992f5e1d54" +"checksum rustc-demangle 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "76d7ba1feafada44f2d38eed812bd2489a03c0f5abb975799251518b68848649" +"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" +"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" +"checksum synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a761d12e6d8dcb4dcf952a7a89b475e3a9d69e4a69307e01a470977642914bd" +"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" +"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" +"checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" +"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" +"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" +"checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" +"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" +"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +"checksum which 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e84a603e7e0b1ce1aa1ee2b109c7be00155ce52df5081590d1ffb93f4f515cb2" +"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 000000000..ae5821dad --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,21 @@ +[workspace] + +members = [ + "cc_binding", + "ccommon_rs", +] + +[profile.release] +opt-level = 3 +debug = true +rpath = false +lto = true +debug-assertions = false +codegen-units = 1 + +[profile.dev] +debug = true +opt-level = 0 + +[term] +verbose = true diff --git a/rust/cc_binding/CMakeLists.txt b/rust/cc_binding/CMakeLists.txt new file mode 100644 index 000000000..6c8be94ae --- /dev/null +++ b/rust/cc_binding/CMakeLists.txt @@ -0,0 +1,4 @@ +cargo_build(NAME cc_binding) + +add_dependencies(cc_binding_static ccommon-static) +add_dependencies(cc_binding_shared ccommon-shared) diff --git a/rust/cc_binding/Cargo.toml b/rust/cc_binding/Cargo.toml new file mode 100644 index 000000000..0ad06b1df --- /dev/null +++ b/rust/cc_binding/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "cc_binding" +version = "0.1.0" +authors = ["Jonathan Simms "] + +[build-dependencies] +bindgen = "0.37.0" +failure = "~0.1.1" + +[lib] +name = "cc_binding" +crate-type = ["rlib", "dylib", "staticlib", "lib"] diff --git a/rust/cc_binding/build.rs b/rust/cc_binding/build.rs new file mode 100644 index 000000000..4e6dc243d --- /dev/null +++ b/rust/cc_binding/build.rs @@ -0,0 +1,108 @@ +extern crate bindgen; +#[macro_use] +extern crate failure; + +use std::env; +use std::fs; +use std::io; +use std::io::BufReader; +use std::io::prelude::*; +use std::ffi::OsString; +use std::os::unix::fs as unix_fs; +use std::path::Path; +use std::path::PathBuf; +use std::result; + +type Result = result::Result; + +fn get_cmake_binary_dir() -> io::Result { + // this file is written by cmake on each run, updated with the location of + // the build directory. + let mut fp = fs::File::open("../CMAKE_BINARY_DIR")?; + let mut buf = String::new(); + let n = fp.read_to_string(&mut buf)?; + assert!(n > 0, "file was empty"); + Ok(String::from(buf.trim_right())) +} + +const CMAKE_CACHE: &str = "CMakeCache.txt"; +const CCOMMON_BINARY_DIR_KEY: &str = "ccommon_BINARY_DIR:STATIC"; + +fn get_cmake_cache_value(binary_dir: &Path, key: &str) -> Result> { + let cache_path = binary_dir.join(CMAKE_CACHE); + let fp = BufReader::new(fs::File::open(cache_path)?); + + for x in fp.lines() { + let line = x?; + let needle = format!("{}=", key); + if line.starts_with(&needle[..]) { + if let Some(v) = line.rsplit("=").take(1).last() { + return Ok(Some(v.to_owned())) + } else { + bail!("bad line: {:#?}", line); + } + } + } + + Ok(None) +} + +fn main() { + println!("cargo:rustc-link-lib=static=ccommon-1.2.0"); + + let include_path = fs::canonicalize("./../../include").unwrap(); + + let cmake_binary_dir = match get_cmake_binary_dir() { + Ok(p) => p, + Err(err) => panic!("Failed locating the CMAKE_BINARY_DIR file: {:#?}", err), + }; + + let cbd = PathBuf::from(cmake_binary_dir); + + let mut config_h_dir = cbd.clone(); + config_h_dir.push("ccommon"); + + let lib_dir: String = { + let cbd = get_cmake_cache_value(&cbd, CCOMMON_BINARY_DIR_KEY) + .map(|o| o.map(OsString::from)) + .unwrap() + .expect( + format!("could not find {} in {}", CCOMMON_BINARY_DIR_KEY, CMAKE_CACHE).as_ref() + ); + + let cbd = Path::new(&cbd); + cbd.join("lib").to_str().unwrap().to_string() + }; + + println!("cargo:rustc-link-search=native={}", lib_dir); + + let bindings = bindgen::Builder::default() + .clang_args(vec![ + "-I", include_path.to_str().unwrap(), + "-I", config_h_dir.to_str().unwrap(), + "-I", cbd.to_str().unwrap(), + "-L", &lib_dir, + ]) + .header("wrapper.h") + .blacklist_type("max_align_t") // https://github.com/rust-lang-nursery/rust-bindgen/issues/550 + .generate() + .expect("Unable to generate bindings"); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("Couldn't write bindings!"); + + // ./target/debug/build/cc_binding-27eac70f0fa2e180/out <<- starts here + + // cc_binding-27eac70f0fa2e180 + let symlink_content = + out_path.parent().unwrap().file_name().unwrap(); + + let build_dir = out_path.parent().and_then(|p| p.parent()).unwrap(); + + let link_location = build_dir.join("cc_binding"); + let _ = fs::remove_file(link_location.as_path()); + unix_fs::symlink(symlink_content, link_location).unwrap(); +} + diff --git a/rust/cc_binding/src/lib.rs b/rust/cc_binding/src/lib.rs new file mode 100644 index 000000000..2cf071191 --- /dev/null +++ b/rust/cc_binding/src/lib.rs @@ -0,0 +1,8 @@ +#![allow(unknown_lints)] +#![allow(clippy)] +#![allow(clippy_pedantic)] +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(dead_code)] +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/rust/cc_binding/wrapper.h b/rust/cc_binding/wrapper.h new file mode 100644 index 000000000..c5b21b148 --- /dev/null +++ b/rust/cc_binding/wrapper.h @@ -0,0 +1,2 @@ +#include +#include diff --git a/rust/ccommon_rs/CMakeLists.txt b/rust/ccommon_rs/CMakeLists.txt new file mode 100644 index 000000000..f373e0938 --- /dev/null +++ b/rust/ccommon_rs/CMakeLists.txt @@ -0,0 +1,4 @@ +file(WRITE CMAKE_BINARY_DIR "${CMAKE_BINARY_DIR}\n") +cargo_build(NAME ccommon_rs) +add_dependencies(ccommon_rs_static ccommon-static) +add_dependencies(ccommon_rs_shared ccommon-shared) diff --git a/rust/ccommon_rs/Cargo.toml b/rust/ccommon_rs/Cargo.toml new file mode 100644 index 000000000..382bf2286 --- /dev/null +++ b/rust/ccommon_rs/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ccommon_rs" +version = "0.1.0" +authors = ["Jonathan Simms "] + +[dependencies] + +cc_binding = { path = "../cc_binding" } + +log = "~0.4" +libc = "~0.2.42" +failure = "~0.1.1" diff --git a/rust/ccommon_rs/src/bstring.rs b/rust/ccommon_rs/src/bstring.rs new file mode 100644 index 000000000..f8588d608 --- /dev/null +++ b/rust/ccommon_rs/src/bstring.rs @@ -0,0 +1,486 @@ +//! BString is a wrapper around a foreign allocated and freed pointer to a cc_bstring. +//! It takes care of creating and freeing the foreign pointer within the normal +//! Rust lifetime rules. It has a companion reference object BStr, and the relation +//! of BString to BStr is similar to the relationship between String and &str. +//! +//! # Safety +//! +//! The point of this module is to ensure safe interaction between +//! Rust and C, and to facilitate passing BStrings between the two +//! as the sized buffer of choice. +//! +//! Much like with the standard library collections (Vec, Box), one +//! cannot simply pass their `from_raw` methods any old pointer. +//! You must only pass pointers obtained via the `into_raw` method. +//! +//! # Undefined Behavior +//! +//! Creating a BString from a Rust-allocated `bind::bstring` struct +//! will lead to undefined behavior if it is allowed to Drop. BString's +//! Drop implmentation passes the contained pointer to libc's free method +//! which can lead to memory corruption and [nasal demons]. +//! +//! [nasal demons]: http://www.catb.org/jargon/html/N/nasal-demons.html + +use cc_binding as bind; +use std::borrow::Borrow; +use std::boxed::Box; +use std::cell::UnsafeCell; +use std::fmt; +use std::fmt::Debug; +use std::fmt::Formatter; +use std::mem; +use std::ops::{Deref, DerefMut}; +use std::slice; +use std::str; +use std::borrow::BorrowMut; + + +pub type CCbstring = bind::bstring; + + +#[doc(hidden)] +#[inline] +unsafe fn raw_ptr_to_bytes<'a>(ptr: *const CCbstring) -> &'a [u8] { + slice::from_raw_parts( + (*ptr).data as *const _ as *const u8, + (*ptr).len as usize + ) +} + +#[doc(hidden)] +#[inline] +unsafe fn raw_ptr_to_bytes_mut<'a>(ptr: *mut CCbstring) -> &'a mut [u8] { + slice::from_raw_parts_mut( + (*ptr).data as *mut _ as *mut u8, + (*ptr).len as usize + ) +} + + +// this pattern lifted from https://docs.rs/foreign-types-shared/0.1.1/src/foreign_types_shared/lib.rs.html +struct Opaque(UnsafeCell<()>); + +/// A reference to a BString. String is to &str as BString is to &BStr. +/// This should be used when one does not want to take ownership of the +/// underlying pointer, but wants to access it in a rust-friendly way. +/// +/// This is useful in the case where the caller owns the buffer in the +/// data field and expects it to be filled (as opposed to in BString where +/// *we* own that memory). +/// +pub struct BStr(Opaque); + +impl BStr { + /// Wraps a raw pointer to a cc_bstring struct with a BStr. This is a + /// reference only conversion, and is zero cost. + #[inline] + pub unsafe fn from_ptr<'a>(ptr: *mut CCbstring) -> &'a Self { + &*(ptr as *mut _) + } + + /// Wraps a raw pointer to a cc_bstring struct with a BStr, and returns + /// a mutable reference. This is a reference only conversion, + /// and is zero cost. + #[inline] + pub unsafe fn from_ptr_mut<'a>(ptr: *mut CCbstring) -> &'a mut Self { + &mut *(ptr as *mut _) + } + + #[inline] + pub fn as_ptr(&self) -> *mut CCbstring { + self as *const _ as *mut _ + } +} + +impl Deref for BStr { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &[u8] { + unsafe { raw_ptr_to_bytes(self.as_ptr()) } + } +} + +impl DerefMut for BStr { + #[inline] + fn deref_mut(&mut self) -> &mut [u8] { + unsafe { raw_ptr_to_bytes_mut(self.as_ptr()) } + } +} + +impl AsRef for BStr { + fn as_ref(&self) -> &CCbstring { + unsafe { &*self.as_ptr() } + } +} + +impl AsMut for BStr { + fn as_mut(&mut self) -> &mut CCbstring { + unsafe { &mut *(self.as_ptr() as *mut _)} + } +} + +impl Borrow for BStr { + fn borrow(&self) -> &CCbstring { + unsafe { &*self.as_ptr() } + } +} + +impl BorrowMut for BStr { + fn borrow_mut(&mut self) -> &mut CCbstring { + unsafe { &mut *(self.as_ptr() as *mut _)} + } +} + +impl ToOwned for BStr { + type Owned = BString; + + #[inline] + fn to_owned(&self) -> BString { + unsafe { BString::from_raw(self.as_ptr()).clone() } + } +} + +unsafe impl Send for BStr {} +unsafe impl Sync for BStr {} + +/// An owned BString. By definition, a BString is allocated by +/// cc_bstring and freed by cc_bstring. This is because libc `malloc/free` +/// and Rust's `malloc/free` are two different implementations, and it's +/// important to keep track of who allocated what. To avoid this issue +/// all allocations are done from ccommon. +/// +/// This struct has Drop defined, and will call `bstring_free` when its +/// dropped. You can avoid this by using the `into_raw` method which +/// essentially leaks the memory. +/// +/// BString is safe for interoperation with ccommon libraries. You can pass +/// a pointer you got by using `BString::into_raw()` to C and everything will +/// just work. +/// +/// # Examples +/// +/// Creating and using a BString of a given size: +/// +/// ```rust +/// # use ccommon_rs::bstring::*; +/// +/// let mut bs = BString::new(3); +/// bs.copy_from_slice(&vec![0,1,2]); +/// +/// assert_eq!(&bs[..], &[0,1,2]) +/// ``` +/// +/// Creating an owned BString from Rust collections. If you can dereference it +/// to `&[u8]` you can copy it into a BString. +/// +/// ```rust +/// # use ccommon_rs::bstring::*; +/// +/// let s = BString::from("abc"); +/// assert_eq!(&s[..], "abc".as_bytes()); +/// +/// let v = BString::from(vec![0, 1, 2]); +/// assert_eq!(&v[..], &[0, 1, 2]); +/// ``` +/// +/// Mutating the content of a BString: +/// +/// ```rust +/// # use ccommon_rs::bstring::*; +/// +/// let mut s = BString::from("abc"); +/// s[0] = b'x'; +/// s[1] = b'y'; +/// s[2] = b'z'; +/// assert_eq!(&s[..], "xyz".as_bytes()); +/// ``` +/// +/// Use it as a buffer: +/// +/// ```rust +/// # use ccommon_rs::bstring::*; +/// use std::io::*; +/// use std::str; +/// +/// let mut x = BString::from("mutation is terrible "); +/// { +/// let mut c = Cursor::new(&mut x[..]); +/// let f = "fantastic".as_bytes(); +/// c.seek(SeekFrom::End(-9)); +/// +/// let sz = c.write(&f[..]).unwrap(); +/// assert_eq!(sz, f.len()); +/// } +/// +/// assert_eq!( +/// unsafe { str::from_utf8_unchecked(&x[..]) }, +/// "mutation is fantastic" +/// ); +/// ``` +/// +/// Note: if you're using BString as a buffer, it's important to +/// know that it *will not automatically resize*. If you write past the +/// end it will panic!. +pub struct BString(*mut CCbstring); + +impl BString { + pub fn new(size: u32) -> Self { + let bsp: *mut CCbstring = unsafe { bind::bstring_alloc(size) }; + + assert!(!bsp.is_null()); + BString(bsp) + } + + #[inline] + pub fn into_raw(bs: BString) -> *mut CCbstring { + let unique = bs.0; + mem::forget(bs); + unique + } + + #[inline] + pub unsafe fn from_raw(ptr: *mut CCbstring) -> BString { + assert!(!ptr.is_null()); + BString(ptr) + } + + /// Takes byte slice `&[u8]` and copies it into an owned BString. + #[inline] + pub fn from_bytes(s: &[u8]) -> Self { + let bsp: *mut CCbstring = unsafe { bind::bstring_alloc(s.len() as u32) }; + + assert!(!bsp.is_null()); + + let mut b = BString(bsp); + b.as_bytes_mut().clone_from_slice(&s[..]); + b + } + + /// Copies the contents of `src` into self. + /// + /// # Panics + /// + /// This method will panic if `src.len() != self.len()` + #[inline] + #[allow(dead_code)] + fn copy_from_slice(&mut self, src: &[u8]) { + assert_eq!(src.len(), self.len()); + (&mut (**self)).copy_from_slice(&src[..]); + } + + #[inline] + fn as_bytes(&self) -> &[u8] { + unsafe { raw_ptr_to_bytes(self.0) } + } + + #[inline] + fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { raw_ptr_to_bytes_mut(self.0) } + } + + #[inline] + fn len(&self) -> usize { + unsafe { (*self.0).len as usize } + } +} + +impl Debug for BString { + fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> { + f.debug_struct("BString") + .field("len", &self.len()) + .field("data", &self.as_bytes()) + .finish() + } +} + +impl PartialEq for BString { + #[inline] + fn eq(&self, other: &BString) -> bool { + self.as_bytes().eq(other.as_bytes()) + } +} + +impl Drop for BString { + #[inline] + fn drop(&mut self) { + unsafe { bind::bstring_free(&mut self.0) }; + } +} + +impl Clone for BString { + /// Create a copy of both the underlying struct _and_ the data it points to. + #[inline] + fn clone(&self) -> Self { + BString::from_bytes(self.as_bytes()) + } +} + +impl Deref for BString { + type Target = BStr; + + #[inline] + fn deref(&self) -> &BStr { + unsafe { BStr::from_ptr(self.0) } + } +} + +impl DerefMut for BString { + #[inline] + fn deref_mut(&mut self) -> &mut BStr { + unsafe { BStr::from_ptr_mut(self.0) } + } +} + +impl AsMut for BString { + fn as_mut(&mut self) -> &mut BStr { + &mut (*self) + } +} + +impl AsRef for BString { + #[inline] + fn as_ref(&self) -> &BStr { + &*self + } +} + +impl Borrow for BString { + #[inline] + fn borrow(&self) -> &BStr { + &*self + } +} + +impl From> for BString { + #[inline] + fn from(v: Vec) -> Self { + BString::from_bytes(&v[..]) + } +} + +impl From for Vec { + #[inline] + fn from(bs: BString) -> Self { + let mut v = Vec::with_capacity(bs.len()); + v.copy_from_slice(&**bs); // &**bs is &(BString -> BStr -> [u8]) + v + } +} + +impl From> for BString { + #[inline] + fn from(b: Box<[u8]>) -> Self { + BString::from_bytes(&b[..]) + } +} + +impl<'a> From<&'a str> for BString { + #[inline] + fn from(s: &'a str) -> Self { + BString::from_bytes(s.as_bytes()) + } +} + +unsafe impl Send for BString {} +unsafe impl Sync for BString {} + + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_raw_ptr_to_bytes() { + let bs = CCbstring { + len: 5, + data: String::from("abcde").as_ptr() as *mut i8 + }; + + let ptr: *const CCbstring = &bs as *const CCbstring; + + let slice = unsafe { raw_ptr_to_bytes(ptr) }; + assert_eq!(slice.len(), 5); + assert_eq!(&slice[..], "abcde".as_bytes()); + } + + #[test] + fn test_raw_ptr_to_bytes_mut() { + let mut bs = BString::new(5); + BString::copy_from_slice(&mut bs, "abcde".as_bytes()); + + let ptr: *const CCbstring = BString::into_raw(bs) as *const CCbstring; + + { + let s = unsafe { raw_ptr_to_bytes_mut(ptr as *mut _) }; + s[0] = 0; + } + + let s = unsafe { raw_ptr_to_bytes(ptr) }; + assert_eq!(s[0], 0); + } + + #[test] + fn test_bstring_from_str() { + let bs = BString::from("wat"); + + assert_eq!(bs.as_bytes(), "wat".as_bytes()); + } + + #[test] + fn test_bstring_into_raw_pointer_remains_valid() { + let bsp: *mut CCbstring; + { + let mut bs = BString::new(5); + bs[0] = 12u8; + bsp = BString::into_raw(bs); + } + + // the bsp pointer should still be valid here even though bs has been dropped + let bytes = unsafe { raw_ptr_to_bytes(bsp) }; + assert_eq!(bytes[0], 12u8); + } + + #[test] + fn test_bstring_copy_from_slice() { + let mut bs = BString::new(5); + bs.copy_from_slice("abcde".as_bytes()); + assert_eq!(&bs[..], "abcde".as_bytes()); + } + + fn foreign_code(s: &str) -> *mut CCbstring { + BString::into_raw(BString::from(s)) + } + + #[test] + fn test_bstr_from_ptr() { + let s = "abc"; + let ptr: *mut CCbstring = foreign_code(s); + let bstr = unsafe { BStr::from_ptr(ptr) }; + assert_eq!(bstr.len(), 3); + assert_eq!(&bstr[..], &s.as_bytes()[..]); + + unsafe { BString::from_raw(ptr) }; + } + + #[test] + fn test_bstring_as_io_write() { + use std::io::*; + + let mut x = BString::from("mutation is terrible "); + { + let mut c = Cursor::new(&mut x[..]); + let f = "fantastic".as_bytes(); + c.seek(SeekFrom::End(-9)).unwrap(); + + let sz = c.write(&f[..]).unwrap(); + assert_eq!(sz, f.len()); + } + + assert_eq!( + unsafe { str::from_utf8_unchecked(&x[..]) }, + "mutation is fantastic" + ); + } +} diff --git a/rust/ccommon_rs/src/lib.rs b/rust/ccommon_rs/src/lib.rs new file mode 100644 index 000000000..6d035da4d --- /dev/null +++ b/rust/ccommon_rs/src/lib.rs @@ -0,0 +1,3 @@ +extern crate cc_binding; +extern crate failure; +pub mod bstring; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b96b0b532..43828ce17 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -35,3 +35,10 @@ set_target_properties (${PROJECT_NAME}-shared OUTPUT_NAME ${PROJECT_NAME} VERSION ${${PROJECT_NAME}_VERSION} SOVERSION 0) + +# install instructions +install(TARGETS ${PROJECT_NAME}-static DESTINATION lib) +install(TARGETS ${PROJECT_NAME}-shared DESTINATION lib) +install(DIRECTORY ../include/ + DESTINATION include/${PROJECT_NAME}-${${PROJECT_NAME}_RELEASE_VERSION} + FILES_MATCHING PATTERN "*") diff --git a/src/cc_bstring.c b/src/cc_bstring.c index bb6b1d52a..58f40ca24 100644 --- a/src/cc_bstring.c +++ b/src/cc_bstring.c @@ -142,3 +142,35 @@ bstring_atou64(uint64_t *u64, struct bstring *str) return CC_OK; } + +struct bstring * +bstring_alloc(uint32_t size) +{ + struct bstring *bs = cc_alloc(sizeof(*bs)); + if (bs == NULL) { + return NULL; + } + bstring_init(bs); + + bs->len = size; + bs->data = cc_alloc(size); + if (bs->data == NULL) { + cc_free(bs); + return NULL; + } + + return bs; +} + +void +bstring_free(struct bstring **ptr) +{ + if ((ptr == NULL) || (*ptr == NULL)) { + return; + } + + struct bstring *bs = *ptr; + cc_free(bs->data); + cc_free(bs); + *ptr = NULL; +} diff --git a/src/cc_debug.c b/src/cc_debug.c index 7aa97dfe2..7c766260b 100644 --- a/src/cc_debug.c +++ b/src/cc_debug.c @@ -105,6 +105,12 @@ _logrotate(int signo) void debug_log_flush(void *arg) { + /* + * arg is unused but necessary for debug_log_flush to be used in conjunction + * with cc_timer and cc_wheel facilities, since to be inserted into a timing + * wheel the function must have the type signature of timeout_cb_fn. + */ + (void)arg; log_flush(dlog->logger); } diff --git a/src/cc_mm.c b/src/cc_mm.c index 4668d8d8e..6a0554289 100644 --- a/src/cc_mm.c +++ b/src/cc_mm.c @@ -34,7 +34,10 @@ _cc_alloc(size_t size, const char *name, int line) { void *p; - ASSERT(size != 0); + if (size == 0) { + log_debug("malloc(0) @ %s:%d", name, line); + return NULL; + } p = malloc(size); if (p == NULL) { @@ -70,7 +73,11 @@ _cc_realloc(void *ptr, size_t size, const char *name, int line) { void *p; - ASSERT(size != 0); + if (size == 0) { + free(ptr); + log_debug("realloc(0) @ %s:%d", name, line); + return NULL; + } p = realloc(ptr, size); if (p == NULL) { @@ -82,10 +89,37 @@ _cc_realloc(void *ptr, size_t size, const char *name, int line) return p; } +void * +_cc_realloc_move(void *ptr, size_t size, const char *name, int line) +{ + void *p = NULL, *pr; + + if (size == 0) { + free(ptr); + log_debug("realloc(0) @ %s:%d", name, line); + return NULL; + } + + /* + * Calling realloc then malloc allows us to force this function call to + * change the address of the allocated memory block. realloc ensures we can + * copy size bytes, and calling malloc before the realloc'd data is free'd + * gives us a new address for the memory object. + */ + if (((pr = realloc(ptr, size)) == NULL || (p = malloc(size)) == NULL)) { + log_error("realloc(%zu) failed @ %s:%d", size, name, line); + } else { + log_vverb("realloc(%zu) at %p @ %s:%d", size, p, name, line); + memcpy(p, pr, size); + } + + free(pr); + return p; +} + void _cc_free(void *ptr, const char *name, int line) { - ASSERT(ptr != NULL); log_vverb("free(%p) @ %s:%d", ptr, name, line); free(ptr); } @@ -103,10 +137,10 @@ _cc_mmap(size_t size, const char *name, int line) * is set appropriately. */ p = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0); + -1, 0); if (p == ((void *) -1)) { log_error("mmap %zu bytes @ %s:%d failed: %s", size, name, line, - strerror(errno)); + strerror(errno)); return NULL; } @@ -128,7 +162,7 @@ _cc_munmap(void *p, size_t size, const char *name, int line) status = munmap(p, size); if (status < 0) { log_error("munmap %p @ %s:%d failed: %s", p, name, line, - strerror(errno)); + strerror(errno)); } return status; diff --git a/src/cc_print.c b/src/cc_print.c index 65243fa41..8a16430f5 100644 --- a/src/cc_print.c +++ b/src/cc_print.c @@ -22,6 +22,9 @@ * implementation as a reference (folly/Conv.h) */ +/* use our own macro instead of llabs() to make sure it works with INT64_MIN */ +#define abs_int64(_x) ((_x) >= 0 ? (_x) : -(_x)) + static inline void _print_uint64(char *buf, size_t d, uint64_t n) { @@ -46,6 +49,22 @@ cc_print_uint64_unsafe(char *buf, uint64_t n) return d; } +size_t +cc_print_int64_unsafe(char *buf, int64_t n) +{ + size_t d; + uint64_t ab = abs_int64(n); + + if (n < 0) { + *buf++ = '-'; + } + + d = digits(ab); + _print_uint64(buf, d, ab); + + return d + (n < 0); +} + size_t cc_print_uint64(char *buf, size_t size, uint64_t n) { @@ -61,6 +80,26 @@ cc_print_uint64(char *buf, size_t size, uint64_t n) return d; } +size_t +cc_print_int64(char *buf, size_t size, int64_t n) +{ + size_t d; + uint64_t ab = abs_int64(n); + + d = digits(ab); + if (size < d + (n < 0)) { + return 0; + } + + if (n < 0) { + *buf++ = '-'; + } + + _print_uint64(buf, d, n); + + return d + (n < 0); +} + size_t _vscnprintf(char *buf, size_t size, const char *fmt, va_list args) { diff --git a/src/cc_ring_array.c b/src/cc_ring_array.c index 2a0561c6a..0d423d931 100644 --- a/src/cc_ring_array.c +++ b/src/cc_ring_array.c @@ -70,7 +70,6 @@ * */ - static inline uint32_t ring_array_nelem(uint32_t rpos, uint32_t wpos, uint32_t cap) { @@ -81,31 +80,12 @@ ring_array_nelem(uint32_t rpos, uint32_t wpos, uint32_t cap) } } -static inline bool -ring_array_empty(uint32_t rpos, uint32_t wpos) -{ - return rpos == wpos; -} - -static inline bool -ring_array_full(uint32_t rpos, uint32_t wpos, uint32_t cap) -{ - return ring_array_nelem(rpos, wpos, cap) == cap; -} - rstatus_i ring_array_push(const void *elem, struct ring_array *arr) { - /** - * Take snapshot of rpos, since another thread might be popping. Note: other - * members of arr do not need to be saved because we assume the other thread - * only pops and does not push; in other words, only one thread updates - * either rpos or wpos. - */ uint32_t new_wpos; - uint32_t rpos = __atomic_load_n(&(arr->rpos), __ATOMIC_RELAXED); - if (ring_array_full(rpos, arr->wpos, arr->cap)) { + if (ring_array_full(arr)) { log_debug("Could not push to ring array %p; array is full", arr); return CC_ERROR; } @@ -119,14 +99,25 @@ ring_array_push(const void *elem, struct ring_array *arr) return CC_OK; } +bool +ring_array_full(const struct ring_array *arr) +{ + /* + * Take snapshot of rpos, since another thread might be popping. Note: other + * members of arr do not need to be saved because we assume the other thread + * only pops and does not push; in other words, only one thread updates + * either rpos or wpos. + */ + uint32_t rpos = __atomic_load_n(&(arr->rpos), __ATOMIC_RELAXED); + return ring_array_nelem(rpos, arr->wpos, arr->cap) == arr->cap; +} + rstatus_i ring_array_pop(void *elem, struct ring_array *arr) { - /* take snapshot of wpos, since another thread might be pushing */ uint32_t new_rpos; - uint32_t wpos = __atomic_load_n(&(arr->wpos), __ATOMIC_RELAXED); - if (ring_array_empty(arr->rpos, wpos)) { + if (ring_array_empty(arr)) { log_debug("Could not pop from ring array %p; array is empty", arr); return CC_ERROR; } @@ -142,11 +133,28 @@ ring_array_pop(void *elem, struct ring_array *arr) return CC_OK; } +bool +ring_array_empty(const struct ring_array *arr) +{ + /* take snapshot of wpos, since another thread might be pushing */ + uint32_t wpos = __atomic_load_n(&(arr->wpos), __ATOMIC_RELAXED); + return ring_array_nelem(arr->rpos, wpos, arr->cap) == 0; +} + +void +ring_array_flush(struct ring_array *arr) +{ + uint32_t wpos = __atomic_load_n(&(arr->wpos), __ATOMIC_RELAXED); + __atomic_store_n(&(arr->rpos), wpos, __ATOMIC_RELAXED); +} + struct ring_array * ring_array_create(size_t elem_size, uint32_t cap) { struct ring_array *arr; + /* underlying array has # items stored + 1, since full is when wpos is 1 + element behind wpos */ arr = cc_alloc(RING_ARRAY_HDR_SIZE + elem_size * (cap + 1)); if (arr == NULL) { @@ -162,8 +170,15 @@ ring_array_create(size_t elem_size, uint32_t cap) } void -ring_array_destroy(struct ring_array *arr) +ring_array_destroy(struct ring_array **arr) { - log_verb("destroying ring array %p and freeing memory"); - cc_free(arr); + log_verb("destroying ring array %p and freeing memory", *arr); + + if ((arr == NULL) || (*arr == NULL)) { + log_warn("destroying NULL ring_array pointer"); + return; + } + + cc_free(*arr); + *arr = NULL; } diff --git a/src/cc_signal.c b/src/cc_signal.c index 222ee7e52..726e2c4c2 100644 --- a/src/cc_signal.c +++ b/src/cc_signal.c @@ -8,6 +8,8 @@ #include #include +struct signal signals[SIGNAL_MAX]; + #ifndef CC_HAVE_SIGNAME const char* sys_signame[SIGNAL_MAX + 1] = { "UNDEFINED", diff --git a/src/channel/cc_tcp.c b/src/channel/cc_tcp.c index 244123c24..d482ac9c0 100644 --- a/src/channel/cc_tcp.c +++ b/src/channel/cc_tcp.c @@ -377,6 +377,11 @@ tcp_accept(struct tcp_conn *sc, struct tcp_conn *c) return true; } + +/* + * due to lack of a direct rejection API in POSIX, tcp_reject accepts the + * frontmost connection and immediately closes it + */ void tcp_reject(struct tcp_conn *sc) { @@ -397,6 +402,45 @@ tcp_reject(struct tcp_conn *sc) } } +/* + * due to lack of a direct rejection API in POSIX, tcp_reject_all accepts + * connections ready on the listening socket, and immediately closes them. + * It does so until there are no more pending connections. + */ +void +tcp_reject_all(struct tcp_conn *sc) +{ + int ret; + int sd; + + for (;;) { + sd = accept(sc->sd, NULL, NULL); + if (sd < 0) { + if (errno == EINTR) { + log_debug("sd %d not ready: eintr", sc->sd); + continue; + } + + if (errno == EAGAIN || errno == EWOULDBLOCK) { + log_debug("sd %d has no more outstanding connections", sc->sd); + return; + } + + log_error("accept on sd %d failed: %s", sc->sd, strerror(errno)); + INCR(tcp_metrics, tcp_reject_ex); + return; + } + + ret = close(sd); + if (ret < 0) { + INCR(tcp_metrics, tcp_reject_ex); + log_warn("close c %d failed, ignored: %s", sd, strerror(errno)); + } + + INCR(tcp_metrics, tcp_reject); + } +} + int tcp_set_blocking(int sd) { diff --git a/src/hash/CMakeLists.txt b/src/hash/CMakeLists.txt index 20fdf65ee..60f178c00 100644 --- a/src/hash/CMakeLists.txt +++ b/src/hash/CMakeLists.txt @@ -1,5 +1,5 @@ set(SOURCE ${SOURCE} - hash/cc_hash.c hash/cc_lookup3.c + hash/cc_murmur3.c PARENT_SCOPE) diff --git a/src/hash/cc_hash.c b/src/hash/cc_hash.c deleted file mode 100644 index 05a6db7f1..000000000 --- a/src/hash/cc_hash.c +++ /dev/null @@ -1,446 +0,0 @@ -/* - * ccommon - a cache common library. - * Copyright (C) 2013 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Hash table - * - * The hash function used here is by Bob Jenkins, 1996: - * - * "By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. - * You may use this code any way you wish, private, educational, - * or commercial. It's free." - * - */ - -/* - * Since the hash function does bit manipulation, it needs to know - * whether it's big or little-endian. HAVE_LITTLE_ENDIAN and HAVE_BIG_ENDIAN - * are set in the configure script. - */ -#include - -#if defined CC_BIG_ENDIAN && CC_BIG_ENDIAN == 1 -# define HASH_LITTLE_ENDIAN 0 -# define HASH_BIG_ENDIAN 1 -#elif defined CC_LITTLE_ENDIAN && CC_LITTLE_ENDIAN == 1 -# define HASH_LITTLE_ENDIAN 1 -# define HASH_BIG_ENDIAN 0 -#else -# define HASH_LITTLE_ENDIAN 0 -# define HASH_BIG_ENDIAN 0 -#endif - -#define rot(x,k) (((x)<<(k)) ^ ((x)>>(32-(k)))) - -/* -------------------------------------------------------------------------------- -mix -- mix 3 32-bit values reversibly. - -This is reversible, so any information in (a,b,c) before mix() is -still in (a,b,c) after mix(). - -If four pairs of (a,b,c) inputs are run through mix(), or through -mix() in reverse, there are at least 32 bits of the output that -are sometimes the same for one pair and different for another pair. -This was tested for: -* pairs that differed by one bit, by two bits, in any combination - of top bits of (a,b,c), or in any combination of bottom bits of - (a,b,c). -* "differ" is defined as +, -, ^, or ~^. For + and -, I transformed - the output delta to a Gray code (a^(a>>1)) so a string of 1's (as - is commonly produced by subtraction) look like a single 1-bit - difference. -* the base values were pseudorandom, all zero but one bit set, or - all zero plus a counter that starts at zero. - -Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that -satisfy this are - 4 6 8 16 19 4 - 9 15 3 18 27 15 - 14 9 3 7 17 3 -Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing -for "differ" defined as + with a one-bit base and a two-bit delta. I -used http://burtleburtle.net/bob/hash/avalanche.html to choose -the operations, constants, and arrangements of the variables. - -This does not achieve avalanche. There are input bits of (a,b,c) -that fail to affect some output bits of (a,b,c), especially of a. The -most thoroughly mixed value is c, but it doesn't really even achieve -avalanche in c. - -This allows some parallelism. Read-after-writes are good at doubling -the number of bits affected, so the goal of mixing pulls in the opposite -direction as the goal of parallelism. I did what I could. Rotates -seem to cost as much as shifts on every machine I could lay my hands -on, and rotates are much kinder to the top and bottom bits, so I used -rotates. -------------------------------------------------------------------------------- -*/ -#define mix(a,b,c) \ -{ \ - a -= c; a ^= rot(c, 4); c += b; \ - b -= a; b ^= rot(a, 6); a += c; \ - c -= b; c ^= rot(b, 8); b += a; \ - a -= c; a ^= rot(c,16); c += b; \ - b -= a; b ^= rot(a,19); a += c; \ - c -= b; c ^= rot(b, 4); b += a; \ -} - -/* -------------------------------------------------------------------------------- -final -- final mixing of 3 32-bit values (a,b,c) into c - -Pairs of (a,b,c) values differing in only a few bits will usually -produce values of c that look totally different. This was tested for -* pairs that differed by one bit, by two bits, in any combination - of top bits of (a,b,c), or in any combination of bottom bits of - (a,b,c). -* "differ" is defined as +, -, ^, or ~^. For + and -, I transformed - the output delta to a Gray code (a^(a>>1)) so a string of 1's (as - is commonly produced by subtraction) look like a single 1-bit - difference. -* the base values were pseudorandom, all zero but one bit set, or - all zero plus a counter that starts at zero. - -These constants passed: - 14 11 25 16 4 14 24 - 12 14 25 16 4 14 24 -and these came close: - 4 8 15 26 3 22 24 - 10 8 15 26 3 22 24 - 11 8 15 26 3 22 24 -------------------------------------------------------------------------------- -*/ -#define final(a,b,c) \ -{ \ - c ^= b; c -= rot(b,14); \ - a ^= c; a -= rot(c,11); \ - b ^= a; b -= rot(a,25); \ - c ^= b; c -= rot(b,16); \ - a ^= c; a -= rot(c,4); \ - b ^= a; b -= rot(a,14); \ - c ^= b; c -= rot(b,24); \ -} - -#if HASH_LITTLE_ENDIAN == 1 -uint32_t hash( - const void *key, /* the key to hash */ - size_t length, /* length of the key */ - const uint32_t initval) /* initval */ -{ - uint32_t a,b,c; /* internal state */ - union { const void *ptr; size_t i; } u; /* needed for Mac Powerbook G4 */ - - /* Set up the internal state */ - a = b = c = 0xdeadbeef + ((uint32_t)length) + initval; - - u.ptr = key; - if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) { - const uint32_t *k = (const uint32_t *)key; /* read 32-bit chunks */ -#ifdef VALGRIND - const uint8_t *k8; -#endif /* ifdef VALGRIND */ - - /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ - while (length > 12) - { - a += k[0]; - b += k[1]; - c += k[2]; - mix(a,b,c); - length -= 12; - k += 3; - } - - /*----------------------------- handle the last (probably partial) block */ - /* - * "k[2]&0xffffff" actually reads beyond the end of the string, but - * then masks off the part it's not allowed to read. Because the - * string is aligned, the masked-off tail is in the same word as the - * rest of the string. Every machine with memory protection I've seen - * does it on word boundaries, so is OK with this. But VALGRIND will - * still catch it and complain. The masking trick does make the hash - * noticably faster for short strings (like English words). - */ -#ifndef VALGRIND - - switch(length) - { - case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; - case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break; - case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break; - case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break; - case 8 : b+=k[1]; a+=k[0]; break; - case 7 : b+=k[1]&0xffffff; a+=k[0]; break; - case 6 : b+=k[1]&0xffff; a+=k[0]; break; - case 5 : b+=k[1]&0xff; a+=k[0]; break; - case 4 : a+=k[0]; break; - case 3 : a+=k[0]&0xffffff; break; - case 2 : a+=k[0]&0xffff; break; - case 1 : a+=k[0]&0xff; break; - case 0 : return c; /* zero length strings require no mixing */ - } - -#else /* make valgrind happy */ - - k8 = (const uint8_t *)k; - switch(length) - { - case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; - case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ - case 10: c+=((uint32_t)k8[9])<<8; /* fall through */ - case 9 : c+=k8[8]; /* fall through */ - case 8 : b+=k[1]; a+=k[0]; break; - case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ - case 6 : b+=((uint32_t)k8[5])<<8; /* fall through */ - case 5 : b+=k8[4]; /* fall through */ - case 4 : a+=k[0]; break; - case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ - case 2 : a+=((uint32_t)k8[1])<<8; /* fall through */ - case 1 : a+=k8[0]; break; - case 0 : return c; /* zero length strings require no mixing */ - } - -#endif /* !valgrind */ - - } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) { - const uint16_t *k = (const uint16_t *)key; /* read 16-bit chunks */ - const uint8_t *k8; - - /*--------------- all but last block: aligned reads and different mixing */ - while (length > 12) - { - a += k[0] + (((uint32_t)k[1])<<16); - b += k[2] + (((uint32_t)k[3])<<16); - c += k[4] + (((uint32_t)k[5])<<16); - mix(a,b,c); - length -= 12; - k += 6; - } - - /*----------------------------- handle the last (probably partial) block */ - k8 = (const uint8_t *)k; - switch(length) - { - case 12: c+=k[4]+(((uint32_t)k[5])<<16); - b+=k[2]+(((uint32_t)k[3])<<16); - a+=k[0]+(((uint32_t)k[1])<<16); - break; - case 11: c+=((uint32_t)k8[10])<<16; /* @fallthrough */ - case 10: c+=k[4]; /* @fallthrough@ */ - b+=k[2]+(((uint32_t)k[3])<<16); - a+=k[0]+(((uint32_t)k[1])<<16); - break; - case 9 : c+=k8[8]; /* @fallthrough */ - case 8 : b+=k[2]+(((uint32_t)k[3])<<16); - a+=k[0]+(((uint32_t)k[1])<<16); - break; - case 7 : b+=((uint32_t)k8[6])<<16; /* @fallthrough */ - case 6 : b+=k[2]; - a+=k[0]+(((uint32_t)k[1])<<16); - break; - case 5 : b+=k8[4]; /* @fallthrough */ - case 4 : a+=k[0]+(((uint32_t)k[1])<<16); - break; - case 3 : a+=((uint32_t)k8[2])<<16; /* @fallthrough */ - case 2 : a+=k[0]; - break; - case 1 : a+=k8[0]; - break; - case 0 : return c; /* zero length strings require no mixing */ - } - - } else { /* need to read the key one byte at a time */ - const uint8_t *k = (const uint8_t *)key; - - /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ - while (length > 12) - { - a += k[0]; - a += ((uint32_t)k[1])<<8; - a += ((uint32_t)k[2])<<16; - a += ((uint32_t)k[3])<<24; - b += k[4]; - b += ((uint32_t)k[5])<<8; - b += ((uint32_t)k[6])<<16; - b += ((uint32_t)k[7])<<24; - c += k[8]; - c += ((uint32_t)k[9])<<8; - c += ((uint32_t)k[10])<<16; - c += ((uint32_t)k[11])<<24; - mix(a,b,c); - length -= 12; - k += 12; - } - - /*-------------------------------- last block: affect all 32 bits of (c) */ - switch(length) /* all the case statements fall through */ - { - case 12: c+=((uint32_t)k[11])<<24; - case 11: c+=((uint32_t)k[10])<<16; - case 10: c+=((uint32_t)k[9])<<8; - case 9 : c+=k[8]; - case 8 : b+=((uint32_t)k[7])<<24; - case 7 : b+=((uint32_t)k[6])<<16; - case 6 : b+=((uint32_t)k[5])<<8; - case 5 : b+=k[4]; - case 4 : a+=((uint32_t)k[3])<<24; - case 3 : a+=((uint32_t)k[2])<<16; - case 2 : a+=((uint32_t)k[1])<<8; - case 1 : a+=k[0]; - break; - case 0 : return c; /* zero length strings require no mixing */ - } - } - - final(a,b,c); - return c; /* zero length strings require no mixing */ -} - -#elif HASH_BIG_ENDIAN == 1 -/* - * hashbig(): - * This is the same as hashword() on big-endian machines. It is different - * from hashlittle() on all machines. hashbig() takes advantage of - * big-endian byte ordering. - */ -uint32_t hash( const void *key, size_t length, const uint32_t initval) -{ - uint32_t a,b,c; - union { const void *ptr; size_t i; } u; /* to cast key to (size_t) happily */ - - /* Set up the internal state */ - a = b = c = 0xdeadbeef + ((uint32_t)length) + initval; - - u.ptr = key; - if (HASH_BIG_ENDIAN && ((u.i & 0x3) == 0)) { - const uint32_t *k = key; /* read 32-bit chunks */ -#ifdef VALGRIND - const uint8_t *k8; -#endif /* ifdef VALGRIND */ - - /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ - while (length > 12) - { - a += k[0]; - b += k[1]; - c += k[2]; - mix(a,b,c); - length -= 12; - k += 3; - } - - /*----------------------------- handle the last (probably partial) block */ - /* - * "k[2]<<8" actually reads beyond the end of the string, but - * then shifts out the part it's not allowed to read. Because the - * string is aligned, the illegal read is in the same word as the - * rest of the string. Every machine with memory protection I've seen - * does it on word boundaries, so is OK with this. But VALGRIND will - * still catch it and complain. The masking trick does make the hash - * noticably faster for short strings (like English words). - */ -#ifndef VALGRIND - - switch(length) - { - case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; - case 11: c+=k[2]&0xffffff00; b+=k[1]; a+=k[0]; break; - case 10: c+=k[2]&0xffff0000; b+=k[1]; a+=k[0]; break; - case 9 : c+=k[2]&0xff000000; b+=k[1]; a+=k[0]; break; - case 8 : b+=k[1]; a+=k[0]; break; - case 7 : b+=k[1]&0xffffff00; a+=k[0]; break; - case 6 : b+=k[1]&0xffff0000; a+=k[0]; break; - case 5 : b+=k[1]&0xff000000; a+=k[0]; break; - case 4 : a+=k[0]; break; - case 3 : a+=k[0]&0xffffff00; break; - case 2 : a+=k[0]&0xffff0000; break; - case 1 : a+=k[0]&0xff000000; break; - case 0 : return c; /* zero length strings require no mixing */ - } - -#else /* make valgrind happy */ - - k8 = (const uint8_t *)k; - switch(length) /* all the case statements fall through */ - { - case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; - case 11: c+=((uint32_t)k8[10])<<8; /* fall through */ - case 10: c+=((uint32_t)k8[9])<<16; /* fall through */ - case 9 : c+=((uint32_t)k8[8])<<24; /* fall through */ - case 8 : b+=k[1]; a+=k[0]; break; - case 7 : b+=((uint32_t)k8[6])<<8; /* fall through */ - case 6 : b+=((uint32_t)k8[5])<<16; /* fall through */ - case 5 : b+=((uint32_t)k8[4])<<24; /* fall through */ - case 4 : a+=k[0]; break; - case 3 : a+=((uint32_t)k8[2])<<8; /* fall through */ - case 2 : a+=((uint32_t)k8[1])<<16; /* fall through */ - case 1 : a+=((uint32_t)k8[0])<<24; break; - case 0 : return c; - } - -#endif /* !VALGRIND */ - - } else { /* need to read the key one byte at a time */ - const uint8_t *k = key; - - /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ - while (length > 12) - { - a += ((uint32_t)k[0])<<24; - a += ((uint32_t)k[1])<<16; - a += ((uint32_t)k[2])<<8; - a += ((uint32_t)k[3]); - b += ((uint32_t)k[4])<<24; - b += ((uint32_t)k[5])<<16; - b += ((uint32_t)k[6])<<8; - b += ((uint32_t)k[7]); - c += ((uint32_t)k[8])<<24; - c += ((uint32_t)k[9])<<16; - c += ((uint32_t)k[10])<<8; - c += ((uint32_t)k[11]); - mix(a,b,c); - length -= 12; - k += 12; - } - - /*-------------------------------- last block: affect all 32 bits of (c) */ - switch(length) /* all the case statements fall through */ - { - case 12: c+=k[11]; - case 11: c+=((uint32_t)k[10])<<8; - case 10: c+=((uint32_t)k[9])<<16; - case 9 : c+=((uint32_t)k[8])<<24; - case 8 : b+=k[7]; - case 7 : b+=((uint32_t)k[6])<<8; - case 6 : b+=((uint32_t)k[5])<<16; - case 5 : b+=((uint32_t)k[4])<<24; - case 4 : a+=k[3]; - case 3 : a+=((uint32_t)k[2])<<8; - case 2 : a+=((uint32_t)k[1])<<16; - case 1 : a+=((uint32_t)k[0])<<24; - break; - case 0 : return c; - } - } - - final(a,b,c); - return c; -} -#else /* HASH_XXX_ENDIAN == 1 */ -#error Must define HASH_BIG_ENDIAN or HASH_LITTLE_ENDIAN -#endif /* HASH_XXX_ENDIAN == 1 */ diff --git a/src/hash/cc_lookup3.c b/src/hash/cc_lookup3.c index 581bc22d1..cef9b1186 100644 --- a/src/hash/cc_lookup3.c +++ b/src/hash/cc_lookup3.c @@ -1,36 +1,50 @@ /* -Excerpt and modified from lookup3.c (http://burtleburtle.net/bob/c/lookup3.c), -originally by Bob Jenkins, May 2006, Public Domain. -*/ - -#include - -#include + * ccommon - a cache common library. + * Copyright (C) 2013 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -#include /* defines uint32_t etc */ -#include /* attempt to define endianness */ +/* + * Hash table + * + * The hash function used here is by Bob Jenkins, 1996: + * + * "By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. + * You may use this code any way you wish, private, educational, + * or commercial. It's free." + * + */ /* - * My best guess at if you are big-endian or little-endian. This may - * need adjustment. + * Since the hash function does bit manipulation, it needs to know + * whether it's big or little-endian. HAVE_LITTLE_ENDIAN and HAVE_BIG_ENDIAN + * are set in the configure script. */ -#if (defined(CC_LITTLE_ENDIAN)) || \ - (defined(i386) || defined(__i386__) || defined(__i486__) || \ - defined(__i586__) || defined(__i686__) || defined(vax) || defined(MIPSEL)) -# define HASH_LITTLE_ENDIAN 1 -# define HASH_BIG_ENDIAN 0 -#elif (defined(CC_BIG_ENDIAN)) || \ - (defined(sparc) || defined(POWERPC) || defined(mc68000) || defined(sel)) +#include + +#if defined CC_BIG_ENDIAN && CC_BIG_ENDIAN == 1 # define HASH_LITTLE_ENDIAN 0 -# define HASH_BIG_ENDIAN 1 +# define HASH_BIG_ENDIAN 1 +#elif defined CC_LITTLE_ENDIAN && CC_LITTLE_ENDIAN == 1 +# define HASH_LITTLE_ENDIAN 1 +# define HASH_BIG_ENDIAN 0 #else # define HASH_LITTLE_ENDIAN 0 -# define HASH_BIG_ENDIAN 0 +# define HASH_BIG_ENDIAN 0 #endif -#define hashsize(n) ((uint32_t)1<<(n)) -#define hashmask(n) (hashsize(n)-1) -#define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k)))) +#define rot(x,k) (((x)<<(k)) ^ ((x)>>(32-(k)))) /* ------------------------------------------------------------------------------- @@ -122,35 +136,11 @@ and these came close: c ^= b; c -= rot(b,24); \ } -/* -------------------------------------------------------------------------------- -hashlittle() -- hash a variable-length key into a 32-bit value - k : the key (the unaligned variable-length array of bytes) - length : the length of the key, counting by bytes - initval : can be any 4-byte value -Returns a 32-bit value. Every bit of the key affects every bit of -the return value. Two keys differing by one or two bits will have -totally different hash values. - -The best hash table sizes are powers of 2. There is no need to do -mod a prime (mod is sooo slow!). If you need less than 32 bits, -use a bitmask. For example, if you need only 10 bits, do - h = (h & hashmask(10)); -In which case, the hash table should have hashsize(10) elements. - -If you are hashing n strings (uint8_t **)k, do it like this: - for (i=0, h=0; i 12) @@ -173,6 +166,7 @@ hashlittle( const void *key, size_t length, uint32_t initval) k += 3; } + /*----------------------------- handle the last (probably partial) block */ /* * "k[2]&0xffffff" actually reads beyond the end of the string, but * then masks off the part it's not allowed to read. Because the @@ -182,6 +176,8 @@ hashlittle( const void *key, size_t length, uint32_t initval) * still catch it and complain. The masking trick does make the hash * noticably faster for short strings (like English words). */ +#ifndef VALGRIND + switch(length) { case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; @@ -196,9 +192,31 @@ hashlittle( const void *key, size_t length, uint32_t initval) case 3 : a+=k[0]&0xffffff; break; case 2 : a+=k[0]&0xffff; break; case 1 : a+=k[0]&0xff; break; - case 0 : return c; /* zero length strings require no mixing */ + case 0 : return c; /* zero length strings require no mixing */ + } + +#else /* make valgrind happy */ + + k8 = (const uint8_t *)k; + switch(length) + { + case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; + case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ + case 10: c+=((uint32_t)k8[9])<<8; /* fall through */ + case 9 : c+=k8[8]; /* fall through */ + case 8 : b+=k[1]; a+=k[0]; break; + case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ + case 6 : b+=((uint32_t)k8[5])<<8; /* fall through */ + case 5 : b+=k8[4]; /* fall through */ + case 4 : a+=k[0]; break; + case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ + case 2 : a+=((uint32_t)k8[1])<<8; /* fall through */ + case 1 : a+=k8[0]; break; + case 0 : return c; /* zero length strings require no mixing */ } +#endif /* !valgrind */ + } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) { const uint16_t *k = (const uint16_t *)key; /* read 16-bit chunks */ const uint8_t *k8; @@ -222,28 +240,28 @@ hashlittle( const void *key, size_t length, uint32_t initval) b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; - case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ - case 10: c+=k[4]; + case 11: c+=((uint32_t)k8[10])<<16; /* @fallthrough */ + case 10: c+=k[4]; /* @fallthrough@ */ b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; - case 9 : c+=k8[8]; /* fall through */ + case 9 : c+=k8[8]; /* @fallthrough */ case 8 : b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; - case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ + case 7 : b+=((uint32_t)k8[6])<<16; /* @fallthrough */ case 6 : b+=k[2]; a+=k[0]+(((uint32_t)k[1])<<16); break; - case 5 : b+=k8[4]; /* fall through */ + case 5 : b+=k8[4]; /* @fallthrough */ case 4 : a+=k[0]+(((uint32_t)k[1])<<16); break; - case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ + case 3 : a+=((uint32_t)k8[2])<<16; /* @fallthrough */ case 2 : a+=k[0]; break; case 1 : a+=k8[0]; break; - case 0 : return c; /* zero length requires no mixing */ + case 0 : return c; /* zero length strings require no mixing */ } } else { /* need to read the key one byte at a time */ @@ -285,6 +303,137 @@ hashlittle( const void *key, size_t length, uint32_t initval) case 2 : a+=((uint32_t)k[1])<<8; case 1 : a+=k[0]; break; + case 0 : return c; /* zero length strings require no mixing */ + } + } + + final(a,b,c); + return c; /* zero length strings require no mixing */ +} + +#elif HASH_BIG_ENDIAN == 1 +/* + * hashbig(): + * This is the same as hashword() on big-endian machines. It is different + * from hashlittle() on all machines. hashbig() takes advantage of + * big-endian byte ordering. + */ +uint32_t hash_lookup3( const void *key, size_t length, const uint32_t initval) +{ + uint32_t a,b,c; + union { const void *ptr; size_t i; } u; /* to cast key to (size_t) happily */ + + /* Set up the internal state */ + a = b = c = 0xdeadbeef + ((uint32_t)length) + initval; + + u.ptr = key; + if (HASH_BIG_ENDIAN && ((u.i & 0x3) == 0)) { + const uint32_t *k = key; /* read 32-bit chunks */ +#ifdef VALGRIND + const uint8_t *k8; +#endif /* ifdef VALGRIND */ + + /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ + while (length > 12) + { + a += k[0]; + b += k[1]; + c += k[2]; + mix(a,b,c); + length -= 12; + k += 3; + } + + /*----------------------------- handle the last (probably partial) block */ + /* + * "k[2]<<8" actually reads beyond the end of the string, but + * then shifts out the part it's not allowed to read. Because the + * string is aligned, the illegal read is in the same word as the + * rest of the string. Every machine with memory protection I've seen + * does it on word boundaries, so is OK with this. But VALGRIND will + * still catch it and complain. The masking trick does make the hash + * noticably faster for short strings (like English words). + */ +#ifndef VALGRIND + + switch(length) + { + case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; + case 11: c+=k[2]&0xffffff00; b+=k[1]; a+=k[0]; break; + case 10: c+=k[2]&0xffff0000; b+=k[1]; a+=k[0]; break; + case 9 : c+=k[2]&0xff000000; b+=k[1]; a+=k[0]; break; + case 8 : b+=k[1]; a+=k[0]; break; + case 7 : b+=k[1]&0xffffff00; a+=k[0]; break; + case 6 : b+=k[1]&0xffff0000; a+=k[0]; break; + case 5 : b+=k[1]&0xff000000; a+=k[0]; break; + case 4 : a+=k[0]; break; + case 3 : a+=k[0]&0xffffff00; break; + case 2 : a+=k[0]&0xffff0000; break; + case 1 : a+=k[0]&0xff000000; break; + case 0 : return c; /* zero length strings require no mixing */ + } + +#else /* make valgrind happy */ + + k8 = (const uint8_t *)k; + switch(length) /* all the case statements fall through */ + { + case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; + case 11: c+=((uint32_t)k8[10])<<8; /* fall through */ + case 10: c+=((uint32_t)k8[9])<<16; /* fall through */ + case 9 : c+=((uint32_t)k8[8])<<24; /* fall through */ + case 8 : b+=k[1]; a+=k[0]; break; + case 7 : b+=((uint32_t)k8[6])<<8; /* fall through */ + case 6 : b+=((uint32_t)k8[5])<<16; /* fall through */ + case 5 : b+=((uint32_t)k8[4])<<24; /* fall through */ + case 4 : a+=k[0]; break; + case 3 : a+=((uint32_t)k8[2])<<8; /* fall through */ + case 2 : a+=((uint32_t)k8[1])<<16; /* fall through */ + case 1 : a+=((uint32_t)k8[0])<<24; break; + case 0 : return c; + } + +#endif /* !VALGRIND */ + + } else { /* need to read the key one byte at a time */ + const uint8_t *k = key; + + /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ + while (length > 12) + { + a += ((uint32_t)k[0])<<24; + a += ((uint32_t)k[1])<<16; + a += ((uint32_t)k[2])<<8; + a += ((uint32_t)k[3]); + b += ((uint32_t)k[4])<<24; + b += ((uint32_t)k[5])<<16; + b += ((uint32_t)k[6])<<8; + b += ((uint32_t)k[7]); + c += ((uint32_t)k[8])<<24; + c += ((uint32_t)k[9])<<16; + c += ((uint32_t)k[10])<<8; + c += ((uint32_t)k[11]); + mix(a,b,c); + length -= 12; + k += 12; + } + + /*-------------------------------- last block: affect all 32 bits of (c) */ + switch(length) /* all the case statements fall through */ + { + case 12: c+=k[11]; + case 11: c+=((uint32_t)k[10])<<8; + case 10: c+=((uint32_t)k[9])<<16; + case 9 : c+=((uint32_t)k[8])<<24; + case 8 : b+=k[7]; + case 7 : b+=((uint32_t)k[6])<<8; + case 6 : b+=((uint32_t)k[5])<<16; + case 5 : b+=((uint32_t)k[4])<<24; + case 4 : a+=k[3]; + case 3 : a+=((uint32_t)k[2])<<8; + case 2 : a+=((uint32_t)k[1])<<16; + case 1 : a+=((uint32_t)k[0])<<24; + break; case 0 : return c; } } @@ -292,3 +441,6 @@ hashlittle( const void *key, size_t length, uint32_t initval) final(a,b,c); return c; } +#else /* HASH_XXX_ENDIAN == 1 */ +#error Must define HASH_BIG_ENDIAN or HASH_LITTLE_ENDIAN +#endif /* HASH_XXX_ENDIAN == 1 */ diff --git a/src/hash/cc_murmur3.c b/src/hash/cc_murmur3.c new file mode 100644 index 000000000..7152523d5 --- /dev/null +++ b/src/hash/cc_murmur3.c @@ -0,0 +1,328 @@ +/* + * ccommon - a cache common library. + * Copyright (C) 2013 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +// Note - The x86 and x64 versions do _not_ produce the same results, as the +// algorithms are optimized for their respective platforms. You can still +// compile and run any of them on any platform, but your performance with the +// non-native version will be less than optimal. + +#include "hash/cc_murmur3.h" + +#define FORCE_INLINE inline __attribute__((always_inline)) + +static inline uint32_t rotl32 ( uint32_t x, int8_t r ) +{ + return (x << r) | (x >> (32 - r)); +} + +static inline uint64_t rotl64 ( uint64_t x, int8_t r ) +{ + return (x << r) | (x >> (64 - r)); +} + +#define ROTL32(x,y) rotl32(x,y) +#define ROTL64(x,y) rotl64(x,y) + +#define BIG_CONSTANT(x) (x##LLU) + +//----------------------------------------------------------------------------- +// Block read - if your platform needs to do endian-swapping or can only +// handle aligned reads, do the conversion here + +static FORCE_INLINE uint32_t getblock32 ( const uint32_t * p, int i ) +{ + return p[i]; +} + +static FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, int i ) +{ + return p[i]; +} + +//----------------------------------------------------------------------------- +// Finalization mix - force all bits of a hash block to avalanche + +static FORCE_INLINE uint32_t fmix32 ( uint32_t h ) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + + return h; +} + +//---------- + +static FORCE_INLINE uint64_t fmix64 ( uint64_t k ) +{ + k ^= k >> 33; + k *= BIG_CONSTANT(0xff51afd7ed558ccd); + k ^= k >> 33; + k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); + k ^= k >> 33; + + return k; +} + +//----------------------------------------------------------------------------- + +void hash_murmur3_32 ( const void * key, int len, + uint32_t seed, void * out ) +{ + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 4; + + uint32_t h1 = seed; + + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + + //---------- + // body + + const uint32_t * blocks = (const uint32_t *)(data + nblocks*4); + + for(int i = -nblocks; i; i++) + { + uint32_t k1 = getblock32(blocks,i); + + k1 *= c1; + k1 = ROTL32(k1,15); + k1 *= c2; + + h1 ^= k1; + h1 = ROTL32(h1,13); + h1 = h1*5+0xe6546b64; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*)(data + nblocks*4); + + uint32_t k1 = 0; + + switch(len & 3) + { + case 3: k1 ^= tail[2] << 16; + case 2: k1 ^= tail[1] << 8; + case 1: k1 ^= tail[0]; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + + h1 = fmix32(h1); + + *(uint32_t*)out = h1; +} + +//----------------------------------------------------------------------------- + +void hash_murmur3_128_x86 ( const void * key, const int len, + uint32_t seed, void * out ) +{ + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 16; + + uint32_t h1 = seed; + uint32_t h2 = seed; + uint32_t h3 = seed; + uint32_t h4 = seed; + + const uint32_t c1 = 0x239b961b; + const uint32_t c2 = 0xab0e9789; + const uint32_t c3 = 0x38b34ae5; + const uint32_t c4 = 0xa1e38b93; + + //---------- + // body + + const uint32_t * blocks = (const uint32_t *)(data + nblocks*16); + + for(int i = -nblocks; i; i++) + { + uint32_t k1 = getblock32(blocks,i*4+0); + uint32_t k2 = getblock32(blocks,i*4+1); + uint32_t k3 = getblock32(blocks,i*4+2); + uint32_t k4 = getblock32(blocks,i*4+3); + + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + + h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b; + + k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; + + h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747; + + k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; + + h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35; + + k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; + + h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*)(data + nblocks*16); + + uint32_t k1 = 0; + uint32_t k2 = 0; + uint32_t k3 = 0; + uint32_t k4 = 0; + + switch(len & 15) + { + case 15: k4 ^= tail[14] << 16; + case 14: k4 ^= tail[13] << 8; + case 13: k4 ^= tail[12] << 0; + k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; + + case 12: k3 ^= tail[11] << 24; + case 11: k3 ^= tail[10] << 16; + case 10: k3 ^= tail[ 9] << 8; + case 9: k3 ^= tail[ 8] << 0; + k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; + + case 8: k2 ^= tail[ 7] << 24; + case 7: k2 ^= tail[ 6] << 16; + case 6: k2 ^= tail[ 5] << 8; + case 5: k2 ^= tail[ 4] << 0; + k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; + + case 4: k1 ^= tail[ 3] << 24; + case 3: k1 ^= tail[ 2] << 16; + case 2: k1 ^= tail[ 1] << 8; + case 1: k1 ^= tail[ 0] << 0; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len; + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + h1 = fmix32(h1); + h2 = fmix32(h2); + h3 = fmix32(h3); + h4 = fmix32(h4); + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + ((uint32_t*)out)[0] = h1; + ((uint32_t*)out)[1] = h2; + ((uint32_t*)out)[2] = h3; + ((uint32_t*)out)[3] = h4; +} + +//----------------------------------------------------------------------------- + +void hash_murmur3_128_x64 ( const void * key, const int len, + const uint32_t seed, void * out ) +{ + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 16; + + uint64_t h1 = seed; + uint64_t h2 = seed; + + const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5); + const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f); + + //---------- + // body + + const uint64_t * blocks = (const uint64_t *)(data); + + for(int i = 0; i < nblocks; i++) + { + uint64_t k1 = getblock64(blocks,i*2+0); + uint64_t k2 = getblock64(blocks,i*2+1); + + k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1; + + h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729; + + k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2; + + h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*)(data + nblocks*16); + + uint64_t k1 = 0; + uint64_t k2 = 0; + + switch(len & 15) + { + case 15: k2 ^= ((uint64_t)tail[14]) << 48; + case 14: k2 ^= ((uint64_t)tail[13]) << 40; + case 13: k2 ^= ((uint64_t)tail[12]) << 32; + case 12: k2 ^= ((uint64_t)tail[11]) << 24; + case 11: k2 ^= ((uint64_t)tail[10]) << 16; + case 10: k2 ^= ((uint64_t)tail[ 9]) << 8; + case 9: k2 ^= ((uint64_t)tail[ 8]) << 0; + k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2; + + case 8: k1 ^= ((uint64_t)tail[ 7]) << 56; + case 7: k1 ^= ((uint64_t)tail[ 6]) << 48; + case 6: k1 ^= ((uint64_t)tail[ 5]) << 40; + case 5: k1 ^= ((uint64_t)tail[ 4]) << 32; + case 4: k1 ^= ((uint64_t)tail[ 3]) << 24; + case 3: k1 ^= ((uint64_t)tail[ 2]) << 16; + case 2: k1 ^= ((uint64_t)tail[ 1]) << 8; + case 1: k1 ^= ((uint64_t)tail[ 0]) << 0; + k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; h2 ^= len; + + h1 += h2; + h2 += h1; + + h1 = fmix64(h1); + h2 = fmix64(h2); + + h1 += h2; + h2 += h1; + + ((uint64_t*)out)[0] = h1; + ((uint64_t*)out)[1] = h2; +} + +//----------------------------------------------------------------------------- + + diff --git a/src/stream/cc_sockio.c b/src/stream/cc_sockio.c index 906fc463e..a44f61001 100644 --- a/src/stream/cc_sockio.c +++ b/src/stream/cc_sockio.c @@ -42,7 +42,9 @@ FREEPOOL(buf_sock_pool, buf_sockq, buf_sock); struct buf_sock_pool bsp; +static bool sockio_init = false; static bool bsp_init = false; +static sockio_metrics_st *sockio_metrics = NULL; rstatus_i buf_tcp_read(struct buf_sock *s) @@ -55,8 +57,9 @@ buf_tcp_read(struct buf_sock *s) rstatus_i status = CC_OK; ssize_t cap, n; - ASSERT(c != NULL && h != NULL && buf != NULL); - ASSERT(h->recv != NULL); + ASSERT(c != NULL); + ASSERT(buf != NULL); + ASSERT(h != NULL && h->recv != NULL); cap = buf_wsize(buf); @@ -212,6 +215,7 @@ buf_sock_create(void) s = (struct buf_sock *)cc_alloc(sizeof(struct buf_sock)); if (s == NULL) { + INCR(sockio_metrics, buf_sock_create_ex); return NULL; } STAILQ_NEXT(s, next) = NULL; @@ -235,12 +239,16 @@ buf_sock_create(void) goto error; } + INCR(sockio_metrics, buf_sock_create); + INCR(sockio_metrics, buf_sock_curr); + log_verb("created buffered socket %p", s); return s; error: log_info("buffered socket creation failed"); + INCR(sockio_metrics, buf_sock_create_ex); buf_sock_destroy(&s); return NULL; @@ -261,6 +269,8 @@ buf_sock_destroy(struct buf_sock **s) cc_free(*s); *s = NULL; + INCR(sockio_metrics, buf_sock_destroy); + DECR(sockio_metrics, buf_sock_curr); } static void @@ -331,10 +341,13 @@ buf_sock_borrow(void) FREEPOOL_BORROW(s, &bsp, next, buf_sock_create); if (s == NULL) { log_debug("borrow buffered socket failed: OOM or over limit"); + INCR(sockio_metrics, buf_sock_borrow_ex); return NULL; } buf_sock_reset(s); + INCR(sockio_metrics, buf_sock_borrow); + INCR(sockio_metrics, buf_sock_active); log_verb("borrowed buffered socket %p", s); @@ -354,18 +367,29 @@ buf_sock_return(struct buf_sock **s) FREEPOOL_RETURN(*s, &bsp, next); *s = NULL; + INCR(sockio_metrics, buf_sock_return); + DECR(sockio_metrics, buf_sock_active); } void -sockio_setup(sockio_options_st *options) +sockio_setup(sockio_options_st *options, sockio_metrics_st *metrics) { uint32_t max = BUFSOCK_POOLSIZE; + log_info("set up the %s module", SOCKIO_MODULE_NAME); + + if (sockio_init) { + log_warn("%s has already been setup, overwrite", SOCKIO_MODULE_NAME); + } + + sockio_metrics = metrics; + if (options != NULL) { max = option_uint(&options->buf_sock_poolsize); } buf_sock_pool_create(max); + sockio_init = true; } void diff --git a/src/time/cc_timer_darwin.c b/src/time/cc_timer_darwin.c index a4984f9f1..3127643be 100644 --- a/src/time/cc_timer_darwin.c +++ b/src/time/cc_timer_darwin.c @@ -52,6 +52,17 @@ duration_reset(struct duration *d) d->stop = 0; } +void +duration_snapshot(struct duration *s, const struct duration *d) +{ + ASSERT(s != 0 && d != NULL); + + s->started = true; + s->start = d->start; + s->stopped = true; + s->stop = mach_absolute_time(); +} + void duration_start(struct duration *d) { diff --git a/src/time/cc_timer_linux.c b/src/time/cc_timer_linux.c index 1b8dddb26..778fedf98 100644 --- a/src/time/cc_timer_linux.c +++ b/src/time/cc_timer_linux.c @@ -80,6 +80,17 @@ duration_start(struct duration *d) d->started = true; } +void +duration_snapshot(struct duration *s, const struct duration *d) +{ + ASSERT(s != 0 && d != NULL); + + s->started = true; + s->start = d->start; + s->stopped = true; + _gettime(&s->stop); +} + void duration_stop(struct duration *d) { diff --git a/test/bstring/check_bstring.c b/test/bstring/check_bstring.c index 9286732df..352b04398 100644 --- a/test/bstring/check_bstring.c +++ b/test/bstring/check_bstring.c @@ -130,6 +130,26 @@ START_TEST(test_atou64) } END_TEST +START_TEST(test_bstring_alloc_and_free) +{ +#define BSTRING_SIZE 9000 + + struct bstring *bs; + + bs = bstring_alloc(BSTRING_SIZE); + ck_assert_uint_eq(bs->len, BSTRING_SIZE); + for (int i = 0; i < BSTRING_SIZE; i++) { + bs->data[i] = 'a'; + } + + /* great! we didn't segfault! */ + bstring_free(&bs); + ck_assert_ptr_null(bs); + +#undef BSTRING_SIZE +} +END_TEST + /* * test suite */ @@ -146,6 +166,7 @@ bstring_suite(void) tcase_add_test(tc_bstring, test_copy); tcase_add_test(tc_bstring, test_compare); tcase_add_test(tc_bstring, test_atou64); + tcase_add_test(tc_bstring, test_bstring_alloc_and_free); return s; } diff --git a/test/channel/tcp/check_tcp.c b/test/channel/tcp/check_tcp.c index e185d4733..1e39156ba 100644 --- a/test/channel/tcp/check_tcp.c +++ b/test/channel/tcp/check_tcp.c @@ -72,7 +72,7 @@ find_port_listen(struct tcp_conn **_conn_listen, struct addrinfo **_ai, uint16_t freeaddrinfo(ai); } /* for some reason this line is needed, I would appreciate some insight */ - ck_assert_int_eq(tcp_connect(ai, conn_client), true); + ck_assert(tcp_connect(ai, conn_client)); tcp_reject(conn_listen); if (_conn_listen) { @@ -157,7 +157,7 @@ START_TEST(test_client_send_server_recv) conn_server = tcp_conn_create(); ck_assert_ptr_ne(conn_server, NULL); - ck_assert_int_eq(tcp_accept(conn_listen, conn_server), true); + ck_assert(tcp_accept(conn_listen, conn_server)); ck_assert_int_eq(tcp_send(conn_client, send_data, LEN), LEN); while ((recv = tcp_recv(conn_server, recv_data, LEN + 1)) == CC_EAGAIN) {} ck_assert_int_eq(recv, LEN); @@ -199,7 +199,7 @@ START_TEST(test_server_send_client_recv) conn_server = tcp_conn_create(); ck_assert_ptr_ne(conn_server, NULL); - ck_assert_int_eq(tcp_accept(conn_listen, conn_server), true); + ck_assert(tcp_accept(conn_listen, conn_server)); ck_assert_int_eq(tcp_send(conn_server, send_data, LEN), LEN); while ((recv = tcp_recv(conn_client, recv_data, LEN + 1)) == CC_EAGAIN) {} ck_assert_int_eq(recv, LEN); @@ -252,7 +252,7 @@ START_TEST(test_client_sendv_server_recvv) conn_server = tcp_conn_create(); ck_assert_ptr_ne(conn_server, NULL); - ck_assert_int_eq(tcp_accept(conn_listen, conn_server), true); + ck_assert(tcp_accept(conn_listen, conn_server)); ck_assert_int_eq(tcp_sendv(conn_client, send_array, LEN), LEN); while ((recv = tcp_recvv(conn_server, recv_array, LEN + 1)) == CC_EAGAIN) {} ck_assert_int_eq(recv, LEN); @@ -315,7 +315,7 @@ START_TEST(test_nonblocking) conn_server = tcp_conn_create(); ck_assert_ptr_ne(conn_server, NULL); - ck_assert_int_eq(tcp_accept(conn_listen, conn_server), true); + ck_assert(tcp_accept(conn_listen, conn_server)); task.usleep = SLEEP_TIME; task.c = conn_server; diff --git a/test/ring_array/check_ring_array.c b/test/ring_array/check_ring_array.c index 2bf212c01..c4bcc0845 100644 --- a/test/ring_array/check_ring_array.c +++ b/test/ring_array/check_ring_array.c @@ -2,6 +2,7 @@ #include +#include #include #include @@ -25,13 +26,57 @@ START_TEST(test_create_push_pop_destroy) ck_assert_int_eq(*test_elem, ELEM_VALUE); - ring_array_destroy(arr); + ring_array_destroy(&arr); #undef ELEM_SIZE #undef CAP #undef ELEM_VALUE } END_TEST +START_TEST(test_empty) +{ +#define ELEM_SIZE sizeof(uint8_t) +#define CAP 10 + struct ring_array *arr; + uint8_t data = 0; + + arr = ring_array_create(ELEM_SIZE, CAP); + ck_assert(ring_array_empty(arr)); + + ring_array_push(&data, arr); + ck_assert(!ring_array_empty(arr)); + + ring_array_pop(NULL, arr); + ck_assert(ring_array_empty(arr)); + + ring_array_destroy(&arr); +#undef ELEM_SIZE +#undef CAP +} +END_TEST + +START_TEST(test_full) +{ +#define ELEM_SIZE sizeof(uint8_t) +#define CAP 1 + struct ring_array *arr; + uint8_t data = 0; + + arr = ring_array_create(ELEM_SIZE, CAP); + ck_assert(!ring_array_full(arr)); + + ring_array_push(&data, arr); + ck_assert(ring_array_full(arr)); + + ring_array_pop(NULL, arr); + ck_assert(!ring_array_full(arr)); + + ring_array_destroy(&arr); +#undef ELEM_SIZE +#undef CAP +} +END_TEST + START_TEST(test_pop_empty) { #define ELEM_SIZE sizeof(uint8_t) @@ -41,7 +86,7 @@ START_TEST(test_pop_empty) arr = ring_array_create(ELEM_SIZE, CAP); ck_assert_int_eq(ring_array_pop(NULL, arr), CC_ERROR); - ring_array_destroy(arr); + ring_array_destroy(&arr); #undef ELEM_SIZE #undef CAP } @@ -60,7 +105,7 @@ START_TEST(test_push_full) } ck_assert_int_eq(ring_array_push(&i, arr), CC_ERROR); - ring_array_destroy(arr); + ring_array_destroy(&arr); #undef ELEM_SIZE #undef CAP } @@ -83,9 +128,89 @@ START_TEST(test_push_pop_many) ck_assert_int_eq(ring_array_push(&i, arr), CC_OK); } - ring_array_destroy(arr); + ring_array_destroy(&arr); +#undef ELEM_SIZE +#undef CAP +} +END_TEST + +START_TEST(test_flush) +{ +#define ELEM_SIZE sizeof(uint8_t) +#define CAP 10 + struct ring_array *arr; + uint8_t i; + + arr = ring_array_create(ELEM_SIZE, CAP); + for (i = 0; i < CAP; i++) { + ring_array_push(&i, arr); + } + + ck_assert(ring_array_full(arr)); + ring_array_flush(arr); + ck_assert(ring_array_empty(arr)); + + ring_array_destroy(&arr); +#undef ELEM_SIZE +#undef CAP +} +END_TEST + +/* + * Threading test + */ +struct test_ring_array_arg { + uint32_t n; + struct ring_array *arr; +}; + +static void * +test_produce(void *arg) +{ + uint32_t i, n = ((struct test_ring_array_arg *)arg)->n; + struct ring_array *arr = ((struct test_ring_array_arg *)arg)->arr; + + for (i = 0; i < n;) { + if (!ring_array_full(arr)) { + ring_array_push(&i, arr); + ++i; + } + } + return NULL; +} + +START_TEST(test_thread) +{ +#define ELEM_SIZE sizeof(uint32_t) +#define CAP 1000 +#define NUM_REPS 5000 + struct ring_array *arr = NULL; + pthread_t producer = NULL; + struct test_ring_array_arg arg; + uint32_t i; + + arr = ring_array_create(ELEM_SIZE, CAP); + ck_assert_ptr_ne(arr, NULL); + + arg.n = NUM_REPS; + arg.arr = arr; + + /* create producer thread */ + ck_assert_int_eq(pthread_create(&producer, NULL, &test_produce, &arg), 0); + + /* parent is consumer thread */ + for (i = 0; i < NUM_REPS;) { + if (!ring_array_empty(arr)) { + uint32_t val; + ck_assert_int_eq(ring_array_pop(&val, arr), CC_OK); + ck_assert_int_eq(val, i++); + } + } + + ring_array_destroy(&arr); #undef ELEM_SIZE #undef CAP +#undef NUM_REPS } END_TEST @@ -101,9 +226,13 @@ ring_array_suite(void) suite_add_tcase(s, tc_ring_array); tcase_add_test(tc_ring_array, test_create_push_pop_destroy); + tcase_add_test(tc_ring_array, test_empty); + tcase_add_test(tc_ring_array, test_full); tcase_add_test(tc_ring_array, test_pop_empty); tcase_add_test(tc_ring_array, test_push_full); tcase_add_test(tc_ring_array, test_push_pop_many); + tcase_add_test(tc_ring_array, test_flush); + tcase_add_test(tc_ring_array, test_thread); return s; } diff --git a/test/time/timer/check_timer.c b/test/time/timer/check_timer.c index de186a378..9822b475f 100644 --- a/test/time/timer/check_timer.c +++ b/test/time/timer/check_timer.c @@ -31,18 +31,25 @@ START_TEST(test_duration) { #define DURATION_NS 100000 - struct duration d; - double d_ns, d_us, d_ms, d_sec; + struct duration d, s; + double d_ns, d_us, d_ms, d_sec, s_ns; struct timespec ts = (struct timespec){0, DURATION_NS}; duration_reset(&d); duration_start(&d); + nanosleep(&ts, NULL); + duration_snapshot(&s, &d); + + /* snapshot is as expected */ + s_ns = duration_ns(&s); + ck_assert_uint_ge((unsigned int)s_ns, DURATION_NS); + nanosleep(&ts, NULL); duration_stop(&d); - /* duration is as expected */ + /* final duration is as expected */ d_ns = duration_ns(&d); - ck_assert_uint_ge((unsigned int)d_ns, DURATION_NS); + ck_assert_uint_ge((unsigned int)d_ns, 2 * DURATION_NS); /* readings of different units are consistent */ d_us = duration_us(&d); diff --git a/test/time/wheel/check_wheel.c b/test/time/wheel/check_wheel.c index c2631d1eb..4bd4df0ee 100644 --- a/test/time/wheel/check_wheel.c +++ b/test/time/wheel/check_wheel.c @@ -116,7 +116,7 @@ END_TEST START_TEST(test_timing_wheel_recur) { -#define TICK_NS 10000000 +#define TICK_NS 50000000 #define NSLOT 3 #define NTICK 2 @@ -136,11 +136,13 @@ START_TEST(test_timing_wheel_recur) timing_wheel_insert(tw, &delay, true, _incr_cb, &i); - nanosleep(&ts, NULL); + /* tick unchanged */ timing_wheel_execute(tw); ck_assert_int_eq(tw->nprocess, 0); ck_assert_int_eq(tw->nevent, 1); + /* next 2 tick */ + nanosleep(&ts, NULL); nanosleep(&ts, NULL); timing_wheel_execute(tw); ck_assert_int_eq(tw->nevent, 1); @@ -152,6 +154,7 @@ START_TEST(test_timing_wheel_recur) ck_assert_int_eq(tw->nprocess, 2); ck_assert_int_eq(i, 2); + /* flush events */ timing_wheel_stop(tw); timing_wheel_flush(tw); ck_assert_int_eq(tw->nevent, 0);