diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5cf5ef6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,62 @@ +name: Benchmarking + +on: + push: + pull_request: + schedule: + - cron: '13 8 * * 0' + +jobs: + build: + strategy: + fail-fast: false + matrix: + runs-on: [ubuntu-latest] + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + fetch-depth: 0 + - uses: goto-bus-stop/setup-zig@v2 + - uses: lukka/get-cmake@latest + with: + cmakeVersion: latest + + - name: Tigerbeetle Installation + run: | + chmod +x $PWD/scripts/install.sh; + chmod +x $PWD/scripts/run_tigerbeetle.sh; + chmod +x $PWD/zig/run.sh; + $PWD/scripts/install.sh; + $PWD/scripts/run_tigerbeetle.sh + + - name: Run - C Client + run: | + cd c + $PWD/run.sh + cd .. + ## need refactoring + # - name: Run - Go Client + # run: | + # cd go + # $PWD/run.sh + # cd .. + # - name: Run - .Net Client + # run: | + # cd dotnet + # $PWD/run.sh + # cd .. + # - name: Run - Java Client + # run: | + # cd java + # $PWD/run.sh + # cd .. + - name: Run - Zig Client + run: | + cd zig + $PWD/run.sh + cd .. + + - name: Kill TB Process + run: pkill -f tigerbeetle diff --git a/.gitignore b/.gitignore index f3c26b8..628e75a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ c/zig-cache c/zig-out zig/zig-cache zig/zig-out +tigerbeetle/ +build/ \ No newline at end of file diff --git a/c/CMakeLists.txt b/c/CMakeLists.txt new file mode 100644 index 0000000..7932bc7 --- /dev/null +++ b/c/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 3.14) +project(bench LANGUAGES C) + +include(FetchContent) + + +find_package(tigerbeetle 0.3.0 QUIET) +if(NOT tigerbeetle_FOUND) + FetchContent_Declare(tigerbeetle GIT_REPOSITORY https://github.com/kassane/tigerbeetle-cpp.git + GIT_TAG main) + FetchContent_GetProperties(tigerbeetle) + set(APP_TARGETS ${PROJECT_NAME}) + FetchContent_MakeAvailable(tigerbeetle) +endif() +find_package(Threads REQUIRED) + +add_executable(${PROJECT_NAME} bench.c) +# Link the executable target with the tigerbeetle client library and Threads library +target_link_libraries(${PROJECT_NAME} + PRIVATE tb_client Threads::Threads +) +# Include the tigerbeetle headers +target_include_directories(${PROJECT_NAME} PUBLIC ${tigerbeetle_SOURCE_DIR}/include) +# Link against the tigerbeetle library directory +target_link_directories(${PROJECT_NAME} PUBLIC ${tigerbeetle_BINARY_DIR}) +# Add a custom target to run the executable +add_custom_target(run + COMMAND ${PROJECT_NAME} + DEPENDS ${PROJECT_NAME} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} +) +# Set the "bench" target as the default target when building the project +set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY VS_STARTUP_PROJECT ${PROJECT_NAME}) diff --git a/c/bench.c b/c/bench.c index 58c8469..f569e6f 100644 --- a/c/bench.c +++ b/c/bench.c @@ -1,139 +1,152 @@ -#include +#include #include +#include #include -#include #include -#include "tb_client.h" +#include // Synchronization context between the callback and the main thread. typedef struct completion_context { - void* reply; - int size; - pthread_mutex_t lock; - pthread_cond_t cv; + void *reply; + int size; + pthread_mutex_t lock; + pthread_cond_t cv; } completion_context_t; -// Completion function, called by tb_client no notify that a request as completed. -void on_completion(uintptr_t context, tb_client_t client, tb_packet_t* packet, const uint8_t* data, uint32_t size) { - - // The user_data gives context to a request: - completion_context_t* ctx = (completion_context_t*)packet->user_data; - - // Signaling the main thread we received the reply: - pthread_mutex_lock(&ctx->lock); - ctx->size = size; - ctx->reply = (void*)data; - pthread_cond_signal(&ctx->cv); - pthread_mutex_unlock(&ctx->lock); +// Completion function, called by tb_client no notify that a request as +// completed. +void on_completion(uintptr_t context, tb_client_t client, tb_packet_t *packet, + const uint8_t *data, uint32_t size) { + + // The user_data gives context to a request: + completion_context_t *ctx = (completion_context_t *)packet->user_data; + + // Signaling the main thread we received the reply: + pthread_mutex_lock(&ctx->lock); + ctx->size = size; + ctx->reply = (void *)data; + pthread_cond_signal(&ctx->cv); + pthread_mutex_unlock(&ctx->lock); } // For benchmarking purposes. long long get_time_ms(void) { - struct timeval tv; - gettimeofday(&tv,NULL); - return (((long long)tv.tv_sec)*1000)+(tv.tv_usec/1000); + struct timeval tv; + gettimeofday(&tv, NULL); + return (((long long)tv.tv_sec) * 1000) + (tv.tv_usec / 1000); } int main(int argc, char **argv) { - tb_client_t client; - tb_packet_list_t packets; - const char* address = "127.0.0.1:3000"; - TB_STATUS status = tb_client_init( - &client, // Output client. - &packets, // Output packet list. - 0, // Cluster ID. - address, // Cluster addresses. - strlen(address), // - 1, // MaxConcurrency == 1, this is a single-threaded program - NULL, // No need for a global context. - &on_completion // Completion callback. - ); - - if (status != TB_STATUS_SUCCESS) { - printf("Failed to initialize tb_client\n"); + tb_client_t client; + tb_packet_t *packet; + const char *address = "127.0.0.1:3000"; + TB_STATUS status = tb_client_init( + &client, // Output client. + 0, // Cluster ID. + address, // Cluster addresses. + strlen(address), // + 1, // MaxConcurrency == 1, this is a single-threaded program + NULL, // No need for a global context. + &on_completion // Completion callback. + ); + + if (status != TB_STATUS_SUCCESS) { + printf("Failed to initialize tb_client\n"); + exit(-1); + } + + // Initializing the mutex and condvar + completion_context_t ctx; + + if (pthread_mutex_init(&ctx.lock, NULL) != 0) { + printf("Failed to initialize mutex\n"); + exit(-1); + } + + if (pthread_cond_init(&ctx.cv, NULL)) { + printf("Failed to initialize condition\n"); + exit(-1); + } + + pthread_mutex_lock(&ctx.lock); + + const int SAMPLES = 1000000; + const int BATCH_SIZE = 8191; + + // Repeat the same test 10 times and pick the best execution + for (int tries = 0; tries < 10; tries++) { + + long max_latency_ms = 0; + long total_time_ms = 0; + + for (int i = 0; i < SAMPLES; i += BATCH_SIZE) { + + tb_transfer_t batch[BATCH_SIZE]; + memset(&batch, 0, BATCH_SIZE); + + for (int j = 0; (j < BATCH_SIZE) && (i + j < SAMPLES); j++) { + batch[j].id = 0; + batch[j].debit_account_id = 0; + batch[j].credit_account_id = 0; + batch[j].code = 1; + batch[j].ledger = 1; + batch[j].amount = 10; + } + + // Acquiring a packet for this request: + if (tb_client_acquire_packet(client, &packet) != TB_PACKET_ACQUIRE_OK) { + printf("Too many concurrent packets\n"); exit(-1); - } + } - // Initializing the mutex and condvar - completion_context_t ctx; - - if (pthread_mutex_init(&ctx.lock, NULL) != 0) { - printf("Failed to initialize mutex\n"); - exit(-1); - } + packet->operation = + TB_OPERATION_CREATE_TRANSFERS; // The operation to be performed. + packet->data = &batch; // The data to be sent. + packet->data_size = BATCH_SIZE * sizeof(tb_transfer_t); + packet->user_data = &ctx; // User-defined context. + packet->status = TB_PACKET_OK; // Will be set when the reply arrives. - if (pthread_cond_init(&ctx.cv, NULL)) { - printf("Failed to initialize condition\n"); - exit(-1); - } + long long now = get_time_ms(); + + tb_client_submit(client, packet); + pthread_cond_wait(&ctx.cv, &ctx.lock); - pthread_mutex_lock(&ctx.lock); - - const int SAMPLES = 1000000; - const int BATCH_SIZE = 8191; - - // Repeat the same test 10 times and pick the best execution - for (int tries = 0; tries < 10; tries++) { - - long max_latency_ms = 0; - long total_time_ms = 0; - - for (int i = 0; i < SAMPLES; i += BATCH_SIZE) { - - tb_transfer_t batch[BATCH_SIZE]; - memset(&batch, 0, BATCH_SIZE); - - for (int j = 0; (j < BATCH_SIZE) && (i + j < SAMPLES); j++) { - batch[j].id = 0; - batch[j].debit_account_id = 0; - batch[j].credit_account_id = 0; - batch[j].code = 1; - batch[j].ledger = 1; - batch[j].amount = 10; - } - - packets.head->operation = TB_OPERATION_CREATE_TRANSFERS; // The operation to be performed. - packets.head->data = &batch; // The data to be sent. - packets.head->data_size = BATCH_SIZE * sizeof(tb_transfer_t); - packets.head->user_data = &ctx; // User-defined context. - packets.head->status = TB_PACKET_OK; // Will be set when the reply arrives. - - long long now = get_time_ms(); - - tb_client_submit(client, &packets); - pthread_cond_wait(&ctx.cv, &ctx.lock); - - long elapsed_ms = get_time_ms() - now; - - if (elapsed_ms > max_latency_ms) max_latency_ms = elapsed_ms; - total_time_ms += elapsed_ms; - - if (packets.head->status != TB_PACKET_OK) { - // Checking if the request failed: - printf("Error calling create_transfers (ret=%d)\n", packets.head->status); - exit(-1); - } - - // Since we are using invalid IDs, - // it is expected to all transfers to be rejected. - tb_create_transfers_result_t* results = (tb_create_transfers_result_t*)ctx.reply; - int results_len = ctx.size / sizeof(tb_create_transfers_result_t); - if (results_len != BATCH_SIZE) { - printf("Unexpected result %d\n", i); - exit(-1); - } - } - - printf("Total time: %d ms\n", total_time_ms); - printf("Max time per batch: %d ms\n", max_latency_ms); - printf("Transfers per second: %d\n", SAMPLES * 1000 / total_time_ms); - printf("\n"); + long elapsed_ms = get_time_ms() - now; + + if (elapsed_ms > max_latency_ms) + max_latency_ms = elapsed_ms; + total_time_ms += elapsed_ms; + + if (packet->status != TB_PACKET_OK) { + // Checking if the request failed: + printf("Error calling create_transfers (ret=%d)\n", packet->status); + exit(-1); + } + + // Releasing the packet, so it can be used in a next request. + tb_client_release_packet(client, packet); + + // Since we are using invalid IDs, + // it is expected to all transfers to be rejected. + tb_create_transfers_result_t *results = + (tb_create_transfers_result_t *)ctx.reply; + int results_len = ctx.size / sizeof(tb_create_transfers_result_t); + if (results_len != BATCH_SIZE) { + printf("Unexpected result %d\n", i); + exit(-1); + } } - // Cleanup - pthread_mutex_unlock(&ctx.lock); - pthread_cond_destroy(&ctx.cv); - pthread_mutex_destroy(&ctx.lock); - tb_client_deinit(client); + printf("Total time: %d ms\n", total_time_ms); + printf("Max time per batch: %d ms\n", max_latency_ms); + printf("Transfers per second: %d\n", SAMPLES * 1000 / total_time_ms); + printf("\n"); + } + + // Cleanup + pthread_mutex_unlock(&ctx.lock); + pthread_cond_destroy(&ctx.cv); + pthread_mutex_destroy(&ctx.lock); + tb_client_deinit(client); } \ No newline at end of file diff --git a/c/run.sh b/c/run.sh index c6e6f3e..e83cc8c 100755 --- a/c/run.sh +++ b/c/run.sh @@ -1 +1,6 @@ -../tigerbeetle/zig/zig build run -Drelease-safe \ No newline at end of file +#!/usr/bin/env bash + +# zig build run -Doptimize=ReleaseSafe +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build --target run + diff --git a/go/go.mod b/go/go.mod index cb8830b..df14bc7 100644 --- a/go/go.mod +++ b/go/go.mod @@ -2,4 +2,4 @@ module tigerbeetle-bench go 1.17 -require github.com/tigerbeetledb/tigerbeetle-go v0.0.0-20230515174739-c787f1046cd0 +require github.com/tigerbeetledb/tigerbeetle-go v0.13.57 diff --git a/go/go.sum b/go/go.sum index bf6fd1c..5495592 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,2 +1,2 @@ -github.com/tigerbeetledb/tigerbeetle-go v0.0.0-20230515174739-c787f1046cd0 h1:5ygk+xhJ2p03J+viEZJ6GwKgw5BfQ2Dm3W/Q2h/063M= -github.com/tigerbeetledb/tigerbeetle-go v0.0.0-20230515174739-c787f1046cd0/go.mod h1:3Tjdd459YDuWJM1mHqafj/6r1lMXkqAJJGrhZ6Hsrv8= +github.com/tigerbeetledb/tigerbeetle-go v0.13.57 h1:Sn5SUXGXbE3mm5bzJZ+sQ7tq5Os8EoUcKx2nmzllXiQ= +github.com/tigerbeetledb/tigerbeetle-go v0.13.57/go.mod h1:3Tjdd459YDuWJM1mHqafj/6r1lMXkqAJJGrhZ6Hsrv8= diff --git a/scripts/install.sh b/scripts/install.sh index fcdc4c2..e9d5653 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2,8 +2,8 @@ set -eEuo pipefail echo "Installing TigerBeetle..." -git submodule init && git submodule update -(cd tigerbeetle && git submodule init && git submodule update && ./scripts/install_zig.sh) +git clone --recursive https://github.com/tigerbeetle/tigerbeetle.git +(cd tigerbeetle && ./scripts/install_zig.sh) echo "Building TigerBeetle Dotnet..." (cd tigerbeetle/src/clients/dotnet && dotnet build -c Release && dotnet pack -c Release) diff --git a/scripts/run_tigerbeetle.sh b/scripts/run_tigerbeetle.sh index f911b1b..a02d2ea 100755 --- a/scripts/run_tigerbeetle.sh +++ b/scripts/run_tigerbeetle.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash set -eEuo pipefail +PATH=$PATH:$PWD/tigerbeetle/zig-out/bin + echo "Building TigerBeetle..." -(cd tigerbeetle && ./zig/zig build install -Dcpu=baseline -Drelease-safe) +(cd tigerbeetle && zig build -Dcpu=baseline -Doptimize=ReleaseSafe) echo "Formatting replica ..." @@ -11,7 +13,7 @@ if [ -f "$FILE" ]; then rm "$FILE" fi -./tigerbeetle/tigerbeetle format --cluster=0 --replica=0 --replica-count=1 "$FILE" +tigerbeetle format --cluster=0 --replica=0 --replica-count=1 "$FILE" echo "Starting tigerbeetle ..." -./tigerbeetle/tigerbeetle start --addresses=3000 "$FILE" +tigerbeetle start --addresses=3000 "$FILE"& diff --git a/tigerbeetle b/tigerbeetle deleted file mode 160000 index 3851db9..0000000 --- a/tigerbeetle +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3851db94aa4414f1078d9f6141faba4fb98cffd3 diff --git a/zig/bench.zig b/zig/bench.zig index f87045c..666327c 100644 --- a/zig/bench.zig +++ b/zig/bench.zig @@ -1,20 +1,20 @@ const std = @import("std"); const panic = std.debug.panic; -const constants = @import("constants.zig"); -const stdx = @import("stdx.zig"); -const IO = @import("io.zig").IO; -const Storage = @import("storage.zig").Storage; -const MessagePool = @import("message_pool.zig").MessagePool; -const MessageBus = @import("message_bus.zig").MessageBusClient; -const StateMachine = @import("state_machine.zig").StateMachineType(Storage, constants.state_machine_config); -const RingBuffer = @import("ring_buffer.zig").RingBuffer; const vsr = @import("vsr.zig"); +const RingBuffer = vsr.ring_buffer.RingBuffer; +const StateMachine = vsr.state_machine.StateMachineType(Storage, constants.state_machine_config); +const MessageBus = vsr.message_bus.MessageBusClient; +const MessagePool = vsr.message_pool.MessagePool; +const Storage = vsr.storage.Storage; +const tb = vsr.tigerbeetle; const Client = vsr.Client(StateMachine, MessageBus); -const tb = @import("tigerbeetle.zig"); +const constants = vsr.constants; +const IO = vsr.io.IO; +const stdx = vsr.stdx; const Result = struct { - reply: Client.Error![]const u8, + reply: anyerror![]const u8, reply_ms: i64, }; @@ -40,7 +40,7 @@ pub fn main() !void { allocator, client_id, cluster_id, - @intCast(u8, address.len), + @as(u8, @intCast(address.len)), &message_pool, .{ .configuration = address[0..], @@ -67,8 +67,7 @@ pub fn main() !void { batch[j].id = 0; batch[j].debit_account_id = 0; batch[j].credit_account_id = 0; - batch[j].user_data = 0; - batch[j].reserved = 0; + batch[j].user_data_128 = 0; batch[j].pending_id = 0; batch[j].timeout = 0; batch[j].ledger = 2; @@ -117,7 +116,7 @@ fn send( const start_ms = std.time.milliTimestamp(); client.request( - @intCast(u128, @ptrToInt(&result)), + @as(u128, @intCast(@intFromPtr(&result))), send_complete, .create_transfers, message, @@ -149,11 +148,11 @@ fn send( fn send_complete( user_data: u128, operation: StateMachine.Operation, - reply: Client.Error![]const u8, + reply: []const u8, ) void { _ = operation; - const result_ptr = @intToPtr(*?Result, @intCast(u64, user_data)); + const result_ptr = @as(*?Result, @ptrFromInt(@as(u64, @intCast(user_data)))); result_ptr.* = Result{ .reply = reply, .reply_ms = std.time.milliTimestamp(), diff --git a/zig/build.zig b/zig/build.zig index be92e18..7672dc4 100644 --- a/zig/build.zig +++ b/zig/build.zig @@ -3,12 +3,15 @@ const builtin = @import("builtin"); pub fn build(b: *std.build.Builder) void { const target = b.standardTargetOptions(.{}); - const mode = b.standardReleaseOptions(); + const optimize = b.standardOptimizeOption(.{}); - var bench = b.addExecutable("bench", "bench.zig"); - bench.setBuildMode(mode); - bench.setTarget(target); - add_zig_files(bench, &.{ + var bench = b.addExecutable(.{ + .name = "bench", + .target = target, + .optimize = optimize, + .root_source_file = .{ .path = "bench.zig" }, + }); + add_zig_files(b, bench, &.{ "tigerbeetle.zig", "constants.zig", "storage.zig", @@ -21,15 +24,15 @@ pub fn build(b: *std.build.Builder) void { "stdx.zig", }); - const run_cmd = bench.run(); + const run_cmd = b.addRunArtifact(bench); run_cmd.step.dependOn(b.getInstallStep()); const bench_build = b.step("run", "Run the Zig client bench"); bench_build.dependOn(&run_cmd.step); } -fn add_zig_files(exe: *std.build.LibExeObjStep, comptime files: []const []const u8) void { - const options = exe.builder.addOptions(); +fn add_zig_files(b: *std.Build, exe: *std.Build.Step.Compile, comptime files: []const []const u8) void { + const options = b.addOptions(); const ConfigBase = enum { production, development, @@ -43,6 +46,12 @@ fn add_zig_files(exe: *std.build.LibExeObjStep, comptime files: []const []const .default, ); + options.addOption( + std.log.Level, + "config_log_level", + .info, + ); + const TracerBackend = enum { none, perfetto, @@ -50,8 +59,8 @@ fn add_zig_files(exe: *std.build.LibExeObjStep, comptime files: []const []const }; options.addOption(TracerBackend, "tracer_backend", .none); - const aof_record_enable = exe.builder.option(bool, "config-aof-record", "Enable AOF Recording.") orelse false; - const aof_recovery_enable = exe.builder.option(bool, "config-aof-recovery", "Enable AOF Recovery mode.") orelse false; + const aof_record_enable = b.option(bool, "config-aof-record", "Enable AOF Recording.") orelse false; + const aof_recovery_enable = b.option(bool, "config-aof-recovery", "Enable AOF Recovery mode.") orelse false; options.addOption(bool, "config_aof_record", aof_record_enable); options.addOption(bool, "config_aof_recovery", aof_recovery_enable); @@ -61,14 +70,18 @@ fn add_zig_files(exe: *std.build.LibExeObjStep, comptime files: []const []const check, }; options.addOption(HashLogMode, "hash_log_mode", .none); - const vsr_options = options.getPackage("vsr_options"); + const vsr_options = options.createModule(); inline for (files) |file| { - var pkg = std.build.Pkg{ - .name = file, - .path = .{ .path = "../tigerbeetle/src/" ++ file }, - .dependencies = &.{vsr_options}, - }; - exe.addPackage(pkg); + var pkg = b.createModule(.{ + .source_file = .{ .path = "../tigerbeetle/src/" ++ file }, + .dependencies = &.{ + .{ + .name = "vsr_options", + .module = vsr_options, + }, + }, + }); + exe.addModule(file, pkg); } } diff --git a/zig/run.sh b/zig/run.sh index c6e6f3e..333982e 100755 --- a/zig/run.sh +++ b/zig/run.sh @@ -1 +1,2 @@ -../tigerbeetle/zig/zig build run -Drelease-safe \ No newline at end of file +#!/usr/bin/env bash +zig build run -Doptimize=ReleaseSafe \ No newline at end of file