From fa0b8ae97d4455ae66347775528079d5fcde479f Mon Sep 17 00:00:00 2001 From: Vito Date: Sat, 20 Aug 2022 21:18:53 +0200 Subject: [PATCH] Redis integration tests launcher (#183) Co-authored-by: Daniele Salvatore Albano --- src/config.c | 4 +- src/module/redis/module_redis_command.c | 39 +- src/network/channel/network_channel.h | 12 +- src/storage/db/storage_db.h | 2 +- .../integration_tests/redis_server/client.tcl | 167 ++++++++ .../redis_server/examples/test-example.tcl | 6 + .../redis_server/helpers.tcl | 126 ++++++ tests/integration_tests/redis_server/main.tcl | 135 +++++++ .../integration_tests/redis_server/redis.tcl | 381 ++++++++++++++++++ .../integration_tests/redis_server/server.tcl | 179 ++++++++ tests/integration_tests/redis_server/test.tcl | 294 ++++++++++++++ 11 files changed, 1322 insertions(+), 23 deletions(-) create mode 100644 tests/integration_tests/redis_server/client.tcl create mode 100644 tests/integration_tests/redis_server/examples/test-example.tcl create mode 100644 tests/integration_tests/redis_server/helpers.tcl create mode 100755 tests/integration_tests/redis_server/main.tcl create mode 100644 tests/integration_tests/redis_server/redis.tcl create mode 100644 tests/integration_tests/redis_server/server.tcl create mode 100644 tests/integration_tests/redis_server/test.tcl diff --git a/src/config.c b/src/config.c index b256d5ee0..6ada4f3f1 100644 --- a/src/config.c +++ b/src/config.c @@ -136,8 +136,8 @@ bool config_validate_after_load( // Validate ad-hoc protocol settings (redis) if (module.type == CONFIG_MODULE_TYPE_REDIS) { - if (module.redis->max_key_length > SLAB_OBJECT_SIZE_MAX) { - LOG_E(TAG, "The allowed maximum value of max_key_length is <%u>", SLAB_OBJECT_SIZE_MAX); + if (module.redis->max_key_length > SLAB_OBJECT_SIZE_MAX - 1) { + LOG_E(TAG, "The allowed maximum value of max_key_length is <%u>", SLAB_OBJECT_SIZE_MAX - 1); return_result = false; } } diff --git a/src/module/redis/module_redis_command.c b/src/module/redis/module_redis_command.c index 17d89257a..11b23847d 100644 --- a/src/module/redis/module_redis_command.c +++ b/src/module/redis/module_redis_command.c @@ -349,19 +349,28 @@ bool module_redis_command_process_argument_stream_data( size_t written_data = 0; do { + // When all the data have been written current_chunk.index will always match chunk_sequence_count, this assert + // catch the cases when this function is invoked to write data even if all the chunks have been written. + assert(string->current_chunk.index < chunk_sequence->count); + storage_db_chunk_info_t *chunk_info = storage_db_chunk_sequence_get( chunk_sequence, string->current_chunk.index); + size_t chunk_length_to_write = chunk_length - written_data; size_t chunk_available_size = chunk_info->chunk_length - string->current_chunk.offset; size_t chunk_data_to_write_length = - chunk_length > chunk_available_size ? chunk_available_size : chunk_length; + chunk_length_to_write > chunk_available_size ? chunk_available_size : chunk_length_to_write; + + // There should always be something to write + assert(chunk_length_to_write > 0); + assert(chunk_data_to_write_length > 0); bool res = storage_db_chunk_write( connection_context->db, chunk_info, string->current_chunk.offset, - chunk_data, + chunk_data + written_data, chunk_data_to_write_length); if (!res) { @@ -378,6 +387,7 @@ bool module_redis_command_process_argument_stream_data( if (string->current_chunk.offset == chunk_info->chunk_length) { string->current_chunk.index++; + string->current_chunk.offset = 0; } } while(written_data < chunk_length); @@ -870,14 +880,23 @@ bool module_redis_command_stream_entry_with_multiple_chunks( buffer_to_send_length = chunk_info->chunk_length; } - // TODO: check if it's the last chunk and, if yes, if it would fit in the send buffer with the protocol - // bits that have to be sent later without doing an implicit flush - if (network_send_direct( - network_channel, - buffer_to_send, - buffer_to_send_length) != NETWORK_OP_RESULT_OK) { - return false; - } + size_t sent_data = 0; + do { + size_t data_available_to_send_length = buffer_to_send_length - sent_data; + size_t data_to_send_length = data_available_to_send_length > NETWORK_CHANNEL_PACKET_SIZE + ? NETWORK_CHANNEL_PACKET_SIZE : data_available_to_send_length; + + // TODO: check if it's the last chunk and, if yes, if it would fit in the send buffer with the protocol + // bits that have to be sent later without doing an implicit flush + if (network_send_direct( + network_channel, + buffer_to_send + sent_data, + data_to_send_length) != NETWORK_OP_RESULT_OK) { + return false; + } + + sent_data += data_to_send_length; + } while (sent_data < buffer_to_send_length); } send_buffer = send_buffer_start = network_send_buffer_acquire_slice( diff --git a/src/network/channel/network_channel.h b/src/network/channel/network_channel.h index c6ff370cc..99473ba7b 100644 --- a/src/network/channel/network_channel.h +++ b/src/network/channel/network_channel.h @@ -5,17 +5,9 @@ extern "C" { #endif -#define NETWORK_CHANNEL_PACKET_SIZE (8 * 1024) - -// The NETWORK_CHANNEL_RECV_BUFFER_SIZE has to be twice the NETWORK_CHANNEL_PACKET_SIZE to ensure that it's always -// possible to read a full packet in addition to any partially received data while processing the buffer and that there -// is enough room to avoid copying continuously the data at the beginning (a streaming parser is being used so there -// maybe data that need still to be parsed) +#define NETWORK_CHANNEL_PACKET_SIZE (32 * 1024) #define NETWORK_CHANNEL_RECV_BUFFER_SIZE (NETWORK_CHANNEL_PACKET_SIZE * 2) - -// Do not lower, to improve the performances the code expects to be able to send up to this amount of data, and do -// not increase as the slab allocator supports only up to 64kb. -#define NETWORK_CHANNEL_SEND_BUFFER_SIZE (64 * 1024) +#define NETWORK_CHANNEL_SEND_BUFFER_SIZE NETWORK_CHANNEL_PACKET_SIZE typedef char network_channel_buffer_data_t; diff --git a/src/storage/db/storage_db.h b/src/storage/db/storage_db.h index c8b958692..3a3d5b862 100644 --- a/src/storage/db/storage_db.h +++ b/src/storage/db/storage_db.h @@ -10,7 +10,7 @@ extern "C" { //#define STORAGE_DB_SHARD_MAGIC_NUMBER_LOW 0x5241000000000000 #define STORAGE_DB_SHARD_VERSION 1 -#define STORAGE_DB_CHUNK_MAX_SIZE (64 * 1024) +#define STORAGE_DB_CHUNK_MAX_SIZE ((64 * 1024) - 1) // This magic value defines the size of the ring buffer used to keep in memory data long enough to be sure they are not // being in use anymore. diff --git a/tests/integration_tests/redis_server/client.tcl b/tests/integration_tests/redis_server/client.tcl new file mode 100644 index 000000000..c47c763e1 --- /dev/null +++ b/tests/integration_tests/redis_server/client.tcl @@ -0,0 +1,167 @@ +# Copyright (C) 2018-2022 Vito Castellano +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +proc spawn_client {} { + puts "\n** Spawn Client" + cleanup + + # Set global keys + set ::idle_clients {} + set ::active_clients {} + array set ::active_clients_task {} + array set ::clients_start_time {} + set ::clients_time_history {} + set ::failed_tests {} + set ::clients_pids {} + + # Finding free socketport + set client_socket_port [find_available_port [expr {$::socket_port - 32}] 32] + + if {$::verbose} { puts -nonewline "Starting socket server at port $client_socket_port... "} + socket -server accept_test_clients -myaddr $::socket_host $client_socket_port + if {$::verbose} {puts "OK"} + + if {$::verbose} { puts "Will be spawned: $::numclients clients... "} + set start_port $::socket_port + set port_count [expr {$::portcount / $::numclients}] + for {set j 0} {$j < $::numclients} {incr j} { + set p [exec $::tclsh [info script] {*}$::argv --client $client_socket_port &] + if {$::verbose} { puts "- Creating instances with PID: $p"} + lappend ::clients_pids $p + incr start_port $port_count + } + if {$::verbose} {puts "Spawning Clients... OK \n"} + + # Enter the event loop to handle clients I/O + after 100 client_watcher + vwait forever +} + +proc client_watcher {} { + set elapsed [expr {[clock seconds]-$::last_progress}] + + if {$elapsed > $::timeout} { + puts "\[TIMEOUT\]: clients state report follows." + kill_clients + the_end + } + + after 100 client_watcher +} + +proc accept_test_clients {fd addr port} { + fconfigure $fd -encoding binary + fileevent $fd readable [list read_from_test_client $fd] +} + +proc read_from_test_client fd { + set bytes [gets $fd] + set payload [read $fd $bytes] + foreach {status data elapsed} $payload break + set ::last_progress [clock seconds] + + if {$status eq {ready}} { + if {$::verbose} { puts "Client with pid \[$data\] recieve \[$status\] status" } + signal_idle_client $fd + + } elseif {$status eq {done}} { + set elapsed [expr {[clock seconds]-$::clients_start_time($fd)}] + set all_tests_count [llength $::all_tests] + set running_tests_count [expr {[llength $::active_clients]-1}] + set completed_tests_count [expr {$::next_test-$running_tests_count}] + + puts "\[$completed_tests_count/$all_tests_count $status\]: $data ($elapsed seconds)" + lappend ::clients_time_history $elapsed $data + + signal_idle_client $fd + set ::active_clients_task($fd) "(DONE) $data" + + } elseif {$status eq {ok}} { + if {$::verbose} { puts "\[$status\]: $data ($elapsed ms)" } + set ::active_clients_task($fd) "(OK) $data" + + } elseif {$status eq {skip}} { + if {$::verbose} { puts "\[$status\]: $data" } + + } elseif {$status eq {ignore}} { + if {$::verbose} { puts "\[$status\]: $data" } + + } elseif {$status eq {err}} { + set err "\[$status\]: $data" + puts $err + lappend ::failed_tests $err + set ::active_clients_task($fd) "(ERR) $data" + + } elseif {$status eq {exception}} { + puts "\[$status\]: $data" + kill_clients + kill_server $::srv + exit 1 + } elseif {$status eq {testing}} { + set ::active_clients_task($fd) "(IN PROGRESS) $data" + + } elseif {$status eq {server-spawning}} { + set ::active_clients_task($fd) "(SPAWNING SERVER) $data" + + } elseif {$status eq {server-spawned}} { + lappend ::active_servers $data + set ::active_clients_task($fd) "(SPAWNED SERVER) pid:$data" + + } elseif {$status eq {server-killing}} { + set ::active_clients_task($fd) "(KILLING SERVER) pid:$data" + + } elseif {$status eq {server-killed}} { + set ::active_servers [lsearch -all -inline -not -exact $::active_servers $data] + set ::active_clients_task($fd) "(KILLED SERVER) pid:$data" + + } elseif {$status eq {run_solo}} { + lappend ::run_solo_tests $data + + } else { + if {$::verbose} { puts "\[$status\]: $data" } + } +} + +proc signal_idle_client fd { + # Remove this fd from the list of active clients. + set ::active_clients \ + [lsearch -all -inline -not -exact $::active_clients $fd] + + # New unit to process? + if {$::next_test != [llength $::all_tests]} { + if {$::verbose} { + puts "Testing \[[lindex $::all_tests $::next_test]\] and ASSIGNED to client : $fd" + set ::active_clients_task($fd) "ASSIGNED: $fd ([lindex $::all_tests $::next_test])" + } + + set ::clients_start_time($fd) [clock seconds] + send_data_packet $fd run [lindex $::all_tests $::next_test] + lappend ::active_clients $fd + incr ::next_test + } elseif {[llength $::run_solo_tests] != 0 && [llength $::active_clients] == 0} { + if {$::verbose} { + puts "Testing solo test and ASSIGNED to client : $fd" + set ::active_clients_task($fd) "ASSIGNED: $fd solo test" + } + + set ::clients_start_time($fd) [clock seconds] + send_data_packet $fd run_code [lpop ::run_solo_tests] + lappend ::active_clients $fd + } else { + lappend ::idle_clients $fd + set ::active_clients_task($fd) "SLEEPING, no more units to assign" + if {[llength $::active_clients] == 0} { + linespacer "#" + the_end + } + } +} + +proc kill_clients {} { + foreach p $::clients_pids { + catch {exec kill $p} + } +} \ No newline at end of file diff --git a/tests/integration_tests/redis_server/examples/test-example.tcl b/tests/integration_tests/redis_server/examples/test-example.tcl new file mode 100644 index 000000000..5bf4716a8 --- /dev/null +++ b/tests/integration_tests/redis_server/examples/test-example.tcl @@ -0,0 +1,6 @@ +start_server {tags {"example"}} { + test {SET and GET example item} { + r set x example + r get x + } {example} +} \ No newline at end of file diff --git a/tests/integration_tests/redis_server/helpers.tcl b/tests/integration_tests/redis_server/helpers.tcl new file mode 100644 index 000000000..9282f3df4 --- /dev/null +++ b/tests/integration_tests/redis_server/helpers.tcl @@ -0,0 +1,126 @@ +# Copyright (C) 2018-2022 Vito Castellano +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +proc cleanup {} { + if {$::verbose} {puts -nonewline "Cleanup: take a seconds... "} + flush stdout + catch {exec rm -rf {*}[glob $::tmproot]} + if {$::verbose} {puts "OK"} +} + +set ::last_port_attempted 0 +proc find_available_port {start count} { + set port [expr $::last_port_attempted + 1] + for {set attempts 0} {$attempts < $count} {incr attempts} { + if {$port < $start || $port >= $start+$count} { + set port $start + } + + set fd1 -1 + if {[catch {set fd1 [socket -server $::socket_host $port]}] || + [catch {set fd2 [socket -server $::server_host [expr $port+10000]]}]} { + if {$fd1 != -1} { + close $fd1 + } + } else { + close $fd1 + close $fd2 + set ::last_port_attempted $port + return $port + } + incr port + } + + error "Can't find a non busy port in the $start-[expr {$start+$count-1}] range." +} + +proc linespacer {chr} { + puts [string repeat $chr 50] +} + +proc is_alive config { + set pid [dict get $config pid] + if {[catch {exec kill -0 $pid} err]} { + return 0 + } else { + return 1 + } +} + +proc send_data_packet {fd status data {elapsed 0}} { + set payload [list $status $data $elapsed] + puts $fd [string length $payload] + puts -nonewline $fd $payload + flush $fd +} + +proc lpop {listVar {count 1}} { + upvar 1 $listVar l + set ele [lindex $l 0] + set l [lrange $l 1 end] + set ele +} + +proc randomInt {max} { + expr {int(rand()*$max)} +} + +proc tags_acceptable {tags err_return} { + upvar $err_return err + + if {[llength $::allowed_tags] > 0} { + set matched 0 + foreach tag $::allowed_tags { + if {[lsearch $tags $tag] >= 0} { + incr matched + } + } + + if {$matched < 1} { + set err "Tag: none of the tags allowed" + return 0 + } + } + + return 1 +} + +proc dump_server_log {srv} { + set pid [dict get $srv "pid"] + linespacer "=" + puts "STDOUT LOG" + puts [exec cat [dict get $srv "stdout"]] + linespacer "=" + + linespacer "=" + puts "STDERR LOG" + puts [exec cat [dict get $srv "stderr"]] + linespacer "=" +} + +proc the_end {} { + linespacer "=" + puts "End Report" + puts "Execution time of different units:" + foreach {time name} $::clients_time_history { + puts "- $time seconds - $name" + } + + if {[llength $::failed_tests]} { + puts "\n :( [llength $::failed_tests] tests failed:" + foreach failed $::failed_tests { + puts "-> $failed" + } + if {!$::dont_clean} cleanup + linespacer "=" + exit 1 + } else { + puts "\n :) All tests passed without errors!" + if {!$::dont_clean} cleanup + linespacer "=" + exit 0 + } +} \ No newline at end of file diff --git a/tests/integration_tests/redis_server/main.tcl b/tests/integration_tests/redis_server/main.tcl new file mode 100755 index 000000000..15c82c495 --- /dev/null +++ b/tests/integration_tests/redis_server/main.tcl @@ -0,0 +1,135 @@ +#!/usr/bin/tclsh + +# Copyright (C) 2018-2022 Vito Castellano +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +package require Tcl 8.5 +source helpers.tcl +source server.tcl +source client.tcl +source redis.tcl +source test.tcl + +######################## +# Suite CFG +######################## +set ::tclsh [info nameofexecutable] +set ::dont_clean 0 +set ::dump_logs 0 +set ::verbose 1 ; # Use this static value for the moment +set ::tmproot "./tmp" +file mkdir $::tmproot + +######################## +# Server +######################## +set ::tls 0 +set ::srv {} +set ::server_host 127.0.0.1 +set ::server_port 6380; +set ::server_pid 0; + +set ::socket_host 127.0.0.1 +set ::socket_port 21111 + +set ::server_path "../../../cmake-build-debug/src/cachegrand-server" +set ::server_cfg "../../../etc/cachegrand.yaml.skel" + +######################## +# Client +######################## +set ::client 0 +set ::numclients 1; # Use static value for the moment +set ::timeout 1200; # If this limit will reach quit the test. +set ::portcount 8000 + +######################## +# Tests +######################## +set ::test_path "./examples" +set ::num_tests 0 +set ::num_passed 0 +set ::num_failed 0 +set ::num_skipped 0 +set ::tests_failed {} +set ::cur_test "" +set ::curfile ""; +set ::durable 1 +set ::skiptests {} +set ::run_solo_tests {} +set ::last_progress [clock seconds] + +set ::tags {} +set ::allowed_tags {} + +set ::next_test 0 +set ::all_tests {} + +for {set j 0} {$j < [llength $argv]} {incr j} { + set opt [lindex $argv $j] + set arg [lindex $argv [expr $j+1]] + if {$opt eq {--server_host}} { + set ::server_host $arg + incr j + } elseif {$opt eq {--server_path}} { + set ::server_path $arg + incr j + } elseif {$opt eq {--server_cfg}} { + set ::server_cfg $arg + incr j + } elseif {$opt eq {--server_port}} { + set ::server_port $arg + incr j + } elseif {$opt eq {--client}} { + set ::client 1 + set ::socket_port $arg + incr j + } elseif {$opt eq {--test_path}} { + set ::test_path $arg + incr j + } elseif {$opt eq {--tests}} { + foreach test $arg { + lappend ::all_tests $test + } + incr j + } elseif {$opt eq {--allowed_tags}} { + foreach tag $arg { + lappend ::allowed_tags $tag + } + incr j + } elseif {$opt eq {--dont_clean}} { + set ::dont_clean 1 + incr j + } else { + puts "Wrong argument: $opt" + exit 1 + } +} + +if {$::client} { + if {[catch { test_launcher $::socket_port } err]} { + set estr "Executing test client: $err.\n$::errorInfo" + if {[catch {send_data_packet $::test_server_fd exception $estr}]} { + puts $estr + } + exit 1 + } + + exit 0 +} + +puts "************************************************" +puts "** Cachegrand - Commands Redis Tests Launcher **" +puts "- Founded [llength $::all_tests] test files to run!" +puts "************************************************" +if {[catch { spawn_client } err]} { + if {[string length $err] > 0} { + if {$err ne "exception"} { + puts $::errorInfo + } + exit 1 + } +} diff --git a/tests/integration_tests/redis_server/redis.tcl b/tests/integration_tests/redis_server/redis.tcl new file mode 100644 index 000000000..549631761 --- /dev/null +++ b/tests/integration_tests/redis_server/redis.tcl @@ -0,0 +1,381 @@ +# Copyright (C) 2018-2022 Vito Castellano +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +set ::redis_id 0 +array set ::redis_fd {} +array set ::redis_addr {} +array set ::redis_blocking {} +array set ::redis_deferred {} +array set ::redis_readraw {} +array set ::redis_attributes {} +array set ::redis_reconnect {} +array set ::redis_tls {} +array set ::redis_callback {} +array set ::redis_state {} +array set ::redis_statestack {} + +proc redis {{server 127.0.0.1} {port 6380} {defer 0} {tls 0} {tlsoptions {}} {readraw 0}} { + set fd [socket $server $port] + fconfigure $fd -translation binary + + set id [incr ::redis_id] + set ::redis_fd($id) $fd + set ::redis_addr($id) [list $server $port] + set ::redis_blocking($id) 1 + set ::redis_deferred($id) $defer + set ::redis_readraw($id) $readraw + set ::redis_reconnect($id) 0 + set ::redis_tls($id) $tls + + redis_reset_state $id + interp alias {} ::redis_redisHandle$id {} ::redis___dispatch__ $id +} + +proc redis_safe_read {fd len} { + if {$len == -1} { + set err [catch {set val [read $fd]} msg] + } else { + set err [catch {set val [read $fd $len]} msg] + } + + if {!$err} { + return $val + } + + if {[string match "*connection abort*" $msg]} { + return {} + } + + error $msg +} + +proc redis_safe_gets {fd} { + if {[catch {set val [gets $fd]} msg]} { + if {[string match "*connection abort*" $msg]} { + return {} + } + error $msg + } + return $val +} + + +proc redis___dispatch__ {id method args} { + set errorcode [catch {::redis___dispatch__raw__ $id $method $args} retval] + if {$errorcode && $::redis_reconnect($id) && $::redis_fd($id) eq {}} { + set errorcode [catch {::redis___dispatch__raw__ $id $method $args} retval] + } + + return -code $errorcode $retval +} + +proc redis___dispatch__raw__ {id method argv} { + set fd $::redis_fd($id) + + # Reconnect the link if needed. + if {$fd eq {} && $method ne {close}} { + lassign $::redis_addr($id) host port + set ::redis_fd($id) [socket $host $port] + + fconfigure $::redis_fd($id) -translation binary + set fd $::redis_fd($id) + } + + set blocking $::redis_blocking($id) + set deferred $::redis_deferred($id) + if {$blocking == 0} { + if {[llength $argv] == 0} { + error "Please provide a callback in non-blocking mode" + } + set callback [lindex $argv end] + set argv [lrange $argv 0 end-1] + } + + if {[info command ::redis___method__$method] eq {}} { + catch {unset ::redis_attributes($id)} + set cmd "*[expr {[llength $argv]+1}]\r\n" + append cmd "$[string length $method]\r\n$method\r\n" + foreach a $argv { + append cmd "$[string length $a]\r\n$a\r\n" + } + + redis_write $fd $cmd + if {[catch {flush $fd}]} { + catch {close $fd} + set ::redis_fd($id) {} + return -code error "I/O error reading reply" + } + + if {!$deferred} { + if {$blocking} { + redis_read_reply $id $fd + } else { + # Every well formed reply read will pop an element from this + # list and use it as a callback. So pipelining is supported + # in non blocking mode. + lappend ::redis_callback($id) $callback + fileevent $fd readable [list redis_readable $fd $id] + } + } + } else { + uplevel 1 [list ::redis___method__$method $id $fd] $argv + } +} + +proc redis___method__blocking {id fd val} { + set ::redis_blocking($id) $val + fconfigure $fd -blocking $val +} + +proc redis___method__reconnect {id fd val} { + set ::redis_reconnect($id) $val +} + +proc redis___method__read {id fd} { + redis_read_reply $id $fd +} + +proc redis___method__rawread {id fd {len -1}} { + return [redis_safe_read $fd $len] +} + +proc redis___method__write {id fd buf} { + redis_write $fd $buf +} + +proc redis___method__flush {id fd} { + flush $fd +} + +proc redis___method__close {id fd} { + catch {close $fd} + catch {unset ::redis_fd($id)} + catch {unset ::redis_addr($id)} + catch {unset ::redis_blocking($id)} + catch {unset ::redis_deferred($id)} + catch {unset ::redis_readraw($id)} + catch {unset ::redis_attributes($id)} + catch {unset ::redis_reconnect($id)} + catch {unset ::redis_tls($id)} + catch {unset ::redis_state($id)} + catch {unset ::redis_statestack($id)} + catch {unset ::redis_callback($id)} + catch {interp alias {} ::redis_redisHandle$id {}} +} + +proc redis___method__channel {id fd} { + return $fd +} + +proc redis___method__deferred {id fd val} { + set ::redis_deferred($id) $val +} + +proc redis___method__readraw {id fd val} { + set ::redis_readraw($id) $val +} + +proc redis___method__readingraw {id fd} { + return $::redis_readraw($id) +} + +proc redis___method__attributes {id fd} { + set _ $::redis_attributes($id) +} + +proc redis_write {fd buf} { + puts -nonewline $fd $buf +} + +proc redis_writenl {fd buf} { + redis_write $fd $buf + redis_write $fd "\r\n" + flush $fd +} + +proc redis_readnl {fd len} { + set buf [redis_safe_read $fd $len] + redis_safe_read $fd 2 ; # discard CR LF + return $buf +} + +proc redis_bulk_read {fd} { + set count [redis_read_line $fd] + if {$count == -1} return {} + set buf [redis_readnl $fd $count] + return $buf +} + +proc redis_multi_bulk_read {id fd} { + set count [redis_read_line $fd] + if {$count == -1} return {} + set l {} + set err {} + for {set i 0} {$i < $count} {incr i} { + if {[catch { + lappend l [redis_read_reply $id $fd] + } e] && $err eq {}} { + set err $e + } + } + if {$err ne {}} {return -code error $err} + return $l +} + +proc redis_read_map {id fd} { + set count [redis_read_line $fd] + if {$count == -1} return {} + set d {} + set err {} + for {set i 0} {$i < $count} {incr i} { + if {[catch { + set k [redis_read_reply $id $fd] ; # key + set v [redis_read_reply $id $fd] ; # value + dict set d $k $v + } e] && $err eq {}} { + set err $e + } + } + if {$err ne {}} {return -code error $err} + return $d +} + +proc redis_read_line fd { + string trim [redis_safe_gets $fd] +} + +proc redis_read_null fd { + redis_safe_gets $fd + return {} +} + +proc redis_read_bool fd { + set v [redis_read_line $fd] + if {$v == "t"} {return 1} + if {$v == "f"} {return 0} + return -code error "Bad protocol, '$v' as bool type" +} + +proc redis_read_verbatim_str fd { + set v [redis_bulk_read $fd] + # strip the first 4 chars ("txt:") + return [string range $v 4 end] +} + +proc redis_read_reply {id fd} { + if {$::redis_readraw($id)} { + return [redis_read_line $fd] + } + + while {1} { + set type [redis_safe_read $fd 1] + switch -exact -- $type { + _ {return [redis_read_null $fd]} + : - + ( - + + {return [redis_read_line $fd]} + , {return [expr {double([redis_read_line $fd])}]} + # {return [redis_read_bool $fd]} + = {return [redis_read_verbatim_str $fd]} + - {return -code error [redis_read_line $fd]} + $ {return [redis_bulk_read $fd]} + > - + ~ - + * {return [redis_multi_bulk_read $id $fd]} + % {return [redis_read_map $id $fd]} + | { + set attrib [redis_read_map $id $fd] + set ::redis_attributes($id) $attrib + continue + } + default { + if {$type eq {}} { + catch {close $fd} + set ::redis_fd($id) {} + return -code error "I/O error reading reply" + } + return -code error "Bad protocol, '$type' as reply type byte" + } + } + } +} + +proc redis_reset_state id { + set ::redis_state($id) [dict create buf {} mbulk -1 bulk -1 reply {}] + set ::redis_statestack($id) {} +} + +proc redis_call_callback {id type reply} { + set cb [lindex $::redis_callback($id) 0] + set ::redis_callback($id) [lrange $::redis_callback($id) 1 end] + uplevel #0 $cb [list ::redis_redisHandle$id $type $reply] + redis_reset_state $id +} + +proc redis_readable {fd id} { + if {[eof $fd]} { + redis_call_callback $id eof {} + ::redis___method__close $id $fd + return + } + + if {[dict get $::redis_state($id) bulk] == -1} { + set line [gets $fd] + if {$line eq {}} return ;# No complete line available, return + switch -exact -- [string index $line 0] { + : - + + {redis_call_callback $id reply [string range $line 1 end-1]} + - {redis_call_callback $id err [string range $line 1 end-1]} + ( {redis_call_callback $id reply [string range $line 1 end-1]} + $ { + dict set ::redis_state($id) bulk \ + [expr [string range $line 1 end-1]+2] + if {[dict get $::redis_state($id) bulk] == 1} { + # We got a $-1, hack the state to play well with this. + dict set ::redis_state($id) bulk 2 + dict set ::redis_state($id) buf "\r\n" + redis_readable $fd $id + } + } + * { + dict set ::redis_state($id) mbulk [string range $line 1 end-1] + # Handle *-1 + if {[dict get $::redis_state($id) mbulk] == -1} { + redis_call_callback $id reply {} + } + } + default { + redis_call_callback $id err \ + "Bad protocol, $type as reply type byte" + } + } + } else { + set totlen [dict get $::redis_state($id) bulk] + set buflen [string length [dict get $::redis_state($id) buf]] + set toread [expr {$totlen-$buflen}] + set data [read $fd $toread] + set nread [string length $data] + dict append ::redis_state($id) buf $data + # Check if we read a complete bulk reply + if {[string length [dict get $::redis_state($id) buf]] == + [dict get $::redis_state($id) bulk]} { + if {[dict get $::redis_state($id) mbulk] == -1} { + redis_call_callback $id reply \ + [string range [dict get $::redis_state($id) buf] 0 end-2] + } else { + dict with ::redis_state($id) { + lappend reply [string range $buf 0 end-2] + incr mbulk -1 + set bulk -1 + } + if {[dict get $::redis_state($id) mbulk] == 0} { + redis_call_callback $id reply \ + [dict get $::redis_state($id) reply] + } + } + } + } +} diff --git a/tests/integration_tests/redis_server/server.tcl b/tests/integration_tests/redis_server/server.tcl new file mode 100644 index 000000000..bc90b0474 --- /dev/null +++ b/tests/integration_tests/redis_server/server.tcl @@ -0,0 +1,179 @@ +# Copyright (C) 2018-2022 Vito Castellano +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +proc start_server {options {code undefined}} { + puts "\n** Start Server" + + # setup defaults + set overrides {} + set tags {} + set args {} + set config_lines {} + + # parse options + foreach {option value} $options { + switch $option { + "overrides" { + set overrides $value + } + "args" { + set args $value + } + "tags" { + set tags [string map { \" "" } $value] + set ::tags [concat $::tags $tags] + } + default { + error "Unknown option $option" + } + } + } + + # Starting server + set server_started 0 + while {$server_started == 0} { + if {$code ne "undefined"} { + set server_started [spawn_server] + } else { + set server_started 1 + } + } + + # Run Test + test_controller $code $tags +} + +proc spawn_server {} { + puts "\n** Spawn server \[$::server_host@$::server_port\]" + + set stdout [format "%s/%s" $::tmproot "stdout"] + set stderr [format "%s/%s" $::tmproot "stderr"] + + # Emit event server-spawning + send_data_packet $::test_server_fd "server-spawning" "port $::server_port" + + if {$::verbose} {puts -nonewline "Spawing... "} + set pid [exec $::server_path -c $::server_cfg >> $stdout 2>> $stderr &] + set ::server_pid $pid + if {$::verbose} {puts "OK Server spawned with pid \[$pid\]"} + + # Emit event server-spawned + send_data_packet $::test_server_fd server-spawned $pid + + # Check if server is truly up + set serverisup [server_is_up $::server_host $::server_port 100] + if {!$serverisup} { + error "Probably there are some errors with server :(" + } + + # Struct server's info + set client [redis $::server_host $::server_port] + dict set ::srv "client" $client + dict set ::srv "pid" $::server_pid + dict set ::srv "host" $::server_host + dict set ::srv "port" $::server_port + dict set ::srv "stdout" $stdout + dict set ::srv "stderr" $stderr + + puts "" + return $serverisup +} + +proc server_is_up {host port retrynum} { + if {$::verbose} {puts -nonewline "Check if server is up..."} + after 10 ; + set retval 0 + while {[incr retrynum -1]} { + if {[catch {ping_server $host $port} ping]} { + set ping 0 + } + if {$ping} {return 1} + after 50 + } + return 0 +} + +proc ping_server {host port} { + set retval 0 + if {[catch { + set fd [socket $host $port] + fconfigure $fd -translation binary + puts $fd "PING\r\n" + flush $fd + + set reply [gets $fd] + if {[string range $reply 0 0] eq {+} || + [string range $reply 0 0] eq {-}} { + set retval 1 + } + close $fd + } e]} { + if {$::verbose} { + puts -nonewline "." + } + } else { + if {$::verbose} { + puts -nonewline " Communication was good.\n" + } + } + + return $retval +} + +proc kill_server config { + if {[dict exists $config "client"]} { + [dict get $config "client"] close + } + + if {![dict exists $config "pid"]} { + return + } + + if {![is_alive $config]} { + return + } + + set pid [dict get $config pid] + + # Emit event server-killing + if {$::verbose} {puts -nonewline "Server killing... "} + send_data_packet $::test_server_fd server-killing $pid + catch {exec kill $pid} + + # Node might have been stopped in the test + catch {exec kill -SIGCONT $pid} + + set max_wait 10000 + while {[is_alive $config]} { + incr wait 10 + if {$wait == $max_wait} { + puts "Forcing process $pid to crash..." + catch {exec kill -SEGV $pid} + } elseif {$wait >= $max_wait * 2} { + puts "Forcing process $pid to exit..." + catch {exec kill -KILL $pid} + } elseif {$wait % 1000 == 0} { + puts "Waiting for process $pid to exit..." + } + after 10 + } + + # Emit event server killed + send_data_packet $::test_server_fd server-killed $pid + if {$::verbose} {puts "OK"} +} + +proc srv {args} { + set level 0 + if {[string is integer [lindex $args 0]]} { + set level [lindex $args 0] + set property [lindex $args 1] + } else { + set property [lindex $args 0] + } + + dict get $::srv $property +} \ No newline at end of file diff --git a/tests/integration_tests/redis_server/test.tcl b/tests/integration_tests/redis_server/test.tcl new file mode 100644 index 000000000..57ae6063a --- /dev/null +++ b/tests/integration_tests/redis_server/test.tcl @@ -0,0 +1,294 @@ +# Copyright (C) 2018-2022 Vito Castellano +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +proc test_launcher socket_port { + puts "** Test Launcher" + + if {$::verbose} { puts "Trying to comunicate using socket client... "} + set ::test_server_fd [socket $::socket_host $socket_port] + fconfigure $::test_server_fd -encoding binary + + if {$::verbose} { puts "Sending ready status... "} + send_data_packet $::test_server_fd ready [pid] + + while 1 { + set bytes [gets $::test_server_fd] + set payload [read $::test_server_fd $bytes] + foreach {cmd data} $payload break + + if {$cmd eq {run}} { + execute_test_file $data + } elseif {$cmd eq {run_code}} { + foreach {name filename code} $data break + execute_test_code $name $filename $code + } else { + error "Unknown test client command: $cmd" + } + } +} + +proc execute_test_file __testname { + set path "$::test_path/$__testname.tcl" + set ::curfile $path + + if {![file exists $path]} { + set path_not_exists "File test not found in this path: $path" + send_data_packet $::test_server_fd exception $path_not_exists + } + + linespacer "#" + puts "** Executing test file: $::curfile" + source $path + send_data_packet $::test_server_fd done "$__testname" +} + +proc execute_test_code {__testname filename code} { + set ::curfile $filename + + linespacer "#" + puts "\n** Executing test code..." + eval $code + send_data_packet $::test_server_fd done "$__testname" +} + +proc test_controller {code tags} { + if {$code ne "undefined"} { + set prev_num_failed $::num_failed + set num_tests $::num_tests + + # Run test and catch! + puts "\n** Ready to test!" + if {[catch { uplevel 1 $code } error]} { + set backtrace $::errorInfo + if {$::durable} { + lappend error_details $error + lappend error_details $backtrace + + # Emit event error with details + send_data_packet $::test_server_fd err [join $error_details "\n"] + } else { + if {$::dump_logs} { + dump_server_log $::srv + } + + error $error $backtrace + } + } + + set ::tags [lrange $::tags 0 end-[llength $tags]] + + kill_server $::srv + set _ "" + } else { + set ::tags [lrange $::tags 0 end-[llength $tags]] + set _ $::srv + } + + puts "" +} + +proc test {name code {okpattern undefined} {tags {}}} { + set tags [concat $::tags $tags] + if {![tags_acceptable $tags err]} { + incr ::num_aborted + # Emit event ingore for logging [skipped at the moment] +# send_data_packet $::test_server_fd ignore "$name: $err" + return + } + + incr ::num_tests + set details {} + lappend details "$name in $::curfile" + + set prev_test $::cur_test + set ::cur_test "$name in $::curfile" + + if {$::verbose} { + set stdout [dict get $::srv stdout] + set fd [open $stdout "a+"] + puts $fd "### Starting test $::cur_test" + close $fd + } + + # Emit event testing + send_data_packet $::test_server_fd testing $name + + set test_start_time [clock milliseconds] + if {[catch {set retval [uplevel 1 $code]} error]} { + # We are here if for example a command is not implemented + set assertion [string match "assertion:*" $error] + set cmd_not_implemented [string match "Command not implemented!" $error] + + # Durable permit to continue to run other tests in case of error + if {$assertion || $::durable} { + lappend details $error + lappend ::tests_failed $details + incr ::num_failed + + # Emit event error + send_data_packet $::test_server_fd err [join $details "\n"] + + if {!$cmd_not_implemented} { + linespacer "+" + puts "Owhh No! Bad error occurred!" + puts "Check if the server is still alive..." + set serverisup [server_is_up $::server_host $::server_port 10] + if {!$serverisup} { + puts "Server goes away" + set server_started 0 + while {$server_started == 0} { + if {$code ne "undefined"} { + set server_started [spawn_server] + } else { + set server_started 1 + } + } + } + linespacer "+" + } + } else { + error $error $::errorInfo + } + } else { + if {$okpattern eq "undefined" || $okpattern eq $retval || [string match $okpattern $retval]} { + incr ::num_passed + set elapsed [expr {[clock milliseconds]-$test_start_time}] + + # Emit event ok + send_data_packet $::test_server_fd ok $name $elapsed + } else { + set msg "Expected '$okpattern' to equal or match '$retval'" + lappend details $msg + lappend ::tests_failed $details + + incr ::num_failed + + # Emit event error + send_data_packet $::test_server_fd err [join $details "\n"] + } + } + + set ::cur_test $prev_test +} + +proc r {args} { + set level 0 + if {[string is integer [lindex $args 0]]} { + set level [lindex $args 0] + set args [lrange $args 1 end] + } + + [srv $level "client"] {*}$args +} + +proc tags {tags code} { + set tags [string map { \" "" } $tags] + set ::tags [concat $::tags $tags] + + uplevel 1 $code + set ::tags [lrange $::tags 0 end-[llength $tags]] +} + +####################################################################### +# Asserts Check List +####################################################################### +proc fail {msg} { + error "assertion:$msg" +} + +proc assert {condition} { + if {![uplevel 1 [list expr $condition]]} { + set context "(context: [info frame -1])" + error "assertion:Expected [uplevel 1 [list subst -nocommands $condition]] $context" + } +} + +proc assert_no_match {pattern value} { + if {[string match $pattern $value]} { + set context "(context: [info frame -1])" + error "assertion:Expected '$value' to not match '$pattern' $context" + } +} + +proc assert_match {pattern value {detail ""}} { + if {![string match $pattern $value]} { + set context "(context: [info frame -1])" + error "assertion:Expected '$value' to match '$pattern' $context $detail" + } +} + +proc assert_failed {expected_err detail} { + if {$detail ne ""} { + set detail "(detail: $detail)" + } else { + set detail "(context: [info frame -2])" + } + error "assertion:$expected_err $detail" +} + +proc assert_not_equal {value expected {detail ""}} { + if {!($expected ne $value)} { + assert_failed "Expected '$value' not equal to '$expected'" $detail + } +} + +proc assert_equal {value expected {detail ""}} { + if {$expected ne $value} { + assert_failed "Expected '$value' to be equal to '$expected'" $detail + } +} + +proc assert_lessthan {value expected {detail ""}} { + if {!($value < $expected)} { + assert_failed "Expected '$value' to be less than '$expected'" $detail + } +} + +proc assert_lessthan_equal {value expected {detail ""}} { + if {!($value <= $expected)} { + assert_failed "Expected '$value' to be less than or equal to '$expected'" $detail + } +} + +proc assert_morethan {value expected {detail ""}} { + if {!($value > $expected)} { + assert_failed "Expected '$value' to be more than '$expected'" $detail + } +} + +proc assert_morethan_equal {value expected {detail ""}} { + if {!($value >= $expected)} { + assert_failed "Expected '$value' to be more than or equal to '$expected'" $detail + } +} + +proc assert_range {value min max {detail ""}} { + if {!($value <= $max && $value >= $min)} { + assert_failed "Expected '$value' to be between to '$min' and '$max'" $detail + } +} + +proc assert_error {pattern code {detail ""}} { + if {[catch {uplevel 1 $code} error]} { + assert_match $pattern $error $detail + } else { + assert_failed "Expected an error matching '$pattern' but got '$error'" $detail + } +} + +proc assert_encoding {enc key} { + set val [r object encoding $key] + assert_match $enc $val +} + +proc assert_type {type key} { + assert_equal $type [r type $key] +} + +proc assert_refcount {ref key} { + set val [r object refcount $key] + assert_equal $ref $val +} \ No newline at end of file