diff --git a/src/data_structures/hashtable/mcmp/hashtable_op_rmw.c b/src/data_structures/hashtable/mcmp/hashtable_op_rmw.c index 3b7903d01..c9a2d0107 100644 --- a/src/data_structures/hashtable/mcmp/hashtable_op_rmw.c +++ b/src/data_structures/hashtable/mcmp/hashtable_op_rmw.c @@ -35,7 +35,7 @@ bool hashtable_mcmp_op_rmw_begin( hashtable_key_data_t *key, hashtable_key_size_t key_size, hashtable_value_data_t *current_value) { - bool created_new = true; + bool created_new = false; hashtable_hash_t hash; hashtable_chunk_index_t chunk_index = 0; hashtable_half_hashes_chunk_volatile_t *half_hashes_chunk = 0; diff --git a/src/module/redis/command/module_redis_command_keys.c b/src/module/redis/command/module_redis_command_keys.c index bde02ed97..a7731b160 100644 --- a/src/module/redis/command/module_redis_command_keys.c +++ b/src/module/redis/command/module_redis_command_keys.c @@ -45,15 +45,21 @@ MODULE_REDIS_COMMAND_FUNCPTR_COMMAND_END(keys) { bool return_res = false; uint64_t keys_count = 0; + uint64_t cursor_next = 0; module_redis_command_keys_context_t *context = connection_context->command.context; storage_db_key_and_key_length_t *keys = storage_db_op_get_keys( connection_context->db, + 0, + 0, context->pattern.value.pattern, context->pattern.value.length, - &keys_count); + &keys_count, + &cursor_next); - if (!module_redis_connection_send_array_header(connection_context, keys_count)) { + assert(cursor_next == 0); + + if (unlikely(!module_redis_connection_send_array_header(connection_context, keys_count))) { goto end; } diff --git a/src/module/redis/command/module_redis_command_scan.c b/src/module/redis/command/module_redis_command_scan.c new file mode 100644 index 000000000..799710bcb --- /dev/null +++ b/src/module/redis/command/module_redis_command_scan.c @@ -0,0 +1,116 @@ +/** + * Copyright (C) 2018-2022 Daniele Salvatore Albano + * All rights reserved. + * + * This software may be modified and distributed under the terms + * of the BSD license. See the LICENSE file for details. + **/ + +#include +#include +#include +#include +#include +#include + +#include "misc.h" +#include "exttypes.h" +#include "log/log.h" +#include "clock.h" +#include "spinlock.h" +#include "data_structures/small_circular_queue/small_circular_queue.h" +#include "data_structures/double_linked_list/double_linked_list.h" +#include "data_structures/hashtable/mcmp/hashtable.h" +#include "data_structures/hashtable/spsc/hashtable_spsc.h" +#include "protocol/redis/protocol_redis.h" +#include "protocol/redis/protocol_redis_reader.h" +#include "protocol/redis/protocol_redis_writer.h" +#include "module/module.h" +#include "network/io/network_io_common.h" +#include "config.h" +#include "fiber.h" +#include "network/channel/network_channel.h" +#include "storage/io/storage_io_common.h" +#include "storage/channel/storage_channel.h" +#include "storage/db/storage_db.h" +#include "module/redis/module_redis.h" +#include "module/redis/module_redis_connection.h" +#include "module/redis/module_redis_command.h" +#include "network/network.h" +#include "worker/worker_stats.h" +#include "worker/worker_context.h" + +#define TAG "module_redis_command_scan" + +MODULE_REDIS_COMMAND_FUNCPTR_COMMAND_END(scan) { + bool return_res = false; + uint64_t keys_count = 0; + storage_db_key_and_key_length_t *keys = NULL; + char* pattern = NULL; + size_t pattern_length = 0; + uint64_t count = 10000; + uint64_t cursor_next = 0; + module_redis_command_scan_context_t *context = connection_context->command.context; + + if (unlikely(context->type_type.has_token)) { + return_res = module_redis_connection_error_message_printf_noncritical( + connection_context, + "ERR the TYPE parameter is not yet supported"); + goto end; + } + + if (likely(context->cursor.value >= 0 && + (!context->count_count.has_token || + (context->count_count.has_token && context->count_count.value > 0)))) { + if (context->match_pattern.has_token) { + pattern = context->match_pattern.value.pattern; + pattern_length = context->match_pattern.value.length; + } + + if (context->count_count.has_token) { + count = context->count_count.value; + } + + keys = storage_db_op_get_keys( + connection_context->db, + context->cursor.value, + count, + pattern, + pattern_length, + &keys_count, + &cursor_next); + } + + if (unlikely(!module_redis_connection_send_array_header(connection_context, 2))) { + goto end; + } + + if (unlikely(!module_redis_connection_send_number(connection_context, (int64_t)cursor_next))) { + goto end; + } + + if (unlikely(!module_redis_connection_send_array_header(connection_context, keys_count))) { + goto end; + } + + if (likely(keys && keys_count > 0)) { + for(uint64_t index = 0; index < keys_count; index++) { + if (!module_redis_connection_send_string( + connection_context, + keys[index].key, + keys[index].key_size)) { + goto end; + } + } + } + + return_res = true; + +end: + + if (keys) { + storage_db_free_key_and_key_length_list(keys, keys_count); + } + + return return_res; +} diff --git a/src/storage/db/storage_db.c b/src/storage/db/storage_db.c index 9179b4377..5cfb6e975 100644 --- a/src/storage/db/storage_db.c +++ b/src/storage/db/storage_db.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "misc.h" #include "exttypes.h" @@ -1237,7 +1238,7 @@ bool storage_db_op_rmw_commit_update( hashtable_mcmp_op_rmw_commit_update( &rmw_status->hashtable, - (uintptr_t) entry_index); + (uintptr_t)entry_index); if (rmw_status->hashtable.current_value != 0) { storage_db_worker_mark_deleted_or_deleting_previous_entry_index( @@ -1355,13 +1356,32 @@ bool storage_db_op_flush_sync( storage_db_key_and_key_length_t *storage_db_op_get_keys( storage_db_t *db, + uint64_t cursor, + uint64_t count, char *pattern, size_t pattern_length, - uint64_t *keys_count) { + uint64_t *keys_count, + uint64_t *cursor_next) { hashtable_key_data_t *key; hashtable_key_size_t key_size; - + bool end_reached = false; + hashtable_bucket_index_t bucket_index = cursor; + storage_db_entry_index_t *entry_index = NULL; *keys_count = 0; + *cursor_next = 0; + + if (unlikely(storage_db_op_get_size(db)) == 0) { + return NULL; + } + + if (cursor >= db->hashtable->ht_current->buckets_count_real) { + return NULL; + } + + if (count == 0) { + count = db->hashtable->ht_current->buckets_count_real; + } + uint64_t keys_allocated_count = 8; storage_db_key_and_key_length_t *keys = xalloc_alloc(sizeof(storage_db_key_and_key_length_t) * keys_allocated_count); @@ -1371,12 +1391,15 @@ storage_db_key_and_key_length_t *storage_db_op_get_keys( int64_t scan_start_ms = clock_monotonic_int64_ms(); // Iterates over the hashtable to free up the entry index - hashtable_bucket_index_t bucket_index = 0; - for( - void *data = hashtable_mcmp_op_iter(db->hashtable, &bucket_index); - data; - ++bucket_index && (data = hashtable_mcmp_op_iter(db->hashtable, &bucket_index))) { - storage_db_entry_index_t *entry_index = data; + do { + entry_index = hashtable_mcmp_op_iter(db->hashtable, &bucket_index); + + if (unlikely(entry_index == NULL)) { + end_reached = true; + break; + } + + *cursor_next = bucket_index + 1; if (unlikely(entry_index->created_time_ms > scan_start_ms)) { continue; @@ -1402,6 +1425,10 @@ storage_db_key_and_key_length_t *storage_db_op_get_keys( keys[*keys_count].key = key; keys[*keys_count].key_size = key_size; (*keys_count)++; + } while(likely(entry_index && ++bucket_index < cursor + count)); + + if (unlikely(end_reached)) { + *cursor_next = 0; } return keys; diff --git a/src/storage/db/storage_db.h b/src/storage/db/storage_db.h index 589a280ca..5e63c1391 100644 --- a/src/storage/db/storage_db.h +++ b/src/storage/db/storage_db.h @@ -359,9 +359,12 @@ bool storage_db_op_flush_sync( storage_db_key_and_key_length_t *storage_db_op_get_keys( storage_db_t *db, + uint64_t cursor, + uint64_t count, char *pattern, size_t pattern_length, - uint64_t *keys_count); + uint64_t *keys_count, + uint64_t *cursor_next); void storage_db_free_key_and_key_length_list( storage_db_key_and_key_length_t *keys, diff --git a/src/utils_string.c b/src/utils_string.c index 04e2b4f8b..b3567b241 100644 --- a/src/utils_string.c +++ b/src/utils_string.c @@ -109,13 +109,11 @@ uint32_t utils_string_utf8_decode_char( } // Derived from https://www.codeproject.com/Articles/5163931/Fast-String-Matching-with-Wildcards-Globs-and-Giti -// TODO: should handle UTF-8? Redis code seems doesn't do it; bool utils_string_glob_match( char *string, size_t string_length, char *pattern, size_t pattern_length) { - int last_char; bool matched, reverse; char *string_backup = NULL, *pattern_backup = NULL; size_t string_length_backup = 0, pattern_length_backup = 0; @@ -195,7 +193,7 @@ bool utils_string_glob_match( } } - prev_char = *pattern; + prev_char = (int)*pattern; pattern++; pattern_length--; break; diff --git a/tests/unit_tests/modules/redis/command/test-modules-redis-command-scan.cpp b/tests/unit_tests/modules/redis/command/test-modules-redis-command-scan.cpp new file mode 100644 index 000000000..3f88fed0e --- /dev/null +++ b/tests/unit_tests/modules/redis/command/test-modules-redis-command-scan.cpp @@ -0,0 +1,302 @@ +/** + * Copyright (C) 2018-2022 Daniele Salvatore Albano + * All rights reserved. + * + * This software may be modified and distributed under the terms + * of the BSD license. See the LICENSE file for details. + **/ + +#include + +#include +#include + +#include + +#include "clock.h" +#include "exttypes.h" +#include "spinlock.h" +#include "data_structures/small_circular_queue/small_circular_queue.h" +#include "data_structures/double_linked_list/double_linked_list.h" +#include "data_structures/hashtable/mcmp/hashtable.h" +#include "config.h" +#include "fiber.h" +#include "worker/worker_stats.h" +#include "worker/worker_context.h" +#include "signal_handler_thread.h" +#include "storage/io/storage_io_common.h" +#include "storage/channel/storage_channel.h" +#include "storage/db/storage_db.h" + +#include "program.h" + +#include "test-modules-redis-command-fixture.hpp" + +#pragma GCC diagnostic ignored "-Wwrite-strings" + +TEST_CASE_METHOD(TestModulesRedisCommandFixture, "Redis - command - SCAN", "[redis][command][SCAN]") { + SECTION("Unsupported TYPE token") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "TYPE"}, + "-ERR the TYPE parameter is not yet supported\r\n")); + } + + SECTION("Empty database") { + SECTION("No pattern and no count") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0"}, + "*2\r\n:0\r\n*0\r\n")); + } + + SECTION("No matching pattern and no count") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "nomatch"}, + "*2\r\n:0\r\n*0\r\n")); + } + + SECTION("No matching pattern and with count") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "nomatch", "COUNT", "10"}, + "*2\r\n:0\r\n*0\r\n")); + } + } + + SECTION("One key") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"MSET", "a_key", "b_value"}, + "+OK\r\n")); + + SECTION("With count") { + SECTION("No match") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "nomatch", "COUNT", "100"}, + "*2\r\n:687\r\n*0\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "687", "MATCH", "nomatch", "COUNT", "100"}, + "*2\r\n:0\r\n*0\r\n")); + } + + SECTION("Match - simple") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "a_key", "COUNT", "100"}, + "*2\r\n:687\r\n*1\r\n$5\r\na_key\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "687", "MATCH", "a_key", "COUNT", "100"}, + "*2\r\n:0\r\n*0\r\n")); + } + + SECTION("Only count") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "COUNT", "100"}, + "*2\r\n:687\r\n*1\r\n$5\r\na_key\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "687", "COUNT", "100"}, + "*2\r\n:0\r\n*0\r\n")); + } + } + + SECTION("With pattern") { + SECTION("No match") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "nomatch"}, + "*2\r\n:0\r\n*0\r\n")); + } + + SECTION("Match - simple") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "a_key"}, + "*2\r\n:0\r\n*1\r\n$5\r\na_key\r\n")); + } + + SECTION("Match - star") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "a_*"}, + "*2\r\n:0\r\n*1\r\n$5\r\na_key\r\n")); + } + + SECTION("Match - question mark") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "a_???"}, + "*2\r\n:0\r\n*1\r\n$5\r\na_key\r\n")); + } + + SECTION("Match - backslash") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "a\\_key"}, + "*2\r\n:0\r\n*1\r\n$5\r\na_key\r\n")); + } + + SECTION("Match - brackets") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "[a-z]_key"}, + "*2\r\n:0\r\n*1\r\n$5\r\na_key\r\n")); + } + + SECTION("Match - everything") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "*"}, + "*2\r\n:0\r\n*1\r\n$5\r\na_key\r\n")); + } + } + } + + SECTION("Multiple keys") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{ + "MSET", + "a_key", "a_value", + "b_key", "b_value", + "c_key", "c_value", + "d_key", "d_value", + "key_zzz", "value_z"}, + "+OK\r\n")); + + SECTION("With count") { + SECTION("No match") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "nomatch", "COUNT", "100"}, + "*2\r\n:281\r\n*0\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "281", "MATCH", "nomatch", "COUNT", "100"}, + "*2\r\n:687\r\n*0\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "687", "MATCH", "nomatch", "COUNT", "100"}, + "*2\r\n:841\r\n*0\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "841", "MATCH", "nomatch", "COUNT", "100"}, + "*2\r\n:0\r\n*0\r\n")); + } + + SECTION("Match - simple") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "a_key", "COUNT", "100"}, + "*2\r\n:281\r\n*0\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "281", "MATCH", "a_key", "COUNT", "100"}, + "*2\r\n:687\r\n*1\r\n$5\r\na_key\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "687", "MATCH", "a_key", "COUNT", "100"}, + "*2\r\n:841\r\n*0\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "841", "MATCH", "nomatch", "COUNT", "100"}, + "*2\r\n:0\r\n*0\r\n")); + } + + SECTION("Only count") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "COUNT", "100"}, + "*2\r\n:281\r\n*2\r\n$7\r\nkey_zzz\r\n$5\r\nb_key\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "281", "COUNT", "100"}, + "*2\r\n:687\r\n*1\r\n$5\r\na_key\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "687", "COUNT", "100"}, + "*2\r\n:841\r\n*2\r\n$5\r\nd_key\r\n$5\r\nc_key\r\n")); + + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "841", "COUNT", "100"}, + "*2\r\n:0\r\n*0\r\n")); + } + } + + SECTION("With pattern") { + SECTION("No match") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "nomatch"}, + "*2\r\n:0\r\n*0\r\n")); + } + + SECTION("Match - simple") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "a_key"}, + "*2\r\n:0\r\n*1\r\n$5\r\na_key\r\n")); + } + + SECTION("Match - star - 1 result") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "a_*"}, + "*2\r\n:0\r\n*1\r\n$5\r\na_key\r\n")); + } + + SECTION("Match - star - multiple results") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "*key"}, + "*2\r\n:0\r\n*4\r\n$5\r\nb_key\r\n$5\r\na_key\r\n$5\r\nd_key\r\n$5\r\nc_key\r\n")); + } + + SECTION("Match - question mark") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "a_???"}, + "*2\r\n:0\r\n*1\r\n$5\r\na_key\r\n")); + } + + SECTION("Match - backslash") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "a\\_key"}, + "*2\r\n:0\r\n*1\r\n$5\r\na_key\r\n")); + } + + SECTION("Match - brackets") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "[a-z]_key"}, + "*2\r\n:0\r\n*4\r\n$5\r\nb_key\r\n$5\r\na_key\r\n$5\r\nd_key\r\n$5\r\nc_key\r\n")); + } + + SECTION("Match - everything") { + REQUIRE(send_recv_resp_command_text( + client_fd, + std::vector{"SCAN", "0", "MATCH", "*"}, + "*2\r\n:0\r\n*5\r\n$7\r\nkey_zzz\r\n$5\r\nb_key\r\n$5\r\na_key\r\n$5\r\nd_key\r\n$5\r\nc_key\r\n")); + } + } + } +}