BitMap support in slimredis [single-bit version]

src/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_subdirectory(core ${PROJECT_BINARY_DIR}/core)
+add_subdirectory(data_structure ${PROJECT_BINARY_DIR}/data_structure)
 add_subdirectory(hotkey ${PROJECT_BINARY_DIR}/hotkey)
 add_subdirectory(protocol ${PROJECT_BINARY_DIR}/protocol)
 add_subdirectory(storage ${PROJECT_BINARY_DIR}/storage)

src/data_structure/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(bitmap)

src/data_structure/bitmap/CMakeLists.txt

@@ -0,0 +1 @@
+add_library(ds_bitmap bitset.c)

src/data_structure/bitmap/README.md

@@ -0,0 +1,7 @@
+Many common bitmap functionalities are implemented in [CRoaring](https://github.com/RoaringBitmap/CRoaring).
+
+For small bitmaps (<1k columns), many of the techniques in more sophisticated
+algorithms, such as Roaring simply aren't applicable, hence there is still
+value in using a simple bitset for such cases.
+However, if we ever need sophisticated bitmaps, we should consider linking
+against the Roaring library or others instead of implementing our own.

src/data_structure/bitmap/bitset.c

@@ -0,0 +1,43 @@
+#include "bitset.h"
+
+#include <stddef.h>
+#include <string.h>
+
+
+#define DATA_OFFSET(_bs) ((uint8_t *)(_bs) + offsetof(struct bitset, data))
+#define SEGMENT_OFFSET(_col) ((uint8_t)(_col) >> 5) /* uint32_t, 2^5 bits per segment */
+#define BIT_OFFSET(_col) ((uint8_t)(_col) & 0x1f)
+#define GET_SEGMENT(_bs, _col)  \
+        ((uint32_t *)DATA_OFFSET(_bs) + SEGMENT_OFFSET(_col))
+#define GET_COL(_v, _offset) (((uint32_t)(_v) >> (_offset)) & 1UL)
+
+void
+bitset_init(struct bitset *bs, uint16_t ncol)
+{
+    uint8_t *d = (uint8_t *)DATA_OFFSET(bs);
+
+    bs->size = (uint8_t)bit2long(ncol);
+    bs->col_w = 1;
+    bs->count = 0;
+    memset(d, 0, size2byte(bs->size));
+}
+
+uint8_t
+bitset_get(struct bitset *bs, uint16_t col)
+{
+    return GET_COL(*GET_SEGMENT(bs, col), BIT_OFFSET(col));
+}
+
+void
+bitset_set(struct bitset *bs, uint16_t col, uint8_t val)
+{
+    uint8_t offset = BIT_OFFSET(col);
+    uint32_t *d = (uint32_t *)GET_SEGMENT(bs, col);
+
+    bs->count += (val != 0) - (bitset_get(bs, col) != 0);
+
+    /* clear column */
+    *d &= ~(1UL << offset);
+    /* set column */
+    *d |= (uint32_t)val << offset;
+}

src/data_structure/bitmap/bitset.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <stdint.h>
+
+/* References:
+ * https://github.com/RoaringBitmap/CRoaring/blob/master/include/roaring/bitset_util.h
+ * https://stackoverflow.com/questions/47981/how-do-you-set-clear-and-toggle-a-single-bit
+ */
+
+/* fit in a 255-byte cuckoo cell with cuckoo header (6) and cas (8), bitset header (4) */
+#define BITSET_COL_MAX (250 * 32)
+
+/* NOTE: bitset must be allocated as 32-bit aligned */
+struct bitset {
+    uint8_t size;   /* in uint32_t => bitset can at most be 255*4 bytes */
+    uint8_t col_w;  /* column width, defaults to 1 (bit), up to 8 (1 byte) */
+    uint16_t count; /* non-zero column count */
+    char data[1];   /* actual bitset data */
+};
+
+#define bit2byte(_col) ((((_col) - 1) >> 3) + 1)
+#define bit2long(_col) ((((_col) - 1) >> 5) + 1)
+#define size2bit(_sz) ((_sz) << 5)
+#define size2byte(_sz) ((_sz) << 2)
+
+void bitset_init(struct bitset *bs, uint16_t ncol);
+
+uint8_t bitset_get(struct bitset *bs, uint16_t col);
+/* Note: the interface is written as a generic set function with a val parameter
+ * instead of two functions, set & clear, because we want to later support
+ * multi-bit columns (up to a byte), so the values may go beyond 0 & 1
+ */
+void bitset_set(struct bitset *bs, uint16_t col, uint8_t val);

src/protocol/data/redis/cmd_bitmap.h

@@ -0,0 +1,41 @@
+#pragma once
+
+/**
+ * create: create a bitmap of certain size, all bits reset unless value given
+ * BitMap.create KEY size
+ * Note: if size is not a multiple of the internal allocation unit (e.g. byte),
+ * it will be rounded up internally
+ * TODO: how to transfer value w/o being misrepresented due to endianness?
+ * until we figure that out we shouldn't allow user to initialize w/ value
+ *
+ * delete: delete a bitmap
+ * BitMap.delete KEY
+ *
+ * get: get value of a column in a bitmap
+ * BitMap.get KEY columnId
+ *
+ * set: set value of a column in a bitmap
+ * BitMap.set KEY columnId val
+ */
+
+/* TODO:
+ * - variable-width columns. this PR will implement only 1-bit columns
+ * - metadata: this will allow simple customization such as softTTL, timestamp
+ *   or other info. This is the same idea behind memcached's `flag' field, but
+ *   it's better to make it optional instead of allocating a fixed sized region
+ *   for all commands and all data types.
+ */
+
+/*          type                string              #arg    #opt */
+#define REQ_BITMAP(ACTION)                                      \
+    ACTION( REQ_BITMAP_CREATE,  "BitMap.create",    3,      0  )\
+    ACTION( REQ_BITMAP_DELETE,  "BitMap.delete",    2,      0  )\
+    ACTION( REQ_BITMAP_GET,     "BitMap.get",       3,      0  )\
+    ACTION( REQ_BITMAP_SET,     "BitMap.set",       4,      0  )
+
+typedef enum bitmap_elem {
+    BITMAP_VERB = 0,
+    BITMAP_KEY = 1,
+    BITMAP_COL = 2,
+    BITMAP_VAL = 3,
+} bitmap_elem_e;

src/protocol/data/redis/cmd_hash.h

@@ -2,7 +2,7 @@
 
 
 /*          type                string          # of args */
-#define REQ_HASH(ACTION)                            \
+#define REQ_HASH(ACTION)                                \
     ACTION( REQ_HDEL,           "hdel",         3,  -1 )\
     ACTION( REQ_HDELALL,        "hdelall",      2,   0 )\
     ACTION( REQ_HEXISTS,        "hexists",      3,   0 )\

src/protocol/data/redis/request.c

@@ -18,6 +18,7 @@ static request_metrics_st *request_metrics = NULL;
     .nopt = _nopt },
 struct command command_table[REQ_SENTINEL] = {
     { .type = REQ_UNKNOWN, .bstr = { 0, NULL }, .narg = 0, .nopt = 0 },
+    REQ_BITMAP(CMD_INIT)
     REQ_HASH(CMD_INIT)
     REQ_ZSET(CMD_INIT)
     REQ_MISC(CMD_INIT)

src/protocol/data/redis/request.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "cmd_bitmap.h"
 #include "cmd_hash.h"
 #include "cmd_misc.h"
 #include "cmd_zset.h"
@@ -42,6 +43,7 @@ typedef struct {
 #define GET_TYPE(_type, _str, narg, nopt) _type,
 typedef enum cmd_type {
     REQ_UNKNOWN,
+    REQ_BITMAP(GET_TYPE)
     REQ_HASH(GET_TYPE)
     REQ_ZSET(GET_TYPE)
     REQ_MISC(GET_TYPE)

src/protocol/data/redis/response.h

@@ -40,8 +40,16 @@ typedef struct {
  * - a dummy entry RSP_UNKNOWN so we can use it as the initial type value;
  * - a RSP_NUMERIC type that doesn't have a corresponding message body.
  */
-#define RSP_STR_OK "OK"
-#define RSP_PONG "pong"
+#define RSP_OK "OK"
+#define RSP_NOTFOUND "NOT_FOUND"
+#define RSP_PONG "PONG"
+#define RSP_EXIST "EXIST" /* key already exists and op is non-overwriting */
+
+#define RSP_ERR_ARG "Err invalid argument"
+#define RSP_ERR_NOSUPPORT "Err command not supported"
+#define RSP_ERR_OUTOFRANGE "Err index out of range"
+#define RSP_ERR_STORAGE "Err storage failure"
+#define RSP_ERR_TYPE "Err type mismatch"
 
 /*
  * NOTE(yao): we store fields as location in rbuf, this assumes the data will

src/server/CMakeLists.txt

@@ -2,7 +2,7 @@ if(TARGET_PINGSERVER)
     add_subdirectory(pingserver)
 endif()
 
-if(TARGET_REDIS)
+if(TARGET_SLIMREDIS)
     add_subdirectory(slimredis)
 endif()
 

src/server/slimredis/CMakeLists.txt

@@ -9,6 +9,8 @@ set(SOURCE
 
 set(MODULES
     core
+    cuckoo
+    ds_bitmap
     protocol_admin
     protocol_redis
     slab

src/server/slimredis/data/CMakeLists.txt

@@ -2,4 +2,5 @@ set(SOURCE
     ${SOURCE}
     ${CMAKE_CURRENT_SOURCE_DIR}/process.c
     ${CMAKE_CURRENT_SOURCE_DIR}/cmd_misc.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/cmd_bitmap.c
     PARENT_SCOPE)