Skip to content

Commit

Permalink
ENG-2520 Additional data types for jsonb serialization.
Browse files Browse the repository at this point in the history 
Summary:
In this diff, I've added support for additional datatypes (int32, uint32, float, double,
int64 and uint32) in the jsonb serialization format. This diff completes all the numeric datatypes
that we need to support for json.

We still need to support arrays and I'll implement that in a separate diff.

Test Plan: Added new tests in jsonb-test

Reviewers: mikhail, robert, mihnea

Reviewed By: mihnea

Subscribers: ybase, yql

Differential Revision: https://phabricator.dev.yugabyte.com/D4204
  • Loading branch information
@pritamdamania87
pritamdamania87 committed Apr 5, 2018
1 parent 907243a commit 05d100c
Show file tree
Hide file tree
Showing 4 changed files with 149 additions and 18 deletions.
5 changes: 5 additions & 0 deletions src/yb/docdb/doc_kv_util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ inline int64_t DecodeInt64FromKey(const rocksdb::Slice& slice) {
return v ^ kInt64SignBitFlipMask;
}

inline int64_t DecodeInt32FromKey(const rocksdb::Slice& slice) {
uint32_t v = BigEndian::Load32(slice.data());
return v ^ kInt32SignBitFlipMask;
}

inline void AppendInt32ToKey(int32_t val, std::string* dest) {
char buf[sizeof(int32_t)];
BigEndian::Store32(buf, val ^ kInt32SignBitFlipMask);
Expand Down
65 changes: 62 additions & 3 deletions src/yb/docdb/jsonb-test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,42 @@
#include "yb/util/test_macros.h"
#include "yb/util/test_util.h"

using std::to_string;
using std::numeric_limits;

namespace yb {
namespace docdb {

TEST(JsonbTest, TestJsonbSerialization) {
std::string jsonb;
ASSERT_OK(Jsonb::ToJsonb(
"{ \"b\" : 1, \"a\" : { \"d\" : true, \"g\" : -100, \"c\" : false, \"f\" : \"hello\", "
"\"e\" : null} }", &jsonb));
ASSERT_OK(Jsonb::ToJsonb(R"#(
{
"b" : 1,
"a" :
{
"d" : true,
"q" :
{
"p" : 4294967295,
"r" : -2147483648,
"s" : 2147483647
},
"g" : -100,
"c" : false,
"f" : "hello",
"x" : 2.1,
"y" : 9223372036854775807,
"z" : -9223372036854775808,
"u" : 18446744073709551615,
"l" : 2147483647.123123e+75,
"e" : null
}
})#", &jsonb));
ASSERT_FALSE(jsonb.empty());

rapidjson::Document document;
ASSERT_OK(Jsonb::FromJsonb(jsonb, &document));

rapidjson::StringBuffer buffer;
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
document.Accept(writer);
Expand All @@ -52,6 +76,41 @@ TEST(JsonbTest, TestJsonbSerialization) {
ASSERT_TRUE(document["a"]["g"].IsInt64());
ASSERT_EQ(-100, document["a"]["g"].GetInt64());

ASSERT_TRUE(document["a"].HasMember("q"));
ASSERT_TRUE(document["a"]["q"].IsObject());

ASSERT_TRUE(document["a"]["q"].HasMember("p"));
ASSERT_TRUE(document["a"]["q"]["p"].IsUint());
ASSERT_EQ(4294967295, document["a"]["q"]["p"].GetUint());

ASSERT_TRUE(document["a"]["q"].HasMember("r"));
ASSERT_TRUE(document["a"]["q"]["r"].IsInt());
ASSERT_EQ(-2147483648, document["a"]["q"]["r"].GetInt());

ASSERT_TRUE(document["a"]["q"].HasMember("s"));
ASSERT_TRUE(document["a"]["q"]["s"].IsInt());
ASSERT_EQ(2147483647, document["a"]["q"]["s"].GetInt());

ASSERT_TRUE(document["a"].HasMember("x"));
ASSERT_TRUE(document["a"]["x"].IsFloat());
ASSERT_FLOAT_EQ(2.1f, document["a"]["x"].GetFloat());

ASSERT_TRUE(document["a"].HasMember("u"));
ASSERT_TRUE(document["a"]["u"].IsUint64());
ASSERT_EQ(18446744073709551615ULL, document["a"]["u"].GetUint64());

ASSERT_TRUE(document["a"].HasMember("y"));
ASSERT_TRUE(document["a"]["y"].IsInt64());
ASSERT_EQ(9223372036854775807LL, document["a"]["y"].GetInt64());

ASSERT_TRUE(document["a"].HasMember("z"));
ASSERT_TRUE(document["a"]["z"].IsInt64());
ASSERT_EQ(-9223372036854775808ULL, document["a"]["z"].GetInt64());

ASSERT_TRUE(document["a"].HasMember("l"));
ASSERT_TRUE(document["a"]["l"].IsDouble());
ASSERT_DOUBLE_EQ(2147483647.123123e+75, document["a"]["l"].GetDouble());

ASSERT_TRUE(document.HasMember("b"));
ASSERT_TRUE(document["b"].IsInt64());
ASSERT_EQ(1, document["b"].GetInt64());
Expand Down
90 changes: 76 additions & 14 deletions src/yb/docdb/jsonb.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,21 @@ CHECKED_STATUS Jsonb::ToJsonbInternal(const rapidjson::Value& document, std::str
RETURN_NOT_OK(ToJsonbInternal(value, jsonb));
break;
case rapidjson::Type::kNumberType:
// TODO: Need to support float, doubles and unsigned ints.
AppendBigEndianUInt64(value.GetInt64(), jsonb);
if (value.IsInt()) {
AppendInt32ToKey(value.GetInt(), jsonb);
} else if (value.IsUint()) {
AppendBigEndianUInt32(value.GetUint(), jsonb);
} else if (value.IsInt64()) {
AppendInt64ToKey(value.GetInt64(), jsonb);
} else if (value.IsUint64()) {
AppendBigEndianUInt64(value.GetUint64(), jsonb);
} else if (value.IsFloat()) {
AppendFloatToKey(value.GetFloat(), jsonb);
} else if (value.IsDouble()) {
AppendDoubleToKey(value.GetDouble(), jsonb);
} else {
return STATUS(NotSupported, "Numeric type is not supported");
}
break;
case rapidjson::Type::kStringType:
jsonb->append(value.GetString());
Expand All @@ -99,6 +112,7 @@ CHECKED_STATUS Jsonb::ToJsonbInternal(const rapidjson::Value& document, std::str
DCHECK_EQ(kv_pairs.size(), value_offsets.size());
int index = 0;
for (const auto& entry : kv_pairs) {
const rapidjson::Value& value = entry.second;
JEntry jentry = value_offsets[index] & kJEOffsetMask;
switch (entry.second.GetType()) {
case rapidjson::Type::kNullType:
Expand All @@ -117,7 +131,21 @@ CHECKED_STATUS Jsonb::ToJsonbInternal(const rapidjson::Value& document, std::str
jentry |= kJEIsContainer;
break;
case rapidjson::Type::kNumberType:
jentry |= kJEIsNumeric;
if (value.IsInt()) {
jentry |= kJEIsInt;
} else if (value.IsUint()) {
jentry |= kJEIsUInt;
} else if (value.IsInt64()) {
jentry |= kJEIsInt64;
} else if (value.IsUint64()) {
jentry |= kJEIsUInt64;
} else if (value.IsFloat()) {
jentry |= kJEIsFloat;
} else if (value.IsDouble()) {
jentry |= kJEIsDouble;
} else {
return STATUS(NotSupported, "Numeric type is not supported");
}
break;
case rapidjson::Type::kStringType:
jentry |= kJEIsString;
Expand All @@ -135,6 +163,16 @@ CHECKED_STATUS Jsonb::ToJsonbInternal(const rapidjson::Value& document, std::str
return Status::OK();
}

namespace {

template <typename T>
void AddNumericMember(rapidjson::Document* document, const string& key, T value) {
document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()),
rapidjson::Value(value),
document->GetAllocator());
}
} // anonymous namespace

Status Jsonb::FromJsonbInternal(const std::string& jsonb, size_t offset,
rapidjson::Document* document) {
DCHECK_LT(offset, jsonb.size());
Expand Down Expand Up @@ -179,35 +217,59 @@ Status Jsonb::FromJsonbInternal(const std::string& jsonb, size_t offset,
metadata_begin_offset);
DCHECK_LE(value_offset + value_length, jsonb.size());

rapidjson::Value json_key(key.c_str(), key.size(), document->GetAllocator());
switch (value_metadata & kJETypeMask) {
case kJEIsString: {
const std::string &value = jsonb.substr(value_offset, value_length);
document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()),
document->AddMember(json_key,
rapidjson::Value(value.c_str(), value.size(), document->GetAllocator()),
document->GetAllocator());
break;
}
case kJEIsNumeric: {
int64_t value = BigEndian::Load64(&jsonb[value_offset]);
document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()),
rapidjson::Value(value),
document->GetAllocator());
case kJEIsInt: {
int32_t value = DecodeInt32FromKey(&jsonb[value_offset]);
AddNumericMember(document, key, value);
break;
}
case kJEIsUInt: {
uint32_t value = BigEndian::Load32(&jsonb[value_offset]);
AddNumericMember(document, key, value);
break;
}
case kJEIsInt64: {
int64_t value = DecodeInt64FromKey(&jsonb[value_offset]);
AddNumericMember(document, key, value);
break;
}
case kJEIsUInt64: {
uint64_t value = BigEndian::Load64(&jsonb[value_offset]);
AddNumericMember(document, key, value);
break;
}
case kJEIsDouble: {
double value = DecodeDoubleFromKey(&jsonb[value_offset]);
AddNumericMember(document, key, value);
break;
}
case kJEIsFloat: {
float value = DecodeFloatFromKey(&jsonb[value_offset]);
AddNumericMember(document, key, value);
break;
}
case kJEIsBoolFalse: {
document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()),
document->AddMember(json_key,
rapidjson::Value(false),
document->GetAllocator());
break;
}
case kJEIsBoolTrue: {
document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()),
document->AddMember(json_key,
rapidjson::Value(true),
document->GetAllocator());
break;
}
case kJEIsNull: {
document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()),
document->AddMember(json_key,
rapidjson::Value(rapidjson::Type::kNullType),
document->GetAllocator());
break;
Expand All @@ -216,8 +278,8 @@ Status Jsonb::FromJsonbInternal(const std::string& jsonb, size_t offset,
rapidjson::Document nested_container(&document->GetAllocator());
nested_container.SetObject();
RETURN_NOT_OK(FromJsonbInternal(jsonb, value_offset, &nested_container));
document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()),
nested_container,
document->AddMember(json_key,
std::move(nested_container),
document->GetAllocator());
break;
}
Expand Down
7 changes: 6 additions & 1 deletion src/yb/docdb/jsonb.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,16 @@ class Jsonb {

// Values stored in the type bits.
static constexpr uint32_t kJEIsString = 0x00000000;
static constexpr uint32_t kJEIsNumeric = 0x10000000;
static constexpr uint32_t kJEIsBoolFalse = 0x20000000;
static constexpr uint32_t kJEIsBoolTrue = 0x30000000;
static constexpr uint32_t kJEIsNull = 0x40000000;
static constexpr uint32_t kJEIsContainer = 0x50000000; // could be array or object.
static constexpr uint32_t kJEIsInt = 0x60000000;
static constexpr uint32_t kJEIsUInt = 0x70000000;
static constexpr uint32_t kJEIsInt64 = 0x80000000;
static constexpr uint32_t kJEIsUInt64 = 0x90000000;
static constexpr uint32_t kJEIsFloat = 0xA0000000;
static constexpr uint32_t kJEIsDouble = 0xB0000000;
};

} // namespace docdb
Expand Down

0 comments on commit 05d100c

Please sign in to comment.