From 05d100cb257fa125a7a9a67010b26227f2ac8275 Mon Sep 17 00:00:00 2001 From: Pritam Damania Date: Thu, 22 Feb 2018 18:51:35 +0530 Subject: [PATCH] ENG-2520 Additional data types for jsonb serialization. Summary: In this diff, I've added support for additional datatypes (int32, uint32, float, double, int64 and uint32) in the jsonb serialization format. This diff completes all the numeric datatypes that we need to support for json. We still need to support arrays and I'll implement that in a separate diff. Test Plan: Added new tests in jsonb-test Reviewers: mikhail, robert, mihnea Reviewed By: mihnea Subscribers: ybase, yql Differential Revision: https://phabricator.dev.yugabyte.com/D4204 --- src/yb/docdb/doc_kv_util.h | 5 +++ src/yb/docdb/jsonb-test.cc | 65 +++++++++++++++++++++++++-- src/yb/docdb/jsonb.cc | 90 ++++++++++++++++++++++++++++++++------ src/yb/docdb/jsonb.h | 7 ++- 4 files changed, 149 insertions(+), 18 deletions(-) diff --git a/src/yb/docdb/doc_kv_util.h b/src/yb/docdb/doc_kv_util.h index c9d555dd0185..9a7b00ee15ae 100644 --- a/src/yb/docdb/doc_kv_util.h +++ b/src/yb/docdb/doc_kv_util.h @@ -90,6 +90,11 @@ inline int64_t DecodeInt64FromKey(const rocksdb::Slice& slice) { return v ^ kInt64SignBitFlipMask; } +inline int64_t DecodeInt32FromKey(const rocksdb::Slice& slice) { + uint32_t v = BigEndian::Load32(slice.data()); + return v ^ kInt32SignBitFlipMask; +} + inline void AppendInt32ToKey(int32_t val, std::string* dest) { char buf[sizeof(int32_t)]; BigEndian::Store32(buf, val ^ kInt32SignBitFlipMask); diff --git a/src/yb/docdb/jsonb-test.cc b/src/yb/docdb/jsonb-test.cc index 5ce25db19d60..91b6d6f5b474 100644 --- a/src/yb/docdb/jsonb-test.cc +++ b/src/yb/docdb/jsonb-test.cc @@ -18,18 +18,42 @@ #include "yb/util/test_macros.h" #include "yb/util/test_util.h" +using std::to_string; +using std::numeric_limits; + namespace yb { namespace docdb { TEST(JsonbTest, TestJsonbSerialization) { std::string jsonb; - ASSERT_OK(Jsonb::ToJsonb( - "{ \"b\" : 1, \"a\" : { \"d\" : true, \"g\" : -100, \"c\" : false, \"f\" : \"hello\", " - "\"e\" : null} }", &jsonb)); + ASSERT_OK(Jsonb::ToJsonb(R"#( + { + "b" : 1, + "a" : + { + "d" : true, + "q" : + { + "p" : 4294967295, + "r" : -2147483648, + "s" : 2147483647 + }, + "g" : -100, + "c" : false, + "f" : "hello", + "x" : 2.1, + "y" : 9223372036854775807, + "z" : -9223372036854775808, + "u" : 18446744073709551615, + "l" : 2147483647.123123e+75, + "e" : null + } + })#", &jsonb)); ASSERT_FALSE(jsonb.empty()); rapidjson::Document document; ASSERT_OK(Jsonb::FromJsonb(jsonb, &document)); + rapidjson::StringBuffer buffer; rapidjson::PrettyWriter writer(buffer); document.Accept(writer); @@ -52,6 +76,41 @@ TEST(JsonbTest, TestJsonbSerialization) { ASSERT_TRUE(document["a"]["g"].IsInt64()); ASSERT_EQ(-100, document["a"]["g"].GetInt64()); + ASSERT_TRUE(document["a"].HasMember("q")); + ASSERT_TRUE(document["a"]["q"].IsObject()); + + ASSERT_TRUE(document["a"]["q"].HasMember("p")); + ASSERT_TRUE(document["a"]["q"]["p"].IsUint()); + ASSERT_EQ(4294967295, document["a"]["q"]["p"].GetUint()); + + ASSERT_TRUE(document["a"]["q"].HasMember("r")); + ASSERT_TRUE(document["a"]["q"]["r"].IsInt()); + ASSERT_EQ(-2147483648, document["a"]["q"]["r"].GetInt()); + + ASSERT_TRUE(document["a"]["q"].HasMember("s")); + ASSERT_TRUE(document["a"]["q"]["s"].IsInt()); + ASSERT_EQ(2147483647, document["a"]["q"]["s"].GetInt()); + + ASSERT_TRUE(document["a"].HasMember("x")); + ASSERT_TRUE(document["a"]["x"].IsFloat()); + ASSERT_FLOAT_EQ(2.1f, document["a"]["x"].GetFloat()); + + ASSERT_TRUE(document["a"].HasMember("u")); + ASSERT_TRUE(document["a"]["u"].IsUint64()); + ASSERT_EQ(18446744073709551615ULL, document["a"]["u"].GetUint64()); + + ASSERT_TRUE(document["a"].HasMember("y")); + ASSERT_TRUE(document["a"]["y"].IsInt64()); + ASSERT_EQ(9223372036854775807LL, document["a"]["y"].GetInt64()); + + ASSERT_TRUE(document["a"].HasMember("z")); + ASSERT_TRUE(document["a"]["z"].IsInt64()); + ASSERT_EQ(-9223372036854775808ULL, document["a"]["z"].GetInt64()); + + ASSERT_TRUE(document["a"].HasMember("l")); + ASSERT_TRUE(document["a"]["l"].IsDouble()); + ASSERT_DOUBLE_EQ(2147483647.123123e+75, document["a"]["l"].GetDouble()); + ASSERT_TRUE(document.HasMember("b")); ASSERT_TRUE(document["b"].IsInt64()); ASSERT_EQ(1, document["b"].GetInt64()); diff --git a/src/yb/docdb/jsonb.cc b/src/yb/docdb/jsonb.cc index 014f68d9a170..0e292d34760e 100644 --- a/src/yb/docdb/jsonb.cc +++ b/src/yb/docdb/jsonb.cc @@ -78,8 +78,21 @@ CHECKED_STATUS Jsonb::ToJsonbInternal(const rapidjson::Value& document, std::str RETURN_NOT_OK(ToJsonbInternal(value, jsonb)); break; case rapidjson::Type::kNumberType: - // TODO: Need to support float, doubles and unsigned ints. - AppendBigEndianUInt64(value.GetInt64(), jsonb); + if (value.IsInt()) { + AppendInt32ToKey(value.GetInt(), jsonb); + } else if (value.IsUint()) { + AppendBigEndianUInt32(value.GetUint(), jsonb); + } else if (value.IsInt64()) { + AppendInt64ToKey(value.GetInt64(), jsonb); + } else if (value.IsUint64()) { + AppendBigEndianUInt64(value.GetUint64(), jsonb); + } else if (value.IsFloat()) { + AppendFloatToKey(value.GetFloat(), jsonb); + } else if (value.IsDouble()) { + AppendDoubleToKey(value.GetDouble(), jsonb); + } else { + return STATUS(NotSupported, "Numeric type is not supported"); + } break; case rapidjson::Type::kStringType: jsonb->append(value.GetString()); @@ -99,6 +112,7 @@ CHECKED_STATUS Jsonb::ToJsonbInternal(const rapidjson::Value& document, std::str DCHECK_EQ(kv_pairs.size(), value_offsets.size()); int index = 0; for (const auto& entry : kv_pairs) { + const rapidjson::Value& value = entry.second; JEntry jentry = value_offsets[index] & kJEOffsetMask; switch (entry.second.GetType()) { case rapidjson::Type::kNullType: @@ -117,7 +131,21 @@ CHECKED_STATUS Jsonb::ToJsonbInternal(const rapidjson::Value& document, std::str jentry |= kJEIsContainer; break; case rapidjson::Type::kNumberType: - jentry |= kJEIsNumeric; + if (value.IsInt()) { + jentry |= kJEIsInt; + } else if (value.IsUint()) { + jentry |= kJEIsUInt; + } else if (value.IsInt64()) { + jentry |= kJEIsInt64; + } else if (value.IsUint64()) { + jentry |= kJEIsUInt64; + } else if (value.IsFloat()) { + jentry |= kJEIsFloat; + } else if (value.IsDouble()) { + jentry |= kJEIsDouble; + } else { + return STATUS(NotSupported, "Numeric type is not supported"); + } break; case rapidjson::Type::kStringType: jentry |= kJEIsString; @@ -135,6 +163,16 @@ CHECKED_STATUS Jsonb::ToJsonbInternal(const rapidjson::Value& document, std::str return Status::OK(); } +namespace { + +template +void AddNumericMember(rapidjson::Document* document, const string& key, T value) { + document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()), + rapidjson::Value(value), + document->GetAllocator()); +} +} // anonymous namespace + Status Jsonb::FromJsonbInternal(const std::string& jsonb, size_t offset, rapidjson::Document* document) { DCHECK_LT(offset, jsonb.size()); @@ -179,35 +217,59 @@ Status Jsonb::FromJsonbInternal(const std::string& jsonb, size_t offset, metadata_begin_offset); DCHECK_LE(value_offset + value_length, jsonb.size()); + rapidjson::Value json_key(key.c_str(), key.size(), document->GetAllocator()); switch (value_metadata & kJETypeMask) { case kJEIsString: { const std::string &value = jsonb.substr(value_offset, value_length); - document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()), + document->AddMember(json_key, rapidjson::Value(value.c_str(), value.size(), document->GetAllocator()), document->GetAllocator()); break; } - case kJEIsNumeric: { - int64_t value = BigEndian::Load64(&jsonb[value_offset]); - document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()), - rapidjson::Value(value), - document->GetAllocator()); + case kJEIsInt: { + int32_t value = DecodeInt32FromKey(&jsonb[value_offset]); + AddNumericMember(document, key, value); + break; + } + case kJEIsUInt: { + uint32_t value = BigEndian::Load32(&jsonb[value_offset]); + AddNumericMember(document, key, value); + break; + } + case kJEIsInt64: { + int64_t value = DecodeInt64FromKey(&jsonb[value_offset]); + AddNumericMember(document, key, value); + break; + } + case kJEIsUInt64: { + uint64_t value = BigEndian::Load64(&jsonb[value_offset]); + AddNumericMember(document, key, value); + break; + } + case kJEIsDouble: { + double value = DecodeDoubleFromKey(&jsonb[value_offset]); + AddNumericMember(document, key, value); + break; + } + case kJEIsFloat: { + float value = DecodeFloatFromKey(&jsonb[value_offset]); + AddNumericMember(document, key, value); break; } case kJEIsBoolFalse: { - document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()), + document->AddMember(json_key, rapidjson::Value(false), document->GetAllocator()); break; } case kJEIsBoolTrue: { - document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()), + document->AddMember(json_key, rapidjson::Value(true), document->GetAllocator()); break; } case kJEIsNull: { - document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()), + document->AddMember(json_key, rapidjson::Value(rapidjson::Type::kNullType), document->GetAllocator()); break; @@ -216,8 +278,8 @@ Status Jsonb::FromJsonbInternal(const std::string& jsonb, size_t offset, rapidjson::Document nested_container(&document->GetAllocator()); nested_container.SetObject(); RETURN_NOT_OK(FromJsonbInternal(jsonb, value_offset, &nested_container)); - document->AddMember(rapidjson::Value(key.c_str(), key.size(), document->GetAllocator()), - nested_container, + document->AddMember(json_key, + std::move(nested_container), document->GetAllocator()); break; } diff --git a/src/yb/docdb/jsonb.h b/src/yb/docdb/jsonb.h index fb6621fb6afe..3c2bd143b7a8 100644 --- a/src/yb/docdb/jsonb.h +++ b/src/yb/docdb/jsonb.h @@ -83,11 +83,16 @@ class Jsonb { // Values stored in the type bits. static constexpr uint32_t kJEIsString = 0x00000000; - static constexpr uint32_t kJEIsNumeric = 0x10000000; static constexpr uint32_t kJEIsBoolFalse = 0x20000000; static constexpr uint32_t kJEIsBoolTrue = 0x30000000; static constexpr uint32_t kJEIsNull = 0x40000000; static constexpr uint32_t kJEIsContainer = 0x50000000; // could be array or object. + static constexpr uint32_t kJEIsInt = 0x60000000; + static constexpr uint32_t kJEIsUInt = 0x70000000; + static constexpr uint32_t kJEIsInt64 = 0x80000000; + static constexpr uint32_t kJEIsUInt64 = 0x90000000; + static constexpr uint32_t kJEIsFloat = 0xA0000000; + static constexpr uint32_t kJEIsDouble = 0xB0000000; }; } // namespace docdb