feat(Ranger): Support encode and decode Ranger policies (#1378)

#1054

This patch is to prepare for parse policies and dump policies:
- 'DEFINE_JSON_SERIALIZATION' for data structure.
- Preparations for json parsing
- add unit test for 'parse_policies_from_json'

src/common/json_helper.h

@@ -25,14 +25,15 @@
  */
 #pragma once
 
-#include <vector>
+#include <cctype>
 #include <map>
-#include <unordered_map>
 #include <set>
 #include <sstream>
 #include <string>
 #include <type_traits>
-#include <cctype>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
 
 #include <rapidjson/ostreamwrapper.h>
 #include <rapidjson/prettywriter.h>
@@ -57,6 +58,7 @@
 #include "consensus_types.h"
 #include "replica_admin_types.h"
 #include "common/replication_enums.h"
+#include "runtime/ranger/access_type.h"
 
 #define JSON_ENCODE_ENTRY(out, prefix, T)                                                          \
     out.Key(#T);                                                                                   \
@@ -335,6 +337,23 @@ INT_TYPE_SERIALIZATION(int16_t)
 INT_TYPE_SERIALIZATION(int32_t)
 INT_TYPE_SERIALIZATION(int64_t)
 
+template <typename Writer>
+inline void json_encode(Writer &out, dsn::ranger::access_type t)
+{
+    out.Uint64(static_cast<uint64_t>(t));
+}
+
+inline bool json_decode(const JsonObject &in, dsn::ranger::access_type &t)
+{
+    dverify(in.IsUint64());
+    auto ans = in.GetUint64();
+    auto act_none = static_cast<uint64_t>(dsn::ranger::kAccessTypeNone);
+    auto act_all = static_cast<uint64_t>(dsn::ranger::kAccessTypeAll);
+    dverify(ans >= act_none && ans <= act_all);
+    t = static_cast<dsn::ranger::access_type>(ans);
+    return true;
+}
+
 // json serialization for uint types
 #define UINT_TYPE_SERIALIZATION(TName)                                                             \
     template <typename Writer>                                                                     \
@@ -452,6 +471,19 @@ inline bool json_decode_map(const JsonObject &in, TMap &t)
     return true;
 }
 
+template <typename TSet>
+inline bool json_decode_set(const JsonObject &in, TSet &t)
+{
+    dverify(in.IsArray());
+    t.clear();
+    for (rapidjson::Value::ConstValueIterator it = in.Begin(); it != in.End(); ++it) {
+        typename TSet::value_type value;
+        dverify(json_forwarder<decltype(value)>::decode(*it, value));
+        dverify(t.emplace(std::move(value)).second);
+    }
+    return true;
+}
+
 template <typename T>
 inline void json_encode(JsonWriter &out, const std::vector<T> &t)
 {
@@ -482,15 +514,19 @@ inline void json_encode(JsonWriter &out, const std::set<T> &t)
 template <typename T>
 inline bool json_decode(const JsonObject &in, std::set<T> &t)
 {
-    dverify(in.IsArray());
-    t.clear();
+    return json_decode_set(in, t);
+}
 
-    for (rapidjson::Value::ConstValueIterator it = in.Begin(); it != in.End(); ++it) {
-        T value;
-        dverify(json_forwarder<T>::decode(*it, value));
-        dverify(t.emplace(std::move(value)).second);
-    }
-    return true;
+template <typename T>
+inline void json_encode(JsonWriter &out, const std::unordered_set<T> &t)
+{
+    json_encode_iterable(out, t);
+}
+
+template <typename T>
+inline bool json_decode(const JsonObject &in, std::unordered_set<T> &t)
+{
+    return json_decode_set(in, t);
 }
 
 template <typename T1, typename T2>

src/runtime/ranger/access_type.cpp

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <type_traits>
+
+#include "access_type.h"
+
+namespace dsn {
+namespace ranger {
+
+access_type operator|(access_type lhs, access_type rhs)
+{
+    return access_type(static_cast<act>(lhs) | static_cast<act>(rhs));
+}
+
+access_type operator&(access_type lhs, access_type rhs)
+{
+    return access_type(static_cast<act>(lhs) & static_cast<act>(rhs));
+}
+
+access_type &operator|=(access_type &lhs, access_type rhs) { return lhs = lhs | rhs; }
+
+} // namespace ranger
+} // namespace dsn

src/runtime/ranger/access_type.h

@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstdint>
+
+namespace dsn {
+namespace ranger {
+
+// ACL type defined in Range service for RPC matching policy
+enum class access_type : uint8_t
+{
+    kInvalid = 0,
+    kRead = 1,
+    kWrite = 1 << 1,
+    kCreate = 1 << 2,
+    kDrop = 1 << 3,
+    kList = 1 << 4,
+    kMetadata = 1 << 5,
+    kControl = 1 << 6
+};
+
+using act = std::underlying_type<access_type>::type;
+
+access_type operator|(access_type lhs, access_type rhs);
+
+access_type operator&(access_type lhs, access_type rhs);
+
+access_type &operator|=(access_type &lhs, access_type rhs);
+
+const access_type kAccessTypeNone = access_type::kInvalid;
+const access_type kAccessTypeAll = access_type::kRead | access_type::kWrite | access_type::kCreate |
+                                   access_type::kDrop | access_type::kList |
+                                   access_type::kMetadata | access_type::kControl;
+
+} // namespace ranger
+} // namespace dsn

src/runtime/ranger/ranger_resource_policy.cpp

@@ -20,18 +20,6 @@
 namespace dsn {
 namespace ranger {
 
-/*extern*/ access_type operator|(access_type lhs, access_type rhs)
-{
-    using act = std::underlying_type<access_type>::type;
-    return access_type(static_cast<act>(lhs) | static_cast<act>(rhs));
-}
-
-/*extern*/ access_type operator&(access_type lhs, access_type rhs)
-{
-    using act = std::underlying_type<access_type>::type;
-    return access_type(static_cast<act>(lhs) & static_cast<act>(rhs));
-}
-
 bool policy_item::match(const access_type &ac_type, const std::string &user_name) const
 {
     return static_cast<bool>(access_types & ac_type) && users.count(user_name) != 0;

src/runtime/ranger/ranger_resource_policy.h

@@ -24,34 +24,21 @@
 
 #include <rapidjson/document.h>
 
+#include "access_type.h"
 #include "common/json_helper.h"
 #include "utils/fmt_logging.h"
 
 namespace dsn {
 namespace ranger {
 
-// ACL type defined in Range service for RPC matching policy
-enum class access_type : uint8_t
-{
-    KRead = 1,
-    KWrite = 1 << 1,
-    KCreate = 1 << 2,
-    KDrop = 1 << 3,
-    KList = 1 << 4,
-    KMetadata = 1 << 5,
-    KControl = 1 << 6
-};
-
-extern access_type operator|(access_type lhs, access_type rhs);
-
-extern access_type operator&(access_type lhs, access_type rhs);
-
 // Ranger policy data structure
 struct policy_item
 {
-    access_type access_types;
+    access_type access_types = access_type::kInvalid;
     std::unordered_set<std::string> users;
 
+    DEFINE_JSON_SERIALIZATION(access_types, users);
+
     // Check if the 'acl_type' - 'user_name' pair is matched to the policy.
     // Return true if it is matched, otherwise return false.
     // TODO(wanghao): add benchmark test
@@ -68,6 +55,11 @@ struct acl_policies
     std::vector<policy_item> deny_policies;
     std::vector<policy_item> deny_policies_exclude;
 
+    DEFINE_JSON_SERIALIZATION(allow_policies,
+                              allow_policies_exclude,
+                              deny_policies,
+                              deny_policies_exclude);
+
     // Check whether the 'user_name' is allowed to access the resource by type of 'ac_type'.
     bool allowed(const access_type &ac_type, const std::string &user_name) const;
 };
@@ -79,6 +71,8 @@ struct ranger_resource_policy
     std::unordered_set<std::string> database_names;
     std::unordered_set<std::string> table_names;
     acl_policies policies;
+
+    DEFINE_JSON_SERIALIZATION(name, database_names, table_names, policies)
 };
 
 } // namespace ranger

src/runtime/ranger/ranger_resource_policy_manager.cpp

@@ -22,6 +22,34 @@
 namespace dsn {
 namespace ranger {
 
+#define RETURN_ERR_IF_MISSING_MEMBER(obj, member)                                                  \
+    do {                                                                                           \
+        if (!obj.IsObject() || !obj.HasMember(member)) {                                           \
+            return dsn::ERR_RANGER_PARSE_ACL;                                                      \
+        }                                                                                          \
+    } while (0)
+
+#define CONTINUE_IF_MISSING_MEMBER(obj, member)                                                    \
+    do {                                                                                           \
+        if (!obj.IsObject() || !obj.HasMember(member)) {                                           \
+            continue;                                                                              \
+        }                                                                                          \
+    } while (0)
+
+#define RETURN_ERR_IF_NOT_ARRAY(obj)                                                               \
+    do {                                                                                           \
+        if (!obj.IsArray() || obj.Empty()) {                                                       \
+            return dsn::ERR_RANGER_PARSE_ACL;                                                      \
+        }                                                                                          \
+    } while (0)
+
+#define RETURN_VOID_IF_NOT_ARRAY(obj)                                                              \
+    do {                                                                                           \
+        if (!obj.IsArray() || obj.Empty()) {                                                       \
+            return;                                                                                \
+        }                                                                                          \
+    } while (0)
+
 namespace {
 // Register access types of 'rpc_codes' as 'ac_type' to 'ac_type_of_rpc'.
 // TODO(wanghao): A better way is to define the ac_type when defining rpc, and traverse all RPCs to
@@ -36,6 +64,15 @@ void register_rpc_access_type(access_type ac_type,
         ac_type_of_rpc.emplace(code, ac_type);
     }
 }
+
+// Used to map access_type matched resources policies json string.
+const std::map<std::string, access_type> kAccessTypeMaping({{"READ", access_type::kRead},
+                                                            {"WRITE", access_type::kWrite},
+                                                            {"CREATE", access_type::kCreate},
+                                                            {"DROP", access_type::kDrop},
+                                                            {"LIST", access_type::kList},
+                                                            {"METADATA", access_type::kMetadata},
+                                                            {"CONTROL", access_type::kControl}});
 } // anonymous namespace
 
 ranger_resource_policy_manager::ranger_resource_policy_manager(
@@ -47,11 +84,11 @@ ranger_resource_policy_manager::ranger_resource_policy_manager(
 
     // GLOBAL - KMetadata
     register_rpc_access_type(
-        access_type::KMetadata,
+        access_type::kMetadata,
         {"RPC_CM_LIST_NODES", "RPC_CM_CLUSTER_INFO", "RPC_CM_LIST_APPS", "RPC_QUERY_DISK_INFO"},
         _ac_type_of_global_rpcs);
     // GLOBAL - KControl
-    register_rpc_access_type(access_type::KControl,
+    register_rpc_access_type(access_type::kControl,
                              {"RPC_HTTP_SERVICE",
                               "RPC_CM_CONTROL_META",
                               "RPC_CM_START_RECOVERY",
@@ -61,15 +98,15 @@ ranger_resource_policy_manager::ranger_resource_policy_manager(
                               "RPC_CLI_CLI_CALL_ACK"},
                              _ac_type_of_global_rpcs);
     // DATABASE - KList
-    register_rpc_access_type(access_type::KList, {"RPC_CM_LIST_APPS"}, _ac_type_of_database_rpcs);
+    register_rpc_access_type(access_type::kList, {"RPC_CM_LIST_APPS"}, _ac_type_of_database_rpcs);
     // DATABASE - KCreate
     register_rpc_access_type(
-        access_type::KCreate, {"RPC_CM_CREATE_APP"}, _ac_type_of_database_rpcs);
+        access_type::kCreate, {"RPC_CM_CREATE_APP"}, _ac_type_of_database_rpcs);
     // DATABASE - KDrop
     register_rpc_access_type(
-        access_type::KDrop, {"RPC_CM_DROP_APP", "RPC_CM_RECALL_APP"}, _ac_type_of_database_rpcs);
+        access_type::kDrop, {"RPC_CM_DROP_APP", "RPC_CM_RECALL_APP"}, _ac_type_of_database_rpcs);
     // DATABASE - KMetadata
-    register_rpc_access_type(access_type::KMetadata,
+    register_rpc_access_type(access_type::kMetadata,
                              {"RPC_CM_QUERY_BACKUP_STATUS",
                               "RPC_CM_QUERY_RESTORE_STATUS",
                               "RPC_CM_QUERY_DUPLICATION",
@@ -79,7 +116,7 @@ ranger_resource_policy_manager::ranger_resource_policy_manager(
                               "RPC_CM_GET_MAX_REPLICA_COUNT"},
                              _ac_type_of_database_rpcs);
     // DATABASE - KControl
-    register_rpc_access_type(access_type::KControl,
+    register_rpc_access_type(access_type::kControl,
                              {"RPC_CM_START_BACKUP_APP",
                               "RPC_CM_START_RESTORE",
                               "RPC_CM_PROPOSE_BALANCER",
@@ -97,5 +134,34 @@ ranger_resource_policy_manager::ranger_resource_policy_manager(
                               "RPC_CM_RENAME_APP"},
                              _ac_type_of_database_rpcs);
 }
+
+void ranger_resource_policy_manager::parse_policies_from_json(const rapidjson::Value &data,
+                                                              std::vector<policy_item> &policies)
+{
+    CHECK(policies.empty(), "Ranger policy list should not be empty.");
+    RETURN_VOID_IF_NOT_ARRAY(data);
+    for (const auto &item : data.GetArray()) {
+        CONTINUE_IF_MISSING_MEMBER(item, "accesses");
+        policy_item pi;
+        for (const auto &access : item["accesses"].GetArray()) {
+            CONTINUE_IF_MISSING_MEMBER(access, "isAllowed");
+            CONTINUE_IF_MISSING_MEMBER(access, "type");
+            if (access["isAllowed"].GetBool()) {
+                std::string type = access["type"].GetString();
+                std::transform(type.begin(), type.end(), type.begin(), toupper);
+                const auto &at = kAccessTypeMaping.find(type);
+                // ignore invalid access_type
+                if (kAccessTypeMaping.end() != at) {
+                    pi.access_types |= at->second;
+                }
+            }
+        }
+        CONTINUE_IF_MISSING_MEMBER(item, "users");
+        for (const auto &user : item["users"].GetArray()) {
+            pi.users.emplace(user.GetString());
+        }
+        policies.emplace_back(pi);
+    }
+}
 } // namespace ranger
 } // namespace dsn