Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Store some data in ES as objects instead of just as strings #2565

Merged
merged 21 commits into from
Dec 26, 2021
Merged

Store some data in ES as objects instead of just as strings #2565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
9d99fce
Move some code from hpp file to cpp file
abitmore Dec 17, 2021
91675d7
Wrap long lines
abitmore Dec 17, 2021
03b9057
Adapt static_variant and map for ElasticSearch
abitmore Dec 17, 2021
0947fed
Add logging
abitmore Dec 17, 2021
d03ef2d
Fix adaption of proposed_ops for ElasticSearch
abitmore Dec 17, 2021
7152dbb
Fix adaption of fee schedule update op for ES
abitmore Dec 17, 2021
aff8e72
Fix adaption of arrays for ES
abitmore Dec 17, 2021
345942b
Fix adaption of maps for ES
abitmore Dec 18, 2021
2e62d8e
Remove unnecessary code
abitmore Dec 18, 2021
09b9116
Merge develop branch
abitmore Dec 22, 2021
ab64829
Fix code smells
abitmore Dec 22, 2021
38d5b28
Simplify code about adaption of nonce for ES
abitmore Dec 22, 2021
ba39422
Adapt more maps as objects for ES
abitmore Dec 24, 2021
e28cb30
Fix adaption of strings in maps or static variants
abitmore Dec 24, 2021
79993e2
Move es_data_adaptor into utilities namespace
abitmore Dec 24, 2021
0eed923
Merge adaptor_struct in es_objects into utilities
abitmore Dec 24, 2021
67e6733
Try to fix genesis data process in es_objects
abitmore Dec 24, 2021
952a326
Insert genesis objects into ES in smaller bulks
abitmore Dec 25, 2021
4f4f8a4
Insert bitasset data in genesis into ES
abitmore Dec 25, 2021
695187c
Move program options of es_objects into a struct
abitmore Dec 25, 2021
b02337e
Adapt htlc_preimage_hash as an object for ES
abitmore Dec 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 63 additions & 4 deletions libraries/plugins/elasticsearch/elasticsearch_plugin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,20 +264,24 @@ void elasticsearch_plugin_impl::getOperationType(const optional <operation_histo
}

void elasticsearch_plugin_impl::doOperationHistory(const optional <operation_history_object>& oho)
{
{ try {
os.trx_in_block = oho->trx_in_block;
os.op_in_trx = oho->op_in_trx;
os.operation_result = fc::json::to_string(oho->result);
os.virtual_op = oho->virtual_op;

if(_elasticsearch_operation_object) {
// op
oho->op.visit(fc::from_static_variant(os.op_object, FC_PACK_MAX_DEPTH));
adaptor_struct adaptor;
os.op_object = adaptor.adapt(os.op_object.get_object());
os.op_object = graphene::utilities::es_data_adaptor::adapt( os.op_object.get_object() );
// operation_result
variant v;
fc::to_variant( oho->result, v, FC_PACK_MAX_DEPTH );
os.operation_result_object = graphene::utilities::es_data_adaptor::adapt_static_variant( v.get_array() );
}
if(_elasticsearch_operation_string)
os.op = fc::json::to_string(oho->op);
}
} FC_CAPTURE_LOG_AND_RETHROW( (oho) ) }

void elasticsearch_plugin_impl::doBlock(uint32_t trx_in_block, const signed_block& b)
{
Expand All @@ -289,6 +293,61 @@ void elasticsearch_plugin_impl::doBlock(uint32_t trx_in_block, const signed_bloc
bs.trx_id = trx_id;
}

struct operation_visitor
{
using result_type = void;

share_type fee_amount;
asset_id_type fee_asset;

asset_id_type transfer_asset_id;
share_type transfer_amount;
account_id_type transfer_from;
account_id_type transfer_to;

void operator()( const graphene::chain::transfer_operation& o )
{
fee_asset = o.fee.asset_id;
fee_amount = o.fee.amount;

transfer_asset_id = o.amount.asset_id;
transfer_amount = o.amount.amount;
transfer_from = o.from;
transfer_to = o.to;
}

object_id_type fill_order_id;
account_id_type fill_account_id;
asset_id_type fill_pays_asset_id;
share_type fill_pays_amount;
asset_id_type fill_receives_asset_id;
share_type fill_receives_amount;
double fill_fill_price;
bool fill_is_maker;

void operator()( const graphene::chain::fill_order_operation& o )
{
fee_asset = o.fee.asset_id;
fee_amount = o.fee.amount;

fill_order_id = o.order_id;
fill_account_id = o.account_id;
fill_pays_asset_id = o.pays.asset_id;
fill_pays_amount = o.pays.amount;
fill_receives_asset_id = o.receives.asset_id;
fill_receives_amount = o.receives.amount;
fill_fill_price = o.fill_price.to_real();
fill_is_maker = o.is_maker;
}

template<typename T>
void operator()( const T& o )
{
fee_asset = o.fee.asset_id;
fee_amount = o.fee.amount;
}
};

void elasticsearch_plugin_impl::doVisitor(const optional <operation_history_object>& oho)
{
graphene::chain::database& db = database();
Expand Down
169 changes: 9 additions & 160 deletions libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,68 +80,14 @@ class elasticsearch_plugin : public graphene::app::plugin
};


struct operation_visitor
{
typedef void result_type;

share_type fee_amount;
asset_id_type fee_asset;

asset_id_type transfer_asset_id;
share_type transfer_amount;
account_id_type transfer_from;
account_id_type transfer_to;

void operator()( const graphene::chain::transfer_operation& o )
{
fee_asset = o.fee.asset_id;
fee_amount = o.fee.amount;

transfer_asset_id = o.amount.asset_id;
transfer_amount = o.amount.amount;
transfer_from = o.from;
transfer_to = o.to;
}

object_id_type fill_order_id;
account_id_type fill_account_id;
asset_id_type fill_pays_asset_id;
share_type fill_pays_amount;
asset_id_type fill_receives_asset_id;
share_type fill_receives_amount;
double fill_fill_price;
bool fill_is_maker;

void operator()( const graphene::chain::fill_order_operation& o )
{
fee_asset = o.fee.asset_id;
fee_amount = o.fee.amount;

fill_order_id = o.order_id;
fill_account_id = o.account_id;
fill_pays_asset_id = o.pays.asset_id;
fill_pays_amount = o.pays.amount;
fill_receives_asset_id = o.receives.asset_id;
fill_receives_amount = o.receives.amount;
fill_fill_price = o.fill_price.to_real();
fill_is_maker = o.is_maker;
}

template<typename T>
void operator()( const T& o )
{
fee_asset = o.fee.asset_id;
fee_amount = o.fee.amount;
}
};

struct operation_history_struct {
int trx_in_block;
int op_in_trx;
std::string operation_result;
int virtual_op;
std::string op;
variant op_object;
variant operation_result_object;
};

struct block_struct {
Expand Down Expand Up @@ -197,115 +143,18 @@ struct bulk_struct {
optional<visitor_struct> additional_data;
};

struct adaptor_struct {
variant adapt(const variant_object& op)
{
fc::mutable_variant_object o(op);
vector<string> keys_to_rename;
for (auto i = o.begin(); i != o.end(); ++i)
{
auto& element = (*i).value();
if (element.is_object())
{
const string& name = (*i).key();
auto& vo = element.get_object();
if (vo.contains(name.c_str()))
keys_to_rename.emplace_back(name);
element = adapt(vo);
}
else if (element.is_array())
adapt(element.get_array());
}
for (const auto& i : keys_to_rename)
{
string new_name = i + "_";
o[new_name] = variant(o[i]);
o.erase(i);
}

if (o.find("memo") != o.end())
{
auto& memo = o["memo"];
if (memo.is_string())
{
o["memo_"] = o["memo"];
o.erase("memo");
}
else if (memo.is_object())
{
fc::mutable_variant_object tmp(memo.get_object());
if (tmp.find("nonce") != tmp.end())
{
tmp["nonce"] = tmp["nonce"].as_string();
o["memo"] = tmp;
}
}
}
if (o.find("new_parameters") != o.end())
{
auto& tmp = o["new_parameters"];
if (tmp.is_object())
{
fc::mutable_variant_object tmp2(tmp.get_object());
if (tmp2.find("current_fees") != tmp2.end())
{
tmp2.erase("current_fees");
o["new_parameters"] = tmp2;
}
}
}
if (o.find("owner") != o.end() && o["owner"].is_string())
{
o["owner_"] = o["owner"].as_string();
o.erase("owner");
}

vector<string> to_string_fields = {
"proposed_ops",
"initializer",
"policy",
"predicates",
"active_special_authority",
"owner_special_authority",
"acceptable_collateral",
"acceptable_borrowers"
};
for( const auto& name : to_string_fields )
{
if (o.find(name) != o.end())
{
o[name] = fc::json::to_string(o[name]);
}
}

variant v;
fc::to_variant(o, v, FC_PACK_MAX_DEPTH);
return v;
}

void adapt(fc::variants& v)
{
for (auto& array_element : v)
{
if (array_element.is_object())
array_element = adapt(array_element.get_object());
else if (array_element.is_array())
adapt(array_element.get_array());
else
array_element = array_element.as_string();
}
}
};

} } //graphene::elasticsearch

FC_REFLECT_ENUM( graphene::elasticsearch::mode, (only_save)(only_query)(all) )
FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op)(op_object) )
FC_REFLECT( graphene::elasticsearch::operation_history_struct,
(trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op)(op_object)(operation_result_object) )
FC_REFLECT( graphene::elasticsearch::block_struct, (block_num)(block_time)(trx_id) )
FC_REFLECT( graphene::elasticsearch::fee_struct, (asset)(asset_name)(amount)(amount_units) )
FC_REFLECT( graphene::elasticsearch::transfer_struct, (asset)(asset_name)(amount)(amount_units)(from)(to) )
FC_REFLECT( graphene::elasticsearch::fill_struct, (order_id)(account_id)(pays_asset_id)(pays_asset_name)(pays_amount)(pays_amount_units)
(receives_asset_id)(receives_asset_name)(receives_amount)(receives_amount_units)(fill_price)
(fill_price_units)(is_maker))
FC_REFLECT( graphene::elasticsearch::fill_struct,
(order_id)(account_id)(pays_asset_id)(pays_asset_name)(pays_amount)(pays_amount_units)
(receives_asset_id)(receives_asset_name)(receives_amount)(receives_amount_units)(fill_price)
(fill_price_units)(is_maker) )
FC_REFLECT( graphene::elasticsearch::visitor_struct, (fee_data)(transfer_data)(fill_data) )
FC_REFLECT( graphene::elasticsearch::bulk_struct, (account_history)(operation_history)(operation_type)(operation_id_num)(block_data)(additional_data) )
FC_REFLECT( graphene::elasticsearch::bulk_struct,
(account_history)(operation_history)(operation_type)(operation_id_num)(block_data)(additional_data) )
Loading