From f3f3ee9366a0908f4faab9614416a597a54822ef Mon Sep 17 00:00:00 2001 From: brave-builds Date: Wed, 22 Apr 2020 15:53:53 +0000 Subject: [PATCH] Uplift of #5301 (squashed) to release --- .../database_server_publisher_amounts.cc | 55 +++-- .../database_server_publisher_amounts.h | 5 +- .../database_server_publisher_banner.cc | 9 +- .../database_server_publisher_info.cc | 42 ++-- .../database_server_publisher_links.cc | 61 ++++-- .../database_server_publisher_links.h | 5 +- .../ledger/internal/database/database_util.h | 2 + .../publisher/publisher_server_list.cc | 197 +++++++----------- .../publisher/publisher_server_list.h | 22 +- 9 files changed, 219 insertions(+), 179 deletions(-) diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_amounts.cc b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_amounts.cc index 77b19720cf0e..9766b8904067 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_amounts.cc +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_amounts.cc @@ -5,7 +5,6 @@ #include #include -#include #include "base/strings/stringprintf.h" #include "bat/ledger/internal/database/database_server_publisher_amounts.h" @@ -173,30 +172,56 @@ bool DatabaseServerPublisherAmounts::MigrateToV15( return true; } -void DatabaseServerPublisherAmounts::InsertOrUpdate( +void DatabaseServerPublisherAmounts::InsertOrUpdateList( ledger::DBTransaction* transaction, - const ledger::PublisherBanner& info) { + const std::vector& list) { DCHECK(transaction); - // It's ok if social links are empty - if (info.amounts.empty()) { + if (list.empty()) { return; } - for (const auto& amount : info.amounts) { - const std::string query = base::StringPrintf( - "INSERT OR REPLACE INTO %s " - "(publisher_key, amount) VALUES (?, ?)", + const std::string base_query = base::StringPrintf( + "INSERT OR REPLACE INTO %s VALUES ", table_name_); - auto command = ledger::DBCommand::New(); - command->type = ledger::DBCommand::Type::RUN; - command->command = query; + size_t i = 0; + std::string query; + for (const auto& info : list) { + // It's ok if amounts are empty + if (info.amounts.empty()) { + continue; + } + + if (i == 0) { + query += base_query; + } - BindString(command.get(), 0, info.publisher_key); - BindDouble(command.get(), 1, amount); - transaction->commands.push_back(std::move(command)); + for (const auto& amount : info.amounts) { + if (i == kBatchLimit) { + query += base_query; + i = 0; + } + + query += base::StringPrintf( + R"(("%s",%g))", + info.publisher_key.c_str(), + amount); + query += (i == kBatchLimit - 1) ? ";" : ","; + i++; + } + } + + if (query.empty()) { + return; } + + query.pop_back(); + + auto command = ledger::DBCommand::New(); + command->type = ledger::DBCommand::Type::EXECUTE; + command->command = query; + transaction->commands.push_back(std::move(command)); } void DatabaseServerPublisherAmounts::GetRecord( diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_amounts.h b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_amounts.h index 1d86050e1adf..3764d865a980 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_amounts.h +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_amounts.h @@ -7,6 +7,7 @@ #define BRAVELEDGER_DATABASE_DATABASE_SERVER_PUBLISHER_AMOUNTS_H_ #include +#include #include "bat/ledger/internal/database/database_table.h" @@ -19,9 +20,9 @@ class DatabaseServerPublisherAmounts: public DatabaseTable { bool Migrate(ledger::DBTransaction* transaction, const int target) override; - void InsertOrUpdate( + void InsertOrUpdateList( ledger::DBTransaction* transaction, - const ledger::PublisherBanner& info); + const std::vector& list); void GetRecord( const std::string& publisher_key, diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_banner.cc b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_banner.cc index d894db5e6a37..20b4f434ed5c 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_banner.cc +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_banner.cc @@ -208,8 +208,9 @@ void DatabaseServerPublisherBanner::InsertOrUpdateList( "VALUES (?, ?, ?, ?, ?)", table_name_); + ledger::DBCommandPtr command; for (const auto& info : list) { - auto command = ledger::DBCommand::New(); + command = ledger::DBCommand::New(); command->type = ledger::DBCommand::Type::RUN; command->command = query; @@ -220,11 +221,11 @@ void DatabaseServerPublisherBanner::InsertOrUpdateList( BindString(command.get(), 4, info.logo); transaction->commands.push_back(std::move(command)); - - links_->InsertOrUpdate(transaction.get(), info); - amounts_->InsertOrUpdate(transaction.get(), info); } + links_->InsertOrUpdateList(transaction.get(), list); + amounts_->InsertOrUpdateList(transaction.get(), list); + auto transaction_callback = std::bind(&OnResultCallback, _1, callback); diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_info.cc b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_info.cc index d20bfed9ad95..ac1a4e5e6a34 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_info.cc +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_info.cc @@ -129,26 +129,42 @@ void DatabaseServerPublisherInfo::InsertOrUpdatePartialList( return; } - const std::string query = base::StringPrintf( - "INSERT OR REPLACE INTO %s " - "(publisher_key, status, excluded, address) " - "VALUES (?, ?, ?, ?)", + const std::string base_query = base::StringPrintf( + "INSERT OR REPLACE INTO %s VALUES ", table_name_); - auto transaction = ledger::DBTransaction::New(); + size_t i = 0; + std::string main_query = base_query; for (const auto& info : list) { - auto command = ledger::DBCommand::New(); - command->type = ledger::DBCommand::Type::RUN; - command->command = query; + if (i == kBatchLimit) { + main_query += base_query; + i = 0; + } - BindString(command.get(), 0, info.publisher_key); - BindInt(command.get(), 1, static_cast(info.status)); - BindBool(command.get(), 2, info.excluded); - BindString(command.get(), 3, info.address); + main_query += base::StringPrintf( + R"(("%s",%d,%d,"%s"))", + info.publisher_key.c_str(), + static_cast(info.status), + info.excluded, + info.address.c_str()); + main_query += (i == kBatchLimit - 1) ? ";" : ","; + i++; + } - transaction->commands.push_back(std::move(command)); + if (main_query.empty()) { + callback(ledger::Result::LEDGER_ERROR); + return; } + main_query.pop_back(); + + auto transaction = ledger::DBTransaction::New(); + auto command = ledger::DBCommand::New(); + command->type = ledger::DBCommand::Type::EXECUTE; + command->command = main_query; + + transaction->commands.push_back(std::move(command)); + auto transaction_callback = std::bind(&OnResultCallback, _1, callback); diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_links.cc b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_links.cc index 9438e5275a5f..d351a20f7501 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_links.cc +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_links.cc @@ -175,36 +175,61 @@ bool DatabaseServerPublisherLinks::MigrateToV15( return true; } -void DatabaseServerPublisherLinks::InsertOrUpdate( +void DatabaseServerPublisherLinks::InsertOrUpdateList( ledger::DBTransaction* transaction, - const ledger::PublisherBanner& info) { + const std::vector& list) { DCHECK(transaction); - // It's ok if links are empty - if (info.links.empty()) { + if (list.empty()) { return; } - for (const auto& link : info.links) { - if (link.second.empty()) { + const std::string base_query = base::StringPrintf( + "INSERT OR REPLACE INTO %s VALUES ", + table_name_); + + size_t i = 0; + std::string query; + for (const auto& info : list) { + // It's ok if links are empty + if (info.links.empty()) { continue; } - const std::string query = base::StringPrintf( - "INSERT OR REPLACE INTO %s " - "(publisher_key, provider, link) " - "VALUES (?, ?, ?)", - table_name_); + if (i == 0) { + query += base_query; + } - auto command = ledger::DBCommand::New(); - command->type = ledger::DBCommand::Type::RUN; - command->command = query; + for (const auto& link : info.links) { + if (link.second.empty()) { + continue; + } + + if (i == kBatchLimit) { + query += base_query; + i = 0; + } + + query += base::StringPrintf( + R"(("%s","%s","%s"))", + info.publisher_key.c_str(), + link.first.c_str(), + link.second.c_str()); + query += (i == kBatchLimit - 1) ? ";" : ","; + i++; + } + } - BindString(command.get(), 0, info.publisher_key); - BindString(command.get(), 1, link.first); - BindString(command.get(), 2, link.second); - transaction->commands.push_back(std::move(command)); + if (query.empty()) { + return; } + + query.pop_back(); + + auto command = ledger::DBCommand::New(); + command->type = ledger::DBCommand::Type::EXECUTE; + command->command = query; + transaction->commands.push_back(std::move(command)); } void DatabaseServerPublisherLinks::GetRecord( diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_links.h b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_links.h index 1f93adccc007..79077e9e3f40 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_links.h +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_server_publisher_links.h @@ -8,6 +8,7 @@ #include #include +#include #include "bat/ledger/internal/database/database_table.h" @@ -20,9 +21,9 @@ class DatabaseServerPublisherLinks: public DatabaseTable { bool Migrate(ledger::DBTransaction* transaction, const int target) override; - void InsertOrUpdate( + void InsertOrUpdateList( ledger::DBTransaction* transaction, - const ledger::PublisherBanner& info); + const std::vector& list); void GetRecord( const std::string& publisher_key, diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_util.h b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_util.h index 2834ccbae663..9791003e293c 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_util.h +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/database/database_util.h @@ -15,6 +15,8 @@ namespace braveledger_database { +const size_t kBatchLimit = 999; + bool DropTable( ledger::DBTransaction* transaction, const std::string& table_name); diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/publisher/publisher_server_list.cc b/vendor/bat-native-ledger/src/bat/ledger/internal/publisher/publisher_server_list.cc index 66fa166eafda..4724eba79b58 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/publisher/publisher_server_list.cc +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/publisher/publisher_server_list.cc @@ -5,7 +5,6 @@ #include #include -#include #include "base/json/json_reader.h" #include "base/time/time.h" @@ -188,73 +187,56 @@ ledger::PublisherStatus PublisherServerList::ParsePublisherStatus( void PublisherServerList::ParsePublisherList( const std::string& data, ParsePublisherListCallback callback) { - std::vector list_publisher; - std::vector list_banner; + auto list_publisher = + std::make_shared>(); + auto list_banner = std::make_shared>(); base::Optional value = base::JSONReader::Read(data); if (!value || !value->is_list()) { + BLOG(ledger_, ledger::LogLevel::LOG_ERROR) << "Data is not correct"; callback(ledger::Result::LEDGER_ERROR); return; } - base::ListValue* publishers = nullptr; - if (!value->GetAsList(&publishers)) { - callback(ledger::Result::LEDGER_ERROR); - return; - } + list_publisher->reserve(value->GetList().size()); + list_banner->reserve(value->GetList().size()); - for (auto& item : *publishers) { - base::ListValue* values = nullptr; - if (!item.GetAsList(&values)) { + for (auto& item : value->GetList()) { + if (!item.is_list()) { continue; } - ledger::ServerPublisherPartial publisher; - - // Publisher key - std::string publisher_key = ""; - if (!values->GetList()[0].is_string()) { - continue; - } - publisher_key = values->GetList()[0].GetString(); + const auto& list = item.GetList(); - if (publisher_key.empty()) { + if (list.size() != 5) { continue; } - publisher.publisher_key = publisher_key; - - // Status - if (!values->GetList()[1].is_string()) { + if (!list[0].is_string() || list[0].GetString().empty() // Publisher key + || !list[1].is_string() // Status + || !list[2].is_bool() // Excluded + || !list[3].is_string()) { // Address continue; } - publisher.status = ParsePublisherStatus(values->GetList()[1].GetString()); - // Excluded - if (!values->GetList()[2].is_bool()) { - continue; - } - publisher.excluded = values->GetList()[2].GetBool(); - - // Address - if (!values->GetList()[3].is_string()) { - continue; - } - publisher.address = values->GetList()[3].GetString(); + list_publisher->emplace_back( + list[0].GetString(), + ParsePublisherStatus(list[1].GetString()), + list[2].GetBool(), + list[3].GetString()); // Banner - base::DictionaryValue* banner = nullptr; - if (values->GetList()[4].GetAsDictionary(&banner)) { - auto parsed_banner = ParsePublisherBanner(publisher_key, banner); - if (!parsed_banner.publisher_key.empty()) { - list_banner.push_back(parsed_banner); - } + if (!list[4].is_dict() || list[4].DictEmpty()) { + continue; } - list_publisher.push_back(publisher); + list_banner->push_back(ledger::PublisherBanner()); + auto& banner = list_banner->back(); + ParsePublisherBanner(&banner, &list[4]); + banner.publisher_key = list[0].GetString(); } - if (list_publisher.empty()) { + if (list_publisher->empty()) { callback(ledger::Result::LEDGER_ERROR); return; } @@ -269,97 +251,70 @@ void PublisherServerList::ParsePublisherList( ledger_->ClearServerPublisherList(clear_callback); } -ledger::PublisherBanner PublisherServerList::ParsePublisherBanner( - const std::string& publisher_key, - base::DictionaryValue* dictionary) { - ledger::PublisherBanner banner; +void PublisherServerList::ParsePublisherBanner( + ledger::PublisherBanner* banner, + base::Value* dictionary) { + DCHECK(dictionary && banner); if (!dictionary->is_dict()) { - return banner; + return; } - bool empty = true; - auto* title = dictionary->FindKey("title"); - if (title && title->is_string()) { - banner.title = title->GetString(); - if (!banner.title.empty()) { - empty = false; - } + const auto* title = dictionary->FindStringKey("title"); + if (title) { + banner->title = *title; } - auto* description = dictionary->FindKey("description"); - if (description && description->is_string()) { - banner.description = description->GetString(); - if (!banner.description.empty()) { - empty = false; - } + const auto* description = dictionary->FindStringKey("description"); + if (description) { + banner->description = *description; } - auto* background = dictionary->FindKey("backgroundUrl"); - if (background && background->is_string()) { - banner.background = background->GetString(); - - if (!banner.background.empty()) { - banner.background = "chrome://rewards-image/" + banner.background; - empty = false; - } + const auto* background = dictionary->FindStringKey("backgroundUrl"); + if (background && !background->empty()) { + banner->background = "chrome://rewards-image/" + *background; } - auto* logo = dictionary->FindKey("logoUrl"); - if (logo && logo->is_string()) { - banner.logo = logo->GetString(); - - if (!banner.logo.empty()) { - banner.logo = "chrome://rewards-image/" + banner.logo; - empty = false; - } + const auto* logo = dictionary->FindStringKey("logoUrl"); + if (logo && !logo->empty()) { + banner->logo = "chrome://rewards-image/" + *logo; } auto* amounts = dictionary->FindKey("donationAmounts"); if (amounts && amounts->is_list()) { for (const auto& it : amounts->GetList()) { - banner.amounts.push_back(it.GetInt()); - } - - if (banner.amounts.size() != 0) { - empty = false; + if (it.is_int()) { + banner->amounts.push_back(it.GetInt()); + } } } auto* links = dictionary->FindKey("socialLinks"); if (links && links->is_dict()) { for (const auto& it : links->DictItems()) { - banner.links.insert(std::make_pair(it.first, it.second.GetString())); - } - - if (banner.links.size() != 0) { - empty = false; + if (it.second.is_string()) { + banner->links.insert(std::make_pair(it.first, it.second.GetString())); + } } } - - if (!empty) { - banner.publisher_key = publisher_key; - } - - return banner; } void PublisherServerList::SaveParsedData( const ledger::Result result, - const std::vector& list_publisher, - const std::vector& list_banner, + const SharedServerPublisherPartial& list_publisher, + const SharedPublisherBanner& list_banner, ParsePublisherListCallback callback) { if (result != ledger::Result::LEDGER_OK) { callback(result); return; } - if (!list_publisher.empty()) { + if (list_publisher && !list_publisher->empty()) { SavePublishers(list_publisher, list_banner, callback); return; } - if (!list_banner.empty()) { - SaveBanners({}, list_banner, callback); + if (list_banner && !list_banner->empty()) { + SaveBanners(list_banner, callback); return; } @@ -367,23 +322,29 @@ void PublisherServerList::SaveParsedData( } void PublisherServerList::SavePublishers( - const std::vector& list_publisher, - const std::vector& list_banner, + const SharedServerPublisherPartial& list_publisher, + const SharedPublisherBanner& list_banner, ParsePublisherListCallback callback) { + if (!list_publisher) { + callback(ledger::Result::LEDGER_OK); + return; + } + const int max_insert_records_ = 100000; int32_t interval = max_insert_records_; - const auto list_size = list_publisher.size(); + const auto list_size = list_publisher->size(); if (list_size < max_insert_records_) { interval = list_size; } std::vector save_list( - list_publisher.begin(), - list_publisher.begin() + interval); - std::vector new_list_publisher( - list_publisher.begin() + interval, - list_publisher.end()); + list_publisher->begin(), + list_publisher->begin() + interval); + auto new_list_publisher = + std::make_shared>( + list_publisher->begin() + interval, + list_publisher->end()); auto save_callback = std::bind(&PublisherServerList::SaveParsedData, this, @@ -396,28 +357,32 @@ void PublisherServerList::SavePublishers( } void PublisherServerList::SaveBanners( - const std::vector& list_publisher, - const std::vector& list_banner, + const SharedPublisherBanner& list_banner, ParsePublisherListCallback callback) { + if (!list_banner) { + callback(ledger::Result::LEDGER_OK); + return; + } + const int max_insert_records_ = 80000; int32_t interval = max_insert_records_; - const auto list_size = list_banner.size(); + const auto list_size = list_banner->size(); if (list_size < max_insert_records_) { interval = list_size; } std::vector save_list( - list_banner.begin(), - list_banner.begin() + interval); - std::vector new_list_banner( - list_banner.begin() + interval, - list_banner.end()); + list_banner->begin(), + list_banner->begin() + interval); + auto new_list_banner = std::make_shared>( + list_banner->begin() + interval, + list_banner->end()); auto save_callback = std::bind(&PublisherServerList::SaveParsedData, this, _1, - list_publisher, + nullptr, new_list_banner, callback); diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/publisher/publisher_server_list.h b/vendor/bat-native-ledger/src/bat/ledger/internal/publisher/publisher_server_list.h index fe7ef499b8e5..b585c26ebd88 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/publisher/publisher_server_list.h +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/publisher/publisher_server_list.h @@ -23,6 +23,11 @@ class LedgerImpl; namespace braveledger_publisher { +using SharedServerPublisherPartial = + std::shared_ptr>; +using SharedPublisherBanner = + std::shared_ptr>; + class PublisherServerList { public: explicit PublisherServerList(bat_ledger::LedgerImpl* ledger); @@ -56,24 +61,23 @@ class PublisherServerList { const std::string& data, ParsePublisherListCallback callback); - ledger::PublisherBanner ParsePublisherBanner( - const std::string& publisher_key, - base::DictionaryValue* dictionary); + void ParsePublisherBanner( + ledger::PublisherBanner* banner, + base::Value* dictionary); void SaveParsedData( const ledger::Result result, - const std::vector& list_publisher, - const std::vector& list_banner, + const SharedServerPublisherPartial& list_publisher, + const SharedPublisherBanner& list_banner, ParsePublisherListCallback callback); void SavePublishers( - const std::vector& list_publisher, - const std::vector& list_banner, + const SharedServerPublisherPartial& list_publisher, + const SharedPublisherBanner& list_banner, ParsePublisherListCallback callback); void SaveBanners( - const std::vector& list_publisher, - const std::vector& list_banner, + const SharedPublisherBanner& list_banner, ParsePublisherListCallback callback); bat_ledger::LedgerImpl* ledger_; // NOT OWNED