Skip to content

Commit

Permalink
[MRG] check duplicate params (#961)
Browse files Browse the repository at this point in the history 
* add get params

clean codes

* check duplicate params

* Revert "add get params"

This reverts commit ec1d8dd.

* set priority by length & check duplicate

* rename function
  • Loading branch information
@wxchan @guolinke
wxchan authored and guolinke committed Oct 7, 2017
1 parent a6af7a1 commit d3e88ef
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 42 deletions.
36 changes: 30 additions & 6 deletions include/LightGBM/config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ struct ConfigBase {
const std::unordered_map<std::string, std::string>& params,
const std::string& name, bool* out);

static void KV2Map(std::unordered_map<std::string, std::string>& params, const char* kv);
static std::unordered_map<std::string, std::string> Str2Map(const char* parameters);
};

Expand Down Expand Up @@ -473,15 +474,38 @@ struct ParameterAlias {
});
std::unordered_map<std::string, std::string> tmp_map;
for (const auto& pair : *params) {
if (alias_table.count(pair.first) > 0) {
tmp_map[alias_table.at(pair.first)] = pair.second;
} else if (parameter_set.count(pair.first) == 0) {
Log::Fatal("Unknown parameter: %s", pair.first.c_str());
auto alias = alias_table.find(pair.first);
if (alias != alias_table.end()) { // found alias
auto alias_set = tmp_map.find(alias->second);
if (alias_set != tmp_map.end()) { // alias already set
// set priority by length & alphabetically to ensure reproducible behavior
if (alias_set->second.size() < pair.first.size() ||
(alias_set->second.size() == pair.first.size() && alias_set->second < pair.first)) {
Log::Warning("%s is set with %s=%s, %s=%s will be ignored. Current value: %s=%s.",
alias->second.c_str(), alias_set->second.c_str(), params->at(alias_set->second).c_str(),
pair.first.c_str(), pair.second.c_str(), alias->second.c_str(), params->at(alias_set->second).c_str());
} else {
Log::Warning("%s is set with %s=%s, will be overrided by %s=%s. Current value: %s=%s.",
alias->second.c_str(), alias_set->second.c_str(), params->at(alias_set->second).c_str(),
pair.first.c_str(), pair.second.c_str(), alias->second.c_str(), pair.second.c_str());
tmp_map[alias->second] = pair.first;
}
} else { // alias not set
tmp_map.emplace(alias->second, pair.first);
}
} else if (parameter_set.find(pair.first) == parameter_set.end()) {
Log::Warning("Unknown parameter: %s", pair.first.c_str());
}
}
for (const auto& pair : tmp_map) {
if (params->count(pair.first) == 0) {
params->insert(std::make_pair(pair.first, pair.second));
auto alias = params->find(pair.first);
if (alias == params->end()) { // not find
params->emplace(pair.first, params->at(pair.second));
params->erase(pair.second);
} else {
Log::Warning("%s is set=%s, %s=%s will be ignored. Current value: %s=%s.",
pair.first.c_str(), alias->second.c_str(), pair.second.c_str(), params->at(pair.second).c_str(),
pair.first.c_str(), alias->second.c_str());
}
}
}
Expand Down
27 changes: 2 additions & 25 deletions src/application/application.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,7 @@ Application::~Application() {
void Application::LoadParameters(int argc, char** argv) {
std::unordered_map<std::string, std::string> params;
for (int i = 1; i < argc; ++i) {
std::vector<std::string> tmp_strs = Common::Split(argv[i], '=');
if (tmp_strs.size() == 2) {
std::string key = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[0]));
std::string value = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[1]));
if (key.size() <= 0) {
continue;
}
params[key] = value;
} else {
Log::Warning("Unknown parameter in command line: %s", argv[i]);
}
ConfigBase::KV2Map(params, argv[i]);
}
// check for alias
ParameterAlias::KeyAliasTransform(&params);
Expand All @@ -76,20 +66,7 @@ void Application::LoadParameters(int argc, char** argv) {
if (line.size() == 0) {
continue;
}
std::vector<std::string> tmp_strs = Common::Split(line.c_str(), '=');
if (tmp_strs.size() == 2) {
std::string key = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[0]));
std::string value = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[1]));
if (key.size() <= 0) {
continue;
}
// Command-line has higher priority
if (params.count(key) == 0) {
params[key] = value;
}
} else {
Log::Warning("Unknown parameter in config file: %s", line.c_str());
}
ConfigBase::KV2Map(params, line.c_str());
}
} else {
Log::Warning("Config file %s doesn't exist, will ignore",
Expand Down
32 changes: 21 additions & 11 deletions src/io/config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,31 @@

namespace LightGBM {

void ConfigBase::KV2Map(std::unordered_map<std::string, std::string>& params, const char* kv) {
std::vector<std::string> tmp_strs = Common::Split(kv, '=');
if (tmp_strs.size() == 2) {
std::string key = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[0]));
std::string value = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[1]));
if (key.size() > 0) {
auto value_search = params.find(key);
if (value_search == params.end()) { // not set
params.emplace(key, value);
} else {
Log::Warning("%s is set=%s, %s=%s will be ignored. Current value: %s=%s.",
key.c_str(), value_search->second.c_str(), key.c_str(), value.c_str(),
key.c_str(), value_search->second.c_str());
}
}
} else {
Log::Warning("Unknown parameter %s", kv);
}
}

std::unordered_map<std::string, std::string> ConfigBase::Str2Map(const char* parameters) {
std::unordered_map<std::string, std::string> params;
auto args = Common::Split(parameters, " \t\n\r");
for (auto arg : args) {
std::vector<std::string> tmp_strs = Common::Split(arg.c_str(), '=');
if (tmp_strs.size() == 2) {
std::string key = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[0]));
std::string value = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[1]));
if (key.size() <= 0) {
continue;
}
params[key] = value;
} else if (Common::Trim(arg).size() > 0) {
Log::Warning("Unknown parameter %s", arg.c_str());
}
KV2Map(params, Common::Trim(arg).c_str());
}
ParameterAlias::KeyAliasTransform(&params);
return params;
Expand Down

0 comments on commit d3e88ef

Please sign in to comment.