From 799414a6ba1f52b7a012ddaf326d8733352cece2 Mon Sep 17 00:00:00 2001 From: Aleksei Borzenkov Date: Tue, 22 Oct 2024 16:35:27 +0300 Subject: [PATCH] Implement runtime flags codegen in C++ (#10711) --- ydb/core/base/generated/codegen/__main__.py | 84 ---------- ydb/core/base/generated/codegen/main.cpp | 167 ++++++++++++++++++++ ydb/core/base/generated/codegen/ya.make | 7 +- 3 files changed, 170 insertions(+), 88 deletions(-) delete mode 100644 ydb/core/base/generated/codegen/__main__.py create mode 100644 ydb/core/base/generated/codegen/main.cpp diff --git a/ydb/core/base/generated/codegen/__main__.py b/ydb/core/base/generated/codegen/__main__.py deleted file mode 100644 index 809cc96f5fe1..000000000000 --- a/ydb/core/base/generated/codegen/__main__.py +++ /dev/null @@ -1,84 +0,0 @@ -import os -import sys -from ydb.core.protos.feature_flags_pb2 import TFeatureFlags, RequireRestart - -from jinja2 import Environment, FileSystemLoader -from google.protobuf.descriptor import FieldDescriptor - - -class Slot(object): - def __init__(self, name, index): - self.name = name - self.index = index - self.fields = [] - self.default_value = 0 - self.runtime_flags_mask = 0 - - -class Field(object): - def __init__(self, name, slot, has_mask, value_mask, full_mask, default_value, is_runtime): - self.name = name - self.slot = slot - self.has_mask = has_mask - self.value_mask = value_mask - self.full_mask = full_mask - self.default_value = default_value - self.is_runtime = is_runtime - - -class EnvironmentByDir(dict): - def __missing__(self, template_dir): - env = Environment(loader=FileSystemLoader(template_dir)) - self[template_dir] = env - return env - - -def main(): - slots = [] - fields = [] - current_bits = 64 - for field in TFeatureFlags.DESCRIPTOR.fields: - if field.type == FieldDescriptor.TYPE_BOOL: - if current_bits + 2 > 64: - index = len(slots) - slots.append(Slot(name=f'slot{index}', index=index)) - current_bits = 0 - shift = current_bits - current_bits += 2 - slot = slots[-1] - has_mask = 1 << shift - value_mask = 2 << shift - full_mask = 3 << shift - default_value = 0 - if field.default_value: - default_value = 2 << shift - is_runtime = not field.GetOptions().Extensions[RequireRestart] - fields.append(Field( - name=field.name, - slot=slot, - has_mask=has_mask, - value_mask=value_mask, - full_mask=full_mask, - default_value=default_value, - is_runtime=is_runtime, - )) - slot.fields.append(fields[-1]) - slot.default_value |= default_value - if is_runtime: - slot.runtime_flags_mask |= full_mask - - template_files = sys.argv[1::2] - output_files = sys.argv[2::2] - env_by_dir = EnvironmentByDir() - for (template_file, output_file) in zip(template_files, output_files): - (template_dir, template_name) = os.path.split(template_file) - env = env_by_dir[template_dir] - template = env.get_template(template_name) - result = template.render(generator=__file__, slots=slots, fields=fields) - with open(output_file, 'w') as f: - f.write(result) - print(f'Generated {output_file} from {template_name}') - - -if __name__ == '__main__': - main() diff --git a/ydb/core/base/generated/codegen/main.cpp b/ydb/core/base/generated/codegen/main.cpp new file mode 100644 index 000000000000..a18c6aa8c3a2 --- /dev/null +++ b/ydb/core/base/generated/codegen/main.cpp @@ -0,0 +1,167 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +struct TSlot; +struct TField; + +struct TSlot { + TString Name; + size_t Index; + std::vector Fields; + ui64 DefaultValue = 0; + ui64 RuntimeFlagsMask = 0; + + TSlot(const TString& name, size_t index) + : Name(name) + , Index(index) + {} +}; + +struct TField { + TString Name; + const TSlot* Slot; + ui64 HasMask = 0; + ui64 ValueMask = 0; + ui64 FullMask = 0; + ui64 DefaultValue = 0; + bool IsRuntime = false; + + TField(const TString& name, const TSlot* slot) + : Name(name) + , Slot(slot) + {} +}; + +namespace jinja2 { + + template<> + struct TypeReflection : TypeReflected { + static const auto& GetAccessors() { + static std::unordered_map accessors = { + {"name", [](const TSlot& slot) { return Reflect(std::string(slot.Name)); }}, + {"index", [](const TSlot& slot) { return Reflect(slot.Index); }}, + {"fields", [](const TSlot& slot) { return Reflect(slot.Fields); }}, + {"default_value", [](const TSlot& slot) { + return Reflect(std::string(TStringBuilder() << slot.DefaultValue)); + }}, + {"runtime_flags_mask", [](const TSlot& slot) { + return Reflect(std::string(TStringBuilder() << slot.RuntimeFlagsMask)); + }}, + }; + return accessors; + } + }; + + template<> + struct TypeReflection : TypeReflected { + static const auto& GetAccessors() { + static std::unordered_map accessors = { + {"name", [](const TField& field) { return Reflect(std::string(field.Name)); }}, + {"slot", [](const TField& field) { return Reflect(field.Slot); }}, + {"has_mask", [](const TField& field) { + return Reflect(std::string(TStringBuilder() << field.HasMask)); + }}, + {"value_mask", [](const TField& field) { + return Reflect(std::string(TStringBuilder() << field.ValueMask)); + }}, + {"full_mask", [](const TField& field) { + return Reflect(std::string(TStringBuilder() << field.FullMask)); + }}, + {"default_value", [](const TField& field) { + return Reflect(std::string(TStringBuilder() << field.DefaultValue)); + }}, + {"is_runtime", [](const TField& field) { return Reflect(field.IsRuntime); }}, + }; + return accessors; + } + }; + +} // namespace jinja2 + +int main(int argc, char** argv) { + if (argc < 3) { + Cerr << "Usage: " << argv[0] << " INPUT OUTPUT ..." << Endl; + return 1; + } + + std::deque slots; + std::deque fields; + std::vector jinjaSlots; + std::vector jinjaFields; + + TSlot* slot = nullptr; + int currentBits = 0; + + const auto* d = NKikimrConfig::TFeatureFlags::descriptor(); + for (int fieldIndex = 0; fieldIndex < d->field_count(); ++fieldIndex) { + const auto* protoField = d->field(fieldIndex); + if (protoField->type() != google::protobuf::FieldDescriptor::TYPE_BOOL) { + continue; + } + if (!slot || currentBits + 2 > 64) { + size_t index = slots.size(); + TString name = TStringBuilder() << "slot" << index; + slot = &slots.emplace_back(name, index); + currentBits = 0; + jinjaSlots.push_back(slot); + } + int shift = currentBits; + currentBits += 2; + TField* field = &fields.emplace_back(protoField->name(), slot); + jinjaFields.push_back(field); + field->HasMask = 1ULL << shift; + field->ValueMask = 2ULL << shift; + field->FullMask = 3ULL << shift; + if (protoField->default_value_bool()) { + field->DefaultValue = field->ValueMask; + } + field->IsRuntime = !protoField->options().GetExtension(NKikimrConfig::RequireRestart); + + slot->Fields.push_back(field); + slot->DefaultValue |= field->DefaultValue; + if (field->IsRuntime) { + slot->RuntimeFlagsMask |= field->FullMask; + } + } + + jinja2::TemplateEnv env; + env.AddGlobal("generator", jinja2::Reflect(std::string(__SOURCE_FILE__))); + env.AddGlobal("slots", jinja2::Reflect(jinjaSlots)); + env.AddGlobal("fields", jinja2::Reflect(jinjaFields)); + + for (int i = 1; i < argc; i += 2) { + if (!(i + 1 < argc)) { + Cerr << "ERROR: missing output for " << argv[i] << Endl; + return 1; + } + + jinja2::Template t(&env); + auto loaded = t.Load(TFileInput(argv[i]).ReadAll(), argv[i]); + if (!loaded) { + Cerr << "ERROR: " << loaded.error().ToString() << Endl; + return 1; + } + + auto rendered = t.RenderAsString({}); + if (!rendered) { + Cerr << "ERROR: " << rendered.error().ToString() << Endl; + return 1; + } + + TFileOutput(argv[i + 1]).Write(rendered.value()); + Cout << "Generated " << argv[i + 1] << " from " << argv[i] << Endl; + } + + return 0; +} diff --git a/ydb/core/base/generated/codegen/ya.make b/ydb/core/base/generated/codegen/ya.make index 3881cc92ff22..d8b1a352e13a 100644 --- a/ydb/core/base/generated/codegen/ya.make +++ b/ydb/core/base/generated/codegen/ya.make @@ -1,10 +1,9 @@ -PY3_PROGRAM() +PROGRAM() -PY_SRCS(__main__.py) +SRCS(main.cpp) PEERDIR( - contrib/python/MarkupSafe - contrib/python/Jinja2 + contrib/libs/jinja2cpp ydb/core/protos )