diff --git a/compile_thrift.py b/compile_thrift.py index 385f041d88..a0a2756bbb 100755 --- a/compile_thrift.py +++ b/compile_thrift.py @@ -14,10 +14,6 @@ "name": "dsn.layer2", "path": "src", "include_fix": { - ".types.h": { - "add": [""], - "remove": ["\"dsn.layer2_types.h\""] - }, "_types.h": { "add": [""], "remove": ["\"dsn_types.h\""] @@ -28,15 +24,15 @@ } }, "file_move": { - ".types.h _types.h": "include/dsn/cpp/serialization_helper", + "_types.h": "include/dsn/cpp/serialization_helper", "_types.cpp": "src/core/core" } }, { - "name": "fd", - "path": "src/dist/failure_detector", + "name": "fd", + "path": "src/dist/failure_detector", "file_move": { - ".types.h _types.h": "include/dsn/dist/failure_detector" + "_types.h": "include/dsn/dist/failure_detector" }, "include_fix": { "_types.h": { @@ -47,17 +43,13 @@ "add": [""], "remove": ["\"fd_types.h\""] }, - ".types.h": { - "add": [""], - "remove": ["\"fd_types.h\""] - } } - }, + }, { - "name": "replication", - "path": "src/dist/replication", + "name": "replication", + "path": "src/dist/replication", "file_move": { - ".types.h _types.h": "include/dsn/dist/replication", + "_types.h": "include/dsn/dist/replication", "_types.cpp": "src/dist/replication/common" }, "include_fix": { @@ -69,23 +61,14 @@ "add": [""], "remove": ["\"replication_types.h\""] }, - ".types.h": { - "add": [""], - "remove": ["\"replication_types.h\""] - } }, - "add_end_newline": ["replication.types.h"] - }, + }, { - "name": "simple_kv", + "name": "simple_kv", "path": "src/dist/replication/test/simple_kv" - }, - { - "name": "echo", - "path": "src/apps/echo" - }, + }, { - "name": "nfs", + "name": "nfs", "path": "src/dist/nfs", "include_fix": { "_types.h": { @@ -93,16 +76,16 @@ "remove": ["\"dsn_types.h\""] } } - }, + }, { - "name": "simple_kv", + "name": "simple_kv", "path": "src/apps/skv" - }, + }, { - "name": "cli", + "name": "cli", "path": "src/dist/cli", "file_move": { - ".types.h _types.h": "include/dsn/dist/cli" + "_types.h": "include/dsn/dist/cli" }, "include_fix": { "_types.h": { @@ -112,94 +95,25 @@ "add": [""], "remove": ["\"cli_types.h\""] }, - ".types.h": { - "add": [""], - "remove": ["\"cli_types.h\""] - } } } ] -env_tools = { - "dsn_gentool": "", - "thrift_exe": "", - "root_dir": "" -} +thrift_exe = os.getcwd() + "/bin/Linux/thrift" +root_dir = os.getcwd() +print "thrift_exe = " + thrift_exe +print "root_dir = " + root_dir + -class CompileError(Exception): +class CompileError(Exception): """ Raised when dealing with thrift idl have errors""" + def __init__(self, msg): self.msg = msg + def __str__(self): return self.msg -def init_env(): - env_tools["dsn_gentool"] = os.getcwd() + "/bin/dsn.cg.sh" - env_tools["thrift_exe"] = os.getcwd() + "/bin/Linux/thrift" - -def find_struct_define(line, enum_class_list): - if len(line)<7 or line[0:7] != "struct ": - return "" - struct_name = line.strip().split()[1] - if struct_name in enum_class_list: - return struct_name - return "" - -def modify_struct_define(header_file, enum_class): - tmp_result = header_file + ".swapfile" - res_fd, src_fd = open(tmp_result, "w"), open(header_file, "r") - - in_enum_define = 0 - - for line in src_fd: - struct_name = find_struct_define(line, enum_class) - if struct_name != "": - res_fd.write("enum %s {\n"%(struct_name)) - in_enum_define = 1 - elif "enum type" in line and in_enum_define == 1: - in_enum_define = 2 - elif line.strip() == "};" and in_enum_define == 2: - in_enum_define = 1 - elif "VALUES_TO_NAMES" in line or "include \"dsn_types.h\"" in line: - pass - else: - res_fd.write(line) - - src_fd.close() - res_fd.close() - - os.system("rm %s"%(header_file)) - os.system("mv %s %s"%(tmp_result, header_file)) - -def remove_struct_impl(impl_file, enum_class): - os.system("sed -i \'/VALUES_TO_NAMES/\'d %s"%(impl_file)) - -def replace_struct_usage(cpp_file, enum_class): - sed_exp = "sed -i " + " ".join(["-e \'s/%s::type/%s/\'"%(i,i) for i in enum_class]) + " " + cpp_file - os.system(sed_exp) - -def add_end_newline_file(filename): - tmp_result = filename + ".swapfile" - from_fd, to_fd = open(filename, "r"), open(tmp_result, "w") - - for line in from_fd: - to_fd.write(line) - to_fd.write("\n"); - - from_fd.close() - to_fd.close() - - os.remove(filename) - os.rename(tmp_result, filename) - -def add_end_newline(thrift_name, add_end_newline_list): - # current dir is thrift file dir - os.chdir("output") - - for filename in add_end_newline_list: - add_end_newline_file(filename) - - os.chdir("..") def fix_include_file(filename, fix_commands): tmp_result = filename + ".swapfile" @@ -209,18 +123,19 @@ def fix_include_file(filename, fix_commands): for line in from_fd: include_statement = False - if len(line.strip())>0: + if len(line.strip()) > 0: stripped_line = line.strip() - if stripped_line[0]=="#" and "include" in stripped_line: + if stripped_line[0] == "#" and "include" in stripped_line: include_statement = True - if include_statement==True and add_ok==False: - add_includes = "\n".join(["#include %s"%(s) for s in fix_commands["add"]]) + if include_statement == True and add_ok == False: + add_includes = "\n".join(["#include %s" % (s) + for s in fix_commands["add"]]) to_fd.write(add_includes + "\n") add_ok = True - if include_statement==True and ("remove" in fix_commands): - if len(filter(lambda x: x in line, fix_commands["remove"]))==0: + if include_statement == True and ("remove" in fix_commands): + if len(filter(lambda x: x in line, fix_commands["remove"])) == 0: to_fd.write(line) else: to_fd.write(line) @@ -231,6 +146,7 @@ def fix_include_file(filename, fix_commands): os.remove(filename) os.rename(tmp_result, filename) + def fix_include(thrift_name, include_fix_dict): # current dir is thrift file dir os.chdir("output") @@ -241,30 +157,32 @@ def fix_include(thrift_name, include_fix_dict): os.chdir("..") + def compile_thrift_file(thrift_info): thrift_name = thrift_info["name"] - print ">>>compiling thrift file %s.thrift ..."%(thrift_name) + print "\n>>> compiling thrift file %s.thrift ..." % (thrift_name) if "path" not in thrift_info: raise CompileError("can't find thrift file") - os.chdir( env_tools["root_dir"] + "/" + thrift_info["path"]) + # ensure .thrift exists + os.chdir(root_dir + "/" + thrift_info["path"]) if os.path.isfile(thrift_name+".thrift") == False: raise CompileError("can't find thrift file") - ## generate the files + # create tmp directory: /output os.system("rm -rf output") os.system("mkdir output") - #### first generate .types.h - os.system("%s %s.thrift cpp build binary"%(env_tools["dsn_gentool"], thrift_name)) + print "mkdir {}/output".format(os.getcwd()) - os.system("cp build/%s.types.h output"%(thrift_name)) - os.system("cp build/%s_types.h output"%(thrift_name)) - os.system("cp build/%s_types.cpp output"%(thrift_name)) - os.system("rm -rf build") + # generate files + cmd = "{} -gen cpp:moveable_types -out output {}.thrift".format( + thrift_exe, thrift_name) + os.system(cmd) + print cmd - if "add_end_newline" in thrift_info: - add_end_newline(thrift_name, thrift_info["add_end_newline"]) + # TODO(wutao1): code format files + # os.system("clang-format-3.9 -i output/*") if "include_fix" in thrift_info: fix_include(thrift_name, thrift_info["include_fix"]) @@ -277,26 +195,22 @@ def compile_thrift_file(thrift_info): if "file_move" in thrift_info: for pair in thrift_info["file_move"].iteritems(): - dest_path = env_tools["root_dir"] + "/" + pair[1] + dest_path = root_dir + "/" + pair[1] for postfix in pair[0].split(): - src_path = "output/%s%s"%(thrift_name, postfix) - cmd = "mv %s %s"%(src_path, dest_path) + src_path = "output/%s%s" % (thrift_name, postfix) + cmd = "mv %s %s" % (src_path, dest_path) os.system(cmd) + print cmd - if len(os.listdir("output"))>0: - os.system("mv output/* ./") os.system("rm -rf output") + print "rm -rf {}/output".format(os.getcwd()) - os.chdir( env_tools["root_dir"] ) + os.chdir(root_dir) -def find_desc_from_path(path): - ans = [] - for i in thrift_description: - if i["path"] == path: - ans.append(i) - return ans # special hooks for thrift, all these are executed in the output dir + + def constructor_hook(args): generated_fname = args[0] class_name = args[1] @@ -325,51 +239,6 @@ def constructor_hook(args): os.remove(generated_fname) os.rename(target_fname, generated_fname) -def remove_all_enums_define_hook(args): - generated_fname = args[0] - target_fname = generated_fname + ".swapfile" - src_fd, dst_fd = open(generated_fname, "r"), open(target_fname, "w") - - in_enums = False - for line in src_fd: - if in_enums: - if line.strip().startswith("};"): - in_enums = False - line = "" - else: - tokens = line.strip().split() - if len(tokens)>1 and tokens[0]=="enum": - in_enums = True - line = "" - dst_fd.write(line) - - src_fd.close() - dst_fd.close() - - os.remove(generated_fname) - os.rename(target_fname, generated_fname) - -def remove_struct_define_hook(args): - generated_fname = args[0] - struct_defines = ["struct "+name+" {" for name in args[1:]] - - target_fname = generated_fname + ".swapfile" - src_fd, dst_fd = open(generated_fname, "r"), open(target_fname, "w") - - in_class = 0 - for line in src_fd: - if in_class==0 and line.strip() in struct_defines: - in_class = 1 - if in_class==0: - dst_fd.write(line) - if in_class==1 and line.startswith("};"): - in_class = 0 - - src_fd.close() - dst_fd.close() - - os.remove(generated_fname) - os.rename(target_fname, generated_fname) def replace_hook(args): generated_fname = args[0] @@ -385,10 +254,11 @@ def replace_hook(args): src_fd.close() dst_fd.close() - + os.remove(generated_fname) os.rename(target_fname, generated_fname) + def add_hook(name, path, func, args): for i in thrift_description: if name == i["name"] and path == i["path"]: @@ -397,19 +267,16 @@ def add_hook(name, path, func, args): else: i["hook"].append((func, args)) -if __name__ == "__main__": - init_env() +if __name__ == "__main__": ctor_kv_pair = " kv_pair(const std::string& _key, const std::string& _val): key(_key), value(_val) {\n }" ctor_configuration_proposal_action = " configuration_proposal_action(::dsn::rpc_address t, ::dsn::rpc_address n, config_type::type tp): target(t), node(n), type(tp) {}" - add_hook("simple_kv", "src/apps/skv", constructor_hook, ["simple_kv_types.h", "kv_pair", ctor_kv_pair]) - add_hook("replication", "src/dist/replication", constructor_hook, ["replication_types.h", "configuration_proposal_action", ctor_configuration_proposal_action]) - add_hook("dsn.layer2", "src", replace_hook, ["dsn.layer2_types.h", {r"dsn\.layer2_TYPES_H": 'dsn_layer2_TYPES_H'}]) - - if len(sys.argv)>1: - for i in sys.argv[1:]: - for desc in find_desc_from_path(i): - compile_thrift_file(desc) - else: - for i in thrift_description: - compile_thrift_file(i) + add_hook("simple_kv", "src/apps/skv", constructor_hook, + ["simple_kv_types.h", "kv_pair", ctor_kv_pair]) + add_hook("replication", "src/dist/replication", constructor_hook, + ["replication_types.h", "configuration_proposal_action", ctor_configuration_proposal_action]) + add_hook("dsn.layer2", "src", replace_hook, ["dsn.layer2_types.h", { + r"dsn\.layer2_TYPES_H": 'dsn_layer2_TYPES_H'}]) + + for i in thrift_description: + compile_thrift_file(i)