From 6bd8165cef6bf346dbabc870d10dcdd7d1534285 Mon Sep 17 00:00:00 2001 From: Daniel Nachbaur Date: Thu, 29 Jan 2015 12:09:26 +0100 Subject: [PATCH 1/4] Add base64 encoding/decoding for ubyte vectors to JSON --- CMakeLists.txt | 3 + include/flatbuffers/idl.h | 3 +- src/base64.cpp | 123 ++++++++++++++++++++++++++++++++++++++ src/base64.h | 31 ++++++++++ src/idl_gen_text.cpp | 10 ++++ src/idl_parser.cpp | 17 +++++- 6 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 src/base64.cpp create mode 100644 src/base64.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 0856513a59b..80e6d01a312 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ set(FlatBuffers_Compiler_SRCS include/flatbuffers/flatbuffers.h include/flatbuffers/idl.h include/flatbuffers/util.h + src/base64.cpp src/idl_parser.cpp src/idl_gen_cpp.cpp src/idl_gen_general.cpp @@ -37,6 +38,7 @@ set(FlatBuffers_Tests_SRCS include/flatbuffers/flatbuffers.h include/flatbuffers/idl.h include/flatbuffers/util.h + src/base64.cpp src/idl_parser.cpp src/idl_gen_text.cpp src/idl_gen_fbs.cpp @@ -56,6 +58,7 @@ set(FlatBuffers_Sample_Text_SRCS include/flatbuffers/flatbuffers.h include/flatbuffers/idl.h include/flatbuffers/util.h + src/base64.cpp src/idl_parser.cpp src/idl_gen_text.cpp samples/sample_text.cpp diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 173115037a7..4e178668eb4 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -375,6 +375,7 @@ struct GeneratorOptions { bool output_enum_identifiers; bool prefixed_enums; bool include_dependence_headers; + bool base64_byte_array; // Possible options for the more general generator below. enum Language { kJava, kCSharp, kMAX }; @@ -383,7 +384,7 @@ struct GeneratorOptions { GeneratorOptions() : strict_json(false), indent_step(2), output_enum_identifiers(true), prefixed_enums(true), - include_dependence_headers(false), + include_dependence_headers(false), base64_byte_array(false), lang(GeneratorOptions::kJava) {} }; diff --git a/src/base64.cpp b/src/base64.cpp new file mode 100644 index 00000000000..2a863d161a0 --- /dev/null +++ b/src/base64.cpp @@ -0,0 +1,123 @@ +/* + base64.cpp and base64.h + + Copyright (C) 2004-2008 René Nyffenegger + + This source code is provided 'as-is', without any express or implied + warranty. In no event will the author be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + + 3. This notice may not be removed or altered from any source distribution. + + René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +*/ + +#include "base64.h" +#include + +static const std::string base64_chars = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + + +static inline bool is_base64(unsigned char c) { + return (isalnum(c) || (c == '+') || (c == '/')); +} + +std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) { + std::string ret; + int i = 0; + int j = 0; + unsigned char char_array_3[3]; + unsigned char char_array_4[4]; + + while (in_len--) { + char_array_3[i++] = *(bytes_to_encode++); + if (i == 3) { + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; + + for(i = 0; (i <4) ; i++) + ret += base64_chars[char_array_4[i]]; + i = 0; + } + } + + if (i) + { + for(j = i; j < 3; j++) + char_array_3[j] = '\0'; + + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; + + for (j = 0; (j < i + 1); j++) + ret += base64_chars[char_array_4[j]]; + + while((i++ < 3)) + ret += '='; + + } + + return ret; + +} + +std::string base64_decode(std::string const& encoded_string) { + int in_len = encoded_string.size(); + int i = 0; + int j = 0; + int in_ = 0; + unsigned char char_array_4[4], char_array_3[3]; + std::string ret; + + while (in_len-- && ( encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { + char_array_4[i++] = encoded_string[in_]; in_++; + if (i ==4) { + for (i = 0; i <4; i++) + char_array_4[i] = base64_chars.find(char_array_4[i]); + + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + + for (i = 0; (i < 3); i++) + ret += char_array_3[i]; + i = 0; + } + } + + if (i) { + for (j = i; j <4; j++) + char_array_4[j] = 0; + + for (j = 0; j <4; j++) + char_array_4[j] = base64_chars.find(char_array_4[j]); + + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + + for (j = 0; (j < i - 1); j++) ret += char_array_3[j]; + } + + return ret; +} diff --git a/src/base64.h b/src/base64.h new file mode 100644 index 00000000000..0dd4afc6761 --- /dev/null +++ b/src/base64.h @@ -0,0 +1,31 @@ +/* + base64.cpp and base64.h + + Copyright (C) 2004-2008 René Nyffenegger + + This source code is provided 'as-is', without any express or implied + warranty. In no event will the author be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + + 3. This notice may not be removed or altered from any source distribution. + + René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +*/ + +#include + +std::string base64_encode(unsigned char const* , unsigned int len); +std::string base64_decode(std::string const& s); diff --git a/src/idl_gen_text.cpp b/src/idl_gen_text.cpp index e0299379d37..aec61511189 100644 --- a/src/idl_gen_text.cpp +++ b/src/idl_gen_text.cpp @@ -19,6 +19,7 @@ #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" +#include "base64.h" namespace flatbuffers { @@ -67,6 +68,15 @@ template void Print(T val, Type type, int /*indent*/, template void PrintVector(const Vector &v, Type type, int indent, const GeneratorOptions &opts, std::string *_text) { + // Specialization: Print base64 encoded string for a byte vector + if (opts.base64_byte_array && type.element == BASE_TYPE_UCHAR) + { + std::string &text = *_text; + text += "\""; + text += base64_encode(v.Data(), v.Length()); + text += "\""; + return; + } std::string &text = *_text; text += "["; text += NewLine(opts); diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index e3b7cc09ef3..450c93e7ea0 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -19,6 +19,7 @@ #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" +#include "base64.h" namespace flatbuffers { @@ -452,8 +453,20 @@ void Parser::ParseAnyValue(Value &val, FieldDef *field) { break; } case BASE_TYPE_VECTOR: { - Expect('['); - val.constant = NumToString(ParseVector(val.type.VectorType())); + // binary vectors in JSON can be base64 encoded aka a string + if ('[' != token_ && val.type.element == BASE_TYPE_UCHAR) + { + auto s = attribute_; + Expect(kTokenStringConstant); + auto decoded = base64_decode(s); + val.constant = NumToString(builder_.CreateVector(decoded.data(), decoded.size()).o); + Next(); + } + else + { + Expect('['); + val.constant = NumToString(ParseVector(val.type.VectorType())); + } break; } default: From 5c3fa1886a85e92368bc6d01feea038a99738fcf Mon Sep 17 00:00:00 2001 From: Daniel Nachbaur Date: Thu, 29 Jan 2015 12:10:05 +0100 Subject: [PATCH 2/4] Create and install library to make Parser available outside of flatc --- CMakeLists.txt | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80e6d01a312..e19ffebba18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,7 @@ configure_file(include/flatbuffers/options.in.h "${PROJECT_BINARY_DIR}/include/flatbuffers/options.h") include_directories("${PROJECT_BINARY_DIR}/include") -set(FlatBuffers_Compiler_SRCS +set(FlatBuffers_Library_SRCS "${PROJECT_BINARY_DIR}/include/flatbuffers/options.h" include/flatbuffers/flatbuffers.h include/flatbuffers/idl.h @@ -34,38 +34,29 @@ set(FlatBuffers_Compiler_SRCS src/flatc.cpp ) +set(FlatBuffers_Compiler_SRCS + src/flatc.cpp +) + set(FlatBuffers_Tests_SRCS - include/flatbuffers/flatbuffers.h - include/flatbuffers/idl.h - include/flatbuffers/util.h - src/base64.cpp - src/idl_parser.cpp - src/idl_gen_text.cpp - src/idl_gen_fbs.cpp tests/test.cpp # file generate by running compiler on tests/monster_test.fbs ${CMAKE_CURRENT_BINARY_DIR}/tests/monster_test_generated.h ) set(FlatBuffers_Sample_Binary_SRCS - include/flatbuffers/flatbuffers.h samples/sample_binary.cpp # file generated by running compiler on samples/monster.fbs ${CMAKE_CURRENT_BINARY_DIR}/samples/monster_generated.h ) set(FlatBuffers_Sample_Text_SRCS - include/flatbuffers/flatbuffers.h - include/flatbuffers/idl.h - include/flatbuffers/util.h - src/base64.cpp - src/idl_parser.cpp - src/idl_gen_text.cpp samples/sample_text.cpp # file generated by running compiler on samples/monster.fbs ${CMAKE_CURRENT_BINARY_DIR}/samples/monster_generated.h ) +# source_group(Compiler FILES ${FlatBuffers_Library_SRCS}) # source_group(Compiler FILES ${FlatBuffers_Compiler_SRCS}) # source_group(Tests FILES ${FlatBuffers_Tests_SRCS}) @@ -85,6 +76,10 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments") endif() +if(CMAKE_SYSTEM_NAME MATCHES "Linux") + add_definitions(-fPIC) +endif() + if(FLATBUFFERS_CODE_COVERAGE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fprofile-arcs -ftest-coverage") set(CMAKE_EXE_LINKER_FLAGS @@ -102,17 +97,22 @@ function(compile_flatbuffers_schema_to_cpp SRC_FBS) DEPENDS flatc) endfunction() +add_library(flatbuffers STATIC ${FlatBuffers_Library_SRCS}) add_executable(flatc ${FlatBuffers_Compiler_SRCS}) +target_link_libraries(flatc flatbuffers) if(FLATBUFFERS_BUILD_TESTS) compile_flatbuffers_schema_to_cpp(tests/monster_test.fbs) include_directories(${CMAKE_CURRENT_BINARY_DIR}/tests) add_executable(flattests ${FlatBuffers_Tests_SRCS}) + target_link_libraries(flattests flatbuffers) compile_flatbuffers_schema_to_cpp(samples/monster.fbs) include_directories(${CMAKE_CURRENT_BINARY_DIR}/samples) add_executable(flatsamplebinary ${FlatBuffers_Sample_Binary_SRCS}) + target_link_libraries(flatsamplebinary flatbuffers) add_executable(flatsampletext ${FlatBuffers_Sample_Text_SRCS}) + target_link_libraries(flatsampletext flatbuffers) enable_testing() file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/tests" DESTINATION @@ -153,6 +153,12 @@ install(TARGETS flatc RUNTIME DESTINATION bin PUBLIC_HEADER DESTINATION include) +install(TARGETS flatbuffers + EXPORT FlatBuffersTargets + ARCHIVE DESTINATION lib COMPONENT dev + RUNTIME DESTINATION bin COMPONENT lib + LIBRARY DESTINATION lib COMPONENT lib) + install(EXPORT FlatBuffersTargets FILE FlatBuffersTargets.cmake DESTINATION .) From 15f35c6192a8cecd79c28edfec5f9a6deab6bfbc Mon Sep 17 00:00:00 2001 From: Daniel Nachbaur Date: Thu, 29 Jan 2015 12:10:22 +0100 Subject: [PATCH 3/4] Fix FBS to string generation --- include/flatbuffers/idl.h | 3 ++- src/idl_gen_fbs.cpp | 27 +++++++++++++++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 4e178668eb4..4e60ea50d80 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -376,6 +376,7 @@ struct GeneratorOptions { bool prefixed_enums; bool include_dependence_headers; bool base64_byte_array; + bool escape_newlines; // Possible options for the more general generator below. enum Language { kJava, kCSharp, kMAX }; @@ -385,7 +386,7 @@ struct GeneratorOptions { GeneratorOptions() : strict_json(false), indent_step(2), output_enum_identifiers(true), prefixed_enums(true), include_dependence_headers(false), base64_byte_array(false), - lang(GeneratorOptions::kJava) {} + escape_newlines(false), lang(GeneratorOptions::kJava) {} }; // Generate text (JSON) from a given FlatBuffer, and a given Parser diff --git a/src/idl_gen_fbs.cpp b/src/idl_gen_fbs.cpp index 87603e60731..e7a9e836dcb 100644 --- a/src/idl_gen_fbs.cpp +++ b/src/idl_gen_fbs.cpp @@ -31,11 +31,12 @@ static std::string GenType(const Type &type) { } } +#define NEWLINE schema += opts.escape_newlines ? "\\n" : "\n" // Generate a flatbuffer schema from the Parser's internal representation. std::string GenerateFBS(const Parser &parser, const std::string &file_name, const GeneratorOptions &opts) { std::string schema; - schema += "// Generated from " + file_name + ".proto\n\n"; + schema += "// Generated from " + file_name + ".proto"; NEWLINE; NEWLINE; if (opts.include_dependence_headers) { int num_includes = 0; for (auto it = parser.included_files_.begin(); @@ -43,11 +44,11 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name, auto basename = flatbuffers::StripPath( flatbuffers::StripExtension(it->first)); if (basename != file_name) { - schema += "include \"" + basename + ".fbs\";\n"; + schema += "include \"" + basename + ".fbs\";"; NEWLINE; num_includes++; } } - if (num_includes) schema += "\n"; + if (num_includes) NEWLINE; } schema += "namespace "; auto name_space = parser.namespaces_.back(); @@ -56,34 +57,40 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name, if (it != name_space->components.begin()) schema += "."; schema += *it; } - schema += ";\n\n"; + schema += ";"; NEWLINE; NEWLINE; // Generate code for all the enum declarations. for (auto it = parser.enums_.vec.begin(); it != parser.enums_.vec.end(); ++it) { EnumDef &enum_def = **it; schema += "enum " + enum_def.name + " : "; - schema += GenType(enum_def.underlying_type) + " {\n"; + schema += GenType(enum_def.underlying_type) + " {"; NEWLINE; for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); ++it) { auto &ev = **it; - schema += " " + ev.name + " = " + NumToString(ev.value) + ",\n"; + schema += " " + ev.name + " = " + NumToString(ev.value) + ","; NEWLINE; } - schema += "}\n\n"; + schema += "}"; NEWLINE; NEWLINE; } // Generate code for all structs/tables. for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end(); ++it) { StructDef &struct_def = **it; - schema += "table " + struct_def.name + " {\n"; + schema += "table " + struct_def.name + " {"; NEWLINE; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { auto &field = **it; schema += " " + field.name + ":" + GenType(field.value.type); if (field.value.constant != "0") schema += " = " + field.value.constant; if (field.required) schema += " (required)"; - schema += ";\n"; + schema += ";"; NEWLINE; } - schema += "}\n\n"; + schema += "}"; NEWLINE; NEWLINE; + } + if (parser.root_struct_def) + { + schema += "root_type "; + schema += parser.root_struct_def->name; + schema += ";"; NEWLINE; } return schema; } From bfb9d6490aa29396842033bb39a2e553a112d99d Mon Sep 17 00:00:00 2001 From: Daniel Nachbaur Date: Thu, 29 Jan 2015 13:53:42 +0100 Subject: [PATCH 4/4] Fix config file to match the behavior of the retired FindFlatBuffers.cmake --- CMake/FlatBuffersConfig.cmake.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMake/FlatBuffersConfig.cmake.in b/CMake/FlatBuffersConfig.cmake.in index 62671abbc73..8c1b09d1750 100644 --- a/CMake/FlatBuffersConfig.cmake.in +++ b/CMake/FlatBuffersConfig.cmake.in @@ -18,6 +18,9 @@ set(FlatBuffers_INCLUDE_DIR @FLATBUFFERS_INCLUDE_DIRS@) set(FLATBUFFERS_INCLUDE_DIRS @FLATBUFFERS_INCLUDE_DIRS@) set(FLATBUFFERS_INCLUDE_DIR @FLATBUFFERS_INCLUDE_DIRS@) set(FLATBUFFERS_FLATC_EXECUTABLE flatc) +set(FLATBUFFERS_LIBRARY flatbuffers) + +include_directories(${PROJECT_BINARY_DIR}) #----------------------------------------------------------------------------- # function needed by dependent projects