Skip to content

Commit

Permalink
Integrate google internal changes.
Browse files Browse the repository at this point in the history
  • Loading branch information
liujisi committed Mar 30, 2016
1 parent 7810589 commit 3b3c8ab
Show file tree
Hide file tree
Showing 269 changed files with 34,407 additions and 4,807 deletions.
8 changes: 8 additions & 0 deletions Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,8 @@ java_EXTRA_DIST=
java/core/src/main/java/com/google/protobuf/BlockingRpcChannel.java \
java/core/src/main/java/com/google/protobuf/BlockingService.java \
java/core/src/main/java/com/google/protobuf/BooleanArrayList.java \
java/core/src/main/java/com/google/protobuf/ByteBufferWriter.java \
java/core/src/main/java/com/google/protobuf/ByteOutput.java \
java/core/src/main/java/com/google/protobuf/ByteString.java \
java/core/src/main/java/com/google/protobuf/CodedInputStream.java \
java/core/src/main/java/com/google/protobuf/CodedOutputStream.java \
Expand Down Expand Up @@ -243,6 +245,8 @@ java_EXTRA_DIST=
java/core/src/main/java/com/google/protobuf/SmallSortedMap.java \
java/core/src/main/java/com/google/protobuf/TextFormat.java \
java/core/src/main/java/com/google/protobuf/TextFormatEscaper.java \
java/core/src/main/java/com/google/protobuf/TextFormatParseInfoTree.java \
java/core/src/main/java/com/google/protobuf/TextFormatParseLocation.java \
java/core/src/main/java/com/google/protobuf/UninitializedMessageException.java \
java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java \
java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java \
Expand All @@ -254,6 +258,7 @@ java_EXTRA_DIST=
java/core/src/test/java/com/google/protobuf/AnyTest.java \
java/core/src/test/java/com/google/protobuf/BooleanArrayListTest.java \
java/core/src/test/java/com/google/protobuf/BoundedByteStringTest.java \
java/core/src/test/java/com/google/protobuf/ByteBufferWriterTest.java \
java/core/src/test/java/com/google/protobuf/ByteStringTest.java \
java/core/src/test/java/com/google/protobuf/CheckUtf8Test.java \
java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java \
Expand All @@ -262,6 +267,7 @@ java_EXTRA_DIST=
java/core/src/test/java/com/google/protobuf/DescriptorsTest.java \
java/core/src/test/java/com/google/protobuf/DoubleArrayListTest.java \
java/core/src/test/java/com/google/protobuf/DynamicMessageTest.java \
java/core/src/test/java/com/google/protobuf/EnumTest.java \
java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java \
java/core/src/test/java/com/google/protobuf/FloatArrayListTest.java \
java/core/src/test/java/com/google/protobuf/ForceFieldBuildersPreRun.java \
Expand Down Expand Up @@ -294,6 +300,8 @@ java_EXTRA_DIST=
java/core/src/test/java/com/google/protobuf/SmallSortedMapTest.java \
java/core/src/test/java/com/google/protobuf/TestBadIdentifiers.java \
java/core/src/test/java/com/google/protobuf/TestUtil.java \
java/core/src/test/java/com/google/protobuf/TextFormatParseInfoTreeTest.java \
java/core/src/test/java/com/google/protobuf/TextFormatParseLocationTest.java \
java/core/src/test/java/com/google/protobuf/TextFormatTest.java \
java/core/src/test/java/com/google/protobuf/UnknownEnumValueTest.java \
java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java \
Expand Down
2 changes: 1 addition & 1 deletion conformance/ConformanceJava.java
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ public void run() throws Exception {
typeRegistry = TypeRegistry.newBuilder().add(
Conformance.TestAllTypes.getDescriptor()).build();
while (doTestIo()) {
// Empty.
this.testCount++;
}

System.err.println("ConformanceJava: received EOF from test runner after " +
Expand Down
125 changes: 125 additions & 0 deletions conformance/ConformanceJavaLite.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@

import com.google.protobuf.conformance.Conformance;
import com.google.protobuf.InvalidProtocolBufferException;

class ConformanceJavaLite {
private int testCount = 0;

private boolean readFromStdin(byte[] buf, int len) throws Exception {
int ofs = 0;
while (len > 0) {
int read = System.in.read(buf, ofs, len);
if (read == -1) {
return false; // EOF
}
ofs += read;
len -= read;
}

return true;
}

private void writeToStdout(byte[] buf) throws Exception {
System.out.write(buf);
}

// Returns -1 on EOF (the actual values will always be positive).
private int readLittleEndianIntFromStdin() throws Exception {
byte[] buf = new byte[4];
if (!readFromStdin(buf, 4)) {
return -1;
}
return (buf[0] & 0xff)
| ((buf[1] & 0xff) << 8)
| ((buf[2] & 0xff) << 16)
| ((buf[3] & 0xff) << 24);
}

private void writeLittleEndianIntToStdout(int val) throws Exception {
byte[] buf = new byte[4];
buf[0] = (byte)val;
buf[1] = (byte)(val >> 8);
buf[2] = (byte)(val >> 16);
buf[3] = (byte)(val >> 24);
writeToStdout(buf);
}

private Conformance.ConformanceResponse doTest(Conformance.ConformanceRequest request) {
Conformance.TestAllTypes testMessage;

switch (request.getPayloadCase()) {
case PROTOBUF_PAYLOAD: {
try {
testMessage = Conformance.TestAllTypes.parseFrom(request.getProtobufPayload());
} catch (InvalidProtocolBufferException e) {
return Conformance.ConformanceResponse.newBuilder().setParseError(e.getMessage()).build();
}
break;
}
case JSON_PAYLOAD: {
return Conformance.ConformanceResponse.newBuilder().setSkipped(
"Lite runtime does not suport Json Formant.").build();
}
case PAYLOAD_NOT_SET: {
throw new RuntimeException("Request didn't have payload.");
}

default: {
throw new RuntimeException("Unexpected payload case.");
}
}

switch (request.getRequestedOutputFormat()) {
case UNSPECIFIED:
throw new RuntimeException("Unspecified output format.");

case PROTOBUF:
return Conformance.ConformanceResponse.newBuilder().setProtobufPayload(testMessage.toByteString()).build();

case JSON:
return Conformance.ConformanceResponse.newBuilder().setSkipped(
"Lite runtime does not suport Json Formant.").build();

default: {
throw new RuntimeException("Unexpected request output.");
}
}
}

private boolean doTestIo() throws Exception {
int bytes = readLittleEndianIntFromStdin();

if (bytes == -1) {
return false; // EOF
}

byte[] serializedInput = new byte[bytes];

if (!readFromStdin(serializedInput, bytes)) {
throw new RuntimeException("Unexpected EOF from test program.");
}

Conformance.ConformanceRequest request =
Conformance.ConformanceRequest.parseFrom(serializedInput);
Conformance.ConformanceResponse response = doTest(request);
byte[] serializedOutput = response.toByteArray();

writeLittleEndianIntToStdout(serializedOutput.length);
writeToStdout(serializedOutput);

return true;
}

public void run() throws Exception {
while (doTestIo()) {
this.testCount++;
}

System.err.println("ConformanceJavaLite: received EOF from test runner after " +
this.testCount + " tests");
}

public static void main(String[] args) throws Exception {
new ConformanceJavaLite().run();
}
}
20 changes: 19 additions & 1 deletion conformance/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ protoc_outputs = \
other_language_protoc_outputs = \
conformance.rb \
com/google/protobuf/conformance/Conformance.java \
lite/com/google/protobuf/conformance/Conformance.java \
conformance_pb2.py \
Conformance.pbobjc.h \
Conformance.pbobjc.m
Expand All @@ -30,6 +31,7 @@ bin_PROGRAMS = conformance-test-runner conformance-cpp
# automatically.
EXTRA_DIST = \
ConformanceJava.java \
ConformanceJavaLite.java \
README.md \
conformance.proto \
conformance_python.py \
Expand Down Expand Up @@ -88,6 +90,7 @@ if USE_EXTERNAL_PROTOC
protoc_middleman: $(conformance_protoc_inputs) $(well_known_type_protoc_inputs)
$(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=. --ruby_out=. --objc_out=. --python_out=. $(conformance_protoc_inputs)
$(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=. --ruby_out=. --python_out=. $(well_known_type_protoc_inputs)
$(PROTOC) -I$(srcdir) -I$(top_srcdir) --java_out=lite:lite $(conformance_protoc_inputs) $(well_known_type_protoc_inputs)
touch protoc_middleman

else
Expand All @@ -98,6 +101,8 @@ else
protoc_middleman: $(top_srcdir)/src/protoc$(EXEEXT) $(conformance_protoc_inputs) $(well_known_type_protoc_inputs)
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd --ruby_out=$$oldpwd --objc_out=$$oldpwd --python_out=$$oldpwd $(conformance_protoc_inputs) )
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd --ruby_out=$$oldpwd --python_out=$$oldpwd $(well_known_type_protoc_inputs) )
@mkdir -p lite
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --java_out=lite:$$oldpwd/lite $(conformance_protoc_inputs) $(well_known_type_protoc_inputs) )
touch protoc_middleman

endif
Expand All @@ -108,7 +113,7 @@ $(other_language_protoc_outputs): protoc_middleman

BUILT_SOURCES = $(protoc_outputs) $(other_language_protoc_outputs)

CLEANFILES = $(protoc_outputs) protoc_middleman javac_middleman conformance-java conformance-csharp $(other_language_protoc_outputs)
CLEANFILES = $(protoc_outputs) protoc_middleman javac_middleman conformance-java javac_middleman_lite conformance-java-lite conformance-csharp $(other_language_protoc_outputs)

MAINTAINERCLEANFILES = \
Makefile.in
Expand All @@ -123,6 +128,16 @@ conformance-java: javac_middleman
@jar=`ls ../java/util/target/*jar-with-dependencies.jar` && echo java -classpath .:../java/target/classes:$$jar ConformanceJava '$$@' >> conformance-java
@chmod +x conformance-java

javac_middleman_lite: ConformanceJavaLite.java protoc_middleman $(other_language_protoc_outputs)
javac -classpath ../java/lite/target/classes:lite ConformanceJavaLite.java lite/com/google/protobuf/conformance/Conformance.java
@touch javac_middleman_lite

conformance-java-lite: javac_middleman_lite
@echo "Writing shortcut script conformance-java-lite..."
@echo '#! /bin/sh' > conformance-java-lite
@echo java -classpath .:../java/lite/target/classes:lite ConformanceJavaLite '$$@' >> conformance-java-lite
@chmod +x conformance-java-lite

# Currently the conformance code is alongside the rest of the C#
# source, as it's easier to maintain there. We assume we've already
# built that, so we just need a script to run it.
Expand All @@ -139,6 +154,9 @@ test_cpp: protoc_middleman conformance-test-runner conformance-cpp
test_java: protoc_middleman conformance-test-runner conformance-java
./conformance-test-runner --failure_list failure_list_java.txt ./conformance-java

test_java_lite: protoc_middleman conformance-test-runner conformance-java-lite
./conformance-test-runner ./conformance-java-lite

test_csharp: protoc_middleman conformance-test-runner conformance-csharp
./conformance-test-runner --failure_list failure_list_csharp.txt ./conformance-csharp

Expand Down
13 changes: 8 additions & 5 deletions generate_descriptor_proto.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
# to make when building protoc. This is particularly useful for passing
# -j4 to run 4 jobs simultaneously.

set -e

if test ! -e src/google/protobuf/stubs/common.h; then
cat >&2 << __EOF__
Could not find source code. Make sure you are running this script from the
Expand Down Expand Up @@ -52,9 +50,14 @@ do
echo "Round $PROCESS_ROUND"
CORE_PROTO_IS_CORRECT=1

make $@ protoc &&
./protoc --cpp_out=dllexport_decl=LIBPROTOBUF_EXPORT:$TMP ${RUNTIME_PROTO_FILES[@]} && \
./protoc --cpp_out=dllexport_decl=LIBPROTOC_EXPORT:$TMP google/protobuf/compiler/plugin.proto
make $@ protoc
if test $? -ne 0; then
echo "Failed to build protoc."
exit 1
fi

./protoc --cpp_out=dllexport_decl=LIBPROTOBUF_EXPORT:$TMP ${RUNTIME_PROTO_FILES[@]} && \
./protoc --cpp_out=dllexport_decl=LIBPROTOC_EXPORT:$TMP google/protobuf/compiler/plugin.proto

for PROTO_FILE in ${RUNTIME_PROTO_FILES[@]}; do
BASE_NAME=${PROTO_FILE%.*}
Expand Down
Loading

1 comment on commit 3b3c8ab

@damienmg
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change broke build with Bazel and sandboxing enabled, see http://ci.bazel.io/job/protobuf/136/ (bisected to find the culprit).

The exact log:

$ bazel build //:protoc
INFO: Found 1 target...
ERROR: /usr/local/google/home/dmarting/toto/protobuf/BUILD:271:1: Linking of rule '//:protoc' failed: namespace-sandbox failed: error executing command <user_home>/.cache/bazel/_bazel_dmarting/c28d9280fb33934092856a1adca4fe91/protobuf/_bin/namespace-sandbox ... (remaining 13 argument(s) skipped).
bazel-out/local_linux-fastbuild/bin/libprotoc_lib.a(java_generator_factory.pic.o): In function `google::protobuf::compiler::java::ImmutableGeneratorFactory::NewExtensionGenerator(google::protobuf::FieldDescriptor const*) const':
java_generator_factory.cc:(.text+0x265): undefined reference to `google::protobuf::compiler::java::ImmutableExtensionLiteGenerator::ImmutableExtensionLiteGenerator(google::protobuf::FieldDescriptor const*, google::protobuf::compiler::java::Context*)'
bazel-out/local_linux-fastbuild/bin/libprotoc_lib.a(java_message_lite.pic.o): In function `google::protobuf::compiler::java::ImmutableMessageLiteGenerator::Generate(google::protobuf::io::Printer*)':
java_message_lite.cc:(.text+0x1b82): undefined reference to `google::protobuf::compiler::java::ImmutableExtensionLiteGenerator::ImmutableExtensionLiteGenerator(google::protobuf::FieldDescriptor const*, google::protobuf::compiler::java::Context*)'
java_message_lite.cc:(.text+0x1b9b): undefined reference to `google::protobuf::compiler::java::ImmutableExtensionLiteGenerator::Generate(google::protobuf::io::Printer*)'
java_message_lite.cc:(.text+0x1baa): undefined reference to `google::protobuf::compiler::java::ImmutableExtensionLiteGenerator::~ImmutableExtensionLiteGenerator()'
java_message_lite.cc:(.text+0x1f41): undefined reference to `google::protobuf::compiler::java::ImmutableExtensionLiteGenerator::~ImmutableExtensionLiteGenerator()'
bazel-out/local_linux-fastbuild/bin/libprotoc_lib.a(java_message_lite.pic.o): In function `google::protobuf::compiler::java::ImmutableMessageLiteGenerator::GenerateExtensionRegistrationCode(google::protobuf::io::Printer*)':
java_message_lite.cc:(.text+0x394e): undefined reference to `google::protobuf::compiler::java::ImmutableExtensionLiteGenerator::ImmutableExtensionLiteGenerator(google::protobuf::FieldDescriptor const*, google::protobuf::compiler::java::Context*)'
java_message_lite.cc:(.text+0x3961): undefined reference to `google::protobuf::compiler::java::ImmutableExtensionLiteGenerator::GenerateRegistrationCode(google::protobuf::io::Printer*)'
java_message_lite.cc:(.text+0x396d): undefined reference to `google::protobuf::compiler::java::ImmutableExtensionLiteGenerator::~ImmutableExtensionLiteGenerator()'
java_message_lite.cc:(.text+0x3a14): undefined reference to `google::protobuf::compiler::java::ImmutableExtensionLiteGenerator::~ImmutableExtensionLiteGenerator()'
collect2: error: ld returned 1 exit status
Target //:protoc failed to build
Use --verbose_failures to see the command lines of failed build steps.
INFO: Elapsed time: 28.883s, Critical Path: 27.78s

Please sign in to comment.