Merge branch 'master' into internal-merge

eeight · Apr 7, 2016 · 4465daa · 4465daa
2 parents 1523936 + 09292d5
commit 4465daa
Show file tree

Hide file tree

Showing 42 changed files with 371 additions and 124 deletions.
diff --git a/BUILD b/BUILD
@@ -15,8 +15,18 @@ COPTS = [
     "-Wno-error=unused-function",
 ]
 
-# Bazel should provide portable link_opts for pthread.
-LINK_OPTS = ["-lpthread"]
+config_setting(
+    name = "android",
+    values = {
+        "crosstool_top": "//external:android/crosstool",
+    },
+)
+
+# Android builds do not need to link in a separate pthread library.
+LINK_OPTS = select({
+    ":android": [],
+    "//conditions:default": ["-lpthread"],
+})
 
 load(
     "protobuf",

diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in
@@ -23,6 +23,7 @@ copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\command_line_
 copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\cpp_generator.h include\google\protobuf\compiler\cpp\cpp_generator.h
 copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_generator.h include\google\protobuf\compiler\csharp\csharp_generator.h
 copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_names.h include\google\protobuf\compiler\csharp\csharp_names.h
+copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\csharp\csharp_options.h include\google\protobuf\compiler\csharp\csharp_options.h
 copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\importer.h include\google\protobuf\compiler\importer.h
 copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\java_generator.h include\google\protobuf\compiler\java\java_generator.h
 copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\java_names.h include\google\protobuf\compiler\java\java_names.h

diff --git a/objectivec/DevTools/full_mac_build.sh b/objectivec/DevTools/full_mac_build.sh
@@ -242,6 +242,14 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then
           -destination "platform=iOS Simulator,name=iPad Air,OS=9.0" # 64bit
       )
       ;;
+    7.3* )
+      XCODEBUILD_TEST_BASE_IOS+=(
+          -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
+          -destination "platform=iOS Simulator,name=iPhone 6,OS=9.3" # 64bit
+          -destination "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit
+          -destination "platform=iOS Simulator,name=iPad Air,OS=9.3" # 64bit
+      )
+      ;;
     7.* )
       XCODEBUILD_TEST_BASE_IOS+=(
           -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit

diff --git a/objectivec/GPBCodedInputStream.m b/objectivec/GPBCodedInputStream.m
@@ -219,15 +219,16 @@ int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state) {
     result = [[NSString alloc] initWithBytes:&state->bytes[state->bufferPos]
                                       length:size
                                     encoding:NSUTF8StringEncoding];
+    state->bufferPos += size;
     if (!result) {
-      result = @"";
 #ifdef DEBUG
       // https://developers.google.com/protocol-buffers/docs/proto#scalar
-      NSLog(@"UTF8 failure, is some field type 'string' when it should be "
+      NSLog(@"UTF-8 failure, is some field type 'string' when it should be "
             @"'bytes'?");
 #endif
+      [NSException raise:NSParseErrorException
+                  format:@"Invalid UTF-8 for a 'string'"];
     }
-    state->bufferPos += size;
   }
   return result;
 }

diff --git a/objectivec/Tests/GPBCodedInputStreamTests.m b/objectivec/Tests/GPBCodedInputStreamTests.m
@@ -283,16 +283,53 @@ - (void)testReadMalformedString {
   [output writeRawData:[NSData dataWithBytes:bytes length:sizeof(bytes)]];
   [output flush];
 
-  NSData* data =
+  NSData *data =
       [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey];
   GPBCodedInputStream* input = [GPBCodedInputStream streamWithData:data];
+  NSError *error = nil;
   TestAllTypes* message = [TestAllTypes parseFromCodedInputStream:input
                                                 extensionRegistry:nil
-                                                            error:NULL];
-  XCTAssertNotNil(message);
-  // Make sure we can read string properties twice without crashing.
-  XCTAssertEqual([message.defaultString length], (NSUInteger)0);
-  XCTAssertEqualObjects(@"", message.defaultString);
+                                                            error:&error];
+  XCTAssertNotNil(error);
+  XCTAssertNil(message);
+}
+
+- (void)testBOMWithinStrings {
+  // We've seen servers that end up with BOMs within strings (not always at the
+  // start, and sometimes in multiple places), make sure they always parse
+  // correctly. (Again, this is inpart incase a custom string class is ever
+  // used again.)
+  const char* strs[] = {
+    "\xEF\xBB\xBF String with BOM",
+    "String with \xEF\xBB\xBF in middle",
+    "String with end bom \xEF\xBB\xBF",
+    "\xEF\xBB\xBF\xe2\x99\xa1",  // BOM White Heart
+    "\xEF\xBB\xBF\xEF\xBB\xBF String with Two BOM",
+  };
+  for (size_t i = 0; i < GPBARRAYSIZE(strs); ++i) {
+    NSOutputStream* rawOutput = [NSOutputStream outputStreamToMemory];
+    GPBCodedOutputStream* output =
+        [GPBCodedOutputStream streamWithOutputStream:rawOutput];
+
+    int32_t tag = GPBWireFormatMakeTag(TestAllTypes_FieldNumber_DefaultString,
+                                       GPBWireFormatLengthDelimited);
+    [output writeRawVarint32:tag];
+    size_t length = strlen(strs[i]);
+    [output writeRawVarint32:(int32_t)length];
+    [output writeRawData:[NSData dataWithBytes:strs[i] length:length]];
+    [output flush];
+
+    NSData* data =
+        [rawOutput propertyForKey:NSStreamDataWrittenToMemoryStreamKey];
+    GPBCodedInputStream* input = [GPBCodedInputStream streamWithData:data];
+    TestAllTypes* message = [TestAllTypes parseFromCodedInputStream:input
+                                                  extensionRegistry:nil
+                                                              error:NULL];
+    XCTAssertNotNil(message, @"Loop %zd", i);
+    // Ensure the string is there. NSString can consume the BOM in some
+    // cases, so don't actually check the string for exact equality.
+    XCTAssertTrue(message.defaultString.length > 0, @"Loop %zd", i);
+  }
 }
 
 @end
diff --git a/src/Makefile.am b/src/Makefile.am
@@ -455,6 +455,7 @@ libprotoc_la_SOURCES =                                         \
   google/protobuf/compiler/csharp/csharp_message.h             \
   google/protobuf/compiler/csharp/csharp_message_field.cc      \
   google/protobuf/compiler/csharp/csharp_message_field.h       \
+  google/protobuf/compiler/csharp/csharp_options.h             \
   google/protobuf/compiler/csharp/csharp_primitive_field.cc    \
   google/protobuf/compiler/csharp/csharp_primitive_field.h     \
   google/protobuf/compiler/csharp/csharp_reflection_class.cc     \

diff --git a/src/google/protobuf/compiler/csharp/csharp_enum.cc b/src/google/protobuf/compiler/csharp/csharp_enum.cc
@@ -49,8 +49,8 @@ namespace protobuf {
 namespace compiler {
 namespace csharp {
 
-EnumGenerator::EnumGenerator(const EnumDescriptor* descriptor) :
-    SourceGeneratorBase(descriptor->file()),
+EnumGenerator::EnumGenerator(const EnumDescriptor* descriptor, const Options* options) :
+    SourceGeneratorBase(descriptor->file(), options),
     descriptor_(descriptor) {
 }
 

diff --git a/src/google/protobuf/compiler/csharp/csharp_enum.h b/src/google/protobuf/compiler/csharp/csharp_enum.h
@@ -43,7 +43,7 @@ namespace csharp {
 
 class EnumGenerator : public SourceGeneratorBase {
  public:
-  EnumGenerator(const EnumDescriptor* descriptor);
+  EnumGenerator(const EnumDescriptor* descriptor, const Options* options);
   ~EnumGenerator();
 
   void Generate(io::Printer* printer);

diff --git a/src/google/protobuf/compiler/csharp/csharp_enum_field.cc b/src/google/protobuf/compiler/csharp/csharp_enum_field.cc
@@ -38,6 +38,7 @@
 #include <google/protobuf/io/zero_copy_stream.h>
 
 #include <google/protobuf/compiler/csharp/csharp_helpers.h>
+#include <google/protobuf/compiler/csharp/csharp_options.h>
 #include <google/protobuf/compiler/csharp/csharp_enum_field.h>
 
 namespace google {
@@ -46,8 +47,8 @@ namespace compiler {
 namespace csharp {
 
 EnumFieldGenerator::EnumFieldGenerator(const FieldDescriptor* descriptor,
-                                       int fieldOrdinal)
-    : PrimitiveFieldGenerator(descriptor, fieldOrdinal) {
+                                       int fieldOrdinal, const Options *options)
+    : PrimitiveFieldGenerator(descriptor, fieldOrdinal, options) {
 }
 
 EnumFieldGenerator::~EnumFieldGenerator() {
@@ -80,9 +81,9 @@ void EnumFieldGenerator::GenerateCodecCode(io::Printer* printer) {
         "pb::FieldCodec.ForEnum($tag$, x => (int) x, x => ($type_name$) x)");
 }
 
-EnumOneofFieldGenerator::EnumOneofFieldGenerator(const FieldDescriptor* descriptor,
-						 int fieldOrdinal)
-  : PrimitiveOneofFieldGenerator(descriptor, fieldOrdinal) {
+EnumOneofFieldGenerator::EnumOneofFieldGenerator(
+    const FieldDescriptor* descriptor, int fieldOrdinal, const Options *options)
+  : PrimitiveOneofFieldGenerator(descriptor, fieldOrdinal, options) {
 }
 
 EnumOneofFieldGenerator::~EnumOneofFieldGenerator() {

diff --git a/src/google/protobuf/compiler/csharp/csharp_enum_field.h b/src/google/protobuf/compiler/csharp/csharp_enum_field.h
@@ -43,7 +43,9 @@ namespace csharp {
 
 class EnumFieldGenerator : public PrimitiveFieldGenerator {
  public:
-  EnumFieldGenerator(const FieldDescriptor* descriptor, int fieldOrdinal);
+  EnumFieldGenerator(const FieldDescriptor* descriptor,
+                     int fieldOrdinal,
+                     const Options *options);
   ~EnumFieldGenerator();
 
   virtual void GenerateCodecCode(io::Printer* printer);
@@ -57,7 +59,9 @@ class EnumFieldGenerator : public PrimitiveFieldGenerator {
 
 class EnumOneofFieldGenerator : public PrimitiveOneofFieldGenerator {
  public:
-  EnumOneofFieldGenerator(const FieldDescriptor* descriptor, int fieldOrdinal);
+  EnumOneofFieldGenerator(const FieldDescriptor* descriptor,
+                          int fieldOrdinal,
+                          const Options *options);
   ~EnumOneofFieldGenerator();
 
   virtual void GenerateParsingCode(io::Printer* printer);

diff --git a/src/google/protobuf/compiler/csharp/csharp_field_base.cc b/src/google/protobuf/compiler/csharp/csharp_field_base.cc
@@ -94,14 +94,15 @@ void FieldGeneratorBase::SetCommonFieldVariables(
 void FieldGeneratorBase::SetCommonOneofFieldVariables(
     map<string, string>* variables) {
   (*variables)["oneof_name"] = oneof_name();
-  (*variables)["has_property_check"] = oneof_name() + "Case_ == " + oneof_property_name() +
+  (*variables)["has_property_check"] =
+    oneof_name() + "Case_ == " + oneof_property_name() +
     "OneofCase." + property_name();
   (*variables)["oneof_property_name"] = oneof_property_name();
 }
 
 FieldGeneratorBase::FieldGeneratorBase(const FieldDescriptor* descriptor,
-                                       int fieldOrdinal)
-    : SourceGeneratorBase(descriptor->file()),
+                                       int fieldOrdinal, const Options* options)
+    : SourceGeneratorBase(descriptor->file(), options),
       descriptor_(descriptor),
       fieldOrdinal_(fieldOrdinal) {
   SetCommonFieldVariables(&variables_);
@@ -158,10 +159,11 @@ std::string FieldGeneratorBase::type_name(const FieldDescriptor* descriptor) {
     case FieldDescriptor::TYPE_MESSAGE:
     case FieldDescriptor::TYPE_GROUP:
       if (IsWrapperType(descriptor)) {
-        const FieldDescriptor* wrapped_field = descriptor->message_type()->field(0);
+        const FieldDescriptor* wrapped_field =
+            descriptor->message_type()->field(0);
         string wrapped_field_type_name = type_name(wrapped_field);
-        // String and ByteString go to the same type; other wrapped types go to the
-        // nullable equivalent.
+        // String and ByteString go to the same type; other wrapped types
+        // go to the nullable equivalent.
         if (wrapped_field->type() == FieldDescriptor::TYPE_STRING ||
             wrapped_field->type() == FieldDescriptor::TYPE_BYTES) {
           return wrapped_field_type_name;

diff --git a/src/google/protobuf/compiler/csharp/csharp_field_base.h b/src/google/protobuf/compiler/csharp/csharp_field_base.h
@@ -44,7 +44,9 @@ namespace csharp {
 
 class FieldGeneratorBase : public SourceGeneratorBase {
  public:
-  FieldGeneratorBase(const FieldDescriptor* descriptor, int fieldOrdinal);
+  FieldGeneratorBase(const FieldDescriptor* descriptor,
+                     int fieldOrdinal,
+                     const Options* options);
   ~FieldGeneratorBase();
 
   virtual void GenerateCloningCode(io::Printer* printer) = 0;

diff --git a/src/google/protobuf/compiler/csharp/csharp_generator.cc b/src/google/protobuf/compiler/csharp/csharp_generator.cc
@@ -41,6 +41,7 @@
 #include <google/protobuf/compiler/csharp/csharp_generator.h>
 #include <google/protobuf/compiler/csharp/csharp_helpers.h>
 #include <google/protobuf/compiler/csharp/csharp_names.h>
+#include <google/protobuf/compiler/csharp/csharp_options.h>
 #include <google/protobuf/compiler/csharp/csharp_reflection_class.h>
 
 using google::protobuf::internal::scoped_ptr;
@@ -51,8 +52,9 @@ namespace compiler {
 namespace csharp {
 
 void GenerateFile(const google::protobuf::FileDescriptor* file,
-                  io::Printer* printer) {
-  ReflectionClassGenerator reflectionClassGenerator(file);
+                  io::Printer* printer,
+                  const Options* options) {
+  ReflectionClassGenerator reflectionClassGenerator(file, options);
   reflectionClassGenerator.Generate(printer);
 }
 
@@ -71,23 +73,27 @@ bool Generator::Generate(
     return false;
   }
 
-  std::string file_extension = ".cs";
-  std::string base_namespace = "";
-  bool generate_directories = false;
+  struct Options cli_options;
+
   for (int i = 0; i < options.size(); i++) {
     if (options[i].first == "file_extension") {
-      file_extension = options[i].second;
+      cli_options.file_extension = options[i].second;
     } else if (options[i].first == "base_namespace") {
-      base_namespace = options[i].second;
-      generate_directories = true;
+      cli_options.base_namespace = options[i].second;
+      cli_options.base_namespace_specified = true;
     } else {
       *error = "Unknown generator option: " + options[i].first;
       return false;
     }
   }
 
   string filename_error = "";
-  std::string filename = GetOutputFile(file, file_extension, generate_directories, base_namespace, &filename_error);
+  std::string filename = GetOutputFile(file,
+      cli_options.file_extension,
+      cli_options.base_namespace_specified,
+      cli_options.base_namespace,
+      &filename_error);
+
   if (filename.empty()) {
     *error = filename_error;
     return false;
@@ -96,7 +102,7 @@ bool Generator::Generate(
       generator_context->Open(filename));
   io::Printer printer(output.get(), '$');
 
-  GenerateFile(file, &printer);
+  GenerateFile(file, &printer, &cli_options);
 
   return true;
 }

diff --git a/src/google/protobuf/compiler/csharp/csharp_helpers.cc b/src/google/protobuf/compiler/csharp/csharp_helpers.cc
@@ -48,6 +48,7 @@
 #include <google/protobuf/compiler/csharp/csharp_enum_field.h>
 #include <google/protobuf/compiler/csharp/csharp_map_field.h>
 #include <google/protobuf/compiler/csharp/csharp_message_field.h>
+#include <google/protobuf/compiler/csharp/csharp_options.h>
 #include <google/protobuf/compiler/csharp/csharp_primitive_field.h>
 #include <google/protobuf/compiler/csharp/csharp_repeated_enum_field.h>
 #include <google/protobuf/compiler/csharp/csharp_repeated_message_field.h>
@@ -127,7 +128,8 @@ std::string GetFileNameBase(const FileDescriptor* descriptor) {
 }
 
 std::string GetReflectionClassUnqualifiedName(const FileDescriptor* descriptor) {
-  // TODO: Detect collisions with existing messages, and append an underscore if necessary.
+  // TODO: Detect collisions with existing messages,
+  // and append an underscore if necessary.
   return GetFileNameBase(descriptor) + "Reflection";
 }
 
@@ -351,49 +353,50 @@ std::string FileDescriptorToBase64(const FileDescriptor* descriptor) {
 }
 
 FieldGeneratorBase* CreateFieldGenerator(const FieldDescriptor* descriptor,
-                                         int fieldOrdinal) {
+                                         int fieldOrdinal,
+                                         const Options* options) {
   switch (descriptor->type()) {
     case FieldDescriptor::TYPE_GROUP:
     case FieldDescriptor::TYPE_MESSAGE:
       if (descriptor->is_repeated()) {
         if (descriptor->is_map()) {
-          return new MapFieldGenerator(descriptor, fieldOrdinal);
+          return new MapFieldGenerator(descriptor, fieldOrdinal, options);
         } else {
-          return new RepeatedMessageFieldGenerator(descriptor, fieldOrdinal);
+          return new RepeatedMessageFieldGenerator(descriptor, fieldOrdinal, options);
         }
       } else {
         if (IsWrapperType(descriptor)) {
           if (descriptor->containing_oneof()) {
-            return new WrapperOneofFieldGenerator(descriptor, fieldOrdinal);
+            return new WrapperOneofFieldGenerator(descriptor, fieldOrdinal, options);
           } else {
-            return new WrapperFieldGenerator(descriptor, fieldOrdinal);
+            return new WrapperFieldGenerator(descriptor, fieldOrdinal, options);
           }
         } else {
           if (descriptor->containing_oneof()) {
-            return new MessageOneofFieldGenerator(descriptor, fieldOrdinal);
+            return new MessageOneofFieldGenerator(descriptor, fieldOrdinal, options);
           } else {
-            return new MessageFieldGenerator(descriptor, fieldOrdinal);
+            return new MessageFieldGenerator(descriptor, fieldOrdinal, options);
           }
         }
       }
     case FieldDescriptor::TYPE_ENUM:
       if (descriptor->is_repeated()) {
-        return new RepeatedEnumFieldGenerator(descriptor, fieldOrdinal);
+        return new RepeatedEnumFieldGenerator(descriptor, fieldOrdinal, options);
       } else {
         if (descriptor->containing_oneof()) {
-          return new EnumOneofFieldGenerator(descriptor, fieldOrdinal);
+          return new EnumOneofFieldGenerator(descriptor, fieldOrdinal, options);
         } else {
-          return new EnumFieldGenerator(descriptor, fieldOrdinal);
+          return new EnumFieldGenerator(descriptor, fieldOrdinal, options);
         }
       }
     default:
       if (descriptor->is_repeated()) {
-        return new RepeatedPrimitiveFieldGenerator(descriptor, fieldOrdinal);
+        return new RepeatedPrimitiveFieldGenerator(descriptor, fieldOrdinal, options);
       } else {
         if (descriptor->containing_oneof()) {
-          return new PrimitiveOneofFieldGenerator(descriptor, fieldOrdinal);
+          return new PrimitiveOneofFieldGenerator(descriptor, fieldOrdinal, options);
         } else {
-          return new PrimitiveFieldGenerator(descriptor, fieldOrdinal);
+          return new PrimitiveFieldGenerator(descriptor, fieldOrdinal, options);
         }
       }
   }