Bazel build: Keep generated sources and Python runtime in the same di…

…rectory.

Users often encounter a Python import error when trying to build Python
protos if protobuf is installed locally on the machine. In this case,
Python ends up looking in the wrong directory when importing files (see
bazelbuild/bazel#1209 and tensorflow/tensorflow#2021). It seems that the
problem is caused by Python getting confused when there are Python
source files that are meant to be part of the same package but are
in separate directories.

Prior to protocolbuffers#1233, the Bazel build setup would copy the Python
runtime sources and all generated sources for the builtin protos into
the root directory (assuming that the protobuf tree is vendored in a
google/protobuf directory).

With protocolbuffers#1233, the two sets of sources are kept in their respective
directories but both `src/` and `python/` are added to the `PYTHONPATH`
using the new `imports` attribute of the Bazel Python rules. However,
both the runtime sources and the generated sources are under the same
package: `google.protobuf`, causing Python to become confused when
trying to import modules that are in the other directory.

This patch adds a workaround to the Bazel build to add a modified
version of the original `internal_copied_filegroup` macro to copy the
`.proto` files under `src/` to `python/` before building the
`py_proto_library` targets for the builtin protos. This ensures that the
generated sources for the builtin protos will be in the same directory
as the corresponding runtime sources.

This patch was tested with the following:
* All Python tests in protobuf
* All Python tests in tensorflow
* All tests in [Skydoc](https://github.com/bazelbuild/skydoc)
* Importing protobuf as `//google/protobuf`
* Importing and binding targets under `//external`
* Importing protobuf as `//third_party/protobuf`

BUILD

@@ -32,6 +32,7 @@ load(
     "protobuf",
     "cc_proto_library",
     "py_proto_library",
+    "internal_copied_filegroup",
     "internal_gen_well_known_protos_java",
     "internal_protobuf_py_tests",
 )
@@ -560,6 +561,8 @@ py_library(
             "python/google/protobuf/**/*.py",
         ],
         exclude = [
+            "python/google/protobuf/__init__.py",
+            "python/google/protobuf/**/__init__.py",
             "python/google/protobuf/internal/*_test.py",
             "python/google/protobuf/internal/test_util.py",
         ],
@@ -622,10 +625,26 @@ config_setting(
     },
 )
 
+# Copy the builtin proto files from src/google/protobuf to
+# python/google/protobuf. This way, the generated Python sources will be in the
+# same directory as the Python runtime sources. This is necessary for the
+# modules to be imported correctly since they are all part of the same Python
+# package.
+internal_copied_filegroup(
+    name = "protos_python",
+    srcs = WELL_KNOWN_PROTOS,
+    strip_prefix = "src",
+    dest = "python",
+)
+
+# TODO(dzc): Remove this once py_proto_library can have labels in srcs, in
+# which case we can simply add :protos_python in srcs.
+COPIED_WELL_KNOWN_PROTOS = ["python/" + s for s in RELATIVE_WELL_KNOWN_PROTOS]
+
 py_proto_library(
     name = "protobuf_python",
-    srcs = WELL_KNOWN_PROTOS,
-    include = "src",
+    srcs = COPIED_WELL_KNOWN_PROTOS,
+    include = "python",
     data = select({
         "//conditions:default": [],
         ":use_fast_cpp_protos": [
@@ -643,10 +662,27 @@ py_proto_library(
     visibility = ["//visibility:public"],
 )
 
+# Copy the test proto files from src/google/protobuf to
+# python/google/protobuf. This way, the generated Python sources will be in the
+# same directory as the Python runtime sources. This is necessary for the
+# modules to be imported correctly by the tests since they are all part of the
+# same Python package.
+internal_copied_filegroup(
+    name = "protos_python_test",
+    srcs = LITE_TEST_PROTOS + TEST_PROTOS,
+    strip_prefix = "src",
+    dest = "python",
+)
+
+# TODO(dzc): Remove this once py_proto_library can have labels in srcs, in
+# which case we can simply add :protos_python_test in srcs.
+COPIED_LITE_TEST_PROTOS = ["python/" + s for s in RELATIVE_LITE_TEST_PROTOS]
+COPIED_TEST_PROTOS = ["python/" + s for s in RELATIVE_TEST_PROTOS]
+
 py_proto_library(
     name = "python_common_test_protos",
-    srcs = LITE_TEST_PROTOS + TEST_PROTOS,
-    include = "src",
+    srcs = COPIED_LITE_TEST_PROTOS + COPIED_TEST_PROTOS,
+    include = "python",
     default_runtime = "",
     protoc = ":protoc",
     srcs_version = "PY2AND3",
@@ -672,6 +708,7 @@ py_library(
         [
             "python/google/protobuf/internal/*_test.py",
             "python/google/protobuf/internal/test_util.py",
+            "python/google/protobuf/internal/import_test_package/__init__.py",
         ],
     ),
     imports = ["python"],

protobuf.bzl

@@ -26,7 +26,7 @@ def _CcOuts(srcs, use_grpc_plugin=False):
 def _PyOuts(srcs):
   return [s[:-len(".proto")] + "_pb2.py" for s in srcs]
 
-def _RelativeOutputPath(path, include):
+def _RelativeOutputPath(path, include, dest=""):
   if include == None:
     return path
 
@@ -35,16 +35,11 @@ def _RelativeOutputPath(path, include):
 
   if include and include[-1] != '/':
     include = include + '/'
+  if dest and dest[-1] != '/':
+    dest = dest + '/'
 
   path = path[len(include):]
-
-  if not path.startswith(PACKAGE_NAME):
-    fail("The package %s is not within the path %s" % (PACKAGE_NAME, path))
-
-  if not PACKAGE_NAME:
-    return path
-
-  return path[len(PACKAGE_NAME)+1:]
+  return dest + path
 
 def _proto_gen_impl(ctx):
   """General implementation for generating protos"""
@@ -53,7 +48,7 @@ def _proto_gen_impl(ctx):
   deps += ctx.files.srcs
   gen_dir = _GenDir(ctx)
   if gen_dir:
-    import_flags = ["-I" + gen_dir]
+    import_flags = ["-I" + gen_dir, "-I" + ctx.var["GENDIR"] + "/" + gen_dir]
   else:
     import_flags = ["-I."]
 
@@ -224,6 +219,36 @@ def internal_gen_well_known_protos_java(srcs):
   )
 
 
+def internal_copied_filegroup(name, srcs, strip_prefix, dest, **kwargs):
+  """Macro to copy files to a different directory and then create a filegroup.
+
+  This is used by the //:protobuf_python py_proto_library target to work around
+  an issue caused by Python source files that are part of the same Python
+  package being in separate directories.
+
+  Args:
+    srcs: The source files to copy and add to the filegroup.
+    strip_prefix: Path to the root of the files to copy.
+    dest: The directory to copy the source files into.
+    **kwargs: extra arguments that will be passesd to the filegroup.
+  """
+  outs = [_RelativeOutputPath(s, strip_prefix, dest) for s in srcs]
+
+  native.genrule(
+      name = name + "_genrule",
+      srcs = srcs,
+      outs = outs,
+      cmd = " && ".join(
+          ["cp $(location %s) $(location %s)" %
+           (s, _RelativeOutputPath(s, strip_prefix, dest)) for s in srcs]),
+  )
+
+  native.filegroup(
+      name = name,
+      srcs = outs,
+      **kwargs)
+
+
 def py_proto_library(
         name,
         srcs=[],