Tests for tools/cpp:cc_configure.bzl

They test ./compile.sh under various configuration using Docker.

Because we miss several stuff from our docker support (docker_pull and
docker_test), those test are highly unhermetic.

This only includes tests for a few OS. We will add tests for specific
use case on-demand.

--
MOS_MIGRATED_REVID=116197057

BUILD

@@ -10,6 +10,12 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 
+filegroup(
+    name = "workspace-file",
+    srcs = [":WORKSPACE"],
+    visibility = ["//tools/cpp/test:__pkg__"],
+)
+
 filegroup(
     name = "srcs",
     srcs = glob(

WORKSPACE

@@ -23,6 +23,10 @@ bind(
     actual = "//:dummy",
 )
 
+# For tools/cpp/test/...
+load("//tools/cpp/test:docker_repository.bzl", "docker_repository")
+docker_repository()
+
 # In order to run the Android integration tests, run
 # scripts/workspace_user.sh and uncomment the next two lines.
 # load("/WORKSPACE.user", "android_repositories")

tools/cpp/BUILD

@@ -127,5 +127,5 @@ cc_toolchain(
 
 filegroup(
     name = "srcs",
-    srcs = glob(["**"]),
+    srcs = glob(["**"]) + ["//tools/cpp/test:srcs"],
 )

tools/cpp/test/BUILD

@@ -0,0 +1,120 @@
+load("//tools/build_defs/docker:docker.bzl", "docker_build")
+load("//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+
+# This is totally non hermetic, we should really replace that by a
+# docker_pull rule.
+FLAVOURS = [f[f.find(".") + 1:] for f in glob(["Dockerfile.*"])]
+
+[
+    # This is totally non hermetic.
+    genrule(
+        name = "docker-" + flavour,
+        srcs = ["Dockerfile." + flavour],
+        outs = ["docker-%s.tar" % flavour],
+        cmd = "\n".join([
+            "DIR=\"$$(dirname $<)\"",
+            "IMG=\"$$(basename $<)\"",
+            "DOCKER=\"$${PWD}/$(location @docker//:docker)\"",
+            "(",
+            "   cd $$DIR",
+            "   $$DOCKER build -f $$IMG -t bazel_tools_cpp_test:%s ." % flavour,
+            ")",
+            "$$DOCKER save bazel_tools_cpp_test:%s > $@" % flavour,
+        ]),
+        tags = ["local"],
+        # Docker needs to knows how to contact the daemon from the environment.
+        # @docker//:docker point to a docker wrapper that copy the environment
+        # of the user.
+        tools = ["@docker//:docker"],
+    )
+    for flavour in FLAVOURS
+]
+
+# Just to avoid re-reading docker images all the time
+[
+    docker_build(
+        name = "base-" + flavour,
+        base = "docker-" + flavour,
+    )
+    for flavour in FLAVOURS
+]
+
+genrule(
+    name = "gen_workspace",
+    srcs = ["//:workspace-file"],
+    outs = ["WORKSPACE"],
+    cmd = """
+    cat <<EOF >$@
+load("//tools/cpp:cc_configure.bzl", "cc_configure")
+cc_configure()
+EOF
+    cat $(location //:workspace-file) >>$@
+""",
+)
+
+pkg_tar(
+    name = "cc_configure_ws",
+    files = [":WORKSPACE"],
+    package_dir = "/opt/workspace",
+    strip_prefix = ".",
+)
+
+pkg_tar(
+    name = "bazel_cc_configure",
+    package_dir = "/opt/workspace",
+    strip_prefix = "/",
+    deps = [
+        # Order matters.
+        ":cc_configure_ws",
+        "//:bazel-srcs",
+    ],
+)
+
+[
+    [docker_build(
+        name = "bazel_cc_configure-%s-%s" % (flavour, mode),
+        base = ":base-" + flavour,
+        entrypoint = "/opt/workspace/compile.sh",
+        env = {
+            "EXTRA_BAZEL_ARGS": "--spawn_strategy=standalone --genrule_strategy=standalone -c %s" % mode,
+        },
+        tars = [":bazel_cc_configure"],
+        workdir = "/opt/workspace",
+    ) for mode in [
+        "dbg",
+        "opt",
+        "fastbuild",
+    ]]
+    for flavour in FLAVOURS
+]
+
+[
+    [py_test(
+        name = "test_cc_configure-%s-%s" % (flavour, mode),
+        size = "large",
+        srcs = ["docker_test.py"],
+        args = [
+            "--main='$(location :bazel_cc_configure-%s-%s)'" % (flavour, mode),
+            "--docker='$(location @docker//:docker)'",
+        ],
+        data = [
+            ":bazel_cc_configure-%s-%s" % (flavour, mode),
+            "@docker//:docker",
+        ],
+        local = 1,
+        main = "docker_test.py",
+        tags = ["local"],
+        deps = ["//third_party/py/gflags"],
+    ) for mode in [
+        "dbg",
+        "opt",
+        "fastbuild",
+    ]]
+    for flavour in FLAVOURS
+]
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["**"]),
+    visibility = ["//tools/cpp:__pkg__"],
+)

tools/cpp/test/Dockerfile.fedora23

@@ -0,0 +1,8 @@
+FROM fedora:23
+
+RUN dnf -y update && dnf clean all
+RUN dnf -y install \
+    which findutils binutils gcc tar gzip \
+    zip unzip java java-devel git clang zlib-devel \
+    && dnf clean all
+ENV JAVA_HOME /usr/lib/jvm/java-openjdk

tools/cpp/test/Dockerfile.ubuntu-15.04

@@ -0,0 +1,8 @@
+FROM ubuntu:15.04
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends curl ca-certificates \
+      git pkg-config zip unzip \
+      g++ gcc openjdk-8-jdk \
+      zlib1g-dev libarchive-dev \
+      ca-certificates-java && \
+    apt-get clean

tools/cpp/test/Dockerfile.ubuntu-15.10

@@ -0,0 +1,8 @@
+FROM ubuntu:15.10
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends curl ca-certificates \
+      git pkg-config zip unzip \
+      g++ gcc openjdk-8-jdk \
+      zlib1g-dev libarchive-dev \
+      ca-certificates-java && \
+    apt-get clean

tools/cpp/test/docker_repository.bzl

@@ -0,0 +1,57 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Rule for importing the docker binary for tests (experimental)."""
+
+def _impl(ctx):
+  docker = ctx.which("docker")
+  if docker == None:
+    # We cannot find docker, we won't be able to run tests depending
+    # on it, silently ignoring.
+    ctx.file("BUILD",
+             "\n".join([
+                 "filegroup(",
+                 "    name = 'docker',",
+                 "    visibility = ['//visibility:public'],",
+                 ")"
+                 ]))
+  else:
+    exports = []
+    for k in ctx.os.environ:
+      # DOCKER* environment variable are used by the docker client
+      # to know how to talk to the docker daemon.
+      if k.startswith("DOCKER"):
+        exports.append("export %s='%s'" % (k, ctx.os.environ[k]))
+    ctx.symlink(docker, "docker-bin")
+    ctx.file("docker.sh", "\n".join([
+        "#!/bin/bash",
+        "\n".join(exports),
+"""BIN="$0"
+while [ -L "${BIN}" ]; do
+  BIN="$(readlink "${BIN}")"
+done
+exec "${BIN%%.sh}-bin" "$@"
+"""]))
+    ctx.file("BUILD", "\n".join([
+        "sh_binary(",
+        "    name = 'docker',",
+        "    srcs = ['docker.sh'],",
+        "    data = [':docker-bin'],",
+        "    visibility = ['//visibility:public'],",
+        ")"]))
+
+docker_repository_ = repository_rule(_impl)
+
+def docker_repository():
+  """Declare a @docker repository that provide a docker binary."""
+  docker_repository_(name = "docker")

tools/cpp/test/docker_test.py

@@ -0,0 +1,158 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A simple test runner for docker (experimental)."""
+
+import copy
+import os
+import os.path
+import shlex
+import StringIO
+import subprocess
+import sys
+import threading
+
+from third_party.py import gflags
+
+gflags.DEFINE_multistring(
+    "image", [],
+    "The list of additional docker image to load (path to a docker_build "
+    "target), optional.")
+
+gflags.DEFINE_string(
+    "main", None,
+    "The main image to run (path to a docker_build target), mandatory.")
+gflags.MarkFlagAsRequired("main")
+
+gflags.DEFINE_string(
+    "cmd", None,
+    "A command to provide to the docker image, optional (default: use the "
+    "entrypoint).")
+
+gflags.DEFINE_string("docker", "docker", "Path to the docker client binary.")
+
+gflags.DEFINE_boolean("verbose", True, "Be verbose.")
+
+FLAGS = gflags.FLAGS
+
+LOCAL_IMAGE_PREFIX = "bazel/docker_test:"
+
+
+def _copy_stream(in_stream, out_stream):
+  for c in iter(lambda: in_stream.read(1), ""):
+    out_stream.write(c)
+    out_stream.flush()
+
+
+def execute(command, stdout=sys.stdout, stderr=sys.stderr, env=None):
+  """Execute a command while redirecting its output streams."""
+  if FLAGS.verbose:
+    print "Executing '%s'" % " ".join(command)
+  p = subprocess.Popen(command,
+                       stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE,
+                       env=env)
+  t1 = threading.Thread(target=_copy_stream, args=[p.stdout, stdout])
+  t2 = threading.Thread(target=_copy_stream, args=[p.stderr, stderr])
+  t1.daemon = True
+  t2.daemon = True
+  t1.start()
+  t2.start()
+  p.wait()
+  t1.join()
+  t2.join()
+  return p.returncode
+
+
+def load_image(image):
+  """Load a docker image using the runner provided by docker_build."""
+  tag = LOCAL_IMAGE_PREFIX + image.replace("/", "_")
+  err = StringIO.StringIO()
+  env = copy.deepcopy(os.environ)
+  env["DOCKER"] = FLAGS.docker
+  ret = execute([image, tag], stderr=err, env=env)
+  if ret != 0:
+    sys.stderr.write("Error loading image %s (return code: %s):\n" %
+                     (image, ret))
+    sys.stderr.write(err.getvalue())
+    return None
+  return tag
+
+
+def load_images(images):
+  """Load a series of docker images using the docker_build's runner."""
+  print "### Image loading ###"
+  return [load_image(image) for image in images]
+
+
+def cleanup_images(tags):
+  """Remove docker tags and images previously loaded."""
+  print "### Image cleanup ###"
+  for tag in tags:
+    if isinstance(tag, basestring):
+      execute([FLAGS.docker, "rmi", tag])
+
+
+def run_image(tag):
+  """Run a docker image, in background."""
+  print "Running " + tag
+  out = StringIO.StringIO()
+  err = StringIO.StringIO()
+  process = execute([FLAGS.docker, "run", "--rm", tag], out, err)
+  if process.wait() != 0:
+    sys.stderr.write("Error running docker run on %s:\n" % tag)
+    sys.stderr.write(err.getvalue())
+    return None
+  else:
+    return out.getvalue().strip()
+
+
+def run_images(tags):
+  """Run a list of docker images, in background."""
+  print "### Running images ###"
+  return [run_image(tag) for tag in tags]
+
+
+def cleanup_containers(containers):
+  """Kill containers."""
+  print "### Containers cleanup ###"
+  for c in containers:
+    if isinstance(c, basestring):
+      execute([FLAGS.docker, "kill", c])
+
+
+def main(unused_argv):
+  tags = load_images([FLAGS.main] + FLAGS.image)
+  if None in tags:
+    cleanup_images(tags)
+    return -1
+  try:
+    containers = run_images(tags[1:])
+    ret = -1
+    if None not in containers:
+      print "### Running main container ###"
+      ret = execute([
+          FLAGS.docker,
+          "run",
+          "--rm",
+          "--attach=STDOUT",
+          "--attach=STDERR", tags[0]
+          ] + ([] if FLAGS.cmd is None else shlex.split(FLAGS.cmd)))
+  finally:
+    cleanup_containers(containers)
+    cleanup_images(tags)
+  return ret
+
+
+if __name__ == "__main__":
+  sys.exit(main(FLAGS(sys.argv)))