add option to singlejar to double-check correct default and static in…

…terface method desugaring for Android.

RELNOTES: none

PiperOrigin-RevId: 171891682

WORKSPACE

@@ -78,6 +78,12 @@ new_local_repository(
     path = "./third_party/protobuf/3.4.0/",
 )
 
+new_local_repository(
+    name = "com_google_protobuf_cc",
+    build_file = "./third_party/protobuf/3.4.0/BUILD",
+    path = "./third_party/protobuf/3.4.0/",
+)
+
 new_local_repository(
     name = "com_google_protobuf_java",
     build_file = "./third_party/protobuf/3.4.0/com_google_protobuf_java.BUILD",

src/tools/singlejar/BUILD

@@ -285,7 +285,10 @@ cc_library(
         ":zip_headers",
     ],
     hdrs = ["combiners.h"],
-    deps = ["//third_party/zlib"],
+    deps = [
+        "//src/main/protobuf:desugar_deps_cc_proto",
+        "//third_party/zlib",
+    ],
 )
 
 cc_library(

src/tools/singlejar/combiners.cc

@@ -14,6 +14,7 @@
 
 #include "src/tools/singlejar/combiners.h"
 #include "src/tools/singlejar/diag.h"
+#include "src/main/protobuf/desugar_deps.pb.h"
 
 Combiner::~Combiner() {}
 
@@ -174,3 +175,139 @@ PropertyCombiner::~PropertyCombiner() {}
 bool PropertyCombiner::Merge(const CDH *cdh, const LH *lh) {
   return false;  // This should not be called.
 }
+
+bool Java8DesugarDepsChecker::Merge(const CDH *cdh, const LH *lh) {
+  // Throw away anything previously read, no need to concatenate
+  buffer_.reset(new TransientBytes());
+  if (Z_NO_COMPRESSION == lh->compression_method()) {
+    buffer_->ReadEntryContents(lh);
+  } else if (Z_DEFLATED == lh->compression_method()) {
+    if (!inflater_.get()) {
+      inflater_.reset(new Inflater());
+    }
+    buffer_->DecompressEntryContents(cdh, lh, inflater_.get());
+  } else {
+    errx(2, "META-INF/desugar_deps is neither stored nor deflated");
+  }
+
+  // TODO(kmb): Wrap buffer_ as ZeroCopyInputStream to avoid copying out.
+  // Note we only copy one file at a time, so overhead should be modest.
+  uint32_t checksum;
+  const size_t data_size = buffer_->data_size();
+  uint8_t *buf = reinterpret_cast<uint8_t *>(malloc(data_size));
+  buffer_->CopyOut(reinterpret_cast<uint8_t *>(buf), &checksum);
+  buffer_.reset();  // release buffer eagerly
+
+  bazel::tools::desugar::DesugarDepsInfo deps_info;
+  google::protobuf::io::CodedInputStream content(buf, data_size);
+  if (!deps_info.ParseFromCodedStream(&content)) {
+    errx(2, "META-INF/desugar_deps: unable to parse");
+  }
+  if (!content.ConsumedEntireMessage()) {
+    errx(2, "META-INF/desugar_deps: unexpected trailing content");
+  }
+  free(buf);
+
+  for (const auto &assume_present : deps_info.assume_present()) {
+    // This means we need file named <target>.class in the output.  Remember
+    // the first origin of this requirement for error messages, drop others.
+    needed_deps_.emplace(assume_present.target().binary_name() + ".class",
+                         assume_present.origin().binary_name());
+  }
+
+  for (const auto &missing : deps_info.missing_interface()) {
+    // Remember the first origin of this requirement for error messages, drop
+    // subsequent ones.
+    missing_interfaces_.emplace(missing.target().binary_name(),
+                                missing.origin().binary_name());
+  }
+
+  for (const auto &extends : deps_info.interface_with_supertypes()) {
+    // Remember interface hierarchy the first time we see this interface, drop
+    // subsequent ones for consistency with how singlejar will keep the first
+    // occurrence of the file defining the interface.  We'll lazily derive
+    // whether missing_interfaces_ inherit default methods with this data later.
+    if (extends.extended_interface_size() > 0) {
+      std::vector<std::string> extended;
+      extended.reserve(extends.extended_interface_size());
+      for (const auto &itf : extends.extended_interface()) {
+        extended.push_back(itf.binary_name());
+      }
+      extended_interfaces_.emplace(extends.origin().binary_name(),
+                                   std::move(extended));
+    }
+  }
+
+  for (const auto &companion : deps_info.interface_with_companion()) {
+    // Only remember interfaces that definitely have default methods for now.
+    // For all other interfaces we'll transitively check extended interfaces
+    // in HasDefaultMethods.
+    if (companion.num_default_methods() > 0) {
+      has_default_methods_[companion.origin().binary_name()] = true;
+    }
+  }
+  return true;
+}
+
+void *Java8DesugarDepsChecker::OutputEntry(bool compress) {
+  if (verbose_) {
+    fprintf(stderr, "Needed deps: %lu\n", needed_deps_.size());
+    fprintf(stderr, "Interfaces to check: %lu\n", missing_interfaces_.size());
+    fprintf(stderr, "Sub-interfaces: %lu\n", extended_interfaces_.size());
+    fprintf(stderr, "Interfaces w/ default methods: %lu\n",
+            has_default_methods_.size());
+  }
+  for (auto needed : needed_deps_) {
+    if (verbose_) {
+      fprintf(stderr, "Looking for %s\n", needed.first.c_str());
+    }
+    if (!known_member_(needed.first)) {
+      if (fail_on_error_) {
+        errx(2, "%s referenced by %s but not found.  Is the former defined in "
+                "a neverlink library?",
+                needed.first.c_str(), needed.second.c_str());
+      } else {
+        error_ = true;
+      }
+    }
+  }
+
+  for (auto missing : missing_interfaces_) {
+    if (verbose_) {
+      fprintf(stderr, "Checking %s\n", missing.first.c_str());
+    }
+    if (HasDefaultMethods(missing.first)) {
+      if (fail_on_error_) {
+        errx(2, "%s needed on the classpath for desugaring %s.  Please add the "
+                "missing dependency to the target containing the latter.",
+                missing.first.c_str(), missing.second.c_str());
+      } else {
+        error_ = true;
+      }
+    }
+  }
+
+  // We don't want these files in the output, just check them for consistency
+  return nullptr;
+}
+
+bool Java8DesugarDepsChecker::HasDefaultMethods(
+    const std::string &interface_name) {
+  auto cached = has_default_methods_.find(interface_name);
+  if (cached != has_default_methods_.end()) {
+    return cached->second;
+  }
+
+  // Prime with false in case there's a cycle.  We'll update with the true value
+  // (ignoring the cycle) below.
+  has_default_methods_.emplace(interface_name, false);
+
+  for (const std::string &extended : extended_interfaces_[interface_name]) {
+    if (HasDefaultMethods(extended)) {
+      has_default_methods_[interface_name] = true;
+      return true;
+    }
+  }
+  has_default_methods_[interface_name] = false;
+  return false;
+}

src/tools/singlejar/combiners.h

@@ -15,8 +15,11 @@
 #ifndef SRC_TOOLS_SINGLEJAR_COMBINERS_H_
 #define SRC_TOOLS_SINGLEJAR_COMBINERS_H_ 1
 
+#include <map>
 #include <memory>
 #include <string>
+#include <unordered_map>
+#include <vector>
 
 #include "src/tools/singlejar/transient_bytes.h"
 #include "src/tools/singlejar/zip_headers.h"
@@ -133,4 +136,61 @@ class PropertyCombiner : public Concatenator {
   }
 };
 
+// Combiner that checks META-INF/desugar_deps files (b/65645388) to ensure
+// correct bytecode desugaring, specifically of default and static interface
+// methods, across an entire binary.  Two checks are performed:
+// 1. Make sure that any dependency assumed by the desugaring process is in
+//    fact part of the binary.  This protects against ill-advised uses of
+//    neverlink, where a library is only on the compile-time classpath but not
+//    the runtime classpath.
+// 2. To paper over incomplete classpaths during desugaring (b/65211436), check
+//    that interfaces that couldn't be found don't declare or inherit default
+//    methods.  Desugar emits extra metadata to avoid us having to open up and
+//    parse .class files for this purpose.
+class Java8DesugarDepsChecker : public Combiner {
+ public:
+  Java8DesugarDepsChecker(std::function<bool(const std::string &)> known_member,
+                          bool verbose)
+      : Java8DesugarDepsChecker(std::move(known_member), verbose, true) {}
+  ~Java8DesugarDepsChecker() override {}
+
+  bool Merge(const CDH *cdh, const LH *lh) override;
+
+  void *OutputEntry(bool compress) override;
+
+ private:
+  Java8DesugarDepsChecker(std::function<bool(const std::string &)> known_member,
+                          bool verbose, bool fail_on_error)
+      : known_member_(std::move(known_member)),
+        verbose_(verbose),
+        fail_on_error_(fail_on_error),
+        error_(false) {}
+  /// Computes and caches whether the given interface has default methods.
+  /// \param interface_name interface name as it would appear in bytecode, e.g.,
+  ///        "java/lang/Runnable"
+  bool HasDefaultMethods(const std::string &interface_name);
+
+  const std::function<bool(const std::string &)> known_member_;
+  const bool verbose_;
+  const bool fail_on_error_;  // For testing
+
+  std::unique_ptr<TransientBytes> buffer_;
+  std::unique_ptr<Inflater> inflater_;
+  /// Reverse mapping from needed dependencies to one of the users.
+  std::map<std::string, std::string> needed_deps_;
+  /// Reverse mapping from missing interfaces to one of the classes that missed
+  /// them.
+  std::map<std::string, std::string> missing_interfaces_;
+  std::unordered_map<std::string, std::vector<std::string> >
+      extended_interfaces_;
+  /// Cache of interfaces known to definitely define or inherit default methods
+  /// or definitely not define and not inherit default methods.  Merge()
+  /// populates initial entries and HasDefaultMethods() adds to the cache as
+  /// needed.
+  std::unordered_map<std::string, bool> has_default_methods_;
+  bool error_;
+
+  friend class CombinersTest;
+};
+
 #endif  //  SRC_TOOLS_SINGLEJAR_COMBINERS_H_