Add support for downloading .tar.xz files to http_archive rules.

Fixes bazelbuild#845 RELNOTES: Add support for .tar.xz archives to http_archive rules. -- MOS_MIGRATED_REVID=113829042
paniag · Feb 4, 2016 · 4134802 · 4134802
1 parent 5e8e5fe
commit 4134802
Show file tree

Hide file tree

Showing 9 changed files with 160 additions and 59 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -604,6 +604,7 @@ java_library(
         "//third_party:maven",
         "//third_party:maven_model",
         "//third_party:plexus_component_annotations",
+        "//third_party:xz",
     ],
 )
 

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/CompressedTarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/CompressedTarFunction.java
@@ -0,0 +1,86 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.base.Optional;
+import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
+import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+/**
+ * Common code for unarchiving a compressed TAR file.
+ */
+public abstract class CompressedTarFunction implements Decompressor {
+  protected abstract InputStream getDecompressorStream(DecompressorDescriptor descriptor)
+      throws IOException;
+
+  @Override
+  public Path decompress(DecompressorDescriptor descriptor) throws RepositoryFunctionException {
+    Optional<String> prefix = descriptor.prefix();
+    boolean foundPrefix = false;
+
+    try (InputStream decompressorStream = getDecompressorStream(descriptor)) {
+      TarArchiveInputStream tarStream = new TarArchiveInputStream(decompressorStream);
+      TarArchiveEntry entry;
+      while ((entry = tarStream.getNextTarEntry()) != null) {
+        StripPrefixedPath entryPath = StripPrefixedPath.maybeDeprefix(entry.getName(), prefix);
+        foundPrefix = foundPrefix || entryPath.foundPrefix();
+        if (entryPath.skip()) {
+          continue;
+        }
+
+        Path filename = descriptor.repositoryPath().getRelative(entryPath.getPathFragment());
+        FileSystemUtils.createDirectoryAndParents(filename.getParentDirectory());
+        if (entry.isDirectory()) {
+          FileSystemUtils.createDirectoryAndParents(filename);
+        } else {
+          if (entry.isSymbolicLink()) {
+            PathFragment linkName = new PathFragment(entry.getLinkName());
+            if (linkName.isAbsolute()) {
+              linkName = linkName.relativeTo(PathFragment.ROOT_DIR);
+              linkName = descriptor.repositoryPath().getRelative(linkName).asFragment();
+            }
+            FileSystemUtils.ensureSymbolicLink(filename, linkName);
+          } else {
+            Files.copy(
+                tarStream, filename.getPathFile().toPath(), StandardCopyOption.REPLACE_EXISTING);
+            filename.chmod(entry.getMode());
+          }
+        }
+      }
+    } catch (IOException e) {
+      throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+    }
+
+    if (prefix.isPresent() && !foundPrefix) {
+      throw new RepositoryFunctionException(
+          new IOException("Prefix " + prefix.get() + " was given, but not found in the archive"),
+          Transience.PERSISTENT);
+    }
+
+    return descriptor.repositoryPath();
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java
@@ -64,10 +64,12 @@ static Decompressor getDecompressor(Path archivePath)
       return ZipFunction.INSTANCE;
     } else if (baseName.endsWith(".tar.gz") || baseName.endsWith(".tgz")) {
       return TarGzFunction.INSTANCE;
+    } else if (baseName.endsWith(".tar.xz")) {
+      return TarXzFunction.INSTANCE;
     } else {
       throw new RepositoryFunctionException(
           new EvalException(null, String.format(
-              "Expected a file with a .zip, .jar, .war, .tar.gz, or .tgz suffix (got %s)",
+              "Expected a file with a .zip, .jar, .war, .tar.gz, .tgz, or .tar.xz suffix (got %s)",
               archivePath)),
           Transience.PERSISTENT);
     }

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/TarGzFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/TarGzFunction.java
@@ -14,77 +14,25 @@
 
 package com.google.devtools.build.lib.bazel.repository;
 
-import com.google.common.base.Optional;
 import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
-import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
-import com.google.devtools.build.lib.vfs.FileSystemUtils;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
-
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
+import java.io.InputStream;
 import java.util.zip.GZIPInputStream;
 
 /**
- * Creates a  repository by unarchiving a .tar.gz file.
+ * Creates a repository by unarchiving a .tar.gz file.
  */
-public class TarGzFunction implements Decompressor {
+public class TarGzFunction extends CompressedTarFunction {
   public static final Decompressor INSTANCE = new TarGzFunction();
 
   private TarGzFunction() {
   }
 
   @Override
-  public Path decompress(DecompressorDescriptor descriptor) throws RepositoryFunctionException {
-    Optional<String> prefix = descriptor.prefix();
-    boolean foundPrefix = false;
-
-    try (GZIPInputStream gzipStream = new GZIPInputStream(
-        new FileInputStream(descriptor.archivePath().getPathFile()))) {
-      TarArchiveInputStream tarStream = new TarArchiveInputStream(gzipStream);
-      TarArchiveEntry entry;
-      while ((entry = tarStream.getNextTarEntry()) != null) {
-        StripPrefixedPath entryPath = StripPrefixedPath.maybeDeprefix(entry.getName(), prefix);
-        foundPrefix = foundPrefix || entryPath.foundPrefix();
-        if (entryPath.skip()) {
-          continue;
-        }
-
-        Path filename = descriptor.repositoryPath().getRelative(entryPath.getPathFragment());
-        FileSystemUtils.createDirectoryAndParents(filename.getParentDirectory());
-        if (entry.isDirectory()) {
-          FileSystemUtils.createDirectoryAndParents(filename);
-        } else {
-          if (entry.isSymbolicLink()) {
-            PathFragment linkName = new PathFragment(entry.getLinkName());
-            if (linkName.isAbsolute()) {
-              linkName = linkName.relativeTo(PathFragment.ROOT_DIR);
-              linkName = descriptor.repositoryPath().getRelative(linkName).asFragment();
-            }
-            FileSystemUtils.ensureSymbolicLink(filename, linkName);
-          } else {
-            Files.copy(
-                tarStream, filename.getPathFile().toPath(), StandardCopyOption.REPLACE_EXISTING);
-            filename.chmod(entry.getMode());
-          }
-        }
-      }
-    } catch (IOException e) {
-      throw new RepositoryFunctionException(e, Transience.TRANSIENT);
-    }
-
-    if (prefix.isPresent() && !foundPrefix) {
-      throw new RepositoryFunctionException(
-          new IOException("Prefix " + prefix.get() + " was given, but not found in the archive"),
-          Transience.PERSISTENT);
-    }
-
-    return descriptor.repositoryPath();
+  protected InputStream getDecompressorStream(DecompressorDescriptor descriptor)
+      throws IOException {
+    return new GZIPInputStream(new FileInputStream(descriptor.archivePath().getPathFile()));
   }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/TarXzFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/TarXzFunction.java
@@ -0,0 +1,39 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
+
+import org.tukaani.xz.XZInputStream;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Creates a repository by unarchiving a .tar.xz file.
+ */
+class TarXzFunction extends CompressedTarFunction {
+  public static final Decompressor INSTANCE = new TarXzFunction();
+
+  private TarXzFunction() {
+  }
+
+  @Override
+  protected InputStream getDecompressorStream(DecompressorDescriptor descriptor)
+      throws IOException {
+    return new XZInputStream(new FileInputStream(descriptor.archivePath().getPathFile()));
+  }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java
@@ -42,6 +42,12 @@ public void testKnownFileExtensionsDoNotThrow() throws Exception {
     DecompressorDescriptor.builder().setArchivePath(path).build();
     path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.zip");
     DecompressorDescriptor.builder().setArchivePath(path).build();
+    path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.gz");
+    DecompressorDescriptor.builder().setArchivePath(path).build();
+    path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tgz");
+    DecompressorDescriptor.builder().setArchivePath(path).build();
+    path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.xz");
+    DecompressorDescriptor.builder().setArchivePath(path).build();
   }
 
   @Test

diff --git a/src/test/shell/bazel/external_integration_test.sh b/src/test/shell/bazel/external_integration_test.sh
@@ -57,6 +57,11 @@ function tar_gz_up() {
   tar czf $repo2_zip WORKSPACE fox
 }
 
+function tar_xz_up() {
+  repo2_zip=$TEST_TMPDIR/fox.tar.xz
+  tar cJf $repo2_zip WORKSPACE fox
+}
+
 # Test downloading a file from a repository.
 # This creates a simple repository containing:
 #
@@ -187,6 +192,10 @@ function test_http_archive_tgz() {
   http_archive_helper tar_gz_up "do_symlink"
 }
 
+function test_http_archive_tar_xz() {
+  http_archive_helper tar_xz_up "do_symlink"
+}
+
 function test_http_archive_no_server() {
   nc_port=$(pick_random_unused_tcp_port) || exit 1
   cat > WORKSPACE <<EOF

diff --git a/third_party/BUILD b/third_party/BUILD
@@ -398,6 +398,11 @@ java_import(
     jars = ["truth/truth-0.28.jar"],
 )
 
+java_import(
+    name = "xz",
+    jars = ["xz/xz-1.5.jar"],
+)
+
 cc_library(
     name = "gtest",
     srcs = [

diff --git a/third_party/README.md b/third_party/README.md
@@ -278,3 +278,8 @@ a minimal set of extra dependencies.
 
 * Version: 0.28
 * License: Apache License 2.0
+
+## [xz](http://tukaani.org/xz/java.html)
+
+* Version: 1.5
+* License: Public Domain