Merge branch 'master' of https://github.com/JohnLangford/vowpal_wabbit

trevorlazarus · Jan 29, 2016 · 5be0fc8 · 5be0fc8
2 parents ebe3ab7 + 9655ef4
commit 5be0fc8
Show file tree

Hide file tree

Showing 8 changed files with 227 additions and 72 deletions.
diff --git a/Makefile b/Makefile
@@ -20,7 +20,7 @@ endif
 UNAME := $(shell uname)
 LIBS = -l boost_program_options -l pthread -l z
 BOOST_INCLUDE = -I /usr/include
-BOOST_LIBRARY = -L /usr/lib
+BOOST_LIBRARY = -L /usr/local/lib -L /usr/lib
 NPROCS := 1
 
 ifeq ($(UNAME), Linux)

diff --git a/README.md b/README.md
@@ -18,6 +18,7 @@ These prerequisites are usually pre-installed on many platforms. However, you ma
 manager (*yum*, *apt*, *MacPorts*, *brew*, ...) to install missing software.
 
 - [Boost](http://www.boost.org) library, with the `Boost::Program_Options` library option enabled.
+- lsb-release  (RedHat/CentOS: redhat-lsb-core, Debian: lsb-release, Ubuntu: you're all set, OSX: not required)
 - GNU *autotools*: *autoconf*, *automake*, *libtool*, *autoheader*, et. al. This is not a strict prereq. On many systems (notably Ubuntu with `libboost-program-options-dev` installed), the provided `Makefile` works fine.
 - (optional) [git](http://git-scm.com) if you want to check out the latest version of *vowpal wabbit*,
   work on the code, or even contribute code to the main project.
@@ -133,13 +134,16 @@ brew install vowpal-wabbit
 ```
 [The homebrew formula for VW is located on github](https://github.com/Homebrew/homebrew/blob/master/Library/Formula/vowpal-wabbit.rb).
 
-### brew install dependencies + manual install of vowpal wabbit
+### Manual install ov Vowpal Wabbit
+#### OSX Dependencies (if using Brew): 
 ```
 brew install libtool
+brew install autoconf
+brew install automake
 brew install boost --with-python
 ```
 
-### MacPorts
+#### OSX Dependencies (if using MacPorts):
 ```
 ## Install glibtool and other GNU autotool friends:
 $ port install libtool autoconf automake
@@ -151,6 +155,7 @@ $ port install boost +no_single +no_static +openmpi +python27 configure.cxx_stdl
 $ port install boost +no_single +no_static +openmpi +python27
 ```
 
+#### OSX Manual compile:
 *Mac OS X 10.8 and below*: ``configure.cxx_stdlib=libc++`` and ``configure.cxx=clang++`` ensure that ``clang++`` uses
 the correct C++11 functionality while building Boost. Ordinarily, ``clang++`` relies on the older GNU ``g++`` 4.2 series
 header files and ``stdc++`` library; ``libc++`` is the ``clang`` replacement that provides newer C++11 functionality. If

diff --git a/java/src/main/bin/build.sh b/java/src/main/bin/build.sh
@@ -8,10 +8,10 @@ __brew_not_installed=2
 
 make_base="cd /vowpal_wabbit;
 make clean;
-make;"
+make vw java;"
 
 ubuntu_base="apt-get update -qq;
-apt-get install -qq software-properties-common g++ make libboost-all-dev default-jdk;"
+apt-get install -qq software-properties-common g++ make libboost-program-options-dev zlib1g-dev default-jdk;"
 
 ubuntu_12="$ubuntu_base
 export JAVA_HOME=/usr/lib/jvm/java-6-openjdk-amd64;
@@ -29,7 +29,7 @@ $make_base
 mv java/target/vw_jni.lib java/target/vw_jni.Ubuntu.14.amd64.lib"
 
 early_red_hat="yum update -q -y
-yum install -q -y wget which zlib-devel java-1.7.0-openjdk-devel perl;
+yum install -q -y wget which zlib-devel java-1.7.0-openjdk-devel perl redhat-lsb-core;
 cd /etc/yum.repos.d;
 wget http://people.centos.org/tru/devtools-2/devtools-2.repo;
 yum clean all;
@@ -44,18 +44,19 @@ yum install -q -y make epel-release;
 yum install -q -y boost141-devel;
 make clean;
 cat Makefile.permissive | sed 's/BOOST_LIBRARY = -L \/usr\/lib/BOOST_LIBRARY = -L \/usr\/lib64\/boost141/g' | sed 's/BOOST_INCLUDE = -I \/usr\/include/BOOST_INCLUDE = -I \/usr\/include\/boost141/g' > Makefile.permissive.boost141;
-make -f Makefile.permissive.boost141;
+make -f Makefile.permissive.boost141 vw java;
 rm -f Makefile.permissive Makefile.permissive.boost141;
 mv java/target/vw_jni.lib java/target/vw_jni.Red_Hat.5.amd64.lib"
 
 red_hat_6="$early_red_hat
 yum install -q -y boost-devel;
 make clean;
-make -f Makefile.permissive;
+make -f Makefile.permissive vw java;
 rm -f Makefile.permissive;
 mv java/target/vw_jni.lib java/target/vw_jni.Red_Hat.6.amd64.lib"
 
-red_hat_7="yum install -q -y gcc-c++ make boost-devel zlib-devel java-1.7.0-openjdk-devel perl;
+red_hat_7="yum update -q -y;
+yum install -q -y gcc-c++ make boost-devel zlib-devel java-1.7.0-openjdk-devel perl clang redhat-lsb-core;
 export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk;
 $make_base
 mv java/target/vw_jni.lib java/target/vw_jni.Red_Hat.7.amd64.lib"
@@ -131,7 +132,7 @@ install_cask_app() {
 run_docker() {
   local machine=$1
   local script=$2
-  docker run -v $(pwd):/vowpal_wabbit $machine /bin/bash -c "$script"
+  docker run --rm -v $(pwd):/vowpal_wabbit $machine /bin/bash -c "$script"
 }
 
 # =============================================================================
@@ -145,8 +146,12 @@ install_brew_app "docker-machine"
 install_brew_app "docker"
 
 docker-machine create --driver virtualbox default
+docker-machine start default
 eval "$(docker-machine env default)"
 
+set -e
+set -u
+set -x
 run_docker "ubuntu:12.04" "$ubuntu_12"
 run_docker "32bit/ubuntu:14.04" "$ubuntu_14_32"
 run_docker "ubuntu:14.04" "$ubuntu_14"
@@ -162,5 +167,5 @@ run_docker "centos:6" "$red_hat_6"
 run_docker "centos:7" "$red_hat_7"
 
 make clean
-make
+make vw java
 mv java/target/vw_jni.lib java/target/vw_jni.$(uname -s).$(uname -m).lib
diff --git a/java/src/main/java/vw/jni/NativeUtils.java b/java/src/main/java/vw/jni/NativeUtils.java
@@ -3,6 +3,10 @@
 import java.io.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
 
 /**
  * Simple library class for working with JNI (Java Native Interface),
@@ -19,40 +23,32 @@ public class NativeUtils {
     private NativeUtils() {
     }
 
-    /**
-     * This will read from /proc/version to attempt to find the Linux distribution.
-     * @return The Linux distribution or null if the version cannot be found.
-     * @throws IOException If an I/O error occurs
-     */
-    public static String getDistroName() throws IOException {
-        Pattern distroRegex = Pattern.compile("[^(]+\\([^(]+\\([^(]+\\(([A-Za-z\\s]+).*");
-        BufferedReader reader = new BufferedReader(new FileReader("/proc/version"));
-        String distro;
-        try {
-            Matcher line = distroRegex.matcher(reader.readLine());
-            distro = line.matches() ? line.group(1) : null;
-        }
-        finally {
-            reader.close();
-        }
-        return distro;
+    // Identifies which OS distributions can share binaries
+    private static final Map<String, List<String>> identicalOSs = new HashMap<String, List<String>>();
+    static {
+        // RedHat binaries
+        List<String> redHatAlternatives = new ArrayList<String>();
+        redHatAlternatives.add("Red_Hat.6");
+        identicalOSs.put("Scientific.6", redHatAlternatives);
     }
 
     /**
-     * This will attempt to find the Linux version by making use of {@code lsb_release -r}
-     * @return The Linux version or null if the version cannot be determined.
-     * @throws IOException If an I/O error occurs
+     * Shell call to lsb_release to get OS info
+     * @param lsb_args parameters to lsb_release.  EX: "-i" or "-a"
+     * @param regexp a regular expression pattern used to parse the response from lsb_release
+     *   For example:  lsb_args = "-i", regexp = "Distributor ID: *(.*)$"
+     * @return A system dependent string identifying the OS.
+     * @throws IOException If an error occurs while running shell command
      */
-    public static String getLinuxVersion() throws IOException {
+    public static String lsbRelease(String lsb_args, Pattern regexp) throws IOException {
         BufferedReader reader = null;
         try {
-            Process process = Runtime.getRuntime().exec("lsb_release -r");
+            Process process = Runtime.getRuntime().exec("lsb_release " + lsb_args);
             reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
             String line;
-            Pattern releasePattern = Pattern.compile("Release:\\s*(\\d+).*");
             Matcher matcher;
             while ((line = reader.readLine()) != null) {
-                matcher = releasePattern.matcher(line);
+                matcher = regexp.matcher(line);
                 if (matcher.matches()) {
                     return matcher.group(1);
                 }
@@ -65,32 +61,62 @@ public static String getLinuxVersion() throws IOException {
     }
 
     /**
+     * Attempt to find the Linux distribution ID.
+     * @return The Linux distribution or null if the version cannot be found.
+     * @throws IOException If an I/O error occurs
+     */
+    public static String getDistroName() throws IOException {
+        return lsbRelease("-i", Pattern.compile("Distributor ID:\\s*(.*)\\s*$"));
+    }
+
+    /**
+     * This will attempt to find the Linux version by making use of {@code lsb_release -r}
+     * @return The Linux version or null if the version cannot be determined.
+     * @throws IOException If an I/O error occurs
+     */
+    public static String getLinuxVersion() throws IOException {
+        return lsbRelease("-r", Pattern.compile("Release:\\s*(.*)\\s*$"));
+    }
+
+   /**
      * Returns the system dependent OS family.  In the case of a Linux OS it will combine
      * {@link NativeUtils#getDistroName()} and {@link NativeUtils#getLinuxVersion()}.
-     * @return A system dependent string identifying the OS.
+     * @return A list system dependent string identifying the OS.  This is a list because there are some OSs which are
+     * functionally identical to the one returned by lsbRelease.
      * @throws UnsupportedEncodingException If an error occurs while determining the Linux specific
      *          information.
      * @throws IOException If an I/O error occurs
      * @throws IllegalStateException If the os.name property returns an unsupported OS.
      */
-    public static String getOsFamily() throws IOException {
+    public static List<String> getOsFamilies() throws IOException {
         final String osName = System.getProperty("os.name");
+        final String primaryName;
         if (osName.toLowerCase().contains("mac")) {
-            return "Darwin";
+            primaryName = "Darwin";
         }
         else if (osName.toLowerCase().contains("linux")) {
             String distro = getDistroName();
             if (distro == null) {
                 throw new UnsupportedEncodingException("Cannot determine linux distribution");
             }
-
             String version = getLinuxVersion();
             if (version == null) {
                 throw new UnsupportedOperationException("Cannot determine linux version");
             }
-            return distro.trim().replaceAll(" ", "_") + "." + version;
+            // get the major version.
+            // don't expect a period because linux version might not have one
+            primaryName = distro + "." + version.split("\\.")[0];
+        }
+        else {
+            throw new IllegalStateException("Unsupported operating system " + osName);
         }
-        throw new IllegalStateException("Unsupported operating system " + osName);
+
+        List<String> viableFamilies = new ArrayList<String>();
+        viableFamilies.add(primaryName);
+        if (identicalOSs.containsKey(primaryName)) {
+            viableFamilies.addAll(identicalOSs.get(primaryName));
+        }
+        return viableFamilies;
     }
 
     /**
@@ -102,9 +128,16 @@ else if (osName.toLowerCase().contains("linux")) {
      * @throws IOException If temporary file creation or read/write operation fails
      */
     public static void loadOSDependentLibrary(String path, String suffix) throws IOException {
-        String osFamily = getOsFamily();
-        String osDependentLib = path + "." + osFamily + "." + System.getProperty("os.arch") + suffix;
-        if (NativeUtils.class.getResource(osDependentLib) != null) {
+        List<String> osFamilies = getOsFamilies();
+        String osDependentLib = null;
+        for (String osFamily : osFamilies) {
+            String currentOsDependentLib = path + "." + osFamily + "." + System.getProperty("os.arch") + suffix;
+            if (NativeUtils.class.getResource(currentOsDependentLib) != null) {
+                osDependentLib = currentOsDependentLib;
+                break;
+            }
+        }
+        if (osDependentLib != null) {
             loadLibraryFromJar(osDependentLib);
         }
         else {