Merge remote-tracking branch 'upstream/master'

Author: jgustave

.classpath

@@ -34,5 +34,6 @@
 	<classpathentry kind="lib" path="lib/jogamp/gluegen-rt-natives-linux-amd64.jar"/>
 	<classpathentry kind="lib" path="lib/jogamp/gluegen-rt.jar" sourcepath="lib/jogamp/gluegen-rt-sources.jar"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/python) interpreter library"/>
+	<classpathentry kind="lib" path="lib/joda/joda-time-2.3.jar"/>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>

Makefile

@@ -74,51 +74,43 @@ default: nightly_build_stuff
 
 nightly_build_stuff:
 	@echo PROJECT_VERSION is $(PROJECT_VERSION)
-	$(MAKE) clean PROJECT_VERSION=$(PROJECT_VERSION)
+	$(MAKE) clean PROJECT_VERSION=$(PROJECT_VERSION) 1> /dev/null
+	@mkdir -p target/logs
 	$(MAKE) build PROJECT_VERSION=$(PROJECT_VERSION)
 	$(MAKE) docs-website PROJECT_VERSION=$(PROJECT_VERSION)
 	@echo
 	@echo Build completed successfully.
 
-MILLIS_SINCE_EPOCH = $(shell date '+%s')
-
 build:
-	@echo
-	@echo "PHASE: Building R inner package..."
-	@echo
-ifeq ($(BUILD_NUMBER),99999)
-	$(MAKE) -C R build_inner PROJECT_VERSION=$(PROJECT_VERSION).$(MILLIS_SINCE_EPOCH)
-else
-	$(MAKE) -C R build_inner PROJECT_VERSION=$(PROJECT_VERSION)
-endif
 	@echo
 	@echo "PHASE: Creating ${BUILD_VERSION_JAVA_FILE}..."
 	@echo
-	$(MAKE) build_version PROJECT_VERSION=$(PROJECT_VERSION)
+	$(MAKE) build_version PROJECT_VERSION=$(PROJECT_VERSION) 1> target/logs/version_build.log
+
 	@echo
 	@echo "PHASE: Building H2O..."
 	@echo
 	$(MAKE) build_h2o PROJECT_VERSION=$(PROJECT_VERSION)
+
 	@echo
-	@echo "PHASE: Building Shalala..."
-	@echo
-	$(MAKE) -C h2o-scala PROJECT_VERSION=$(PROJECT_VERSION)
-	@echo
-	@echo "PHASE: Building R outer package..."
+	@echo "PHASE: Building hadoop driver..."
 	@echo
-	$(MAKE) -C R build_outer PROJECT_VERSION=$(PROJECT_VERSION)
+	$(MAKE) -C hadoop build PROJECT_VERSION=$(PROJECT_VERSION) 1> target/logs/hadoop_build.log
+
 	@echo
-	@echo "PHASE: Building hadoop driver..."
+	@echo "PHASE: Building Shalala..."
 	@echo
-	$(MAKE) -C hadoop build PROJECT_VERSION=$(PROJECT_VERSION)
+	$(MAKE) -C h2o-scala PROJECT_VERSION=$(PROJECT_VERSION)
+
 	@echo
-	@echo "PHASE: Building launcher..."
+	@echo "PHASE: Building R package..."
 	@echo
-	$(MAKE) -C launcher build PROJECT_VERSION=$(PROJECT_VERSION)
+	$(MAKE) -C R PROJECT_VERSION=$(PROJECT_VERSION) 1> target/logs/r_build.log
+
 	@echo
 	@echo "PHASE: Building zip package..."
 	@echo
-	$(MAKE) build_package PROJECT_VERSION=$(PROJECT_VERSION)
+	$(MAKE) build_package PROJECT_VERSION=$(PROJECT_VERSION) 1> target/logs/package_build.log
 
 BUILD_BRANCH=$(shell git branch | grep '*' | sed 's/* //')
 BUILD_HASH=$(shell git log -1 --format="%H")
@@ -162,7 +154,7 @@ build_package:
 	cp -p LICENSE.txt target/h2o-$(PROJECT_VERSION)
 	mkdir target/h2o-$(PROJECT_VERSION)/ec2
 	cp -p ec2/*.py ec2/*.sh ec2/README.txt target/h2o-$(PROJECT_VERSION)/ec2
-	(cd target; zip -r h2o-$(PROJECT_VERSION).zip h2o-$(PROJECT_VERSION))
+	(cd target; zip -q -r h2o-$(PROJECT_VERSION).zip h2o-$(PROJECT_VERSION))
 	rm -fr target/h2o-$(PROJECT_VERSION)
 	rm -fr target/ci
 	cp -rp ci target
@@ -185,7 +177,7 @@ docs-website: dw_announce
 
 docs-website-clean:
 else
-docs-website: dw_announce dw_1 dw_2 dw_3 dw_4
+docs-website: dw_announce dw_1 dw_2 dw_3
 
 docs-website-clean:
 	rm -rf h2o-docs/source/developuser/DocGen
@@ -228,22 +220,7 @@ dw_3:
 	cp -p docs/H2O_on_Hadoop_0xdata.pdf $(BUILD_WEBSITE_DIR)/bits/hadoop
 	mkdir -p $(BUILD_WEBSITE_DIR)/bits/ec2
 	cp -p ec2/README.txt $(BUILD_WEBSITE_DIR)/bits/ec2
-
-# Note:  to get pdfunite on a mac, try:
-#     $ brew install poppler
-#
-PDFLATEX=$(shell which pdflatex)
-PDFUNITE=$(shell which pdfunite)
-dw_4:
-ifeq ($(PDFLATEX),)
-	@echo pdflatex not found, skipping...
-else
-ifeq ($(PDFUNITE),)
-	@echo pdfunite not found, skipping...
-else
-	pdfunite R/h2o-package/h2o_package.pdf R/h2oRClient-package/h2oRClient_package.pdf $(BUILD_WEBSITE_DIR)/bits/h2oRjoin.pdf
-endif
-endif
+	cp -p R/h2o-package/h2o_package.pdf $(BUILD_WEBSITE_DIR)/bits/h2o_package.pdf
 
 #
 # Set appropriately for your data size to quickly try out H2O.

R/.gitignore

@@ -1,8 +1,5 @@
 *.pdf
-h2oRClient-package/DESCRIPTION
-h2oRClient-package/h2oRClient_package.pdf
-h2oRClient-package/man/h2oRClient-package.Rd
 h2o-package/DESCRIPTION
-h2o-package/h2o_package.pdf
 h2o-package/man/h2o-package.Rd
+h2o-package/h2o_package.pdf
 h2o-package/inst/java

R/H2OTestDemo.R

@@ -5,7 +5,7 @@
 source("H2O_Load.R")
 # library(h2o)
 localH2O = new("H2OClient", ip = "localhost", port = 54321)
-h2o.checkClient(localH2O)
+# h2o.checkClient(localH2O)
 
 # Test using prostate cancer data set
 prostate.hex = h2o.importURL(localH2O, path = "https://raw.github.com/0xdata/h2o/master/smalldata/logreg/prostate.csv", key = "prostate.hex")

R/H2O_Load.R

@@ -1,4 +1,4 @@
-source("h2oRClient-package/R/Internal.R")
-source("h2oRClient-package/R/Classes.R")
-source("h2oRClient-package/R/ParseImport.R")
-source("h2oRClient-package/R/Algorithms.R")
+source("h2o-package/R/Internal.R")
+source("h2o-package/R/Classes.R")
+source("h2o-package/R/ParseImport.R")
+source("h2o-package/R/Algorithms.R")

R/Makefile

@@ -6,34 +6,26 @@ REPO_PATH_MAC = bin/macosx/contrib/$(R_VERSION)
 TMP_BUILD_DIR = tmp_build
 ifneq ($(OS),Windows_NT)
 OS := $(shell uname)
-UNAME_P := $(shell uname -p)
+UNAME_P := $(shell uname -p | sed 's/unknown/x86_64/')
 else
 # For windows NT the shell is odd.  Cygwin 'shell uname' returns CYGWIN_NT-6.1-WOW64
 # which is not really what we want.  The normal CMD shell does something else.
 # So don't ask for 'uname' if we already got Windows_NT
 endif
 
-H2O_WRAPPER_SOURCE_FILE = h2o_$(PROJECT_VERSION).tar.gz
-H2O_WRAPPER_LINUX_FILE = h2o_$(PROJECT_VERSION)_R_$(UNAME_P)-pc-linux-gnu.tar.gz
-H2O_WRAPPER_MAC_FILE = h2o_$(PROJECT_VERSION).tgz
-H2O_WRAPPER_WINDOWS_FILE = h2o_$(PROJECT_VERSION).zip
-
-H2O_RCLIENT_SOURCE_FILE = h2oRClient_$(PROJECT_VERSION).tar.gz
-H2O_RCLIENT_LINUX_FILE = h2oRClient_$(PROJECT_VERSION)_R_$(UNAME_P)-pc-linux-gnu.tar.gz
-H2O_RCLIENT_MAC_FILE = h2oRClient_$(PROJECT_VERSION).tgz
-H2O_RCLIENT_WINDOWS_FILE = h2oRClient_$(PROJECT_VERSION).zip
+H2O_R_SOURCE_FILE = h2o_$(PROJECT_VERSION).tar.gz
+H2O_R_LINUX_FILE = h2o_$(PROJECT_VERSION)_R_$(UNAME_P)-pc-linux-gnu.tar.gz
+H2O_R_MAC_FILE = h2o_$(PROJECT_VERSION).tgz
+H2O_R_WINDOWS_FILE = h2o_$(PROJECT_VERSION).zip
 
 ifeq ($(OS),Linux)
-H2O_RCLIENT_PKG_FILE = $(H2O_RCLIENT_LINUX_FILE)
-H2O_WRAPPER_PKG_FILE = $(H2O_WRAPPER_LINUX_FILE)
+H2O_R_PKG_FILE = $(H2O_R_LINUX_FILE)
 endif
 ifeq ($(OS),Darwin)
-H2O_RCLIENT_PKG_FILE = $(H2O_RCLIENT_MAC_FILE)
-H2O_WRAPPER_PKG_FILE = $(H2O_WRAPPER_MAC_FILE)
+H2O_R_PKG_FILE = $(H2O_R_MAC_FILE)
 endif
 ifeq ($(OS),Windows_NT)
-H2O_RCLIENT_PKG_FILE = $(H2O_RCLIENT_WINDOWS_FILE)
-H2O_WRAPPER_PKG_FILE = $(H2O_WRAPPER_WINDOWS_FILE)
+H2O_R_PKG_FILE = $(H2O_R_WINDOWS_FILE)
 endif
 
 # Do a best-effort attempt to see if the R user has pdflatex installed
@@ -42,16 +34,24 @@ endif
 # But use the '-' to ignore failures and continue the build even if it does fail.
 PDFLATEX=$(shell which pdflatex)
 
-build_outer:
+build_rh2o:
 	sed 's/SUBST_PROJECT_VERSION/$(PROJECT_VERSION)/' h2o-DESCRIPTION.template > h2o-package/DESCRIPTION
 	sed 's/SUBST_PROJECT_VERSION/$(PROJECT_VERSION)/' h2o-package.template > h2o-package/man/h2o-package.Rd
 ifeq ($(PDFLATEX),)
 	@echo pdflatex not found, skipping pdf generation...
 else
 	-R CMD Rd2pdf --force --output="h2o-package/h2o_package.pdf" --title="Package 'h2o'" --no-index --no-preview h2o-package/man 1> /dev/null
 endif
-	mkdir -p h2o-package/inst/java/
-	cp -f ../target/h2o.jar h2o-package/inst/java/h2o.jar
+	rm -rf h2o-package/inst/java
+	mkdir -p h2o-package/inst/java/tmp
+	cp -f ../target/h2o.jar h2o-package/inst/java/tmp/h2o_full.jar
+	(cd h2o-package/inst/java/tmp && jar xf h2o_full.jar)
+	(cd h2o-package/inst/java/tmp && rm -fr hadoop/0.* hadoop/1.* hadoop/cdh[35]* hadoop/cdh4_yarn)
+	(cd h2o-package/inst/java/tmp && rm -f h2o_full.jar)
+	(cd h2o-package/inst/java/tmp && cp META-INF/MANIFEST.MF ..)
+	(cd h2o-package/inst/java/tmp && rm -fr META-INF)
+	(cd h2o-package/inst/java/tmp && jar cfm ../h2o.jar ../MANIFEST.MF *)
+	rm -rf h2o-package/inst/java/tmp
 	R CMD build h2o-package
 
 	# Create file with description of all packages
@@ -61,23 +61,23 @@ endif
 	# Add to repo in target directory
 	mkdir -p ../target/R/src/contrib
 	cp -p README.txt ../target/R
-	mv $(H2O_WRAPPER_SOURCE_FILE) ../target/R/src/contrib
+	mv $(H2O_R_SOURCE_FILE) ../target/R/src/contrib
 	cp -p PACKAGES PACKAGES.gz ../target/R/src/contrib
 
 	# Build binary for each OS
 	rm -rf $(TMP_BUILD_DIR)
 	mkdir -p $(TMP_BUILD_DIR)
-	R CMD INSTALL -l $(TMP_BUILD_DIR) --build h2o-package
+	[ -x "`which gnutar 2>/dev/null`" ] || echo 'Note: gnutar not found; package install in R may fail in the next step'
+	R CMD INSTALL -l $(TMP_BUILD_DIR) --build h2o-package || echo 'If you got an error like "Dependency foo is not available for package h2o" you need to install the required R package by running R and executing the R command: install.packages("foo")'
 
 ifneq ($(OS),Windows_NT)
-	# rm -rf h2o
-	tar zxvf $(H2O_WRAPPER_PKG_FILE)
-	zip -r $(H2O_WRAPPER_WINDOWS_FILE) h2o
+	tar zxf $(H2O_R_PKG_FILE)
+	zip -q -r $(H2O_R_WINDOWS_FILE) h2o
 	rm -rf h2o
 
 	# Create repo subdirectory for MacOSX
 	mkdir -p ../target/R/$(REPO_PATH_MAC)
-	mv $(H2O_WRAPPER_PKG_FILE) ../target/R/$(REPO_PATH_MAC)/$(H2O_WRAPPER_MAC_FILE)
+	mv $(H2O_R_PKG_FILE) ../target/R/$(REPO_PATH_MAC)/$(H2O_R_MAC_FILE)
 	cp -p PACKAGES PACKAGES.gz ../target/R/$(REPO_PATH_MAC)
 
 	rm -rf ../target/R/bin/macosx/contrib/2.15 ../target/R/bin/macosx/contrib/2.14 ../target/R/bin/macosx/contrib/2.13
@@ -88,7 +88,7 @@ endif
 
 	# Create repo subdirectory for Windows
 	mkdir -p ../target/R/$(REPO_PATH_WIN)
-	mv $(H2O_WRAPPER_WINDOWS_FILE) ../target/R/$(REPO_PATH_WIN)
+	mv $(H2O_R_WINDOWS_FILE) ../target/R/$(REPO_PATH_WIN)
 	mv PACKAGES PACKAGES.gz ../target/R/$(REPO_PATH_WIN)
 
 	rm -rf ../target/R/bin/windows/contrib/2.15 ../target/R/bin/windows/contrib/2.14 ../target/R/bin/windows/contrib/2.13
@@ -99,75 +99,18 @@ endif
 	R CMD REMOVE -l $(TMP_BUILD_DIR) h2o
 	rm -rf $(TMP_BUILD_DIR)
 
-build_inner:
-	sed 's/SUBST_PROJECT_VERSION/$(PROJECT_VERSION)/' h2oRClient-DESCRIPTION.template > h2oRClient-package/DESCRIPTION
-	sed 's/SUBST_PROJECT_VERSION/$(PROJECT_VERSION)/' h2oRClient-package.template > h2oRClient-package/man/h2oRClient-package.Rd
-ifeq ($(PDFLATEX),)
-	@echo pdflatex not found, skipping pdf generation...
-else
-	-R CMD Rd2pdf --force --output="h2oRClient-package/h2oRClient_package.pdf" --title="Package 'h2oRClient'" --no-index --no-preview h2oRClient-package/man 1> /dev/null
-endif
-	R CMD build h2oRClient-package
-	echo h2oRClient_$(PROJECT_VERSION).tar.gz > info.txt
-# ifeq ($(OS),Windows_NT)
-#	md5deep h2oRClient_$(PROJECT_VERSION).tar.gz | cut -d' ' -f1 >> info.txt
-# else
-#	openssl md5 h2oRClient_$(PROJECT_VERSION).tar.gz | sed 's/.*= //' >> info.txt
-# endif
-
-	# Create file with description of all packages
-	cp -p h2oRClient-package/DESCRIPTION PACKAGES
-	gzip -c PACKAGES > PACKAGES.gz
-
-	# Add to resources directory
-	mkdir -p ../lib/resources/R/src/contrib
-	mv info.txt ../lib/resources/R/info.txt
-	mv $(H2O_RCLIENT_SOURCE_FILE) ../lib/resources/R/src/contrib
-	cp -p PACKAGES PACKAGES.gz ../lib/resources/R/src/contrib
-
-	# Build binary for each OS
-	rm -rf $(TMP_BUILD_DIR)
-	mkdir -p $(TMP_BUILD_DIR)
-	R CMD INSTALL -l $(TMP_BUILD_DIR) --build h2oRClient-package
-
-ifneq ($(OS),Windows_NT)
-	# rm -rf h2oRClient
-	tar zxvf $(H2O_RCLIENT_PKG_FILE)
-	zip -r $(H2O_RCLIENT_WINDOWS_FILE) h2oRClient
-	rm -rf h2oRClient
-
-	# Create repo subdirectory for MacOSX
-	mkdir -p ../lib/resources/R/$(REPO_PATH_MAC)
-	mv $(H2O_RCLIENT_PKG_FILE) ../lib/resources/R/$(REPO_PATH_MAC)/$(H2O_RCLIENT_MAC_FILE)
-	cp -p PACKAGES PACKAGES.gz ../lib/resources/R/$(REPO_PATH_MAC)
-endif
-
-	# Create repo subdirectory for Windows
-	mkdir -p ../lib/resources/R/$(REPO_PATH_WIN)
-	mv $(H2O_RCLIENT_WINDOWS_FILE) ../lib/resources/R/$(REPO_PATH_WIN)
-	mv PACKAGES PACKAGES.gz ../lib/resources/R/$(REPO_PATH_WIN)
-
-	# Create temporary directory for jarring later
-	mkdir -p ../r_pack_tmp/resources
-	cp -r ../lib/resources/R ../r_pack_tmp/resources
-
-	R CMD REMOVE -l $(TMP_BUILD_DIR) h2oRClient
-	rm -rf $(TMP_BUILD_DIR)
+check:
+	find h2o-package -name .DS_Store | xargs rm -f
+	R CMD CHECK h2o-package
+	curl -v http://localhost:54321/Shutdown.json
 
 clean:
-	rm -f h2oRClient-package/DESCRIPTION
-	rm -f h2oRClient-package/h2oRClient_package.pdf
-	rm -f h2oRClient-package/man/h2oRClient-package.Rd
-	rm -f h2oRClient_*.tar.gz
-	rm -f h2oRClient_*.tgz
-	rm -f h2oRClient_*.zip
 	rm -f h2o-package/DESCRIPTION
 	rm -f h2o-package/h2o_package.pdf
 	rm -f h2o-package/man/h2o-package.Rd
-	rm -f h2o-package/inst/java/h2o.jar
+	rm -fr h2o-package/inst/java
 	rm -f h2o_*.tar.gz
 	rm -f h2o_*.tgz
 	rm -f h2o_*.zip
-	rm -rf ../lib/resources/R
 	rm -rf h2o
-	rm -rf h2oRClient
+	rm -rf h2oRClient-package