Merge pull request tensorflow#1472 from caisq/help-1445

pip test-on-install changes for skflow

tensorflow/tools/ci_build/Dockerfile.cpu

@@ -7,6 +7,7 @@ COPY install/*.sh /install/
 RUN /install/install_bootstrap_deb_packages.sh
 RUN add-apt-repository -y ppa:openjdk-r/ppa
 RUN /install/install_deb_packages.sh
+RUN /install/install_pip_packages.sh
 RUN /install/install_bazel.sh
 
 # Set up bazelrc.

tensorflow/tools/ci_build/Dockerfile.gpu

@@ -7,6 +7,7 @@ COPY install/*.sh /install/
 RUN /install/install_bootstrap_deb_packages.sh
 RUN add-apt-repository -y ppa:openjdk-r/ppa
 RUN /install/install_deb_packages.sh
+RUN /install/install_pip_packages.sh
 RUN /install/install_bazel.sh
 
 # Set up bazelrc.

tensorflow/tools/ci_build/builds/pip.sh

@@ -22,11 +22,20 @@
 #   pip.sh CONTAINER_TYPE [--test_tutorials]
 #
 # When executing the Python unit tests, the script obeys the shell
-# variables: TF_BUILD_BAZEL_CLEAN, NO_TEST_ON_INSTALL
+# variables: TF_BUILD_BAZEL_CLEAN, TF_BUILD_INSTALL_EXTRA_PIP_PACKAGES,
+# TF_BUILD_NO_CACHING_VIRTUALENV, NO_TEST_ON_INSTALL
 #
 # TF_BUILD_BAZEL_CLEAN, if set to any non-empty and non-0 value, directs the
 # script to perform bazel clean prior to main build and test steps.
 #
+# TF_BUILD_INSTALL_EXTRA_PIP_PACKAGES overrides the default extra pip packages
+# to be installed in virtualenv before test_installation.sh is called. Multiple
+# pakcage names are separated with spaces.
+#
+# TF_BUILD_NO_CACHING_VIRTUALENV: If set to any non-empty and non-0 value,
+# will cause the script to force remove any existing (cached) virtualenv
+# directory.
+#
 # If NO_TEST_ON_INSTALL has any non-empty and non-0 value, the test-on-install
 # part will be skipped.
 #
@@ -35,6 +44,8 @@
 # installation and the Python unit tests-on-install step.
 #
 
+INSTALL_EXTRA_PIP_PACKAGES=${TF_BUILD_INSTALL_EXTRA_PIP_PACKAGES}
+
 # Helper functions
 # Get the absolute path from a path
 abs_path() {
@@ -111,7 +122,7 @@ PIP_WHL_DIR="${PIP_TEST_ROOT}/whl"
 PIP_WHL_DIR=$(abs_path ${PIP_WHL_DIR})  # Get absolute path
 rm -rf ${PIP_WHL_DIR} && mkdir -p ${PIP_WHL_DIR}
 bazel-bin/tensorflow/tools/pip_package/build_pip_package ${PIP_WHL_DIR} || \
-die "build_pip_package FAILED"
+    die "build_pip_package FAILED"
 
 # Perform installation
 WHL_PATH=$(ls ${PIP_WHL_DIR}/tensorflow*.whl)
@@ -125,27 +136,46 @@ echo "whl file path = ${WHL_PATH}"
 # Install, in user's local home folder
 echo "Installing pip whl file: ${WHL_PATH}"
 
-# Create temporary directory for install test
+# Create virtualenv directory for install test
 VENV_DIR="${PIP_TEST_ROOT}/venv"
-rm -rf "${VENV_DIR}" && mkdir -p "${VENV_DIR}"
-echo "Create directory for virtualenv: ${VENV_DIR}"
+if [[ -d "${VENV_DIR}" ]] &&
+   [[ ! -z "${TF_BUILD_NO_CACHING_VIRTUALENV}" ]] &&
+   [[ "${TF_BUILD_NO_CACHING_VIRTUALENV}" != "0" ]]; then
+  echo "TF_BUILD_NO_CACHING_VIRTUALENV=${TF_BUILD_NO_CACHING_VIRTUALENV}:"
+  echo "Removing existing virtualenv directory: ${VENV_DIR}"
+
+  rm -rf "${VENV_DIR}" || \
+      die "Failed to remove existing virtualenv directory: ${VENV_DIR}"
+fi
+
+mkdir -p ${VENV_DIR} || \
+    die "FAILED to create virtualenv directory: ${VENV_DIR}"
 
 # Verify that virtualenv exists
 if [[ -z $(which virtualenv) ]]; then
   die "FAILED: virtualenv not available on path"
 fi
 
-virtualenv -p "${PYTHON_BIN_PATH}" "${VENV_DIR}" ||
-die "FAILED: Unable to create virtualenv"
+virtualenv --system-site-packages -p "${PYTHON_BIN_PATH}" "${VENV_DIR}" || \
+    die "FAILED: Unable to create virtualenv"
+
+source "${VENV_DIR}/bin/activate" || \
+    die "FAILED: Unable to activate virtualenv"
 
-source "${VENV_DIR}/bin/activate" ||
-die "FAILED: Unable to activate virtualenv"
 
 # Install the pip file in virtual env
-pip install -v ${WHL_PATH} \
+pip install -v --force-reinstall ${WHL_PATH} \
 && echo "Successfully installed pip package ${WHL_PATH}" \
 || die "pip install (without --upgrade) FAILED"
 
+# Install extra pip packages required by the test-on-install
+for PACKAGE in ${INSTALL_EXTRA_PIP_PACKAGES}; do
+  echo "Installing extra pip package required by test-on-install: ${PACKAGE}"
+
+  pip install ${PACKAGE} || \
+      die "pip install ${PACKAGE} FAILED"
+done
+
 # If NO_TEST_ON_INSTALL is set to any non-empty value, skip all Python
 # tests-on-install and exit right away
 if [[ ! -z "${NO_TEST_ON_INSTALL}" ]] &&
@@ -158,14 +188,14 @@ fi
 # Call test_installation.sh to perform test-on-install
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-"${DIR}/test_installation.sh" --virtualenv ||
-die "PIP tests-on-install FAILED"
+"${DIR}/test_installation.sh" --virtualenv || \
+    die "PIP tests-on-install FAILED"
 
 # Optional: Run the tutorial tests
 if [[ "${DO_TEST_TUTORIALS}" == "1" ]]; then
-  "${DIR}/test_tutorials.sh" --virtualenv ||
-die "PIP tutorial tests-on-install FAILED"
+  "${DIR}/test_tutorials.sh" --virtualenv || \
+      die "PIP tutorial tests-on-install FAILED"
 fi
 
-deactivate ||
-die "FAILED: Unable to deactivate virtualenv"
+deactivate || \
+    die "FAILED: Unable to deactivate virtualenv"

tensorflow/tools/ci_build/builds/test_installation.sh

@@ -166,7 +166,8 @@ cp -r tensorflow/core/lib/png ${PY_TEST_DIR}/tensorflow/core/lib
 
 # Run tests
 DIR0=$(pwd)
-ALL_PY_TESTS=$(find tensorflow/{contrib,examples,models,python,tensorboard} -name "*_test.py" | sort)
+ALL_PY_TESTS=$(find tensorflow/{contrib,examples,models,python,tensorboard} \
+    -type f \( -name "*_test.py" -o -name "test_*.py" \) | sort)
 # TODO(cais): Add tests in tensorflow/contrib
 
 PY_TEST_COUNT=$(echo ${ALL_PY_TESTS} | wc -w)

tensorflow/tools/ci_build/install/install_deb_packages.sh

@@ -29,10 +29,12 @@ apt-get install -y \
     python-dev \
     python-numpy \
     python-pip \
+    python-scipy \
     python-virtualenv \
     python3-dev \
     python3-numpy \
     python3-pip \
+    python3-scipy \
     sudo \
     swig \
     unzip \

tensorflow/tools/ci_build/install/install_pip_packages.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -e
+
+pip install sklearn
+pip3 install scikit-learn