diff --git a/.travis.yml b/.travis.yml
index bf50b8aca46a9..70902e056e986 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,26 +22,26 @@ addons:
       - python-scipy
 
 env:
+  global:
+    # Directory where tests are run from
+    - TEST_DIR=/tmp/sklearn
   matrix:
     # This environment tests that scikit-learn can be built against
     # versions of numpy, scipy with ATLAS that comes with Ubuntu Precise 12.04
     - DISTRIB="ubuntu" PYTHON_VERSION="2.7" COVERAGE="true"
-      CYTHON_VERSION="0.23.4" NAME="ubuntu"
+      CYTHON_VERSION="0.23.4" CACHED_BUILD_DIR="$HOME/sklearn_build_ubuntu"
     # This environment tests the oldest supported anaconda env
     - DISTRIB="conda" PYTHON_VERSION="2.6" INSTALL_MKL="false"
       NUMPY_VERSION="1.6.2" SCIPY_VERSION="0.11.0" CYTHON_VERSION="0.21"
-      NAME="oldest"
+      CACHED_BUILD_DIR="$HOME/sklearn_build_oldest"
     # This environment tests the newest supported anaconda env
     - DISTRIB="conda" PYTHON_VERSION="3.5" INSTALL_MKL="true"
       NUMPY_VERSION="1.10.1" SCIPY_VERSION="0.16.0" CYTHON_VERSION="0.23.4"
-      NAME="latest"
+      CACHED_BUILD_DIR="$HOME/sklearn_build_latest"
+
 install: source continuous_integration/install.sh
 script: bash continuous_integration/test_script.sh
-after_success:
-    # Ignore coveralls failures as the coveralls server is not very reliable
-    # but we don't want travis to report a failure in the github UI just
-    # because the coverage report failed to be published.
-    - if [[ "$COVERAGE" == "true" ]]; then coveralls || echo "failed"; fi
+after_success: source continuous_integration/after_success.sh
 notifications:
   webhooks:
     urls:
diff --git a/continuous_integration/after_success.sh b/continuous_integration/after_success.sh
new file mode 100755
index 0000000000000..a4613ccc701e0
--- /dev/null
+++ b/continuous_integration/after_success.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# This script is meant to be called by the "after_success" step defined in
+# .travis.yml. See http://docs.travis-ci.com/ for more details.
+
+# License: 3-clause BSD
+
+set -e
+
+if [[ "$COVERAGE" == "true" ]]; then
+    # Need to run coveralls from a git checkout, so we copy .coverage
+    # from TEST_DIR where nosetests has been run
+    cp $TEST_DIR/.coverage $TRAVIS_BUILD_DIR
+    cd $TRAVIS_BUILD_DIR
+    # Ignore coveralls failures as the coveralls server is not
+    # very reliable but we don't want travis to report a failure
+    # in the github UI just because the coverage report failed to
+    # be published.
+    coveralls || echo "Coveralls upload failed"
+fi
diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh
index 06a847e2fab3d..f361ba5b4830a 100755
--- a/continuous_integration/install.sh
+++ b/continuous_integration/install.sh
@@ -83,17 +83,14 @@ if [[ "$COVERAGE" == "true" ]]; then
     pip install coverage coveralls
 fi
 
-GIT_TRAVIS_REPO=$(pwd)
-echo $GIT_TRAVIS_REPO
-
-cd $HOME
-if [ ! -d "sklearn_build_$NAME" ]; then
-    mkdir sklearn_build_$NAME
+if [ ! -d "$CACHED_BUILD_DIR" ]; then
+    mkdir -p $CACHED_BUILD_DIR
 fi
 
-rsync -av --exclude='.git/' --exclude='testvenv/' $GIT_TRAVIS_REPO \
-    sklearn_build_${NAME}
-cd sklearn_build_${NAME}/scikit-learn
+rsync -av --exclude '.git/' --exclude='testvenv/' \
+      $TRAVIS_BUILD_DIR $CACHED_BUILD_DIR
+
+cd $CACHED_BUILD_DIR/scikit-learn
 
 # Build scikit-learn in the install.sh script to collapse the verbose
 # build output in the travis output when it succeeds.
diff --git a/continuous_integration/test_script.sh b/continuous_integration/test_script.sh
old mode 100644
new mode 100755
index 45dba3525b7b3..3c9a341816504
--- a/continuous_integration/test_script.sh
+++ b/continuous_integration/test_script.sh
@@ -10,8 +10,10 @@ set -e
 
 # Get into a temp directory to run test from the installed scikit learn and
 # check if we do not leave artifacts
-mkdir -p /tmp/sklearn_tmp
-cd /tmp/sklearn_tmp
+mkdir -p $TEST_DIR
+# We need the setup.cfg for the nose settings
+cp setup.cfg $TEST_DIR
+cd $TEST_DIR
 
 python --version
 python -c "import numpy; print('numpy %s' % numpy.__version__)"
@@ -29,8 +31,8 @@ else
 fi
 
 # Is directory still empty ?
-ls
+ls -ltra
 
 # Test doc
-cd $HOME/sklearn_build_$NAME/scikit-learn
+cd $CACHED_BUILD_DIR/scikit-learn
 make test-doc test-sphinxext