Fix bash-completion goal when run from sdist/pex. Also add tests, and…

… beef up ci.sh &

release.sh.

When running the latest pants from a PEX inside of Twitter, I noticed that
the new `bash-completion` goal didn't work.  This sort of error is actually
hard to detect, so I added some more smoke tests to `ci.sh`. On my way, I
also beefed up `release.sh`.

Reviewed at https://rbcommons.com/s/twitter/r/2403/

build-support/bin/ci.sh

@@ -125,10 +125,21 @@ fi
 
 if [[ "${skip_sanity_checks:-false}" == "false" ]]; then
   banner "Sanity checking bootstrapped pants and repo BUILD files"
-  ./pants.pex ${PANTS_ARGS[@]} clean-all || die "Failed to clean-all."
-  ./pants.pex ${PANTS_ARGS[@]} goals || die "Failed to list goals."
-  ./pants.pex ${PANTS_ARGS[@]} list :: || die "Failed to list all targets."
-  ./pants.pex ${PANTS_ARGS[@]} targets || die "Failed to show target help."
+  sanity_tests=(
+    "bash-completion"
+    "builddict"
+    "clean-all"
+    "goals"
+    "list ::"
+    "roots"
+    "targets"
+  )
+  for cur_test in "${sanity_tests[@]}"; do
+    cmd="./pants.pex ${PANTS_ARGS[@]} ${cur_test}"
+    echo "Executing command '${cmd}' as a sanity test:"
+    ${cmd} >/dev/null || die "Failed to execute '${cmd}'."
+    echo ""
+  done
 fi
 
 if [[ "${skip_distribution:-false}" == "false" ]]; then

build-support/bin/release.sh

@@ -32,10 +32,10 @@ PKG_PANTS=(
 )
 function pkg_pants_install_test() {
   PIP_ARGS="$@"
-  pip install ${PIP_ARGS} pantsbuild.pants==$(local_version) && \
-  execute_packaged_pants_with_internal_backends list src:: && \
+  pip install ${PIP_ARGS} pantsbuild.pants==$(local_version) || die "pip install of pantsbuild.pants failed!"
+  execute_packaged_pants_with_internal_backends list src:: || die "'pants list src::' failed in venv!"
   [[ "$(execute_packaged_pants_with_internal_backends --version 2>/dev/null)" \
-     == "$(local_version)" ]]
+     == "$(local_version)" ]] || die "Installed version of pants does match local version!"
 }
 
 PKG_PANTS_TESTINFRA=(
@@ -159,6 +159,21 @@ function pre_install() {
 
 function post_install() {
   # this assume pre_install is called and a new temp venv activation has been done.
+  if [[ "${pause_after_venv_creation}" == "true" ]]; then
+    cat <<EOM
+
+If you want to poke around with the new version of pants that has been built
+and installed in a temporary virtualenv, fire up another shell window and type:
+
+  source ${VENV_DIR}/bin/activate
+  cd ${ROOT}
+
+From there, you can run 'pants' (not './pants') to do some testing.
+
+When you're done testing, press enter to continue.
+EOM
+    read
+  fi
   deactivate
 }
 
@@ -168,18 +183,21 @@ function install_and_test_packages() {
     --quiet
   )
 
+  pre_install || die "Failed to setup virtualenv while testing ${NAME}-$(local_version)!"
+
   for PACKAGE in "${RELEASE_PACKAGES[@]}"
   do
     NAME=$(pkg_name $PACKAGE)
     INSTALL_TEST_FUNC=$(pkg_install_test_func $PACKAGE)
 
     banner "Installing and testing package ${NAME}-$(local_version) ..."
 
-    pre_install && \
-    eval $INSTALL_TEST_FUNC ${PIP_ARGS[@]} && \
-    post_install || \
+    eval $INSTALL_TEST_FUNC ${PIP_ARGS[@]} || \
     die "Failed to install and test package ${NAME}-$(local_version)!"
   done
+
+  post_install || die "Failed to deactivate virtual env while testing ${NAME}-$(local_version)!"
+
 }
 
 function dry_run_install() {
@@ -380,7 +398,8 @@ function usage() {
   echo "PyPi.  Credentials are needed for this as described in the"
   echo "release docs: http://pantsbuild.github.io/release.html"
   echo
-  echo "Usage: $0 (-h|-ntlo)"
+  echo "Usage: $0 [-d] (-h|-n|-t|-l|-o)"
+  echo " -d  Enables debug mode (verbose output, script pauses after venv creation)"
   echo " -h  Prints out this help message."
   echo " -n  Performs a release dry run."
   echo "       All package distributions will be built, installed locally in"
@@ -392,7 +411,7 @@ function usage() {
   echo " -l  Lists all pantsbuild packages that this script releases."
   echo " -o  Lists all pantsbuild package owners."
   echo
-  echo "All options are mutually exclusive."
+  echo "All options (except for '-d') are mutually exclusive."
 
   if (( $# > 0 )); then
     die "$@"
@@ -401,9 +420,10 @@ function usage() {
   fi
 }
 
-while getopts "hntlo" opt; do
+while getopts "hdntlo" opt; do
   case ${opt} in
     h) usage ;;
+    d) debug="true" ;;
     n) dry_run="true" ;;
     t) test_release="true" ;;
     l) list_packages && exit 0 ;;
@@ -412,6 +432,11 @@ while getopts "hntlo" opt; do
   esac
 done
 
+if [[ "${debug}" == "true" ]]; then
+  set -x
+  pause_after_venv_creation="true"
+fi
+
 if [[ "${dry_run}" == "true" && "${test_release}" == "true" ]]; then
   usage "The dry run and test options are mutually exclusive, pick one."
 elif [[ "${dry_run}" == "true" ]]; then

src/python/pants/backend/core/tasks/BUILD

@@ -45,6 +45,7 @@ target(
 python_library(
   name = 'bash_completion',
   sources = ['bash_completion.py'],
+  resources = globs('templates/bash_completion/*.mustache'),
   dependencies = [
     ':console_task',
     'src/python/pants/base:exceptions',

src/python/pants/backend/core/tasks/bash_completion.py

@@ -67,10 +67,12 @@ def generate_scoped_option_strings(option_strings, scope):
 
     return result
 
-  def console_output(self, targets):
-    if targets:
-      raise TaskError('This task does not accept any target addresses.')
+  @staticmethod
+  def parse_all_tasks_and_help(self):
+    """Loads all goals, and mines the options & help text for each one.
 
+    Returns: a set of all_scopes, options_text string, a set of all option strings
+    """
     options = self.context.options
 
     goals = Goal.all()
@@ -107,6 +109,14 @@ def record(scope, option_strings):
       for scope, option_strings in sorted(option_strings_by_scope.items(), key=lambda x: x[0])
     ])
 
+    return all_scopes, options_text, global_option_strings_set
+
+  def console_output(self, targets):
+    if targets:
+      raise TaskError('This task does not accept any target addresses.')
+
+    (all_scopes, options_text, global_option_strings_set) = self.parse_all_tasks_and_help(self)
+
     generator = Generator(
       resource_string(__name__, os.path.join('templates', 'bash_completion', 'autocomplete.sh.mustache')),
       scopes_text=' '.join(sorted(list(all_scopes))),

tests/python/pants_test/backend/core/tasks/BUILD

@@ -47,3 +47,14 @@ python_tests(
     'src/python/pants/backend/core/tasks:scm_publish',
   ]
 )
+
+python_tests(
+  name='bash_completion',
+  sources=['test_bash_completion.py'],
+  coverage=['pants.backend.core.tasks.bash_completion'],
+  dependencies=[
+    '3rdparty/python:mock',
+    'src/python/pants/backend/core/tasks:bash_completion',
+    'tests/python/pants_test/tasks:task_test_base',
+  ]
+)

tests/python/pants_test/backend/core/tasks/test_bash_completion.py

@@ -0,0 +1,24 @@
+# coding=utf-8
+# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.backend.core.tasks.bash_completion import BashCompletionTask
+from pants_test.tasks.task_test_base import ConsoleTaskTestBase
+
+
+class MockedBashCompletionTask(BashCompletionTask):
+  """A version of the BashCompletionTask, with the goal/help parsing mocked out."""
+  def parse_all_tasks_and_help(self, _):
+    return set(), '', set()
+
+
+class BashCompletionTest(ConsoleTaskTestBase):
+  @classmethod
+  def task_type(cls):
+    return MockedBashCompletionTask
+
+  def test_bash_completion_loads_template(self):
+    self.assert_console_output_contains("# Pants Autocompletion Support")

tests/python/pants_test/tasks/task_test_base.py

@@ -149,6 +149,14 @@ def assert_console_output(self, *output, **kwargs):
     """
     self.assertEqual(sorted(output), sorted(self.execute_console_task(**kwargs)))
 
+  def assert_console_output_contains(self, output, **kwargs):
+    """Verifies the expected output string is emitted by the console task under test.
+
+    output:  the expected output entry(ies)
+    **kwargs: additional kwargs passed to execute_console_task.
+    """
+    self.assertIn(output, self.execute_console_task(**kwargs))
+
   def assert_console_output_ordered(self, *output, **kwargs):
     """Verifies the expected output entries are emitted by the console task under test.