Subprocess clean-all

Move current .pants.d contents to a temporary directory and subprocess its removal after task execution

Testing Done:
https://travis-ci.org/ebubae/pants/builds/140860387

Bugs closed: 3590

Reviewed at https://rbcommons.com/s/twitter/r/4011/

src/python/pants/core_tasks/clean.py

@@ -5,12 +5,56 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import logging
+import os
+
 from pants.task.task import Task
-from pants.util.dirutil import safe_rmtree
+from pants.util.contextutil import temporary_dir
+from pants.util.dirutil import safe_concurrent_rename, safe_rmtree
+
+
+logger = logging.getLogger(__name__)
 
 
 class Clean(Task):
-  """Delete all build products, creating a clean workspace."""
+  """Delete all build products, creating a clean workspace.
+
+  The clean-all method allows for both synchronous and asynchronous options with the --async option."""
+
+  @classmethod
+  def register_options(cls, register):
+    super(Clean, cls).register_options(register)
+    register('--async', type=bool, default=False,
+             help='Allows clean-all to run in the background. Can dramatically speed up clean-all '
+                  'for large pants workdirs.')
 
   def execute(self):
-    safe_rmtree(self.get_options().pants_workdir)
+    pants_wd = self.get_options().pants_workdir
+    pants_trash = os.path.join(pants_wd, "trash")
+
+    # Creates, and eventually deletes, trash dir created in .pants_cleanall.
+    with temporary_dir(cleanup=False, root_dir=os.path.dirname(pants_wd), prefix=".pants_cleanall") as tmpdir:
+      logger.debug('Moving trash to {} for deletion'.format(tmpdir))
+
+      tmp_trash = os.path.join(tmpdir, "trash")
+
+      # Moves contents of .pants.d to cleanup dir.
+      safe_concurrent_rename(pants_wd, tmp_trash)
+      safe_concurrent_rename(tmpdir, pants_wd)
+
+      if self.get_options().async:
+        # The trash directory is deleted in a child process.
+        pid = os.fork()
+        if pid == 0:
+          try:
+            safe_rmtree(pants_trash)
+          except (IOError, OSError):
+            logger.warning("Async clean-all failed. Please try again.")
+          finally:
+            os._exit(0)
+        else:
+          logger.debug("Forked an asynchronous clean-all worker at pid: {}".format(pid))
+      else:
+        # Recursively removes pants cache; user waits patiently.

+        logger.info('For async removal, run `./pants clean-all --async`')
+        safe_rmtree(pants_trash)

src/python/pants/util/contextutil.py

@@ -62,7 +62,7 @@ def stdio_as(stdout, stderr, stdin=None):
 
 
 @contextmanager
-def temporary_dir(root_dir=None, cleanup=True, suffix=str(), permissions=None):
+def temporary_dir(root_dir=None, cleanup=True, suffix=str(), permissions=None, prefix=tempfile.template):
   """
     A with-context that creates a temporary directory.
 
@@ -73,7 +73,8 @@ def temporary_dir(root_dir=None, cleanup=True, suffix=str(), permissions=None):
     :param bool cleanup: Whether or not to clean up the temporary directory.
     :param int permissions: If provided, sets the directory permissions to this mode.
   """
-  path = tempfile.mkdtemp(dir=root_dir, suffix=suffix)
+  path = tempfile.mkdtemp(dir=root_dir, suffix=suffix, prefix=prefix)
+
   try:
     if permissions is not None:
       os.chmod(path, permissions)

tests/python/pants_test/tasks/test_clean_all_integration.py

@@ -5,12 +5,34 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import os
+import subprocess
+
 from pants.util.contextutil import temporary_dir
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest
 
 
 class CleanAllTest(PantsRunIntegrationTest):
-
   def test_clean_all_on_wrong_dir(self):
     with temporary_dir() as workdir:
       self.assert_failure(self.run_pants_with_workdir(["clean-all"], workdir))
+      self.assert_failure(self.run_pants_with_workdir(["clean-all", "--async"], workdir))
+
+  # Ensure async clean-all exits normally.

+  def test_clean_all_async(self):
+    self.assert_success(self.run_pants(["clean-all", "--async"]))
+
+  # The tests below check for the existence of trash directories.
+  def test_empty_trash(self):
+    with self.temporary_workdir() as work_dir:
+      trash_dir = os.path.join(work_dir, "trash")
+      subprocess.call(["touch", trash_dir + "foo.txt"])
+      self.assert_success(self.run_pants_with_workdir(["clean-all"], work_dir))
+      self.assertFalse(os._exists(trash_dir))
+
+  def test_empty_trash_async(self):
+    with self.temporary_workdir() as work_dir:
+      trash_dir = os.path.join(work_dir, "trash")
+      subprocess.call(["touch", trash_dir + "foo.txt"])
+      self.assert_success(self.run_pants_with_workdir(["clean-all", "--async"], work_dir))
+      self.assertFalse(os._exists(trash_dir))