Introduce idea-plugin goal to invoke intellij pants plugin via CLI

This change enables calling intellij pants plugin from cli to import the targets/directories specified. pantsbuild/intellij-pants-plugin#58

Similar to `idea` goal, `idea-plugin` goal makes a skeleton `ipr` project file, and in addition fill in the workspace `iws` file with target specs and project path, so the plugin will know how to continue to import the project as if the user is doing so.

Only dep is bootstrap.
```
[tw-mbp-yic pants (from_cli)]$ ./pants --explain idea-plugin
Goal Execution Order:

bootstrap -> idea-plugin

Goal [TaskRegistrar->Task] Order:

bootstrap [jar-dependency-management->JarDependencyManagementSetup_bootstrap_jar_dependency_management, bootstrap-jvm-tools->BootstrapJvmTools_bootstrap_bootstrap_jvm_tools]
idea-plugin [idea-plugin->PluginGen_idea_plugin]
```

Example:
`./pants idea-plugin --no-open examples/src/python/example/hello/main/ examples/tests/java/org/pantsbuild/example/hello/greet/` will generate a `iws` file containing targets and project path. (only one project path because that's the directory intellij is going to zoom into):
```
<?xml version="1.0"?>
<project version="4">
  <component name="PropertiesComponent">
    <property name="targets" value="['/Users/yic/workspace/pants/examples/src/python/example/hello/main/', '/Users/yic/workspace/pants/examples/tests/java/org/pantsbuild/example/hello/greet/']" />
    <property name="project_path" value="/Users/yic/workspace/pants/examples/src/python/example/hello/main/" />
  </component>
</project>
```

Other change:
set default java language level to 8 for IdeGen

Testing Done:
https://travis-ci.org/pantsbuild/pants/builds/128569526

Bugs closed: 3187

Reviewed at https://rbcommons.com/s/twitter/r/3664/

src/python/pants/backend/project_info/register.py

@@ -12,6 +12,7 @@
 from pants.backend.project_info.tasks.export import Export
 from pants.backend.project_info.tasks.filedeps import FileDeps
 from pants.backend.project_info.tasks.idea_gen import IdeaGen
+from pants.backend.project_info.tasks.idea_plugin_gen import IdeaPluginGen
 from pants.goal.task_registrar import TaskRegistrar as task
 
 
@@ -23,6 +24,7 @@ def build_file_aliases():
 def register_goals():
   # IDE support.
   task(name='idea', action=IdeaGen).install()
+  task(name='idea-plugin', action=IdeaPluginGen).install()
   task(name='eclipse', action=EclipseGen).install()
   task(name='ensime', action=EnsimeGen).install()
   task(name='export', action=Export).install()

src/python/pants/backend/project_info/tasks/BUILD

@@ -12,6 +12,7 @@ python_library(
     ':filedeps',
     ':ide_gen',
     ':idea_gen',
+    ':idea_plugin_gen',
   ],
 )
 
@@ -137,19 +138,35 @@ python_library(
   name = 'idea_gen',
   sources = ['idea_gen.py'],
   resource_targets = [
-    ':idea_gen_resources',
+    ':idea_resources',
   ],
   dependencies = [
     ':ide_gen',
+    'src/python/pants/backend/jvm/targets:java',
     'src/python/pants/base:build_environment',
     'src/python/pants/base:generator',
+    'src/python/pants/scm:git',
+    'src/python/pants/util:dirutil',
+  ],
+)
+
+python_library(
+  name = 'idea_plugin_gen',
+  sources = ['idea_plugin_gen.py'],
+  resource_targets = [
+    ':idea_resources',
+  ],
+  dependencies = [
+    ':ide_gen',
     'src/python/pants/backend/jvm/targets:java',
+    'src/python/pants/base:build_environment',
+    'src/python/pants/base:generator',
     'src/python/pants/scm:git',
     'src/python/pants/util:dirutil',
   ],
 )
 
 resources(
-  name = 'idea_gen_resources',
+  name = 'idea_resources',
   sources = globs('templates/idea/*.mustache'),
 )

src/python/pants/backend/project_info/tasks/ide_gen.py

@@ -72,7 +72,7 @@ def register_options(cls, register):
     register('--java', type=bool, default=True,
              help='Includes java sources in the project; otherwise compiles them and adds them '
                   'to the project classpath.')
-    register('--java-language-level', type=int, default=7,
+    register('--java-language-level', type=int, default=8,
              help='Sets the java language and jdk used to compile the project\'s java sources.')
     register('--java-jdk-name', default=None,
              help='Sets the jdk used to compile the project\'s java sources. If unset the default '

src/python/pants/backend/project_info/tasks/idea_plugin_gen.py

@@ -0,0 +1,195 @@
+# coding=utf-8
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import json
+import os
+import pkgutil
+import shutil
+import subprocess
+
+from pants.backend.jvm.targets.jvm_target import JvmTarget
+from pants.backend.project_info.tasks.ide_gen import IdeGen
+from pants.base.build_environment import get_buildroot, get_scm
+from pants.base.exceptions import TaskError
+from pants.base.generator import Generator, TemplateData
+from pants.binaries import binary_util
+from pants.task.console_task import ConsoleTask
+from pants.util.contextutil import temporary_dir, temporary_file
+from pants.util.dirutil import safe_mkdir
+
+
+_TEMPLATE_BASEDIR = 'templates/idea'
+
+# Follow `export.py` for versioning strategy.
+IDEA_PLUGIN_VERSION = '0.0.1'
+
+
+class IdeaPluginGen(IdeGen, ConsoleTask):
+  """Invoke IntelliJ Pants plugin (installation required) to create a project.
+
+  The ideal workflow is to programmatically open idea -> select import -> import as pants project -> select project
+  path, but IDEA does not have CLI support for "select import" and "import as pants project" once it is opened.
+
+  Therefore, this task takes another approach to embed the target specs into a `iws` workspace file along
+  with an skeleton `ipr` project file.
+
+  Sample `iws`:
+  ********************************************************
+    <?xml version="1.0"?>
+    <project version="4">
+      <component name="PropertiesComponent">
+        <property name="targets" value="[&quot;/Users/me/workspace/pants/testprojects/tests/scala/org/pantsbuild/testproject/cp-directories/::&quot;]" />
+        <property name="project_path" value="/Users/me/workspace/pants/testprojects/tests/scala/org/pantsbuild/testproject/cp-directories/" />
+      </component>
+    </project>
+  ********************************************************
+
+  Once pants plugin sees `targets` and `project_path`, it will simulate the import process on and populate the
+  existing skeleton project into a Pants project as if user is importing these targets.
+  """
+
+  @classmethod
+  def prepare(cls, options, round_manager):
+    super(IdeGen, cls).prepare(options, round_manager)
+
+  @classmethod
+  def register_options(cls, register):
+    super(IdeaPluginGen, cls).register_options(register)
+    # TODO: https://github.com/pantsbuild/pants/issues/3198
+    # scala/java-language level should use what Pants already knows.
+    register('--open', type=bool, default=True,
+             help='Attempts to open the generated project in IDEA.')
+    register('--java-encoding', default='UTF-8',
+             help='Sets the file encoding for java files in this project.')
+    register('--open-with', advanced=True, default=None, recursive=True,
+             help='Program used to open the generated IntelliJ project.')
+
+  def __init__(self, *args, **kwargs):
+    super(IdeaPluginGen, self).__init__(*args, **kwargs)
+
+    self.open = self.get_options().open
+
+    self.java_encoding = self.get_options().java_encoding
+    self.project_template = os.path.join(_TEMPLATE_BASEDIR,
+                                         'project-12.mustache')
+    self.workspace_template = os.path.join(_TEMPLATE_BASEDIR,
+                                           'workspace-12.mustache')
+
+    output_dir = os.path.join(get_buildroot(), ".idea", self.__class__.__name__)
+    safe_mkdir(output_dir)
+
+    with temporary_dir(root_dir=output_dir, cleanup=False) as output_project_dir:
+      self.gen_project_workdir = output_project_dir
+      self.project_filename = os.path.join(self.gen_project_workdir,
+                                           '{}.ipr'.format(self.project_name))
+      self.workspace_filename = os.path.join(self.gen_project_workdir,
+                                             '{}.iws'.format(self.project_name))
+      self.intellij_output_dir = os.path.join(self.gen_project_workdir, 'out')
+
+  # TODO: https://github.com/pantsbuild/pants/issues/3198
+  # trim it down or refactor together with IdeaGen
+  def generate_project(self, project):
+    def create_content_root(source_set):
+      root_relative_path = os.path.join(source_set.source_base, source_set.path) \
+        if source_set.path else source_set.source_base
+      if source_set.resources_only:
+        if source_set.is_test:
+          content_type = 'java-test-resource'
+        else:
+          content_type = 'java-resource'
+      else:
+        content_type = ''
+
+      sources = TemplateData(
+        path=root_relative_path,
+        package_prefix=source_set.path.replace('/', '.') if source_set.path else None,
+        is_test=source_set.is_test,
+        content_type=content_type
+      )
+
+      return TemplateData(
+        path=root_relative_path,
+        sources=[sources],
+        exclude_paths=[os.path.join(source_set.source_base, x) for x in source_set.excludes],
+      )
+
+    content_roots = [create_content_root(source_set) for source_set in project.sources]
+    if project.has_python:
+      content_roots.extend(create_content_root(source_set) for source_set in project.py_sources)
+
+    java_language_level = None
+    for target in project.targets:
+      if isinstance(target, JvmTarget):
+        if java_language_level is None or java_language_level < target.platform.source_level:
+          java_language_level = target.platform.source_level
+    if java_language_level is not None:
+      java_language_level = 'JDK_{0}_{1}'.format(*java_language_level.components[:2])
+
+    outdir = os.path.abspath(self.intellij_output_dir)
+    if not os.path.exists(outdir):
+      os.makedirs(outdir)
+
+    scm = get_scm()
+    configured_project = TemplateData(
+      root_dir=get_buildroot(),
+      outdir=outdir,
+      git_root=scm.worktree,
+      java=TemplateData(
+        encoding=self.java_encoding,
+        jdk=self.java_jdk,
+        language_level='JDK_1_{}'.format(self.java_language_level)
+      ),
+      resource_extensions=list(project.resource_extensions),
+      debug_port=project.debug_port,
+      extra_components=[],
+      java_language_level=java_language_level,
+    )
+
+    if not self.context.options.target_specs:
+      raise TaskError("No targets specified.")
+
+    abs_target_specs = [os.path.join(get_buildroot(), spec) for spec in self.context.options.target_specs]
+    configured_workspace = TemplateData(
+      targets=json.dumps(abs_target_specs),
+      project_path=os.path.join(get_buildroot(), abs_target_specs[0].split(':')[0]),
+      idea_plugin_version=IDEA_PLUGIN_VERSION
+    )
+
+    # Generate (without merging in any extra components).
+    safe_mkdir(os.path.abspath(self.intellij_output_dir))
+
+    ipr = self._generate_to_tempfile(
+      Generator(pkgutil.get_data(__name__, self.project_template), project=configured_project))
+    iws = self._generate_to_tempfile(
+      Generator(pkgutil.get_data(__name__, self.workspace_template), workspace=configured_workspace))
+
+    self._outstream.write(self.gen_project_workdir)
+
+    shutil.move(ipr, self.project_filename)
+    shutil.move(iws, self.workspace_filename)
+    return self.project_filename
+
+  def _generate_to_tempfile(self, generator):
+    """Applies the specified generator to a temp file and returns the path to that file.
+    We generate into a temp file so that we don't lose any manual customizations on error."""
+    with temporary_file(cleanup=False) as output:
+      generator.write(output)
+      return output.name
+
+  def execute(self):
+    """Stages IDE project artifacts to a project directory and generates IDE configuration files."""
+    # Grab the targets in-play before the context is replaced by `self._prepare_project()` below.
+    self._prepare_project()
+    ide_file = self.generate_project(self._project)
+
+    if ide_file and self.get_options().open:
+      open_with = self.get_options().open_with
+      if open_with:
+        null = open(os.devnull, 'w')
+        subprocess.Popen([open_with, ide_file], stdout=null, stderr=null)
+      else:
+        binary_util.ui_open(ide_file)

src/python/pants/backend/project_info/tasks/templates/idea/workspace-12.mustache

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<project version="4">
+  <component name="PropertiesComponent">
+    <property name="targets" value="{{workspace.targets}}" />
+    <property name="project_path" value="{{workspace.project_path}}" />
+    <property name="pants_idea_plugin_version" value="{{workspace.idea_plugin_version}}" />
+  </component>
+</project>

tests/python/pants_test/backend/project_info/tasks/BUILD

@@ -131,3 +131,12 @@ python_tests(
   ],
   tags = {'integration'},
 )
+
+python_tests(
+  name = 'idea_plugin_integration',
+  sources = ['test_idea_plugin_integration.py'],
+  dependencies = [
+    'tests/python/pants_test:int-test',
+  ],
+  tags = {'integration'},
+)

tests/python/pants_test/backend/project_info/tasks/test_idea_plugin_integration.py

@@ -0,0 +1,85 @@
+# coding=utf-8
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import json
+import os
+from xml.dom import minidom
+
+from pants.base.build_environment import get_buildroot
+from pants.util.contextutil import temporary_file
+from pants_test.pants_run_integration_test import PantsRunIntegrationTest
+
+
+class IdeaPluginIntegrationTest(PantsRunIntegrationTest):
+  RESOURCE = 'java-resource'
+  TEST_RESOURCE = 'java-test-resource'
+
+  def _do_check(self, project_dir_path, expected_project_path, expected_targets):
+    """Check to see that the project contains the expected source folders."""
+
+    iws_file = os.path.join(project_dir_path, 'project.iws')
+    self.assertTrue(os.path.exists(iws_file))
+    dom = minidom.parse(iws_file)
+    self.assertEqual(1, len(dom.getElementsByTagName("project")))
+    project = dom.getElementsByTagName("project")[0]
+
+    self.assertEqual(1, len(project.getElementsByTagName('component')))
+    component = project.getElementsByTagName('component')[0]
+
+    actual_properties = component.getElementsByTagName('property')
+    # 3 properties: targets, project_path, pants_idea_plugin_version
+    self.assertEqual(3, len(actual_properties))
+
+    self.assertEqual('targets', actual_properties[0].getAttribute('name'))
+    actual_targets = json.loads(actual_properties[0].getAttribute('value'))
+    abs_expected_target_specs = [os.path.join(get_buildroot(), relative_spec) for relative_spec in expected_targets]
+    self.assertEqual(abs_expected_target_specs, actual_targets)
+
+    self.assertEqual('project_path', actual_properties[1].getAttribute('name'))
+    actual_project_path = actual_properties[1].getAttribute('value')
+    self.assertEqual(os.path.join(get_buildroot(), expected_project_path), actual_project_path)
+
+    self.assertEqual('pants_idea_plugin_version', actual_properties[2].getAttribute('name'))
+    self.assertEqual('0.0.1', actual_properties[2].getAttribute('value'))
+
+  def _get_project_dir(self, output_file):
+    with open(output_file, 'r') as result:
+      return result.readlines()[0]
+
+  def _run_and_check(self, project_path, targets):
+    with self.temporary_workdir() as workdir:
+      with temporary_file(root_dir=workdir, cleanup=True) as output_file:
+        pants_run = self.run_pants_with_workdir(
+          ['idea-plugin', '--output-file={}'.format(output_file.name), '--no-open'] + targets, workdir)
+        self.assert_success(pants_run)
+
+        project_dir = self._get_project_dir(output_file.name)
+        self.assertTrue(os.path.exists(project_dir), "{} does not exist".format(project_dir))
+        self._do_check(project_dir, project_path, targets)
+
+  def test_idea_plugin_single_target(self):
+
+    target = 'examples/src/scala/org/pantsbuild/example/hello:hello'
+    project_path = "examples/src/scala/org/pantsbuild/example/hello"
+
+    self._run_and_check(project_path, [target])
+
+  def test_idea_plugin_single_directory(self):
+    target = 'testprojects/src/python/antlr::'
+    project_path = "testprojects/src/python/antlr"
+
+    self._run_and_check(project_path, [target])
+
+  def test_idea_plugin_multiple_targets(self):
+    target_a = 'examples/src/scala/org/pantsbuild/example/hello:'
+    target_b = 'testprojects/src/python/antlr::'
+
+    # project_path is always the directory of the first target,
+    # which is where intellij is going to zoom in at project view.
+    project_path = 'examples/src/scala/org/pantsbuild/example/hello'
+
+    self._run_and_check(project_path, [target_a, target_b])