Document pants.ini configuration options.

Mostly a repost of https://rbcommons.com/s/twitter/r/67/ post move to
pantsbuild.

Add a simple system to register pants.ini options so we can generate a
configuration reference page on our website. Lots of required, undocumented
keys in pants.ini has been identified as an issue for new users and this
change lays the groundwork. Once the pattern has been defined I'll move all
options over to this style in a few future reviews.

Similar to command-line flags, pants developers explicitly define options
rather than fetching keys from the config directly. We register these keys
enabling the generation of a docs site. A few examples are provided here, and
we'll do the rest after reaching consensus on the format. The goal is
generating documentation for our users, so we explicitly avoid going overboard
and rewriting the whole config system (e.g.: unifying pants.ini & command-line
flags which is out of scope for this change).

Eliminate two options that are not obviosuly desiarable to allow users to
tweak. In 67 we agreed to remove these rather than document as tuneable:
tasks.build_invalidator, reporting.reports_dir

Reviewed at https://rbcommons.com/s/twitter/r/183/

pants.ini

@@ -51,10 +51,6 @@ bootstrap_buildfiles: [
   ]
 
 
-[tasks]
-build_invalidator: %(pants_workdir)s/build_invalidator
-
-
 [nailgun]
 bootstrap-tools: [':nailgun-server']
 

src/python/pants/base/config.py

@@ -16,6 +16,78 @@
 from pants.base.build_environment import get_buildroot
 
 
+class ConfigOption(object):
+  """Registry of pants.ini options.
+
+  Options are created in code, typically scoped as close to their use as possible. ::
+
+     my_opt = ConfigOption.create(
+       section='mycache',
+       option='workdir',
+       help='Directory, relative to pants_workdir, of the mycache workdir.',
+       default='mycache')
+
+  Read an option from ``pants.ini`` with ::
+
+     mycache_dir = os.path.join(config.get_option(config.DEFAULT_PANTS_WORKDIR),
+                                config.get_option(_REPORTING_REPORTS_DIR))
+
+  Please note `configparser <http://docs.python.org/2/library/configparser.html>`_
+  is used to retrieve options, so variable interpolation and the default section
+  are used as defined in the configparser docs.
+  """
+
+  class Option(object):
+    """A ``pants.ini`` option."""
+    def __init__(self, section, option, help, valtype, default):
+      """Do not instantiate directly - use ConfigOption.create."""
+      self.section = section
+      self.option = option
+      self.help = help
+      self.valtype = valtype
+      self.default = default
+
+    def __hash__(self):
+      return hash(self.section + self.option)
+
+    def __eq__(self, other):
+      if other is None:
+        return False
+      return True if self.section == other.section and self.option == other.option else False
+
+    def __repr__(self):
+      return '%s(%s.%s)' % (self.__class__.__name__, self.section, self.option)
+
+  _CONFIG_OPTIONS = set()
+
+  @classmethod
+  def all(cls):
+    return cls._CONFIG_OPTIONS
+
+  @classmethod
+  def create(cls, section, option, help, valtype=str, default=None):
+    """Create a new ``pants.ini`` option.
+
+    :param section: Name of section to retrieve option from.
+    :param option: Name of option to retrieve from section.
+    :param help: Description for display in the configuration reference.
+    :param valtype: Type to cast the retrieved option to.
+    :param default: Default value if undefined in the config.
+    :returns: An ``Option`` suitable for use with ``Config.get_option``.
+    :raises: ``ValueError`` if the option already exists.
+    """
+    new_opt = cls.Option(section=section,
+                         option=option,
+                         help=help,
+                         valtype=valtype,
+                         default=default)
+    for existing_opt in cls._CONFIG_OPTIONS:
+      if new_opt.section == existing_opt.section and new_opt.option == existing_opt.option:
+        raise ValueError('Option %s.%s already exists.' % (new_opt.section, new_opt.option))
+    cls._CONFIG_OPTIONS.add(new_opt)
+    return new_opt
+
+
 class Config(object):
   """
     Encapsulates ini-style config file loading and access additionally supporting recursive variable
@@ -25,6 +97,24 @@ class Config(object):
 
   DEFAULT_SECTION = ConfigParser.DEFAULTSECT
 
+  DEFAULT_PANTS_DISTDIR = ConfigOption.create(
+    section=DEFAULT_SECTION,
+    option='pants_distdir',
+    help='Directory where pants will write user visible artifacts.',
+    default=os.path.join(get_buildroot(), 'dist'))
+
+  DEFAULT_PANTS_SUPPORTDIR = ConfigOption.create(
+    section=DEFAULT_SECTION,
+    option='pants_supportdir',
+    help='Directory of pants support files (e.g.: ivysettings.xml).',
+    default=os.path.join(get_buildroot(), 'build-support'))
+
+  DEFAULT_PANTS_WORKDIR = ConfigOption.create(
+    section=DEFAULT_SECTION,
+    option='pants_workdir',
+    help='Directory where pants will write its intermediate output files.',
+    default=os.path.join(get_buildroot(), '.pants.d'))
+
   class ConfigError(Exception):
     pass
 
@@ -42,8 +132,8 @@ def load(configpath=None, defaults=None):
       parser.readfp(ini)
     return Config(parser)
 
-  @staticmethod
-  def create_parser(defaults=None):
+  @classmethod
+  def create_parser(cls, defaults=None):
     """Creates a config parser that supports %([key-name])s value substitution.
 
     Any defaults supplied will act as if specified in the loaded config file's DEFAULT section and
@@ -63,9 +153,9 @@ def create_parser(defaults=None):
       homedir=os.path.expanduser('~'),
       user=getpass.getuser(),
       pants_bootstrapdir=os.path.expanduser('~/.pants.d'),
-      pants_workdir=os.path.join(get_buildroot(), '.pants.d'),
-      pants_supportdir=os.path.join(get_buildroot(), 'build-support'),
-      pants_distdir=os.path.join(get_buildroot(), 'dist')
+      pants_workdir=cls.DEFAULT_PANTS_WORKDIR.default,
+      pants_supportdir=cls.DEFAULT_PANTS_SUPPORTDIR.default,
+      pants_distdir=cls.DEFAULT_PANTS_DISTDIR.default,
     )
     if defaults:
       standard_defaults.update(defaults)
@@ -131,6 +221,17 @@ def get(self, section, option, type=str, default=None):
     """
     return self._getinstance(section, option, type, default=default)
 
+  # TODO(travis): Migrate all config reads to get_option and remove other getters.
+  # TODO(travis): Rename to get when other getters are removed.
+  def get_option(self, option):
+    if not isinstance(option, ConfigOption.Option):
+      raise ValueError('Expected %s but found %s' % (ConfigOption.Option.__class__.__name__,
+                                                     option))
+    return self.get(section=option.section,
+                    option=option.option,
+                    type=option.valtype,
+                    default=option.default)
+
   def get_required(self, section, option, type=str):
     """Retrieves option from the specified section and attempts to parse it as type.
     If the specified section is missing a definition for the option, the value is

src/python/pants/commands/goal.py

@@ -27,7 +27,7 @@
 from pants.base.address import Address
 from pants.base.build_environment import get_buildroot
 from pants.base.build_file import BuildFile
-from pants.base.config import Config
+from pants.base.config import Config, ConfigOption
 from pants.base.parse_context import ParseContext
 from pants.base.rcfile import RcFile
 from pants.base.run_info import RunInfo
@@ -533,7 +533,8 @@ def _async_cautious_rmtree(root):
 
 class Invalidator(ConsoleTask):
   def execute(self, targets):
-    build_invalidator_dir = self.context.config.get('tasks', 'build_invalidator')
+    build_invalidator_dir = os.path.join(
+      self.context.config.get_option(Config.DEFAULT_PANTS_WORKDIR), 'build_invalidator')
     _cautious_rmtree(build_invalidator_dir)
 goal(
   name='invalidate',

src/python/pants/docs/.gitignore

@@ -5,3 +5,4 @@ targets.rst
 tasks.rst
 build_dictionary.rst
 goals_reference.rst
+pants_ini_reference.rst

src/python/pants/docs/conf.py

@@ -20,7 +20,7 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('../../..'))
+sys.path.insert(0, os.path.abspath('../..'))
 
 pants_egg_dir = os.path.abspath('../../../../.pants.d/python/eggs')
 if not os.path.exists(pants_egg_dir):

src/python/pants/docs/gen.py

@@ -57,7 +57,7 @@ def gen_base_reference(rst_filename, dirname):
     fh.write('\n'.join(lines))
 
 def copy_builddict(docs_dir):
-  for filename in ['build_dictionary.rst', 'goals_reference.rst']:
+  for filename in ['build_dictionary.rst', 'goals_reference.rst', 'pants_ini_reference.rst']:
     filepath = os.path.abspath(os.path.join(docs_dir,
         '../../../../dist/builddict', filename))
     try:

src/python/pants/docs/index.rst

@@ -68,6 +68,7 @@ Pants Reference Documentation
 
    build_dictionary
    goals_reference
+   pants_ini_reference
 
 
 Contributing to Pants

src/python/pants/goal/initialize_reporting.py

@@ -10,6 +10,7 @@
 from twitter.common.dirutil import safe_mkdir, safe_rmtree
 from twitter.common.lang import Compatibility
 
+from pants.base.config import Config
 from pants.reporting.html_reporter import HtmlReporter
 from pants.reporting.plaintext_reporter import PlainTextReporter
 from pants.reporting.quiet_reporter import QuietReporter
@@ -25,8 +26,7 @@ def initial_reporting(config, run_tracker):
 
   Will be changed after we parse cmd-line flags.
   """
-  reports_dir = config.get('reporting', 'reports_dir',
-                           default=os.path.join(config.getdefault('pants_workdir'), 'reports'))
+  reports_dir = os.path.join(config.get_option(Config.DEFAULT_PANTS_WORKDIR), 'reports')
   link_to_latest = os.path.join(reports_dir, 'latest')
   if os.path.exists(link_to_latest):
     os.unlink(link_to_latest)

src/python/pants/tasks/builddictionary.py

@@ -7,11 +7,14 @@
 import inspect
 import os
 
+from collections import defaultdict
 from pkg_resources import resource_string
 from twitter.common.dirutil import Fileset, safe_open
 
+from pants.base.build_environment import get_buildroot
 from pants.base.build_file_helpers import maven_layout
 from pants.base.build_manual import get_builddict_info
+from pants.base.config import ConfigOption
 from pants.base.generator import Generator, TemplateData
 from pants.base.parse_context import ParseContext
 from pants.goal.phase import Phase
@@ -237,6 +240,7 @@ def __init__(self, context):
 
   def execute(self, targets):
     self._gen_goals_reference()
+    self._gen_config_reference()
 
     d = assemble()
     template = resource_string(__name__, os.path.join(self._templates_dir, 'page.mustache'))
@@ -274,3 +278,20 @@ def _gen_goals_reference(self):
     with safe_open(filename, 'w') as outfile:
       generator = Generator(template, phases=phases)
       generator.write(outfile)
+
+  def _gen_config_reference(self):
+    options_by_section = defaultdict(list)
+    for option in ConfigOption.all():
+      if isinstance(option.default, unicode):
+        option.default = option.default.replace(get_buildroot(), '%(buildroot)s')
+      options_by_section[option.section].append(option)
+    sections = list()
+    for section, options in options_by_section.items():
+      sections.append(TemplateData(section=section, options=options))
+    template = resource_string(__name__,
+                               os.path.join(self._templates_dir, 'pants_ini_reference.mustache'))
+    filename = os.path.join(self._outdir, 'pants_ini_reference.rst')
+    self.context.log.info('Generating %s' % filename)
+    with safe_open(filename, 'w') as outfile:
+      generator = Generator(template, sections=sections)
+      generator.write(outfile)

src/python/pants/tasks/idea_gen.py

@@ -13,10 +13,8 @@
 from twitter.common.dirutil import safe_mkdir
 
 from pants.base.build_environment import get_buildroot
+from pants.base.config import ConfigOption
 from pants.base.generator import Generator, TemplateData
-from pants.targets.java_tests import JavaTests
-from pants.targets.scala_tests import ScalaTests
-from pants.targets.sources import SourceRoot
 from pants.tasks.ide_gen import IdeGen, Project, SourceSet
 
 
@@ -41,6 +39,20 @@
 
 
 class IdeaGen(IdeGen):
+  _IDEA_JAVA_MAX_HEAP_MB = ConfigOption.create(
+    section='idea',
+    option='java_maximum_heap_size_mb',
+    help='Max heap size for javac in megabytes.',
+    valtype=int,
+    default=512)
+
+  _IDEA_SCALA_MAX_HEAP_MB = ConfigOption.create(
+    section='idea',
+    option='scala_maximum_heap_size_mb',
+    help='Max heap size for scalac in megabytes.',
+    valtype=int,
+    default=512)
+
   @classmethod
   def setup_parser(cls, option_group, args, mkflag):
     IdeGen.setup_parser(option_group, args, mkflag)
@@ -98,14 +110,14 @@ def __init__(self, context):
     self.scala_language_level = _SCALA_VERSIONS.get(context.options.idea_scala_language_level, None)
     self.scala_maximum_heap_size = (
       context.options.idea_gen_scala_maximum_heap_size
-      or context.config.getint('idea', 'scala_maximum_heap_size_mb', default=512)
+      or context.config.get_option(self._IDEA_SCALA_MAX_HEAP_MB)
     )
     self.fsc = context.options.idea_gen_fsc
 
     self.java_encoding = context.options.idea_gen_java_encoding
     self.java_maximum_heap_size = (
       context.options.idea_gen_java_maximum_heap_size
-      or context.config.getint('idea', 'java_maximum_heap_size_mb', default=128)
+      or context.config.get_option(self._IDEA_JAVA_MAX_HEAP_MB)
     )
 
     idea_version = _VERSIONS[context.options.idea_gen_version]

src/python/pants/tasks/templates/builddictionary/pants_ini_reference.mustache

@@ -0,0 +1,27 @@
+*******************
+pants.ini Reference
+*******************
+
+All repos using pants have a ``pants.ini`` file at the root,
+in `configparser <http://docs.python.org/2/library/configparser.html>`_ format.
+Users can override the default config with a ``~/.pants.new.rc`` file.
+
+.. NOTE:: As of April 2014, this configuration reference is incomplete while migrating
+          options to the format used to generate this reference.
+
+
+The following configuration options are provided:
+
+{{#sections}}
+
+.. _configref_section_{{section}}
+
+{{section}} section
+-----------------------------------------------------------------------------
+
+{{#options}}
+    ``{{option}}`` - {{help}} (default: ``{{default}}``)
+
+{{/options}}
+
+{{/sections}}