Support JSON and YAML option fromfiles. (pantsbuild#7500)

For cases when eval'd python is burdensome or inappropriate.

3rdparty/python/requirements.txt

@@ -27,6 +27,7 @@ pystache==0.5.3
 pytest-cov>=2.5,<2.6
 pytest>=3.4,<4.0
 pywatchman==1.4.1
+PyYAML==5.1
 py_zipkin==0.17.0
 requests[security]>=2.20.1
 responses==0.10.4

src/python/pants/option/BUILD

@@ -4,8 +4,9 @@
 python_library(
   dependencies=[
     'src/python/pants/util:collections_abc_backport',
-    '3rdparty/python:future',
     '3rdparty/python:ansicolors',
+    '3rdparty/python:future',
+    '3rdparty/python:PyYAML',
     '3rdparty/python:setuptools',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     'src/python/pants/base:build_environment',

src/python/pants/option/parser.py

@@ -5,13 +5,15 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import copy
+import json
 import os
 import re
 import traceback
 from builtins import next, object, open, str
 from collections import defaultdict
 
 import six
+import yaml
 
 from pants.base.deprecated import validate_deprecation_semver, warn_or_error
 from pants.option.arg_splitter import GLOBAL_SCOPE, GLOBAL_SCOPE_CONFIG_SECTION
@@ -453,6 +455,7 @@ def to_value_type(val_str):
             .format(type_arg.__name__, val_str, dest, self._scope_str(), e))
 
     # Helper function to expand a fromfile=True value string, if needed.
+    # May return a string or a dict/list decoded from a json/yaml file.
     def expand(val_str):
       if kwargs.get('fromfile', False) and val_str and val_str.startswith('@'):
         if val_str.startswith('@@'):   # Support a literal @ for fromfile values via @@.
@@ -461,8 +464,14 @@ def expand(val_str):
           fromfile = val_str[1:]
           try:
             with open(fromfile, 'r') as fp:
-              return fp.read().strip()
-          except IOError as e:
+              s = fp.read().strip()
+              if fromfile.endswith('.json'):
+                return json.loads(s)
+              elif fromfile.endswith('.yml') or fromfile.endswith('.yaml'):
+                return yaml.safe_load(s)
+              else:
+                return s
+          except (IOError, ValueError, yaml.YAMLError) as e:
             raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format(
                 dest, self._scope_str(), fromfile, e))
       else:
@@ -471,8 +480,8 @@ def expand(val_str):
     # Get value from config files, and capture details about its derivation.
     config_details = None
     config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
-    config_default_val_str = expand(self._config.get(Config.DEFAULT_SECTION, dest, default=None))
-    config_val_str = expand(self._config.get(config_section, dest, default=None))
+    config_default_val_or_str = expand(self._config.get(Config.DEFAULT_SECTION, dest, default=None))
+    config_val_or_str = expand(self._config.get(config_section, dest, default=None))
     config_source_file = (self._config.get_source_for_option(config_section, dest) or
         self._config.get_source_for_option(Config.DEFAULT_SECTION, dest))
     if config_source_file is not None:
@@ -495,12 +504,12 @@ def expand(val_str):
       sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', self._scope.upper())
       env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]
 
-    env_val_str = None
+    env_val_or_str = None
     env_details = None
     if self._env:
       for env_var in env_vars:
         if env_var in self._env:
-          env_val_str = expand(self._env.get(env_var))
+          env_val_or_str = expand(self._env.get(env_var))
           env_details = 'from env var {}'.format(env_var)
           break
 
@@ -527,8 +536,8 @@ def expand(val_str):
     # is idempotent, so this is OK.
 
     values_to_rank = [to_value_type(x) for x in
-                      [flag_val, env_val_str, config_val_str,
-                       config_default_val_str, kwargs.get('default'), None]]
+                      [flag_val, env_val_or_str, config_val_or_str,
+                       config_default_val_or_str, kwargs.get('default'), None]]
     # Note that ranked_vals will always have at least one element, and all elements will be
     # instances of RankedValue (so none will be None, although they may wrap a None value).
     ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank))))

src/python/pants/option/ranked_value.py

@@ -89,7 +89,8 @@ def get_names(cls):
     return sorted(cls._RANK_NAMES.values(), key=cls.get_rank_value)
 
   @classmethod
-  def prioritized_iter(cls, flag_val, env_val, config_val, config_default_val, hardcoded_val, default):
+  def prioritized_iter(cls, flag_val, env_val, config_val, config_default_val,
+                       hardcoded_val, default):
     """Yield the non-None values from highest-ranked to lowest, wrapped in RankedValue instances."""
     if flag_val is not None:
       yield RankedValue(cls.FLAG, flag_val)

tests/python/pants_test/option/test_options.py

@@ -5,13 +5,15 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import io
+import json
 import os
 import shlex
 from builtins import open, str
 from contextlib import contextmanager
 from textwrap import dedent
 
 import mock
+import yaml
 from future.utils import text_type
 from packaging.version import Version
 
@@ -1224,6 +1226,28 @@ def parse_func(dest, fromfile):
 
     self.assert_fromfile(parse_func)
 
+  def test_fromfile_json(self):
+    def parse_func(dest, fromfile):
+      return self._parse('./pants fromfile --{}=@{}'.format(dest.replace('_', '-'), fromfile))
+
+    val = {'a': {'b': 1}, 'c': [2, 3]}
+    with temporary_file(suffix='.json', binary_mode=False) as fp:
+      json.dump(val, fp)
+      fp.close()
+      options = self._parse('./pants fromfile --{}=@{}'.format('dictvalue', fp.name))
+      self.assertEqual(val, options.for_scope('fromfile')['dictvalue'])
+
+  def test_fromfile_yaml(self):
+    def parse_func(dest, fromfile):
+      return self._parse('./pants fromfile --{}=@{}'.format(dest.replace('_', '-'), fromfile))
+
+    val = {'a': {'b': 1}, 'c': [2, 3]}
+    with temporary_file(suffix='.yaml', binary_mode=False) as fp:
+      yaml.safe_dump(val, fp)
+      fp.close()
+      options = self._parse('./pants fromfile --{}=@{}'.format('dictvalue', fp.name))
+      self.assertEqual(val, options.for_scope('fromfile')['dictvalue'])
+
   def test_fromfile_error(self):
     options = self._parse('./pants fromfile --string=@/does/not/exist')
     with self.assertRaises(Parser.FromfileError):