diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 75583d00d7f..ce90eee4a51 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -27,6 +27,7 @@ pystache==0.5.3 pytest-cov>=2.5,<2.6 pytest>=3.4,<4.0 pywatchman==1.4.1 +PyYAML==5.1 py_zipkin==0.17.0 requests[security]>=2.20.1 responses==0.10.4 diff --git a/src/python/pants/option/BUILD b/src/python/pants/option/BUILD index d329bb6598a..5d4f022a1ef 100644 --- a/src/python/pants/option/BUILD +++ b/src/python/pants/option/BUILD @@ -4,8 +4,9 @@ python_library( dependencies=[ 'src/python/pants/util:collections_abc_backport', - '3rdparty/python:future', '3rdparty/python:ansicolors', + '3rdparty/python:future', + '3rdparty/python:PyYAML', '3rdparty/python:setuptools', '3rdparty/python/twitter/commons:twitter.common.collections', 'src/python/pants/base:build_environment', diff --git a/src/python/pants/option/parser.py b/src/python/pants/option/parser.py index 5a43f0d107b..a0ff9b0c6e3 100644 --- a/src/python/pants/option/parser.py +++ b/src/python/pants/option/parser.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals import copy +import json import os import re import traceback @@ -12,6 +13,7 @@ from collections import defaultdict import six +import yaml from pants.base.deprecated import validate_deprecation_semver, warn_or_error from pants.option.arg_splitter import GLOBAL_SCOPE, GLOBAL_SCOPE_CONFIG_SECTION @@ -453,6 +455,7 @@ def to_value_type(val_str): .format(type_arg.__name__, val_str, dest, self._scope_str(), e)) # Helper function to expand a fromfile=True value string, if needed. + # May return a string or a dict/list decoded from a json/yaml file. def expand(val_str): if kwargs.get('fromfile', False) and val_str and val_str.startswith('@'): if val_str.startswith('@@'): # Support a literal @ for fromfile values via @@. @@ -461,8 +464,14 @@ def expand(val_str): fromfile = val_str[1:] try: with open(fromfile, 'r') as fp: - return fp.read().strip() - except IOError as e: + s = fp.read().strip() + if fromfile.endswith('.json'): + return json.loads(s) + elif fromfile.endswith('.yml') or fromfile.endswith('.yaml'): + return yaml.safe_load(s) + else: + return s + except (IOError, ValueError, yaml.YAMLError) as e: raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format( dest, self._scope_str(), fromfile, e)) else: @@ -471,8 +480,8 @@ def expand(val_str): # Get value from config files, and capture details about its derivation. config_details = None config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope - config_default_val_str = expand(self._config.get(Config.DEFAULT_SECTION, dest, default=None)) - config_val_str = expand(self._config.get(config_section, dest, default=None)) + config_default_val_or_str = expand(self._config.get(Config.DEFAULT_SECTION, dest, default=None)) + config_val_or_str = expand(self._config.get(config_section, dest, default=None)) config_source_file = (self._config.get_source_for_option(config_section, dest) or self._config.get_source_for_option(Config.DEFAULT_SECTION, dest)) if config_source_file is not None: @@ -495,12 +504,12 @@ def expand(val_str): sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', self._scope.upper()) env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)] - env_val_str = None + env_val_or_str = None env_details = None if self._env: for env_var in env_vars: if env_var in self._env: - env_val_str = expand(self._env.get(env_var)) + env_val_or_str = expand(self._env.get(env_var)) env_details = 'from env var {}'.format(env_var) break @@ -527,8 +536,8 @@ def expand(val_str): # is idempotent, so this is OK. values_to_rank = [to_value_type(x) for x in - [flag_val, env_val_str, config_val_str, - config_default_val_str, kwargs.get('default'), None]] + [flag_val, env_val_or_str, config_val_or_str, + config_default_val_or_str, kwargs.get('default'), None]] # Note that ranked_vals will always have at least one element, and all elements will be # instances of RankedValue (so none will be None, although they may wrap a None value). ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank)))) diff --git a/src/python/pants/option/ranked_value.py b/src/python/pants/option/ranked_value.py index 5ce337aa0cc..be249ebecec 100644 --- a/src/python/pants/option/ranked_value.py +++ b/src/python/pants/option/ranked_value.py @@ -89,7 +89,8 @@ def get_names(cls): return sorted(cls._RANK_NAMES.values(), key=cls.get_rank_value) @classmethod - def prioritized_iter(cls, flag_val, env_val, config_val, config_default_val, hardcoded_val, default): + def prioritized_iter(cls, flag_val, env_val, config_val, config_default_val, + hardcoded_val, default): """Yield the non-None values from highest-ranked to lowest, wrapped in RankedValue instances.""" if flag_val is not None: yield RankedValue(cls.FLAG, flag_val) diff --git a/tests/python/pants_test/option/test_options.py b/tests/python/pants_test/option/test_options.py index 70cd646482c..0ef82115393 100644 --- a/tests/python/pants_test/option/test_options.py +++ b/tests/python/pants_test/option/test_options.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals import io +import json import os import shlex from builtins import open, str @@ -12,6 +13,7 @@ from textwrap import dedent import mock +import yaml from future.utils import text_type from packaging.version import Version @@ -1224,6 +1226,28 @@ def parse_func(dest, fromfile): self.assert_fromfile(parse_func) + def test_fromfile_json(self): + def parse_func(dest, fromfile): + return self._parse('./pants fromfile --{}=@{}'.format(dest.replace('_', '-'), fromfile)) + + val = {'a': {'b': 1}, 'c': [2, 3]} + with temporary_file(suffix='.json', binary_mode=False) as fp: + json.dump(val, fp) + fp.close() + options = self._parse('./pants fromfile --{}=@{}'.format('dictvalue', fp.name)) + self.assertEqual(val, options.for_scope('fromfile')['dictvalue']) + + def test_fromfile_yaml(self): + def parse_func(dest, fromfile): + return self._parse('./pants fromfile --{}=@{}'.format(dest.replace('_', '-'), fromfile)) + + val = {'a': {'b': 1}, 'c': [2, 3]} + with temporary_file(suffix='.yaml', binary_mode=False) as fp: + yaml.safe_dump(val, fp) + fp.close() + options = self._parse('./pants fromfile --{}=@{}'.format('dictvalue', fp.name)) + self.assertEqual(val, options.for_scope('fromfile')['dictvalue']) + def test_fromfile_error(self): options = self._parse('./pants fromfile --string=@/does/not/exist') with self.assertRaises(Parser.FromfileError):