From 0dd24234c4f20a27540e1b02bfff1c21cdeda358 Mon Sep 17 00:00:00 2001 From: Nick Howard Date: Wed, 7 Dec 2016 13:44:48 -0800 Subject: [PATCH] [engine] model snapshots in validation, make root rules a dict instead of a set (#4125) ### Problem I'm working on porting my patch for #3580 to the native engine, (#3960). First, I'm updating the graph construction code with the changes there. It hadn't covered Snapshot processes, and it modeled root rules as a set rather than a graph, which caused problems. ### Solution Include snapshot process' behavior in the rule graph construction used by the validator. Update tests to expect the new visualization format. ### List of changes * rename NodeBuilder to RuleIndex * rename RootRule to RootRuleGraphEntry * support SnapshotProcess in validator * Name snapshot process funcs so they are printable * Include subject and product types in intrinsic rule repr * Include graph generation methods that can construct new graphs from an existing one--these are currently unused * Raise an error if a declared rule is not reachable from the declared root types. * Unify naming for input selector for projection selectors * If field of a SelectDependencies is the default value, don't include it in the repr * Include a map of root rule to the rules that would fulfill it. --- src/python/pants/engine/isolated_process.py | 8 +- src/python/pants/engine/rules.py | 276 +++++++++++++----- src/python/pants/engine/scheduler.py | 12 +- src/python/pants/engine/selectors.py | 17 +- tests/python/pants_test/engine/test_rules.py | 236 +++++++++------ .../pants_test/engine/test_selectors.py | 4 +- 6 files changed, 367 insertions(+), 186 deletions(-) diff --git a/src/python/pants/engine/isolated_process.py b/src/python/pants/engine/isolated_process.py index c4981d7abf8..d71c5afcc86 100644 --- a/src/python/pants/engine/isolated_process.py +++ b/src/python/pants/engine/isolated_process.py @@ -199,7 +199,9 @@ def ptree(func): def create_snapshot_intrinsics(project_tree): def ptree(func): - return functools.partial(func, project_tree, snapshot_directory(project_tree)) + partial = functools.partial(func, project_tree, snapshot_directory(project_tree)) + partial.__name__ = '{}_intrinsic'.format(func.__name__) + return partial return [ (Snapshot, Files, ptree(create_snapshot_archive)), ] @@ -213,7 +215,9 @@ def create_snapshot_tasks(project_tree): uncacheable singleton. """ def ptree(func): - return functools.partial(func, project_tree, snapshot_directory(project_tree)) + partial = functools.partial(func, project_tree, snapshot_directory(project_tree)) + partial.__name__ = '{}_task'.format(func.__name__) + return partial return [ (Snapshot, [Select(Files)], ptree(create_snapshot_archive)), ] diff --git a/src/python/pants/engine/rules.py b/src/python/pants/engine/rules.py index 5593118dde8..32bb2c9e704 100644 --- a/src/python/pants/engine/rules.py +++ b/src/python/pants/engine/rules.py @@ -13,6 +13,7 @@ from twitter.common.collections import OrderedSet from pants.engine.addressable import Exactly +from pants.engine.isolated_process import SnapshottedProcess, SnapshottedProcessRequest from pants.engine.selectors import (Select, SelectDependencies, SelectLiteral, SelectProjection, SelectVariant, type_or_constraint_repr) from pants.util.meta import AbstractClass @@ -74,15 +75,15 @@ def has_errors(self): class RulesetValidator(object): - """Validates that the set of rules used by the node builder has no missing tasks.""" + """Validates that the rule index has no missing tasks.""" - def __init__(self, node_builder, goal_to_product, root_subject_fns): + def __init__(self, rule_index, goal_to_product, root_subject_fns): if not root_subject_fns: raise ValueError('root_subject_fns must not be empty') self._goal_to_product = goal_to_product - self._graph = GraphMaker(node_builder, root_subject_fns).full_graph() + self._graph = GraphMaker(rule_index, root_subject_fns).full_graph() def validate(self): """ Validates that all tasks can be executed based on the declared product types and selectors. @@ -121,7 +122,9 @@ def output_product_type(self): return self.product_type def __repr__(self): - return '{}({})'.format(type(self).__name__, self.product_type.__name__) + return '{}({}, {})'.format(type(self).__name__, + self.product_type.__name__, + self.func.__name__) class IntrinsicRule(datatype('IntrinsicRule', ['subject_type', 'product_type', 'func']), Rule): @@ -136,10 +139,13 @@ def output_product_type(self): return self.product_type def __repr__(self): - return '{}({})'.format(type(self).__name__, self.func.__name__) + return '{}(({}, {}), {})'.format(type(self).__name__, + self.subject_type.__name__, + self.output_product_type.__name__, + self.func.__name__) -class NodeBuilder(datatype('NodeBuilder', ['tasks', 'intrinsics', 'singletons'])): +class RuleIndex(datatype('RuleIndex', ['tasks', 'intrinsics', 'singletons'])): """Holds an index of tasks and intrinsics used to instantiate Nodes.""" @classmethod @@ -171,6 +177,8 @@ def add_task(product_type, rule): # TODO: The heterogenity here has some confusing implications here: # see https://github.com/pantsbuild/pants/issues/4005 for kind in constraint.types: + # NB Ensure that interior types from SelectDependencies / SelectProjections work by + # indexing on the list of types in the constraint. add_task(kind, factory) add_task(constraint, factory) else: @@ -204,10 +212,17 @@ def all_rules(self): def all_produced_product_types(self, subject_type): intrinsic_products = set(prod for subj, prod in self.intrinsics.keys() if subj == subject_type) - return intrinsic_products.union(set(self.tasks.keys())).union(set(self.singletons.keys())) + singleton_products = self.singletons.keys() + task_products = self.tasks.keys() + # Unwrap Exactly's if they only contain one type. + # If we don't do this, then the wrapped and unwrapped products both end up in the graph. + # Heterogeneity of constraint types makes this tough. + task_products = set(t._types[0] if type(t) is Exactly and len(t._types) == 1 else t for t in task_products) + + return intrinsic_products.union(task_products).union(set(singleton_products)) def gen_rules(self, subject_type, product_type): - # Singeltons or intrinsics that provide the requested product for the current subject type. + # Singletons or intrinsics that provide the requested product for the current subject type. singleton_node_factory = self.singletons.get(product_type) intrinsic_node_factory = self.intrinsics.get((subject_type, product_type)) if singleton_node_factory: @@ -219,10 +234,6 @@ def gen_rules(self, subject_type, product_type): for node_factory in self._lookup_tasks(product_type): yield node_factory - def gen_nodes(self, subject, product_type, variants): - for rule in self.gen_rules(type(subject), product_type): - yield rule.as_node(subject, variants) - def _lookup_tasks(self, product_type): for entry in self.tasks.get(product_type, tuple()): yield entry @@ -239,7 +250,8 @@ class CanBeDependency(object): class RuleGraph(datatype('RuleGraph', - ['root_subject_types', + ['graph_maker', + 'root_subject_types', 'root_rules', 'rule_dependencies', 'unfulfillable_rules'])): @@ -252,30 +264,50 @@ class RuleGraph(datatype('RuleGraph', Because in `root_subject_types` the root subject types this graph was generated with. - `root_rules` The rule entries that can produce the root products this graph was generated - with. + `root_rules` A map from root rules, ie rules representing the expected selector / subject types + for requests, to the rules that can fulfill them. `rule_dependencies` A map from rule entries to the rule entries they depend on. The collections of dependencies are contained by RuleEdges objects. Keys must be subclasses of CanHaveDependencies values must be subclasses of CanBeDependency `unfulfillable_rules` A map of rule entries to collections of Diagnostics - containing the reasons why they were eliminated from the graph. + containing the reasons why they were eliminated from the graph. """ - # TODO constructing nodes from the resulting graph. - # Possible approach: - # - walk out from root nodes, constructing each node. - # - when hit a node that can't be constructed yet, ie the subject type changes, - # skip and collect for later. - # - inject the constructed nodes into the product graph. + + def dependency_edges_for_rule(self, rule, subject_type): + if type(rule) is RootRuleGraphEntry: + return self.root_rule_edges(RootRuleGraphEntry(rule.subject_type, rule.selector)) + else: + return self.rule_dependencies.get(RuleGraphEntry(subject_type, rule)) + + def is_unfulfillable(self, rule, subject_type): + return RuleGraphEntry(subject_type, rule) in self.unfulfillable_rules + + def root_rule_matching(self, subject_type, selector): + root_rule = RootRuleGraphEntry(subject_type, selector) + if root_rule in self.root_rules: + return root_rule + + def new_graph_with_root_for(self, subject_type, selector): + return self.graph_maker.new_graph_from_existing(subject_type, selector, self) + + def root_rule_edges(self, root_rule): + try: + return self.root_rules[root_rule] + except KeyError: + logger.error('missing root rule {}'.format(root_rule)) + raise def error_message(self): """Returns a nice error message for errors in the rule graph.""" collated_errors = defaultdict(lambda : defaultdict(set)) for rule_entry, diagnostics in self.unfulfillable_rules.items(): # don't include the root rules in the error - # message since they aren't real. - if type(rule_entry) is RootRule: + # message since they are not part of the task list. + # We could include them, but I think we'd want to have a different format, since they + # represent the execution requests. + if type(rule_entry) is RootRuleGraphEntry: continue for diagnostic in diagnostics: collated_errors[rule_entry.rule][diagnostic.reason].add(diagnostic.subject_type) @@ -292,14 +324,14 @@ def subject_type_str(t): def format_messages(rule, subject_types_by_reasons): errors = '\n '.join(sorted('{} with subject types: {}' - .format(reason, ', '.join(sorted(subject_type_str(t) for t in subject_types))) - for reason, subject_types in subject_types_by_reasons.items())) + .format(reason, ', '.join(sorted(subject_type_str(t) for t in subject_types))) + for reason, subject_types in subject_types_by_reasons.items())) return '{}:\n {}'.format(rule, errors) used_rule_lookup = set(rule_entry.rule for rule_entry in self.rule_dependencies.keys()) formatted_messages = sorted(format_messages(rule, subject_types_by_reasons) - for rule, subject_types_by_reasons in collated_errors.items() - if rule not in used_rule_lookup) + for rule, subject_types_by_reasons in collated_errors.items() + if rule not in used_rule_lookup) if not formatted_messages: return None return 'Rules with errors: {}\n {}'.format(len(formatted_messages), @@ -310,20 +342,21 @@ def __str__(self): return '{empty graph}' root_subject_types_str = ', '.join(x.__name__ for x in self.root_subject_types) - root_rules_str = ', '.join(sorted(str(r) for r in self.root_rules)) return dedent(""" {{ root_subject_types: ({},) - root_rules: {} + root_rules: + {} + all_rules: {} }}""".format(root_subject_types_str, - root_rules_str, - '\n '.join(self._dependency_strs()) - )).strip() + '\n '.join(self._dependency_strs(self.root_rules)), + '\n '.join(self._dependency_strs(self.rule_dependencies)) + )).strip() - def _dependency_strs(self): + def _dependency_strs(self, dependencies): return sorted('{} => ({},)'.format(rule, ', '.join(str(d) for d in deps)) - for rule, deps in self.rule_dependencies.items()) + for rule, deps in dependencies.items()) class RuleGraphSubjectIsProduct(datatype('RuleGraphSubjectIsProduct', ['value']), CanBeDependency): @@ -374,7 +407,7 @@ def __str__(self): return '{} of {}'.format(self.rule, self.subject_type.__name__) -class RootRule(datatype('RootRule', ['subject_type', 'selector']), CanHaveDependencies): +class RootRuleGraphEntry(datatype('RootRule', ['subject_type', 'selector']), CanHaveDependencies): """A synthetic rule representing a root selector.""" @property @@ -452,65 +485,93 @@ def without_rule(self, dep_to_eliminate): class GraphMaker(object): - def __init__(self, nodebuilder, root_subject_fns): + def __init__(self, rule_index, root_subject_fns): + if not root_subject_fns: + raise ValueError('root_subject_fns must not be empty') self.root_subject_selector_fns = root_subject_fns - self.nodebuilder = nodebuilder + self.rule_index = rule_index + + def new_graph_from_existing(self, root_subject_type, root_selector, existing_graph): + root_rule = RootRuleGraphEntry(root_subject_type, root_selector) + root_rule_dependency_edges, edges, unfulfillable = self._construct_graph(root_rule, + root_rule_dependency_edges=existing_graph.root_rules, + rule_dependency_edges=existing_graph.rule_dependencies, + unfulfillable_rules=existing_graph.unfulfillable_rules + ) + root_rule_dependency_edges, edges = self._remove_unfulfillable_rules_and_dependents(root_rule_dependency_edges, + edges, unfulfillable) + return RuleGraph(self, + self.root_subject_selector_fns.keys() + [root_subject_type,], + root_rule_dependency_edges, + edges, + unfulfillable) def generate_subgraph(self, root_subject, requested_product): root_subject_type = type(root_subject) root_selector = self.root_subject_selector_fns[root_subject_type](requested_product) - root_rules, edges, unfulfillable = self._construct_graph(RootRule(root_subject_type, root_selector)) - root_rules, edges = self._remove_unfulfillable_rules_and_dependents(root_rules, - edges, unfulfillable) - return RuleGraph((root_subject_type,), root_rules, edges, unfulfillable) + root_rule = RootRuleGraphEntry(root_subject_type, root_selector) + root_rule_dependency_edges, edges, unfulfillable = self._construct_graph(root_rule) + root_rule_dependency_edges, edges = self._remove_unfulfillable_rules_and_dependents(root_rule_dependency_edges, + edges, unfulfillable) + return RuleGraph(self, + (root_subject_type,), + root_rule_dependency_edges, + edges, + unfulfillable) def full_graph(self): """Produces a full graph based on the root subjects and all of the products produced by rules.""" - full_root_rules = set() + full_root_rule_dependency_edges = dict() full_dependency_edges = {} full_unfulfillable_rules = {} for root_subject_type, selector_fn in self.root_subject_selector_fns.items(): - for product in sorted(self.nodebuilder.all_produced_product_types(root_subject_type)): - beginning_root = RootRule(root_subject_type, selector_fn(product)) + for product in sorted(self.rule_index.all_produced_product_types(root_subject_type)): + beginning_root = RootRuleGraphEntry(root_subject_type, selector_fn(product)) root_dependencies, rule_dependency_edges, unfulfillable_rules = self._construct_graph( beginning_root, - root_rules=full_root_rules, + root_rule_dependency_edges=full_root_rule_dependency_edges, rule_dependency_edges=full_dependency_edges, unfulfillable_rules=full_unfulfillable_rules ) - full_root_rules = set(root_dependencies) + full_root_rule_dependency_edges = dict(root_dependencies) full_dependency_edges = rule_dependency_edges full_unfulfillable_rules = unfulfillable_rules rules_in_graph = set(entry.rule for entry in full_dependency_edges.keys()) - rules_eliminated_during_construction = [entry.rule for entry in full_unfulfillable_rules.keys()] - rules_used = set(rules_eliminated_during_construction + self.nodebuilder.intrinsics.values() + self.nodebuilder.singletons.values()) - - declared_rules = self.nodebuilder.all_rules() - unreachable_rules = declared_rules.difference(rules_in_graph, rules_used) + rules_eliminated_during_construction = set(entry.rule + for entry in full_unfulfillable_rules.keys()) + + declared_rules = self.rule_index.all_rules() + unreachable_rules = declared_rules.difference(rules_in_graph, + rules_eliminated_during_construction, + # NB Singletons and intrinsics are ignored for + # purposes of reachability. + self.rule_index.singletons.values(), + self.rule_index.intrinsics.values()) for rule in sorted(unreachable_rules): full_unfulfillable_rules[UnreachableRule(rule)] = [Diagnostic(None, 'Unreachable')] - full_root_rules, full_dependency_edges = self._remove_unfulfillable_rules_and_dependents( - full_root_rules, + full_root_rule_dependency_edges, full_dependency_edges = self._remove_unfulfillable_rules_and_dependents( + full_root_rule_dependency_edges, full_dependency_edges, full_unfulfillable_rules) - return RuleGraph(self.root_subject_selector_fns, - list(full_root_rules), - full_dependency_edges, - full_unfulfillable_rules) + return RuleGraph(self, + self.root_subject_selector_fns, + dict(full_root_rule_dependency_edges), + full_dependency_edges, + full_unfulfillable_rules) def _construct_graph(self, beginning_rule, - root_rules=None, + root_rule_dependency_edges=None, rule_dependency_edges=None, unfulfillable_rules=None): - root_rules = set() if root_rules is None else root_rules + rules_to_traverse = deque([beginning_rule]) + root_rule_dependency_edges = dict() if root_rule_dependency_edges is None else root_rule_dependency_edges rule_dependency_edges = dict() if rule_dependency_edges is None else rule_dependency_edges unfulfillable_rules = dict() if unfulfillable_rules is None else unfulfillable_rules - rules_to_traverse = deque([beginning_rule]) def _find_rhs_for_select(subject_type, selector): if selector.type_constraint.satisfied_by_type(subject_type): @@ -518,7 +579,7 @@ def _find_rhs_for_select(subject_type, selector): return (RuleGraphSubjectIsProduct(subject_type),) else: return tuple(RuleGraphEntry(subject_type, rule) - for rule in self.nodebuilder.gen_rules(subject_type, selector.product)) + for rule in self.rule_index.gen_rules(subject_type, selector.product)) def mark_unfulfillable(rule, subject_type, reason): if rule not in unfulfillable_rules: @@ -527,11 +588,17 @@ def mark_unfulfillable(rule, subject_type, reason): def add_rules_to_graph(rule, selector_path, dep_rules): unseen_dep_rules = [g for g in dep_rules - if g not in rule_dependency_edges and g not in unfulfillable_rules] + if g not in rule_dependency_edges and + g not in unfulfillable_rules and + g not in root_rule_dependency_edges] rules_to_traverse.extend(unseen_dep_rules) - if type(rule) is RootRule: - root_rules.update(dep_rules) - return + if type(rule) is RootRuleGraphEntry: + if rule in root_rule_dependency_edges: + root_rule_dependency_edges[rule].add_edges_via(selector_path, dep_rules) + else: + new_edges = RuleEdges() + new_edges.add_edges_via(selector_path, dep_rules) + root_rule_dependency_edges[rule] = new_edges elif rule not in rule_dependency_edges: new_edges = RuleEdges() new_edges.add_edges_via(selector_path, dep_rules) @@ -573,7 +640,7 @@ def add_rules_to_graph(rule, selector_path, dep_rules): selector, (RuleGraphLiteral(selector.subject, selector.product),)) elif type(selector) is SelectDependencies: - initial_selector = selector.dep_product_selector + initial_selector = selector.input_product_selector initial_rules_or_literals = _find_rhs_for_select(entry.subject_type, initial_selector) if not initial_rules_or_literals: mark_unfulfillable(entry, @@ -598,7 +665,7 @@ def add_rules_to_graph(rule, selector_path, dep_rules): continue add_rules_to_graph(entry, - (selector, selector.dep_product_selector), + (selector, selector.input_product_selector), initial_rules_or_literals) add_rules_to_graph(entry, (selector, selector.projected_product_selector), @@ -611,7 +678,7 @@ def add_rules_to_graph(rule, selector_path, dep_rules): mark_unfulfillable(entry, entry.subject_type, 'no matches for {} when resolving {}' - .format(selector.input_product_selector, selector)) + .format(selector.input_product_selector, selector)) was_unfulfillable = True continue @@ -633,14 +700,51 @@ def add_rules_to_graph(rule, selector_path, dep_rules): projected_rules) else: raise TypeError('Unexpected type of selector: {}'.format(selector)) - if not was_unfulfillable and entry not in rule_dependency_edges: + + if type(entry.rule) is SnapshottedProcess: + # TODO, this is a copy of the SelectDependencies with some changes + # Need to come up with a better approach here, but this fixes things + # It's also not tested explicitly. + snapshot_selector = entry.rule.snapshot_selector + initial_selector = entry.rule.snapshot_selector.input_product_selector + initial_rules_or_literals = _find_rhs_for_select(SnapshottedProcessRequest, initial_selector) + if not initial_rules_or_literals: + mark_unfulfillable(entry, + entry.subject_type, + 'no matches for {} when resolving {}' + .format(initial_selector, snapshot_selector)) + was_unfulfillable = True + else: + + rules_for_dependencies = [] + for field_type in snapshot_selector.field_types: + rules_for_field_subjects = _find_rhs_for_select(field_type, + snapshot_selector.projected_product_selector) + rules_for_dependencies.extend(rules_for_field_subjects) + + if not rules_for_dependencies: + mark_unfulfillable(entry, + snapshot_selector.field_types, + 'no matches for {} when resolving {}' + .format(snapshot_selector.projected_product_selector, snapshot_selector)) + was_unfulfillable = True + else: + add_rules_to_graph(entry, + (snapshot_selector, snapshot_selector.input_product_selector), + initial_rules_or_literals) + add_rules_to_graph(entry, + (snapshot_selector, snapshot_selector.projected_product_selector), + tuple(rules_for_dependencies)) + + + if not was_unfulfillable: # NB: In this case, there are no selectors. add_rules_to_graph(entry, None, tuple()) - return root_rules, rule_dependency_edges, unfulfillable_rules + return root_rule_dependency_edges, rule_dependency_edges, unfulfillable_rules def _remove_unfulfillable_rules_and_dependents(self, - root_rules, + root_rule_dependency_edges, rule_dependency_edges, unfulfillable_rules): """Removes all unfulfillable rules transitively from the roots and the dependency edges. @@ -661,12 +765,30 @@ def _remove_unfulfillable_rules_and_dependents(self, if dependency_edges.makes_unfulfillable(unfulfillable_entry): unfulfillable_rules[current_entry] = [Diagnostic(current_entry.subject_type, - 'depends on unfulfillable {}'.format(unfulfillable_entry))] + 'depends on unfulfillable {}'.format(unfulfillable_entry))] removal_traversal.append(current_entry) else: rule_dependency_edges[current_entry] = dependency_edges.without_rule(unfulfillable_entry) - rule_dependency_edges = dict((k, v) for k, v in rule_dependency_edges.items() - if k not in unfulfillable_rules) - root_rules = tuple(r for r in root_rules if r not in unfulfillable_rules) - return root_rules, rule_dependency_edges + for current_entry, dependency_edges in tuple(root_rule_dependency_edges.items()): + if current_entry in unfulfillable_rules: + # NB: these are removed at the end + continue + + if dependency_edges.makes_unfulfillable(unfulfillable_entry): + unfulfillable_rules[current_entry] = [Diagnostic(current_entry.subject_type, + 'depends on unfulfillable {}'.format(unfulfillable_entry))] + removal_traversal.append(current_entry) + else: + root_rule_dependency_edges[current_entry] = dependency_edges.without_rule(unfulfillable_entry) + + rule_dependency_edges = {k: v for k, v in rule_dependency_edges.items() + if k not in unfulfillable_rules} + root_rule_dependency_edges = {k: v for k, v in root_rule_dependency_edges.items() + if k not in unfulfillable_rules} + + for root_rule, deps in root_rule_dependency_edges.items(): + for d in deps: + if d not in rule_dependency_edges and isinstance(d, RuleGraphEntry): + raise ValueError('expected all referenced dependencies to have entries in the graph: {}'.format(d)) + return root_rule_dependency_edges, rule_dependency_edges diff --git a/src/python/pants/engine/scheduler.py b/src/python/pants/engine/scheduler.py index 359387680a1..6c1008dd544 100644 --- a/src/python/pants/engine/scheduler.py +++ b/src/python/pants/engine/scheduler.py @@ -18,7 +18,7 @@ from pants.engine.fs import PathGlobs, create_fs_intrinsics, generate_fs_subjects from pants.engine.isolated_process import create_snapshot_intrinsics, create_snapshot_singletons from pants.engine.nodes import Return, Runnable, Throw -from pants.engine.rules import NodeBuilder, RulesetValidator +from pants.engine.rules import RuleIndex, RulesetValidator from pants.engine.selectors import (Select, SelectDependencies, SelectLiteral, SelectProjection, SelectVariant, constraint_for) from pants.engine.struct import HasProducts, Variants @@ -111,11 +111,11 @@ def __init__(self, } intrinsics = create_fs_intrinsics(project_tree) + create_snapshot_intrinsics(project_tree) singletons = create_snapshot_singletons(project_tree) - node_builder = NodeBuilder.create(tasks, intrinsics, singletons) - RulesetValidator(node_builder, goals, root_selector_fns).validate() - self._register_tasks(node_builder.tasks) - self._register_intrinsics(node_builder.intrinsics) - self._register_singletons(node_builder.singletons) + rule_index = RuleIndex.create(tasks, intrinsics, singletons) + RulesetValidator(rule_index, goals, root_selector_fns).validate() + self._register_tasks(rule_index.tasks) + self._register_intrinsics(rule_index.intrinsics) + self._register_singletons(rule_index.singletons) def _to_value(self, obj): return self._context.to_value(obj) diff --git a/src/python/pants/engine/selectors.py b/src/python/pants/engine/selectors.py index d7332877645..424fe3ddc9d 100644 --- a/src/python/pants/engine/selectors.py +++ b/src/python/pants/engine/selectors.py @@ -97,11 +97,15 @@ class SelectDependencies(datatype('Dependencies', `dep_product`. """ - def __new__(cls, product, dep_product, field='dependencies', field_types=tuple(), transitive=False): + DEFAULT_FIELD = 'dependencies' + + optional = False + + def __new__(cls, product, dep_product, field=DEFAULT_FIELD, field_types=tuple(), transitive=False): return super(SelectDependencies, cls).__new__(cls, product, dep_product, field, field_types, transitive) @property - def dep_product_selector(self): + def input_product_selector(self): return Select(self.dep_product) @property @@ -113,12 +117,17 @@ def __repr__(self): field_types_portion = ', field_types=({},)'.format(', '.join(f.__name__ for f in self.field_types)) else: field_types_portion = '' + if self.field is not self.DEFAULT_FIELD: + field_name_portion = ', {}'.format(repr(self.field)) + else: + field_name_portion = '' return '{}({}, {}{}{}{})'.format(type(self).__name__, type_or_constraint_repr(self.product), type_or_constraint_repr(self.dep_product), - ', {}'.format(repr(self.field)) if self.field else '', + field_name_portion, + field_types_portion, ', transitive=True' if self.transitive else '', - field_types_portion) + ) class SelectProjection(datatype('Projection', ['product', 'projected_subject', 'fields', 'input_product']), Selector): diff --git a/tests/python/pants_test/engine/test_rules.py b/tests/python/pants_test/engine/test_rules.py index cb2e6dfe557..676dd6a2099 100644 --- a/tests/python/pants_test/engine/test_rules.py +++ b/tests/python/pants_test/engine/test_rules.py @@ -16,7 +16,7 @@ from pants.engine.build_files import create_graph_tasks from pants.engine.fs import PathGlobs, create_fs_intrinsics, create_fs_tasks from pants.engine.mapper import AddressMapper -from pants.engine.rules import GraphMaker, NodeBuilder, Rule, RulesetValidator +from pants.engine.rules import GraphMaker, Rule, RuleIndex, RulesetValidator from pants.engine.selectors import Select, SelectDependencies, SelectLiteral, SelectProjection from pants_test.engine.examples.parsers import JsonParser from pants_test.engine.examples.planners import Goal @@ -92,26 +92,26 @@ def __repr__(self): return '{}({})'.format(type(self).__name__, self.output_product_type.__name__) -class NodeBuilderTest(unittest.TestCase): +class RuleIndexTest(unittest.TestCase): def test_creation_fails_with_bad_declaration_type(self): with self.assertRaises(TypeError) as cm: - NodeBuilder.create([A()], tuple()) + RuleIndex.create([A()], tuple()) self.assertEquals("Unexpected rule type: ." " Rules either extend Rule, or are 3 elem tuples.", str(cm.exception)) def test_creation_fails_with_intrinsic_that_overwrites_another_intrinsic(self): - a_provider = (A, A, noop) + a_intrinsic = (A, A, noop) with self.assertRaises(ValueError): - NodeBuilder.create([BoringRule(A)], (a_provider, a_provider)) + RuleIndex.create([BoringRule(A)], (a_intrinsic, a_intrinsic)) class RulesetValidatorTest(unittest.TestCase): def test_ruleset_with_missing_product_type(self): rules = [(A, (Select(B),), noop)] - validator = RulesetValidator(NodeBuilder.create(rules, tuple()), - goal_to_product={}, - root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) + validator = RulesetValidator(RuleIndex.create(rules, tuple()), + goal_to_product={}, + root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) with self.assertRaises(ValueError) as cm: validator.validate() @@ -124,7 +124,7 @@ def test_ruleset_with_missing_product_type(self): def test_ruleset_with_rule_with_two_missing_selects(self): rules = [(A, (Select(B), Select(C)), noop)] - validator = RulesetValidator(NodeBuilder.create(rules, tuple()), + validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) with self.assertRaises(ValueError) as cm: @@ -140,30 +140,18 @@ def test_ruleset_with_rule_with_two_missing_selects(self): def test_ruleset_with_selector_only_provided_as_root_subject(self): rules = [(A, (Select(B),), noop)] - validator = RulesetValidator(NodeBuilder.create(rules, tuple()), + validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (B,)}) validator.validate() - def test_fails_if_root_subject_types_empty(self): - rules = [ - (A, (Select(B),), noop), - ] - with self.assertRaises(ValueError) as cm: - RulesetValidator(NodeBuilder.create(rules, tuple()), - goal_to_product={}, - root_subject_fns={}) - self.assertEquals(dedent(""" - root_subject_fns must not be empty - """).strip(), str(cm.exception)) - def test_ruleset_with_superclass_of_selected_type_produced_fails(self): rules = [ (A, (Select(B),), noop), (B, (Select(SubA),), noop) ] - validator = RulesetValidator(NodeBuilder.create(rules, tuple()), + validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (C,)}) @@ -185,7 +173,7 @@ def test_ruleset_with_goal_not_produced(self): (B, (Select(SubA),), noop) ] - validator = RulesetValidator(NodeBuilder.create(rules, tuple()), + validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={'goal-name': AGoal}, root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) with self.assertRaises(ValueError) as cm: @@ -199,7 +187,7 @@ def test_ruleset_with_explicit_type_constraint(self): (Exactly(A), (Select(B),), noop), (B, (Select(A),), noop) ] - validator = RulesetValidator(NodeBuilder.create(rules, tuple()), + validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) @@ -212,13 +200,14 @@ def test_ruleset_with_failure_due_to_incompatible_subject_for_intrinsic(self): intrinsics = [ (B, C, noop), ] - validator = RulesetValidator(NodeBuilder.create(rules, intrinsics), + validator = RulesetValidator(RuleIndex.create(rules, intrinsics), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (A,)}) with self.assertRaises(ValueError) as cm: validator.validate() + # This error message could note near matches like the intrinsic. self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (D, (Select(C),), noop): @@ -233,7 +222,7 @@ def test_ruleset_unreachable_due_to_product_of_select_dependencies(self): intrinsics = [ (B, C, noop), ] - validator = RulesetValidator(NodeBuilder.create(rules, intrinsics), + validator = RulesetValidator(RuleIndex.create(rules, intrinsics), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (A,)}) @@ -242,7 +231,7 @@ def test_ruleset_unreachable_due_to_product_of_select_dependencies(self): self.assert_equal_with_printing(dedent(""" Rules with errors: 1 - (A, (SelectDependencies(B, SubA, u'dependencies', field_types=(D,)),), noop): + (A, (SelectDependencies(B, SubA, field_types=(D,)),), noop): Unreachable with subject types: Any """).strip(), str(cm.exception)) @@ -255,7 +244,7 @@ def test_not_fulfillable_duplicated_dependency(self): (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop), (A, (Select(SubA),), noop) ] - validator = RulesetValidator(NodeBuilder.create(rules, tuple()), + validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns=_suba_root_subject_fns) @@ -265,8 +254,8 @@ def test_not_fulfillable_duplicated_dependency(self): self.assert_equal_with_printing(dedent(""" Rules with errors: 2 (B, (Select(D),), noop): - depends on unfulfillable (D, (Select(A), SelectDependencies(A, SubA, u'dependencies', field_types=(C,))), noop) of SubA with subject types: SubA - (D, (Select(A), SelectDependencies(A, SubA, u'dependencies', field_types=(C,))), noop): + depends on unfulfillable (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop) of SubA with subject types: SubA + (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop): depends on unfulfillable (A, (Select(SubA),), noop) of C with subject types: SubA""").strip(), str(cm.exception)) @@ -274,7 +263,7 @@ def test_initial_select_projection_failure(self): rules = [ (Exactly(A), (SelectProjection(B, D, ('some',), C),), noop), ] - validator = RulesetValidator(NodeBuilder.create(rules, tuple()), + validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns=_suba_root_subject_fns) @@ -294,7 +283,7 @@ def test_secondary_select_projection_failure(self): (C, tuple(), noop) ] - validator = RulesetValidator(NodeBuilder.create(rules, tuple()), + validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns=_suba_root_subject_fns) @@ -317,19 +306,30 @@ class RuleGraphMakerTest(unittest.TestCase): # TODO something with variants # TODO HasProducts? + def test_fails_if_root_subject_types_empty(self): + rules = [ + (A, (Select(B),), noop), + ] + with self.assertRaises(ValueError) as cm: + GraphMaker(RuleIndex.create(rules), tuple()) + self.assertEquals(dedent(""" + root_subject_fns must not be empty + """).strip(), str(cm.exception)) + def test_smallest_full_test(self): rules = [ (Exactly(A), (Select(SubA),), noop) ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), - root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) - fullgraph = graphmaker.full_graph() + fullgraph = GraphMaker(RuleIndex.create(rules, tuple()), + root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}).full_graph() self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (Select(SubA),), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (Select(SubA),), noop) of SubA,) + all_rules: (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) }""").strip(), fullgraph) @@ -339,7 +339,7 @@ def test_full_graph_for_planner_example(self): tasks = create_graph_tasks(address_mapper, symbol_table_cls) + create_fs_tasks() intrinsics = create_fs_intrinsics('Let us pretend that this is a ProjectTree!') - rule_index = NodeBuilder.create(tasks, intrinsics) + rule_index = RuleIndex.create(tasks, intrinsics) graphmaker = GraphMaker(rule_index, root_subject_fns={k: lambda p: Select(p) for k in (Address, # TODO, use the actual fns. PathGlobs, @@ -364,23 +364,31 @@ def test_full_graph_for_planner_example(self): rules_remaining_in_graph_strs ) + # statically assert that the number of dependency keys is fixed + self.assertEquals(41, len(fullgraph.rule_dependencies)) + def test_smallest_full_test_multiple_root_subject_types(self): rules = [ - (Exactly(A), (Select(SubA),), noop), - (Exactly(B), (Select(A),), noop) + (A, (Select(SubA),), noop), + (B, (Select(A),), noop) ] select_p = lambda p: Select(p) - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=OrderedDict([(SubA, select_p), (A, select_p)])) fullgraph = graphmaker.full_graph() self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA, A,) - root_rules: (Exactly(A), (Select(SubA),), noop) of SubA, (Exactly(B), (Select(A),), noop) of A, (Exactly(B), (Select(A),), noop) of SubA, SubjectIsProduct(A) - (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) - (Exactly(B), (Select(A),), noop) of A => (SubjectIsProduct(A),) - (Exactly(B), (Select(A),), noop) of SubA => ((Exactly(A), (Select(SubA),), noop) of SubA,) + root_rules: + Select(A) for A => (SubjectIsProduct(A),) + Select(A) for SubA => ((A, (Select(SubA),), noop) of SubA,) + Select(B) for A => ((B, (Select(A),), noop) of A,) + Select(B) for SubA => ((B, (Select(A),), noop) of SubA,) + all_rules: + (A, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) + (B, (Select(A),), noop) of A => (SubjectIsProduct(A),) + (B, (Select(A),), noop) of SubA => ((A, (Select(SubA),), noop) of SubA,) }""").strip(), fullgraph) @@ -389,14 +397,16 @@ def test_single_rule_depending_on_subject_selection(self): (Exactly(A), (Select(SubA),), noop) ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (Select(SubA),), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (Select(SubA),), noop) of SubA,) + all_rules: (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) }""").strip(), subgraph) @@ -406,14 +416,16 @@ def test_multiple_selects(self): (B, tuple(), noop) ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (Select(SubA), Select(B)), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (Select(SubA), Select(B)), noop) of SubA,) + all_rules: (B, (), noop) of SubA => (,) (Exactly(A), (Select(SubA), Select(B)), noop) of SubA => (SubjectIsProduct(SubA), (B, (), noop) of SubA,) }""").strip(), @@ -425,14 +437,16 @@ def test_one_level_of_recursion(self): (B, (Select(SubA),), noop) ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (Select(B),), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (Select(B),), noop) of SubA,) + all_rules: (B, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) (Exactly(A), (Select(B),), noop) of SubA => ((B, (Select(SubA),), noop) of SubA,) }""").strip(), subgraph) @@ -447,15 +461,17 @@ def test_noop_removal_in_subgraph(self): (B, C, noop), ] - graphmaker = GraphMaker(NodeBuilder.create(rules, - intrinsics), + graphmaker = GraphMaker(RuleIndex.create(rules, + intrinsics), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (), noop) of SubA,) + all_rules: (Exactly(A), (), noop) of SubA => (,) }""").strip(), subgraph) @@ -466,17 +482,19 @@ def test_noop_removal_full_single_subject_type(self): (Exactly(A), tuple(), noop), ] intrinsics = [ - (B, C, BoringRule(C)), + (B, C, noop), ] - graphmaker = GraphMaker(NodeBuilder.create(rules, intrinsics), + graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns) fullgraph = graphmaker.full_graph() self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (), noop) of SubA,) + all_rules: (Exactly(A), (), noop) of SubA => (,) }""").strip(), fullgraph) @@ -491,7 +509,7 @@ def test_noop_removal_transitive(self): intrinsics = [ (D, C, BoringRule(C)) ] - graphmaker = GraphMaker(NodeBuilder.create(rules, intrinsics), + graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns, ) @@ -500,7 +518,9 @@ def test_noop_removal_transitive(self): self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (), noop) of SubA,) + all_rules: (Exactly(A), (), noop) of SubA => (,) }""").strip(), subgraph) @@ -511,17 +531,19 @@ def test_select_dependencies_with_separate_types_for_subselectors(self): (C, (Select(SubA),), noop) ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (SelectDependencies(B, C, u'dependencies', field_types=(D,)),), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, C, field_types=(D,)),), noop) of SubA,) + all_rules: (B, (Select(D),), noop) of D => (SubjectIsProduct(D),) (C, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) - (Exactly(A), (SelectDependencies(B, C, u'dependencies', field_types=(D,)),), noop) of SubA => ((C, (Select(SubA),), noop) of SubA, (B, (Select(D),), noop) of D,) + (Exactly(A), (SelectDependencies(B, C, field_types=(D,)),), noop) of SubA => ((C, (Select(SubA),), noop) of SubA, (B, (Select(D),), noop) of D,) }""").strip(), subgraph) @@ -531,16 +553,18 @@ def test_select_dependencies_with_subject_as_first_subselector(self): (B, (Select(D),), noop), ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(D,)),), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop) of SubA,) + all_rules: (B, (Select(D),), noop) of D => (SubjectIsProduct(D),) - (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(D),), noop) of D,) + (Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(D),), noop) of D,) }""").strip(), subgraph) @@ -550,17 +574,19 @@ def test_select_dependencies_multiple_field_types_all_resolvable(self): (B, (Select(Exactly(C, D)),), noop), ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA,) + all_rules: (B, (Select(Exactly(C, D)),), noop) of C => (SubjectIsProduct(C),) (B, (Select(Exactly(C, D)),), noop) of D => (SubjectIsProduct(D),) - (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(Exactly(C, D)),), noop) of C, (B, (Select(Exactly(C, D)),), noop) of D,) + (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(Exactly(C, D)),), noop) of C, (B, (Select(Exactly(C, D)),), noop) of D,) }""").strip(), subgraph) @@ -572,18 +598,20 @@ def test_select_dependencies_multiple_field_types_all_resolvable_with_deps(self) (C, (Select(D),), noop), ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA,) + all_rules: (B, (Select(C),), noop) of C => (SubjectIsProduct(C),) (B, (Select(C),), noop) of D => ((C, (Select(D),), noop) of D,) (C, (Select(D),), noop) of D => (SubjectIsProduct(D),) - (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(C),), noop) of C, (B, (Select(C),), noop) of D,) + (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(C),), noop) of C, (B, (Select(C),), noop) of D,) }""").strip(), subgraph) @@ -595,19 +623,21 @@ def test_select_dependencies_recurse_with_different_type(self): (SubA, tuple(), noop) ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of SubA - (B, (Select(A),), noop) of C => ((Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of C,) - (B, (Select(A),), noop) of D => ((Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of D,) - (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of C => ((SubA, (), noop) of C, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) - (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of D => ((SubA, (), noop) of D, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) - (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) + root_rules: + Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA,) + all_rules: + (B, (Select(A),), noop) of C => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C,) + (B, (Select(A),), noop) of D => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D,) + (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C => ((SubA, (), noop) of C, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) + (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D => ((SubA, (), noop) of D, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) + (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) (SubA, (), noop) of C => (,) (SubA, (), noop) of D => (,) }""").strip(), @@ -620,15 +650,15 @@ def test_select_dependencies_non_matching_subselector_because_of_intrinsic(self) intrinsics = [ (C, B, noop), ] - graphmaker = GraphMaker(NodeBuilder.create(rules, intrinsics), + graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing('{empty graph}', subgraph) self.assert_equal_with_printing(dedent(""" Rules with errors: 1 - (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(D,)),), noop): - no matches for Select(B) when resolving SelectDependencies(B, SubA, u'dependencies', field_types=(D,)) with subject types: D""").strip(), + (Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop): + no matches for Select(B) when resolving SelectDependencies(B, SubA, field_types=(D,)) with subject types: D""").strip(), subgraph.error_message()) def test_select_dependencies_with_matching_intrinsic(self): @@ -639,16 +669,18 @@ def test_select_dependencies_with_matching_intrinsic(self): (B, C, noop), ] - graphmaker = GraphMaker(NodeBuilder.create(rules, intrinsics), + graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C,)),), noop) of SubA - (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C,)),), noop) of SubA => (SubjectIsProduct(SubA), IntrinsicRule(noop) of C,) - IntrinsicRule(noop) of C => (,) + root_rules: + Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA,) + all_rules: + (Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA => (SubjectIsProduct(SubA), IntrinsicRule((C, B), noop) of C,) + IntrinsicRule((C, B), noop) of C => (,) }""").strip(), subgraph) @@ -659,14 +691,16 @@ def test_depends_on_multiple_one_noop(self): (A, (Select(SubA),), noop) ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=B) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (B, (Select(A),), noop) of SubA + root_rules: + Select(B) for SubA => ((B, (Select(A),), noop) of SubA,) + all_rules: (A, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) (B, (Select(A),), noop) of SubA => ((A, (Select(SubA),), noop) of SubA,) }""").strip(), subgraph) @@ -678,14 +712,18 @@ def test_multiple_depend_on_same_rule(self): (A, (Select(SubA),), noop) ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.full_graph() self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (A, (Select(SubA),), noop) of SubA, (B, (Select(A),), noop) of SubA, (C, (Select(A),), noop) of SubA + root_rules: + Select(A) for SubA => ((A, (Select(SubA),), noop) of SubA,) + Select(B) for SubA => ((B, (Select(A),), noop) of SubA,) + Select(C) for SubA => ((C, (Select(A),), noop) of SubA,) + all_rules: (A, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) (B, (Select(A),), noop) of SubA => ((A, (Select(SubA),), noop) of SubA,) (C, (Select(A),), noop) of SubA => ((A, (Select(SubA),), noop) of SubA,) @@ -697,14 +735,16 @@ def test_select_literal(self): (B, (SelectLiteral(literally_a, A),), noop) ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=B) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (B, (SelectLiteral(A(), A),), noop) of SubA + root_rules: + Select(B) for SubA => ((B, (SelectLiteral(A(), A),), noop) of SubA,) + all_rules: (B, (SelectLiteral(A(), A),), noop) of SubA => (Literal(A(), A),) }""").strip(), subgraph) @@ -714,14 +754,16 @@ def test_select_projection_simple(self): (B, (Select(D),), noop), ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (Exactly(A), (SelectProjection(B, D, (u'some',), SubA),), noop) of SubA + root_rules: + Select(A) for SubA => ((Exactly(A), (SelectProjection(B, D, (u'some',), SubA),), noop) of SubA,) + all_rules: (B, (Select(D),), noop) of D => (SubjectIsProduct(D),) (Exactly(A), (SelectProjection(B, D, (u'some',), SubA),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(D),), noop) of D,) }""").strip(), @@ -734,16 +776,18 @@ def test_successful_when_one_field_type_is_unfulfillable(self): (D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C))), noop) ] - graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), + graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=D) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) - root_rules: (D, (Select(Exactly(B)), SelectDependencies(B, SubA, u'dependencies', field_types=(SubA, C,))), noop) of SubA + root_rules: + Select(D) for SubA => ((D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA,) + all_rules: (B, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) - (D, (Select(Exactly(B)), SelectDependencies(B, SubA, u'dependencies', field_types=(SubA, C,))), noop) of SubA => ((B, (Select(SubA),), noop) of SubA, SubjectIsProduct(SubA), (B, (Select(SubA),), noop) of SubA,) + (D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA => ((B, (Select(SubA),), noop) of SubA, SubjectIsProduct(SubA), (B, (Select(SubA),), noop) of SubA,) }""").strip(), subgraph) diff --git a/tests/python/pants_test/engine/test_selectors.py b/tests/python/pants_test/engine/test_selectors.py index 9b0d7f80620..fd448718f75 100644 --- a/tests/python/pants_test/engine/test_selectors.py +++ b/tests/python/pants_test/engine/test_selectors.py @@ -24,11 +24,13 @@ def test_variant_repr(self): self.assert_repr("SelectVariant(AClass, u'field')", SelectVariant(AClass, 'field')) def test_dependencies_repr(self): - self.assert_repr("SelectDependencies(AClass, AClass, u'dependencies')", SelectDependencies(AClass, AClass)) + self.assert_repr("SelectDependencies(AClass, AClass)", SelectDependencies(AClass, AClass)) self.assert_repr("SelectDependencies(AClass, AClass, u'some_field')", SelectDependencies(AClass, AClass, field='some_field')) self.assert_repr("SelectDependencies(AClass, AClass, u'some_field', field_types=(AClass,))", SelectDependencies(AClass, AClass, field='some_field', field_types=(AClass,))) + self.assert_repr("SelectDependencies(AClass, AClass, transitive=True)", + SelectDependencies(AClass, AClass, transitive=True)) def test_projection_repr(self): self.assert_repr("SelectProjection(AClass, AClass, (u'field',), AClass)",