Skip to content

Commit

Permalink
Improve the performance of v2 changed. (pantsbuild#5571)
Browse files Browse the repository at this point in the history
this is good for a ~40x speedup in my test case:

before:

[omerta pants (kwlzn/faster_v2_changed)]$ time ./pants --changed-parent=HEAD~60 list > a

real    4m26.282s
user    2m59.511s
sys     1m25.936s
after:

[omerta pants (kwlzn/faster_v2_changed)]$ time ./pants --changed-parent=HEAD~60 list > b

real    0m7.668s
user    0m6.251s
sys     0m1.425s

Fixes pantsbuild#5570
  • Loading branch information
kwlzn authored Mar 7, 2018
1 parent 7bf7564 commit eddc698
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 11 deletions.
1 change: 1 addition & 0 deletions src/python/pants/engine/legacy/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ python_library(
'src/python/pants/base:specs',
'src/python/pants/build_graph',
'src/python/pants/source',
'3rdparty/python:six'
]
)

Expand Down
20 changes: 12 additions & 8 deletions src/python/pants/engine/legacy/source_mapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

import os

import six

from pants.base.specs import AscendantAddresses, SingleAddress
from pants.build_graph.address import parse_spec
from pants.build_graph.source_mapper import SourceMapper
Expand Down Expand Up @@ -75,13 +77,15 @@ def iter_target_addresses_for_sources(self, sources):
sources_set = set(sources)
subjects = [AscendantAddresses(directory=d) for d in self._unique_dirs_for_sources(sources_set)]

# Uniqify all transitive hydrated targets.
hydrated_target_to_address = {}
for hydrated_targets in self._scheduler.product_request(HydratedTargets, subjects):
for hydrated_target in hydrated_targets.dependencies:
legacy_address = hydrated_target.adaptor.address

# Handle BUILD files.
if any(LegacyAddressMapper.is_declaring_file(legacy_address, f) for f in sources_set):
yield legacy_address
else:
if any(self._owns_source(source, hydrated_target) for source in sources_set):
yield legacy_address
if hydrated_target not in hydrated_target_to_address:
hydrated_target_to_address[hydrated_target] = hydrated_target.adaptor.address

for hydrated_target, legacy_address in six.iteritems(hydrated_target_to_address):
# Handle BUILD files.
if (any(LegacyAddressMapper.is_declaring_file(legacy_address, f) for f in sources_set) or
any(self._owns_source(source, hydrated_target) for source in sources_set)):
yield legacy_address
6 changes: 3 additions & 3 deletions src/python/pants/source/wrapped_globs.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from twitter.common.dirutil.fileset import Fileset

from pants.base.build_environment import get_buildroot
from pants.util.dirutil import fast_relpath
from pants.util.dirutil import fast_relpath, fast_relpath_optional
from pants.util.memo import memoized_property
from pants.util.meta import AbstractClass

Expand Down Expand Up @@ -104,8 +104,8 @@ def __repr__(self):
return 'EagerFilesetWithSpec(rel_root={!r}, files={!r})'.format(self.rel_root, self.files)

def matches(self, path_from_buildroot):
path_relative_to_rel_root = os.path.relpath(path_from_buildroot, self.rel_root)
return path_relative_to_rel_root in self._files
path_relative_to_rel_root = fast_relpath_optional(path_from_buildroot, self.rel_root)
return path_relative_to_rel_root is not None and path_relative_to_rel_root in self._files


class LazyFilesetWithSpec(FilesetWithSpec):
Expand Down

0 comments on commit eddc698

Please sign in to comment.