From d0a701dc914ddde500bba3fe61bcbe98b395c529 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sun, 30 Aug 2015 14:27:23 -0600 Subject: [PATCH] Initial support for `resolve.npm`. This is pants implementing `npm install` for node projects modeled in pants. The driving concern is primary control by pants of Node.js packages via BUILD files, and this necessitates generating package.json files under `.pants.d/` to describe dependencies (and later more) to `npm` so it can resolve both local and remote dependencies. A second fallout is copying pants controlled node module sourcecode up into `.pants.d/` since `node` has a documented behavior of resolving against `realpath`[1], foiling any attempt to use symlinks. The basic targets needed to support `resolve.npm` - aka `npm install` are added along with basic tests for the functionality. [1] https://nodejs.org/api/modules.html#modules_addenda_package_manager_tips Testing Done: I manually confirmed that with the NpmResolve execution of `npm dedupe` commented out, the dedupe portion of the `NpmResolveTest.test_resolve_simple_graph` test failed like so: ```console E AssertionError: Expected to find exactly 1 de-duped `typ` package, but found these: E node_modules/typ/package.json E node_modules/util/node_modules/typ/package.json ``` CI went green here: https://travis-ci.org/pantsbuild/pants/builds/77852199 Bugs closed: 2073, 2087 Reviewed at https://rbcommons.com/s/twitter/r/2723/ --- contrib/node/BUILD | 3 + contrib/node/examples/3rdparty/node/BUILD | 0 .../node/src/python/pants/contrib/node/BUILD | 18 +++ .../src/python/pants/contrib/node/register.py | 26 ++++ .../python/pants/contrib/node/targets/BUILD | 39 +++++ .../pants/contrib/node/targets/__init__.py | 0 .../pants/contrib/node/targets/node_module.py | 41 +++++ .../node/targets/node_remote_module.py | 35 +++++ .../pants/contrib/node/targets/npm_package.py | 33 +++++ .../src/python/pants/contrib/node/tasks/BUILD | 35 +++++ .../pants/contrib/node/tasks/__init__.py | 0 .../pants/contrib/node/tasks/node_task.py | 83 +++++++++++ .../pants/contrib/node/tasks/npm_resolve.py | 140 ++++++++++++++++++ .../pants_test/contrib/node/targets/BUILD | 28 ++++ .../contrib/node/targets/__init__.py | 0 .../node/targets/test_node_remote_module.py | 31 ++++ .../contrib/node/targets/test_npm_package.py | 26 ++++ .../pants_test/contrib/node/tasks/BUILD | 36 +++++ .../pants_test/contrib/node/tasks/__init__.py | 0 .../contrib/node/tasks/test_node_task.py | 79 ++++++++++ .../contrib/node/tasks/test_npm_resolve.py | 122 +++++++++++++++ pants.ini | 1 + 22 files changed, 776 insertions(+) create mode 100644 contrib/node/examples/3rdparty/node/BUILD create mode 100644 contrib/node/src/python/pants/contrib/node/BUILD create mode 100644 contrib/node/src/python/pants/contrib/node/register.py create mode 100644 contrib/node/src/python/pants/contrib/node/targets/BUILD create mode 100644 contrib/node/src/python/pants/contrib/node/targets/__init__.py create mode 100644 contrib/node/src/python/pants/contrib/node/targets/node_module.py create mode 100644 contrib/node/src/python/pants/contrib/node/targets/node_remote_module.py create mode 100644 contrib/node/src/python/pants/contrib/node/targets/npm_package.py create mode 100644 contrib/node/src/python/pants/contrib/node/tasks/BUILD create mode 100644 contrib/node/src/python/pants/contrib/node/tasks/__init__.py create mode 100644 contrib/node/src/python/pants/contrib/node/tasks/node_task.py create mode 100644 contrib/node/src/python/pants/contrib/node/tasks/npm_resolve.py create mode 100644 contrib/node/tests/python/pants_test/contrib/node/targets/BUILD create mode 100644 contrib/node/tests/python/pants_test/contrib/node/targets/__init__.py create mode 100644 contrib/node/tests/python/pants_test/contrib/node/targets/test_node_remote_module.py create mode 100644 contrib/node/tests/python/pants_test/contrib/node/targets/test_npm_package.py create mode 100644 contrib/node/tests/python/pants_test/contrib/node/tasks/BUILD create mode 100644 contrib/node/tests/python/pants_test/contrib/node/tasks/__init__.py create mode 100644 contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_task.py create mode 100644 contrib/node/tests/python/pants_test/contrib/node/tasks/test_npm_resolve.py diff --git a/contrib/node/BUILD b/contrib/node/BUILD index 936d1ca1595..276e269a256 100644 --- a/contrib/node/BUILD +++ b/contrib/node/BUILD @@ -3,3 +3,6 @@ source_root('src/python', python_library) source_root('tests/python', python_library, python_tests) + +source_root('examples/3rdparty/node', node_remote_module) +source_root('examples/src/node', node_module) diff --git a/contrib/node/examples/3rdparty/node/BUILD b/contrib/node/examples/3rdparty/node/BUILD new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/node/src/python/pants/contrib/node/BUILD b/contrib/node/src/python/pants/contrib/node/BUILD new file mode 100644 index 00000000000..983bfa7a8ed --- /dev/null +++ b/contrib/node/src/python/pants/contrib/node/BUILD @@ -0,0 +1,18 @@ +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# TODO(John Sirois): Tie this into the contrib/release_packages.sh the minute the plugin can do +# one useful thing, for example `./pants test contrib/node/examples::`. +contrib_plugin( + name='plugin', + dependencies=[ + 'contrib/node/src/python/pants/contrib/node/targets', + 'contrib/node/src/python/pants/contrib/node/tasks', + 'src/python/pants/base:build_file_aliases', + 'src/python/pants/goal:task_registrar', + ], + distribution_name='pantsbuild.pants.contrib.node', + description='Node.js support for pants.', + build_file_aliases=True, + register_goals=True, +) diff --git a/contrib/node/src/python/pants/contrib/node/register.py b/contrib/node/src/python/pants/contrib/node/register.py new file mode 100644 index 00000000000..a19ba56aec7 --- /dev/null +++ b/contrib/node/src/python/pants/contrib/node/register.py @@ -0,0 +1,26 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.base.build_file_aliases import BuildFileAliases +from pants.goal.task_registrar import TaskRegistrar as task + +from pants.contrib.node.targets.node_module import NodeModule +from pants.contrib.node.targets.node_remote_module import NodeRemoteModule +from pants.contrib.node.tasks.npm_resolve import NpmResolve + + +def build_file_aliases(): + return BuildFileAliases.create( + targets={ + 'node_module': NodeModule, + 'node_remote_module': NodeRemoteModule, + }, + ) + + +def register_goals(): + task(name='npm', action=NpmResolve).install('resolve') diff --git a/contrib/node/src/python/pants/contrib/node/targets/BUILD b/contrib/node/src/python/pants/contrib/node/targets/BUILD new file mode 100644 index 00000000000..02e79c96f96 --- /dev/null +++ b/contrib/node/src/python/pants/contrib/node/targets/BUILD @@ -0,0 +1,39 @@ +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +target( + name='targets', + dependencies=[ + ':node_module', + ':node_remote_module', + ] +) + +python_library( + name='node_module', + sources=['node_module.py'], + dependencies=[ + ':npm_package', + 'src/python/pants/base:payload', + ], +) + +python_library( + name='node_remote_module', + sources=['node_remote_module.py'], + dependencies=[ + ':npm_package', + 'src/python/pants/base:payload', + 'src/python/pants/base:payload_field', + ], +) + +python_library( + name='npm_package', + sources=['npm_package.py'], + dependencies=[ + 'src/python/pants/base:payload', + 'src/python/pants/base:payload_field', + 'src/python/pants/base:target', + ] +) \ No newline at end of file diff --git a/contrib/node/src/python/pants/contrib/node/targets/__init__.py b/contrib/node/src/python/pants/contrib/node/targets/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/node/src/python/pants/contrib/node/targets/node_module.py b/contrib/node/src/python/pants/contrib/node/targets/node_module.py new file mode 100644 index 00000000000..cfeff6eae27 --- /dev/null +++ b/contrib/node/src/python/pants/contrib/node/targets/node_module.py @@ -0,0 +1,41 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.base.payload import Payload + +from pants.contrib.node.targets.npm_package import NpmPackage + + +class NodeModule(NpmPackage): + """Represents a Node module.""" + + def __init__(self, + sources=None, + sources_rel_path=None, + address=None, + payload=None, + **kwargs): + """ + :param sources: Javascript and other source code files that make up this module; paths are + relative to the BUILD file's directory. + :type sources: `globs` , `rglobs` or a list of strings + """ + # TODO(John Sirois): Support devDependencies, etc. The devDependencies case is not + # clear-cut since pants controlled builds would provide devDependencies as needed to perform + # tasks. The reality is likely to be though that both pants will never cover all cases, and a + # back door to execute new tools during development will be desirable and supporting conversion + # of pre-existing package.json files as node_module targets will require this. + + if sources_rel_path is None: + sources_rel_path = address.spec_path + payload = payload or Payload() + payload.add_fields({ + 'sources': self.create_sources_field(sources=sources, + sources_rel_path=sources_rel_path, + key_arg='sources'), + }) + super(NodeModule, self).__init__(address=address, payload=payload, **kwargs) diff --git a/contrib/node/src/python/pants/contrib/node/targets/node_remote_module.py b/contrib/node/src/python/pants/contrib/node/targets/node_remote_module.py new file mode 100644 index 00000000000..1cbd596b5a0 --- /dev/null +++ b/contrib/node/src/python/pants/contrib/node/targets/node_remote_module.py @@ -0,0 +1,35 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.base.payload import Payload +from pants.base.payload_field import PrimitiveField + +from pants.contrib.node.targets.npm_package import NpmPackage + + +class NodeRemoteModule(NpmPackage): + """Represents a remote Node module.""" + + def __init__(self, version=None, address=None, payload=None, **kwargs): + """ + :param string version: The version constraint for the remote node module. Any of the forms + accepted by npm including '' or '*' for unconstrained (the default) are + acceptable. See: https://docs.npmjs.com/files/package.json#dependencies + """ + payload = payload or Payload() + payload.add_fields({ + 'version': PrimitiveField(version or '*'), # Guard against/allow `None`. + }) + super(NodeRemoteModule, self).__init__(address=address, payload=payload, **kwargs) + + @property + def version(self): + """The version constraint of the remote package. + + :rtype: string + """ + return self.payload.version diff --git a/contrib/node/src/python/pants/contrib/node/targets/npm_package.py b/contrib/node/src/python/pants/contrib/node/targets/npm_package.py new file mode 100644 index 00000000000..871ed00409b --- /dev/null +++ b/contrib/node/src/python/pants/contrib/node/targets/npm_package.py @@ -0,0 +1,33 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.base.payload import Payload +from pants.base.payload_field import PrimitiveField +from pants.base.target import Target + + +class NpmPackage(Target): + """Represents an NPM package.""" + + def __init__(self, package_name=None, address=None, payload=None, **kwargs): + """ + :param string package_name: The remote module package name, if not supplied the target name is + used. + """ + payload = payload or Payload() + payload.add_fields({ + 'package_name': PrimitiveField(package_name or address.target_name), + }) + super(NpmPackage, self).__init__(address=address, payload=payload, **kwargs) + + @property + def package_name(self): + """The name of the remote module package. + + :rtype: string + """ + return self.payload.package_name diff --git a/contrib/node/src/python/pants/contrib/node/tasks/BUILD b/contrib/node/src/python/pants/contrib/node/tasks/BUILD new file mode 100644 index 00000000000..9c39ea6ba1c --- /dev/null +++ b/contrib/node/src/python/pants/contrib/node/tasks/BUILD @@ -0,0 +1,35 @@ +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +target( + name='tasks', + dependencies=[ + ':npm_resolve', + ] +) + +python_library( + name='node_task', + sources=['node_task.py'], + dependencies=[ + 'contrib/node/src/python/pants/contrib/node/subsystems:node_distribution', + 'contrib/node/src/python/pants/contrib/node/targets:node_module', + 'contrib/node/src/python/pants/contrib/node/targets:node_remote_module', + 'src/python/pants/backend/core/tasks:task', + 'src/python/pants/base:workunit', + 'src/python/pants/util:memo', + ] +) + +python_library( + name='npm_resolve', + sources=['npm_resolve.py'], + dependencies=[ + ':node_task', + 'src/python/pants/base:build_environment', + 'src/python/pants/base:exceptions', + 'src/python/pants/base:workunit', + 'src/python/pants/util:contextutil', + 'src/python/pants/util:dirutil', + ] +) diff --git a/contrib/node/src/python/pants/contrib/node/tasks/__init__.py b/contrib/node/src/python/pants/contrib/node/tasks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/node/src/python/pants/contrib/node/tasks/node_task.py b/contrib/node/src/python/pants/contrib/node/tasks/node_task.py new file mode 100644 index 00000000000..24285bde701 --- /dev/null +++ b/contrib/node/src/python/pants/contrib/node/tasks/node_task.py @@ -0,0 +1,83 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.backend.core.tasks.task import Task +from pants.base.workunit import WorkUnit, WorkUnitLabel +from pants.util.memo import memoized_property + +from pants.contrib.node.subsystems.node_distribution import NodeDistribution +from pants.contrib.node.targets.node_module import NodeModule +from pants.contrib.node.targets.node_remote_module import NodeRemoteModule + + +class NodeTask(Task): + + @classmethod + def subsystem_dependencies(cls): + return (NodeDistribution.Factory,) + + @memoized_property + def node_distribution(self): + """A bootstrapped node distribution for use by node tasks.""" + return NodeDistribution.Factory.global_instance().create() + + @classmethod + def is_node_module(cls, target): + """Returns `True` if the given target is a `NodeModule`.""" + return isinstance(target, NodeModule) + + @classmethod + def is_node_remote_module(cls, target): + """Returns `True` if the given target is a `NodeRemoteModule`.""" + return isinstance(target, NodeRemoteModule) + + def execute_node(self, args, workunit_name=None, workunit_labels=None, **kwargs): + """Executes node passing the given args. + + :param list args: The command line args to pass to `node`. + :param string workunit_name: A name for the execution's work unit; defaults to 'node'. + :param list workunit_labels: Any extra :class:`pants.base.workunit.WorkUnitLabel`s to apply. + :param **kwargs: Any extra args to pass to :class:`subprocess.Popen`. + :returns: A tuple of (returncode, command). + :rtype: A tuple of (int, + :class:`pants.contrib.node.subsystems.node_distribution.NodeDistribution.Command`) + """ + npm_command = self.node_distribution.node_command(args=args) + return self._execute_command(npm_command, + workunit_name=workunit_name, + workunit_labels=workunit_labels, + **kwargs) + + def execute_npm(self, args, workunit_name=None, workunit_labels=None, **kwargs): + """Executes npm passing the given args. + + :param list args: The command line args to pass to `npm`. + :param string workunit_name: A name for the execution's work unit; defaults to 'npm'. + :param list workunit_labels: Any extra :class:`pants.base.workunit.WorkUnitLabel`s to apply. + :param **kwargs: Any extra args to pass to :class:`subprocess.Popen`. + :returns: A tuple of (returncode, command). + :rtype: A tuple of (int, + :class:`pants.contrib.node.subsystems.node_distribution.NodeDistribution.Command`) + """ + + npm_command = self.node_distribution.npm_command(args=args) + return self._execute_command(npm_command, + workunit_name=workunit_name, + workunit_labels=workunit_labels, + **kwargs) + + def _execute_command(self, command, workunit_name=None, workunit_labels=None, **kwargs): + workunit_name = workunit_name or command.executable + workunit_labels = {WorkUnitLabel.TOOL} | set(workunit_labels or ()) + with self.context.new_workunit(name=workunit_name, + labels=workunit_labels, + cmd=str(command)) as workunit: + process = command.run(stdout=workunit.output('stdout'), stderr=workunit.output('stderr'), + **kwargs) + returncode = process.wait() + workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE) + return returncode, command diff --git a/contrib/node/src/python/pants/contrib/node/tasks/npm_resolve.py b/contrib/node/src/python/pants/contrib/node/tasks/npm_resolve.py new file mode 100644 index 00000000000..cfd1a8f99eb --- /dev/null +++ b/contrib/node/src/python/pants/contrib/node/tasks/npm_resolve.py @@ -0,0 +1,140 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import errno +import json +import os +import shutil + +from pants.base.build_environment import get_buildroot +from pants.base.exceptions import TaskError +from pants.base.workunit import WorkUnitLabel +from pants.util.contextutil import pushd +from pants.util.dirutil import safe_mkdir + +from pants.contrib.node.tasks.node_task import NodeTask + + +def _safe_copy(source, dest): + safe_mkdir(os.path.dirname(dest)) + try: + os.link(source, dest) + except OSError as e: + if e.errno == errno.EXDEV: + # We can't hard link across devices, fall back on copying + shutil.copyfile(source, dest) + else: + raise + + +def _copy_sources(buildroot, node_module, dest_dir): + source_relative_to = node_module.address.spec_path + for source in node_module.sources_relative_to_buildroot(): + dest = os.path.join(dest_dir, os.path.relpath(source, source_relative_to)) + _safe_copy(os.path.join(buildroot, source), dest) + + +class NpmResolve(NodeTask): + """Resolves node modules to isolated chroots. + + See: see `npm install `_ + """ + + # TODO(John Sirois): UnionProducts? That seems broken though for ranged version constraints, + # which npm has and are widely used in the community. For now stay dumb simple (and slow) and + # resolve each node_module individually. + class NodePaths(object): + """Maps NpmPackage targets to their resolved NODE_PATH chroot.""" + + def __init__(self): + self._paths_by_target = {} + + def resolved(self, target, node_path): + """Identifies the given target as resolved to the given chroot path. + + :param target: The target that was resolved to the `node_path` chroot. + :type target: :class:`pants.contrib.node.targets.npm_package.NpmPackage` + :param string node_path: The chroot path the given `target` was resolved to. + """ + self._paths_by_target[target] = node_path + + def node_path(self, target): + """Returns the path of the resolved chroot for the given NpmPackage. + + Returns `None` if the target has not been resolved to a chroot. + + :rtype string + """ + return self._paths_by_target.get(target) + + @classmethod + def product_types(cls): + return [cls.NodePaths] + + @property + def cache_target_dirs(self): + return True + + def execute(self): + # TODO(John Sirois): Is there a way to avoid a naive re-resolve for each target, ie bulk + # resolve and then post-resolve analyze the results locally to create a separate NODE_PATH + # for each target participating in the bulk resolve? This is unlikely since versions are often + # unconstrained or partially constrained in the npm community. + # See NodePaths TODO above. + targets = set(self.context.targets(predicate=self.is_node_module)) + if not targets: + return + + node_paths = self.context.products.get_data(self.NodePaths, init_func=self.NodePaths) + + # We must have linked local sources for internal dependencies before installing dependees; so, + # `topological_order=True` is critical. + with self.invalidated(targets, + topological_order=True, + invalidate_dependents=True) as invalidation_check: + + with self.context.new_workunit(name='install', labels=[WorkUnitLabel.MULTITOOL]): + for vt in invalidation_check.all_vts: + target = vt.target + node_path = vt.results_dir + if not vt.valid: + safe_mkdir(node_path, clean=True) + self._resolve(target, node_path, node_paths) + node_paths.resolved(target, node_path) + + def _resolve(self, target, node_path, node_paths): + _copy_sources(buildroot=get_buildroot(), node_module=target, dest_dir=node_path) + self._emit_package_descriptor(target, node_path, node_paths) + + with pushd(node_path): + # TODO(John Sirois): Handle dev dependency resolution. + result, npm_install = self.execute_npm(args=['install'], + workunit_name=target.address.reference()) + if result != 0: + raise TaskError('Failed to resolve dependencies for {}:\n\t{} failed with exit code {}' + .format(target.address.reference(), npm_install, result)) + + # TODO(John Sirois): This will be part of install in npm 3.x, detect or control the npm version. + # we use and only conditionally execute this. + result, npm_dedupe = self.execute_npm(args=['dedupe'], + workunit_name=target.address.reference()) + if result != 0: + raise TaskError('Failed to dedupe dependencies for {}:\n\t{} failed with exit code {}' + .format(target.address.reference(), npm_dedupe, result)) + + def _emit_package_descriptor(self, npm_package, node_path, node_paths): + def render_dep(target): + return node_paths.node_path(target) if self.is_node_module(target) else target.version + dependencies = {dep.package_name: render_dep(dep) for dep in npm_package.dependencies} + + package = { + 'name': npm_package.package_name, + 'version': '0.0.0', + 'dependencies': dependencies + } + with open(os.path.join(node_path, 'package.json'), 'wb') as fp: + json.dump(package, fp, indent=2) diff --git a/contrib/node/tests/python/pants_test/contrib/node/targets/BUILD b/contrib/node/tests/python/pants_test/contrib/node/targets/BUILD new file mode 100644 index 00000000000..9a93c10b54a --- /dev/null +++ b/contrib/node/tests/python/pants_test/contrib/node/targets/BUILD @@ -0,0 +1,28 @@ +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +target( + name='targets', + dependencies=[ + ':node_remote_module', + ':npm_package', + ] +) + +python_tests( + name='node_remote_module', + sources=['test_node_remote_module.py'], + dependencies=[ + 'contrib/node/src/python/pants/contrib/node/targets:node_remote_module', + 'tests/python/pants_test:base_test' + ] +) + +python_tests( + name='npm_package', + sources=['test_npm_package.py'], + dependencies=[ + 'contrib/node/src/python/pants/contrib/node/targets:npm_package', + 'tests/python/pants_test:base_test' + ] +) \ No newline at end of file diff --git a/contrib/node/tests/python/pants_test/contrib/node/targets/__init__.py b/contrib/node/tests/python/pants_test/contrib/node/targets/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/node/tests/python/pants_test/contrib/node/targets/test_node_remote_module.py b/contrib/node/tests/python/pants_test/contrib/node/targets/test_node_remote_module.py new file mode 100644 index 00000000000..6fd39d4f32d --- /dev/null +++ b/contrib/node/tests/python/pants_test/contrib/node/targets/test_node_remote_module.py @@ -0,0 +1,31 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants_test.base_test import BaseTest + +from pants.contrib.node.targets.node_remote_module import NodeRemoteModule + + +class NodeRemoteModuleTest(BaseTest): + def test_unconstrained(self): + target1 = self.make_target(spec=':unconstrained1', target_type=NodeRemoteModule) + target2 = self.make_target(spec=':unconstrained2', target_type=NodeRemoteModule, version=None) + target3 = self.make_target(spec=':unconstrained3', target_type=NodeRemoteModule, version='') + target4 = self.make_target(spec=':unconstrained4', target_type=NodeRemoteModule, version='*') + + self.assertEqual('*', target1.version) + self.assertEqual('*', target2.version) + self.assertEqual('*', target3.version) + self.assertEqual('*', target4.version) + + def test_constrained(self): + target1 = self.make_target(spec=':unconstrained1', + target_type=NodeRemoteModule, + package_name='asdf', + version='http://asdf.com/asdf.tar.gz#2.0.0') + self.assertEqual('asdf', target1.package_name) + self.assertEqual('http://asdf.com/asdf.tar.gz#2.0.0', target1.version) diff --git a/contrib/node/tests/python/pants_test/contrib/node/targets/test_npm_package.py b/contrib/node/tests/python/pants_test/contrib/node/targets/test_npm_package.py new file mode 100644 index 00000000000..ad16115a9fe --- /dev/null +++ b/contrib/node/tests/python/pants_test/contrib/node/targets/test_npm_package.py @@ -0,0 +1,26 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants_test.base_test import BaseTest + +from pants.contrib.node.targets.npm_package import NpmPackage + + +class NpmPackageTest(BaseTest): + def test_implicit_package_name(self): + target = self.make_target(spec=':name', target_type=NpmPackage) + self.assertEqual('name', target.address.target_name) + self.assertEqual('name', target.package_name) + + def test_explicit_package_name(self): + target1 = self.make_target(spec=':name', target_type=NpmPackage) + target2 = self.make_target(spec=':name2', target_type=NpmPackage, package_name='name') + self.assertNotEqual(target1, target2) + self.assertEqual('name', target1.address.target_name) + self.assertEqual('name', target1.package_name) + self.assertEqual('name2', target2.address.target_name) + self.assertEqual('name', target2.package_name) diff --git a/contrib/node/tests/python/pants_test/contrib/node/tasks/BUILD b/contrib/node/tests/python/pants_test/contrib/node/tasks/BUILD new file mode 100644 index 00000000000..db7c9ee1378 --- /dev/null +++ b/contrib/node/tests/python/pants_test/contrib/node/tasks/BUILD @@ -0,0 +1,36 @@ +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +target( + name='tasks', + dependencies=[ + ':node_task', + ':npm_resolve', + ] +) + +python_tests( + name='node_task', + sources=['test_node_task.py'], + dependencies=[ + 'contrib/node/src/python/pants/contrib/node/targets:node_module', + 'contrib/node/src/python/pants/contrib/node/targets:node_remote_module', + 'contrib/node/src/python/pants/contrib/node/tasks:node_task', + 'src/python/pants/base:target', + 'src/python/pants/util:contextutil', + 'tests/python/pants_test/tasks:task_test_base', + ] +) + +python_tests( + name='npm_resolve', + sources=['test_npm_resolve.py'], + dependencies=[ + 'contrib/node/src/python/pants/contrib/node/targets:node_module', + 'contrib/node/src/python/pants/contrib/node/targets:node_remote_module', + 'contrib/node/src/python/pants/contrib/node/tasks:npm_resolve', + 'src/python/pants/base:source_root', + 'src/python/pants/base:target', + 'tests/python/pants_test/tasks:task_test_base', + ] +) \ No newline at end of file diff --git a/contrib/node/tests/python/pants_test/contrib/node/tasks/__init__.py b/contrib/node/tests/python/pants_test/contrib/node/tasks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_task.py b/contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_task.py new file mode 100644 index 00000000000..05919cc8279 --- /dev/null +++ b/contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_task.py @@ -0,0 +1,79 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import json +import os +from textwrap import dedent + +from pants.base.target import Target +from pants.util.contextutil import temporary_dir +from pants_test.tasks.task_test_base import TaskTestBase + +from pants.contrib.node.targets.node_module import NodeModule +from pants.contrib.node.targets.node_remote_module import NodeRemoteModule +from pants.contrib.node.tasks.node_task import NodeTask + + +class NodeTaskTest(TaskTestBase): + + class TestNodeTask(NodeTask): + def execute(self): + # We never execute the task, we just want to exercise the helpers it provides subclasses. + raise NotImplementedError() + + @classmethod + def task_type(cls): + return cls.TestNodeTask + + def test_is_node_module(self): + self.assertTrue(NodeTask.is_node_module(self.make_target(':a', NodeModule))) + self.assertFalse(NodeTask.is_node_module(self.make_target(':b', NodeRemoteModule))) + self.assertFalse(NodeTask.is_node_module(self.make_target(':c', Target))) + + def test_is_node_remote_module(self): + self.assertTrue(NodeTask.is_node_remote_module(self.make_target(':a', NodeRemoteModule))) + self.assertFalse(NodeTask.is_node_remote_module(self.make_target(':b', NodeModule))) + self.assertFalse(NodeTask.is_node_remote_module(self.make_target(':c', Target))) + + def test_execute_node(self): + task = self.create_task(self.context()) + with temporary_dir() as chroot: + script = os.path.join(chroot, 'test.js') + proof = os.path.join(chroot, 'path') + with open(script, 'w') as fp: + fp.write(dedent(""" + var fs = require('fs'); + fs.writeFile("{proof}", "Hello World!", function(err) {{}}); + """).format(proof=proof)) + self.assertFalse(os.path.exists(proof)) + returncode, command = task.execute_node(args=[script]) + + self.assertEqual(0, returncode) + self.assertTrue(os.path.exists(proof)) + with open(proof) as fp: + self.assertEqual('Hello World!', fp.read().strip()) + + def test_execute_npm(self): + task = self.create_task(self.context()) + with temporary_dir() as chroot: + proof = os.path.join(chroot, 'proof') + self.assertFalse(os.path.exists(proof)) + package = { + 'name': 'pantsbuild.pants.test', + 'version': '0.0.0', + 'scripts': { + 'proof': 'echo "42" > {}'.format(proof) + } + } + with open(os.path.join(chroot, 'package.json'), 'wb') as fp: + json.dump(package, fp) + returncode, command = task.execute_npm(args=['run-script', 'proof'], cwd=chroot) + + self.assertEqual(0, returncode) + self.assertTrue(os.path.exists(proof)) + with open(proof) as fp: + self.assertEqual('42', fp.read().strip()) diff --git a/contrib/node/tests/python/pants_test/contrib/node/tasks/test_npm_resolve.py b/contrib/node/tests/python/pants_test/contrib/node/tasks/test_npm_resolve.py new file mode 100644 index 00000000000..b7a6fb98458 --- /dev/null +++ b/contrib/node/tests/python/pants_test/contrib/node/tasks/test_npm_resolve.py @@ -0,0 +1,122 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import json +import os +from textwrap import dedent + +from pants.base.source_root import SourceRoot +from pants.base.target import Target +from pants_test.tasks.task_test_base import TaskTestBase + +from pants.contrib.node.targets.node_module import NodeModule +from pants.contrib.node.targets.node_remote_module import NodeRemoteModule +from pants.contrib.node.tasks.npm_resolve import NpmResolve + + +class NpmResolveTest(TaskTestBase): + + @classmethod + def task_type(cls): + return NpmResolve + + def test_noop(self): + task = self.create_task(self.context()) + task.execute() + + def test_noop_na(self): + target = self.make_target(spec=':not_a_node_target', target_type=Target) + task = self.create_task(self.context(target_roots=[target])) + task.execute() + + def test_resolve_simple(self): + SourceRoot.register('3rdparty/node', NodeRemoteModule) + typ = self.make_target(spec='3rdparty/node:typ', target_type=NodeRemoteModule, version='0.6.3') + + SourceRoot.register('src/node', NodeModule) + self.create_file('src/node/util/util.js', contents=dedent(""" + var typ = require('typ'); + console.log("type of boolean is: " + typ.BOOLEAN); + """)) + target = self.make_target(spec='src/node/util', + target_type=NodeModule, + sources=['util.js'], + dependencies=[typ]) + + context = self.context(target_roots=[target]) + task = self.create_task(context) + task.execute() + + node_paths = context.products.get_data(NpmResolve.NodePaths) + node_path = node_paths.node_path(target) + self.assertIsNotNone(node_path) + + script_path = os.path.join(node_path, 'util.js') + out = task.node_distribution.node_command(args=[script_path]).check_output() + self.assertIn('type of boolean is: boolean', out) + + def test_resolve_simple_graph(self): + SourceRoot.register('3rdparty/node', NodeRemoteModule) + typ1 = self.make_target(spec='3rdparty/node:typ1', + target_type=NodeRemoteModule, + package_name='typ', + version='0.6.1') + typ2 = self.make_target(spec='3rdparty/node:typ2', + target_type=NodeRemoteModule, + package_name='typ', + version='0.6.x') + + SourceRoot.register('src/node', NodeModule) + self.create_file('src/node/util/typ.js', contents=dedent(""" + var typ = require('typ'); + module.exports = { + BOOL: typ.BOOLEAN + }; + """)) + util = self.make_target(spec='src/node/util', + target_type=NodeModule, + sources=['typ.js'], + dependencies=[typ1]) + + self.create_file('src/node/leaf/leaf.js', contents=dedent(""" + var typ = require('typ'); + var util_typ = require('util/typ'); + console.log("type of boolean is: " + typ.BOOLEAN); + console.log("type of bool is: " + util_typ.BOOL); + """)) + leaf = self.make_target(spec='src/node/leaf', + target_type=NodeModule, + sources=['leaf.js'], + dependencies=[util, typ2]) + context = self.context(target_roots=[leaf]) + task = self.create_task(context) + task.execute() + + node_paths = context.products.get_data(NpmResolve.NodePaths) + self.assertIsNotNone(node_paths.node_path(util)) + + node_path = node_paths.node_path(leaf) + self.assertIsNotNone(node_paths.node_path(leaf)) + + # Verify dependencies are de-duped + typ_packages = [] + for root, _, files in os.walk(node_path): + for f in files: + if 'package.json' == f: + with open(os.path.join(root, f)) as fp: + package = json.load(fp) + if 'typ' == package['name']: + typ_packages.append(os.path.relpath(os.path.join(root, f), node_path)) + self.assertEqual(1, len(typ_packages), + 'Expected to find exactly 1 de-duped `typ` package, but found these:\n\t{}' + .format('\n\t'.join(sorted(typ_packages)))) + + script_path = os.path.join(node_path, 'leaf.js') + out = task.node_distribution.node_command(args=[script_path]).check_output() + lines = {line.strip() for line in out.splitlines()} + self.assertIn('type of boolean is: boolean', lines) + self.assertIn('type of bool is: boolean', lines) diff --git a/pants.ini b/pants.ini index 4ec22780e13..305ef92e819 100644 --- a/pants.ini +++ b/pants.ini @@ -29,6 +29,7 @@ backend_packages: [ "pants.backend.android", "pants.contrib.cpp", "pants.contrib.go", + "pants.contrib.node", "pants.contrib.scrooge", "pants.contrib.spindle", ]