Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Freshen up old code #9

Merged
merged 1 commit into from
Jun 19, 2019
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,15 +1,14 @@
dist: xenial
sudo: false
language: python
matrix:
include:
- python: 2.7
env: TOX_ENV=py27 DOCOV=false
- python: 3.5
env: TOX_ENV=py35 DOCOV=true
- python: 3.6
env: TOX_ENV=py36 DOCOV=false
- python: pypy
env: TOX_ENV=pypy DOCOV=false
- python: 3.7
env: TOX_ENV=py37 DOCOV=false
- python: 3.6
env: TOX_ENV=flake8 DOCOV=false

2 changes: 1 addition & 1 deletion LICENSE
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
MIT License

Copyright (c) 2018, John Bjorn Nelson
Copyright (c) 2019, John Bjorn Nelson

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
11 changes: 3 additions & 8 deletions README.rst
Original file line number Diff line number Diff line change
@@ -2,10 +2,13 @@
:target: https://pypi.python.org/pypi/modpipe
.. image:: https://img.shields.io/travis/jbn/modpipe.svg
:target: https://travis-ci.org/jbn/modpipe
.. image:: https://ci.appveyor.com/api/projects/status/21l6df8evjepq41s?svg=true
:target: https://ci.appveyor.com/project/jbn/modpipe
.. image:: https://coveralls.io/repos/github/jbn/modpipe/badge.svg?branch=master
:target: https://coveralls.io/github/jbn/modpipe?branch=master



=============================
modpipe: modules as pipelines
=============================
@@ -24,14 +27,6 @@ Installation

pip install modpipe

# For Python 2.7, also do,
pip install git+git://github.com/jbn/funcsigs.git@214840c53529f019638229d72dcf2257fe154458

Python 2.7 requires the `functools <https://github.com/aliles/funcsigs>`_
backport. But, the current master branch doesn't have support for pickling.
Until my `PR <https://github.com/aliles/funcsigs/pull/33>`_ gets
accepted, the pypi package won't work in spark environments. the above pip
command will install my branch.

-------------
Why is this?
5 changes: 2 additions & 3 deletions appveyor.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
environment:
matrix:
- PYTHON: "C:\\Python27"
TOX_ENV: "py27"

- PYTHON: "C:\\Python35"
TOX_ENV: "py35"

- PYTHON: "C:\\Python36"
TOX_ENV: "py36"

- PYTHON: "C:\\Python37"
TOX_ENV: "py37"

install:
- "%PYTHON%/Scripts/easy_install -U pip"
6 changes: 3 additions & 3 deletions modpipe/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from modpipe.modpipe_impl import ModPipe, Result, Done, SkipTo # noqa: F401
from modpipe.modpipe_impl import ModPipe # noqa: F401
from modpipe.results import Result, Done, SkipTo # noqa: F401


__author__ = """John Bjorn Nelson"""
__author__ = 'John Bjorn Nelson'
__email__ = '[email protected]'
__version__ = '0.0.4'
157 changes: 157 additions & 0 deletions modpipe/helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
import re
from collections import OrderedDict
from inspect import getmodule, getsourcelines, getsource, signature, Signature
from typing import Mapping, Callable
from types import ModuleType
from typing import Iterator, Tuple

BindingSeq = Iterator[Tuple[str, object]]


def is_simple_callable(obj) -> bool:
"""
:param obj: Any object
:return: true if the object is callable but class constructor.
"""
return callable(obj) and not isinstance(obj, type)


def iter_defined_in(module: ModuleType) -> BindingSeq:
"""
Iterate over all objects defined in a particular module.

This does not include builtins or anything it imports.

:param module: A loaded module
:type module: ModuleType
:return: A generator of the binding and the bound object for all objects
defined in a module.
"""
pairs = vars(module).items()
return (pair for pair in pairs if getmodule(pair[1]) == module)


def iter_pipelineable(pairs: BindingSeq) -> BindingSeq:
"""
Iterate over every pair that has an object that is a simple callable.

:param module: pairs of names to objects.
:type module: Iterator[Tuple[str, object]]
:return: A generator of the binding and the bound object for all objects
that are simple callables.
"""
return (pair for pair in pairs if is_simple_callable(pair[1]))


def is_pseudo_private(name, _) -> bool:
return name.startswith('_')


def remove_from_pipeline_seq(pipeline_seq: Mapping[str, object], predicate: Callable[[str, object], bool]):
"""

:param pipeline_seq: A Mapping (albeit ordered) of bindings to objects.
:type pipeline_seq: Mapping[str, object]
:param predicate: A callable that filters out entries that should not
be retained in a pipeline.
:type predicate: Callable[[str, object], bool])
:return: None
"""
remove_ks = [k for k, v in pipeline_seq.items() if predicate(k, v)]

for k in remove_ks:
del pipeline_seq[k]


def sequence_objects(module: ModuleType, items: BindingSeq) -> BindingSeq:
"""
:param module: The module to extract callable definitions from.
:type module: ModuleType
:param items: The callables and bindings for extraction.
:type items: Iterator[Tuple[str, object]]
:returns: a list of (name, callable) pairs sorted by source line number
"""
items, linenos, missing = list(items), {}, set()

# First try to get the line numbers via getsourcelines.
for k, obj in items:
try:
if obj.__module__ == module.__name__:
linenos[k] = getsourcelines(obj)[-1]
else:
missing.add(k) # Not in module
except TypeError:
missing.add(k) # i.e. class instance

# Now find any instantiated objects.
missing_re = re.compile("^({})".format("|".join(missing)))
for lineno, line in enumerate(getsource(module).splitlines()):
match = missing_re.match(line)
if match:
linenos[match.group(0)] = lineno

# Verify all objects found.
still_missing = missing - set(linenos)
if still_missing:
raise RuntimeError("Unable to resolve: {}".format(still_missing))

return sorted(items, key=lambda p: linenos[p[0]])


def sigs_equal_case_insensitive(a, b):
a_ = [p.kind for p in a.parameters.values()]
b_ = [p.kind for p in b.parameters.values()]
return a_ == b_


def sigs_equal_case_sensitive(a, b):
return a == b


# XXX: TODO: WRONG SIGNATURE!
def compile_signatures(pipeline_seq: BindingSeq, assert_unif=False, ignore_names=True) -> Mapping[str, Signature]:
"""
:param pipeline_seq: The sequence of name to object bindings.
:param assert_unif: if True raises a RuntimeError if all the functions
don't share the same signature.
:param ignore_names: if True, then the uniform assertion ignores name
differences
:return: a map from the callable to the Signature for each item
"""
signatures = [(f, signature(f)) for f in pipeline_seq.values()]

equal_sigs = sigs_equal_case_sensitive
if ignore_names:
equal_sigs = sigs_equal_case_insensitive

if assert_unif:
last = None
for f, sig in signatures:
# The order is important! It's easier to debug with the first
# inconsistency than an arbitrary one.
if last is not None and not equal_sigs(last, sig):
msg = "Signature for {} is {} which doesn't match {}"
raise RuntimeError(msg.format(f.__name__, sig, last))
last = sig

return dict(signatures)


def load_pipeline_seq(module: ModuleType, elide_helpers=True, *predicates) -> BindingSeq:
"""
:param module: The module to load from
:param elide_helpers: if True, remove all conventionally-designated
helper functions.
:param predicates: Predicates to remove_from_pipeline_seq
:return: an OrderedDict mapping name to callable.
"""
pairs = sequence_objects(module, iter_pipelineable(iter_defined_in(module)))
pipeline_seq = OrderedDict(pairs)

if elide_helpers:
remove_from_pipeline_seq(pipeline_seq, is_pseudo_private)

for predicate in predicates:
remove_from_pipeline_seq(pipeline_seq, predicate)

return pipeline_seq
208 changes: 32 additions & 176 deletions modpipe/modpipe_impl.py
Original file line number Diff line number Diff line change
@@ -1,139 +1,34 @@
import re
from importlib import import_module
from inspect import getmodule, getsourcelines, getsource, getfile
from collections import OrderedDict
from inspect import getfile
from importlib import reload
from types import ModuleType

try:
from importlib import reload
except ImportError:
try:
from imp import reload
except ImportError:
pass # Assuming 2.7


def _defined_in(module):
pairs = vars(module).items()
return (pair for pair in pairs if getmodule(pair[1]) == module)


try:
from inspect import signature

def _is_simple_callable(obj):
return callable(obj) and not isinstance(obj, type)
except ImportError:
from funcsigs import signature as _signature
from types import TypeType, ClassType, InstanceType

def _is_simple_callable(obj):
return callable(obj) and not isinstance(obj, (TypeType, ClassType))

def signature(f):
return _signature(f.__call__ if isinstance(f, InstanceType) else f)


def _pipelineable(pairs):
return (pair for pair in pairs if _is_simple_callable(pair[1]))


def _is_pseudo_private(name, obj):
return name.startswith('_')


def _remove_from_pipeline_seq(pipeline_seq, predicate):
"""
Remove items that match the predicate *in place*.
"""
remove_ks = [k for k, v in pipeline_seq.items() if predicate(k, v)]

for k in remove_ks:
del pipeline_seq[k]


def _sequence_objects(module, items):
"""
:returns: a list of (name, callable) pairs sorted by source line number
"""
items, linenos, missing = list(items), {}, set()

# First try to get the line numbers via getsourcelines.
for k, obj in items:
try:
linenos[k] = getsourcelines(obj)[-1]
except TypeError:
missing.add(k) # i.e. class instance

# Now find any instantiated objects.
missing_re = re.compile("^({})".format("|".join(missing)))
for lineno, line in enumerate(getsource(module).splitlines()):
match = missing_re.match(line)
if match:
linenos[match.group(0)] = lineno

# Verify all objects found.
still_missing = missing - set(linenos)
if still_missing:
raise RuntimeError("Unable to resolve: {}".format(still_missing))

return sorted(items, key=lambda p: linenos[p[0]])


def _compile_signatures(pipeline_seq, assert_unif=False, ignore_names=True):
"""
:param assert_unif: if True raises a runtime if all the functions
don't share the same signature.
:param ignore_names: if True, then the uniform assertion ignores name
differences
:return: a map from the callable to the Signature for each item
"""
signatures = [(f, signature(f)) for f in pipeline_seq.values()]

if ignore_names:
def equal_sigs(a, b):
a_ = [p.kind for p in a.parameters.values()]
b_ = [p.kind for p in b.parameters.values()]
return a_ == b_
else:
def equal_sigs(a, b):
return a == b

if assert_unif:
last = None
for f, sig in signatures:
# The order is important! It's easier to debug with the first
# inconsistency than an arbitrary one.
if last is not None and not equal_sigs(last, sig):
msg = "Signature for {} is {} which doesn't match {}"
raise RuntimeError(msg.format(f.__name__, sig, last))
last = sig

return dict(signatures)


def _load_pipeline_seq(module, elide_helpers=True):
"""
:param elide_helpers: if True, remove all conventionally-designated
helper functions.
:return: an OrderedDict mapping name to callable.
"""
pairs = _sequence_objects(module, _pipelineable(_defined_in(module)))
pipeline_seq = OrderedDict(pairs)

if elide_helpers:
_remove_from_pipeline_seq(pipeline_seq, _is_pseudo_private)

return pipeline_seq
from modpipe.results import Result, Done, SkipTo
from modpipe.helpers import compile_signatures, load_pipeline_seq


class ModPipe:

@classmethod
def on(cls, module_dot_path, unif_sigs=False, ignore_names=True):
"""
:param module: The module to turn into a pipe, as a fully
qualified name or a module instance.
:param unif_sigs: if True, then enforce the expectation that
every callable in the pipeline has the same signature
:param ignore_names: if True, then enforce the expectation that
every callable in the pipeline has the same signature,
including argument names.
:return: an instantiated ModPipe
"""
return ModPipe(module_dot_path, unif_sigs, ignore_names)

def __init__(self, module_dot_path, unif_sigs=False, ignore_names=True):
self._module_dot_path = module_dot_path
def __init__(self, module, unif_sigs=False, ignore_names=True):
if isinstance(module, ModuleType):
module = module.__name__

self._module_dot_path = module
self._unif_sigs = unif_sigs
self._ignore_names = ignore_names

@@ -148,17 +43,20 @@ def abs_module_path(self):
return self._module_path

def reload(self):
"""
Reloads the module and all pipeline elements.
"""
# Don't save a ref to module. It's not picklable.
module = reload(import_module(self._module_dot_path))
self._module_name = module.__name__
self._module_path = getfile(module)
self._pipeline = _load_pipeline_seq(module)
self._pipeline = load_pipeline_seq(module)

assert len(self._pipeline) > 0
assert len(self._pipeline) > 0, "No elements in pipeline."

self._signatures = _compile_signatures(self._pipeline,
self._unif_sigs,
self._ignore_names)
self._signatures = compile_signatures(self._pipeline,
self._unif_sigs,
self._ignore_names)
self._expected_args = {k: len(sig.parameters)
for k, sig in self._signatures.items()}

@@ -195,50 +93,8 @@ def __call__(self, *args):
break

if isinstance(res, SkipTo):
msg = "Pipeline ended before encountering {}"
raise RuntimeError(msg.format(res.target_f.__name__))
target_name = res.target_f.__name__
msg = "Pipeline ended before encountering {}".format(target_name)
raise RuntimeError(msg)

return res.args


class Result(object):

def __init__(self, *args):
if len(args) == 1 and isinstance(args, tuple):
self.args = args[0]
else:
self.args = args

def apply_to(self, f, arity):
res = None

if isinstance(self.args, tuple) and arity == len(self.args):
res = f(*self.args)
else:
res = f(self.args)

if isinstance(res, Result):
return res
elif res is not None:
return Result(res)
else:
return Result(self.args)


class Done(Result):

def apply_to(self, f, arity):
raise RuntimeError("Calling apply to on a completed result.")


class SkipTo(Result):

def __init__(self, target_f, *args):
super(SkipTo, self).__init__(*args)
self.target_f = target_f

def apply_to(self, f, arity):
if f != self.target_f:
return self
else:
return Result.apply_to(self, f, arity)
45 changes: 45 additions & 0 deletions modpipe/results.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
class Result:

def __init__(self, *args):
if len(args) == 1 and isinstance(args, tuple):
# Unpack a single element tuple.
self.args = args[0]
else:
self.args = args

def apply_to(self, f, arity):
res = None

if isinstance(self.args, tuple) and arity == len(self.args):
res = f(*self.args)
else:
res = f(self.args)

if isinstance(res, Result):
return res
elif res is not None:
return Result(res)
else:
return Result(self.args)

def __repr__(self):
return "{}({})".format(self.__class__.__name__, self.args)


class Done(Result):

def apply_to(self, f, arity):
raise RuntimeError("Calling apply to on a completed result.")


class SkipTo(Result):

def __init__(self, target_f, *args):
super(SkipTo, self).__init__(*args)
self.target_f = target_f

def apply_to(self, f, arity):
if f != self.target_f:
return self
else:
return Result.apply_to(self, f, arity)
18 changes: 5 additions & 13 deletions requirements_dev.txt
Original file line number Diff line number Diff line change
@@ -1,13 +1,5 @@
pip==9.0.1
bumpversion==0.5.3
wheel==0.30.0
watchdog==0.8.3
flake8==3.5.0
tox==2.9.1
coverage==4.5.1
Sphinx==1.7.1
twine==1.10.0

pytest==3.4.2
pytest-runner==2.11.1
git+git://github.com/jbn/funcsigs.git@214840c53529f019638229d72dcf2257fe154458#egg=funcsigs
flake8
tox
coverage
pytest
pytest-runner
7 changes: 5 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
from setuptools import setup, find_packages


with open('README.rst') as readme_file:
readme = readme_file.read()


with open('HISTORY.rst') as history_file:
history = history_file.read()


requirements = []

setup_requirements = ['pytest-runner']
@@ -20,14 +23,14 @@
'Intended Audience :: Developers',
'License :: OSI Approved :: MIT License',
'Natural Language :: English',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
],
description="A package that loads a module as a callable pipeline.",
install_requires=requirements,
license="MIT license",
license="MIT",
long_description=readme + '\n\n' + history,
include_package_data=True,
keywords='modpipe',
10 changes: 10 additions & 0 deletions tests/examples/malformed_skipto.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
from modpipe import SkipTo


def f(x):
return x + 1


def g(x):
# This is before in the linearized pipeline!
return SkipTo(f, x)
62 changes: 36 additions & 26 deletions tests/test_helper_functions.py
Original file line number Diff line number Diff line change
@@ -1,49 +1,51 @@
import pytest
from modpipe.modpipe_impl import (_defined_in,
_pipelineable,
_is_simple_callable,
_is_pseudo_private,
_remove_from_pipeline_seq,
_sequence_objects,
_compile_signatures,
_load_pipeline_seq)
from modpipe.helpers import is_simple_callable, iter_defined_in, iter_pipelineable, is_pseudo_private, \
remove_from_pipeline_seq, sequence_objects, compile_signatures, load_pipeline_seq
from tests.examples import math_mod
from collections import OrderedDict


def test_defined_in():
res = {k for k, v in _defined_in(math_mod)}
res = {k for k, v in iter_defined_in(math_mod)}
assert res == {'normed', 'rot90', 'times_ten', 'ScaleAll', '_is_zero'}


def test_class_constructors_arent_simple_callables():
class MyClass:
pass

assert not _is_simple_callable(MyClass)
assert not is_simple_callable(MyClass)


def test_pipelineable():
res = {k for k, v in _pipelineable(_defined_in(math_mod))}
def test_iter_pipelineable():
res = {k for k, v in iter_pipelineable(iter_defined_in(math_mod))}
assert res == {'normed', 'rot90', 'times_ten', '_is_zero'}


def test_underscores_are_pseudo_private():
assert _is_pseudo_private('_name', None)
assert not _is_pseudo_private('name', None)
assert is_pseudo_private('_name', None)
assert not is_pseudo_private('name', None)


def test_remove_from_pipeline_seq():
pipeline_seq = OrderedDict(_pipelineable(_defined_in(math_mod)))
_remove_from_pipeline_seq(pipeline_seq, _is_pseudo_private)
pipeline_seq = OrderedDict(iter_pipelineable(iter_defined_in(math_mod)))
remove_from_pipeline_seq(pipeline_seq, is_pseudo_private)
assert set(pipeline_seq.keys()) == {'normed', 'rot90', 'times_ten'}


def test_sequence_objects():
pairs = _pipelineable(_defined_in(math_mod))
res = [k for k, v in _sequence_objects(math_mod, pairs)]
pairs = iter_pipelineable(iter_defined_in(math_mod))
res = [k for k, v in sequence_objects(math_mod, pairs)]
assert res == ['_is_zero', 'normed', 'rot90', 'times_ten']

pairs = list(iter_pipelineable(iter_defined_in(math_mod)))
class MissingClass: pass
pairs.append(('zed', MissingClass))

with pytest.raises(RuntimeError):
sequence_objects(math_mod, pairs)



def test_compile_signatures():

@@ -60,29 +62,37 @@ def f_prime(a, b):
pass

# Only one callable.
_compile_signatures({'f': f})
res = compile_signatures({'f': f})
assert list(res[f].parameters) == ['x', 'y']

# Two callables with the same signature.
tbl = {func.__name__: func for func in [f, g]}
_compile_signatures(tbl)
res = compile_signatures(tbl)
assert list(res[f].parameters) == ['x', 'y']
assert list(res[g].parameters) == ['x', 'y']

# Two callables with the same signature different names.
tbl = {func.__name__: func for func in [f, f_prime]}
_compile_signatures(tbl)
res = compile_signatures(tbl)
assert list(res[f].parameters) == ['x', 'y']
assert list(res[f_prime].parameters) == ['a', 'b']

# Two callables with the same signature different names.
with pytest.raises(RuntimeError):
_compile_signatures(tbl, assert_unif=True, ignore_names=False)
compile_signatures(tbl, assert_unif=True, ignore_names=False)

# Three with a heterogeneous signature.
with pytest.raises(RuntimeError) as e:
tbl = OrderedDict([(func.__name__, func) for func in [h, f, g]])
_compile_signatures(tbl, assert_unif=True)
compile_signatures(tbl, assert_unif=True)

e.match("Signature for f is \(x, y\) which doesn't match \(x\)")
e.match(r"Signature for f is \(x, y\) which doesn't match \(x\)")


def test_load_pipeline_seq():
pipeline_seq = _load_pipeline_seq(math_mod)
def starts_with_rot(k, _):
return k.startswith('rot')

pipeline_seq = load_pipeline_seq(math_mod, True, starts_with_rot)

assert list(pipeline_seq) == ['normed', 'rot90', 'times_ten']
assert list(pipeline_seq) == ['normed', 'times_ten']
25 changes: 25 additions & 0 deletions tests/test_modpipe.py
Original file line number Diff line number Diff line change
@@ -10,6 +10,11 @@ def math_pipeline():
return ModPipe('tests.examples.math_mod')


def test_from_loaded_module():
from tests.examples import math_mod
assert ModPipe(math_mod).module_name == 'tests.examples.math_mod'


def test_visit_all_elements(math_pipeline):
assert math_pipeline(0, 1) == (10, 0)

@@ -45,3 +50,23 @@ def stripped_pyc(s):

def test_is_picklable(math_pipeline):
pickle.dumps(math_pipeline)


def test_repr(math_pipeline):
assert str(math_pipeline) == 'ModPipe(tests.examples.math_mod)'


def test_get_item(math_pipeline):
assert math_pipeline['normed'].__name__ == 'normed'


def test_ipython_key_completions(math_pipeline):
completions = math_pipeline._ipython_key_completions_()
expected = ['normed', 'rot90', 'times_ten']
assert completions == expected


def test_guards_against_cycles():
pipeline = ModPipe.on("tests.examples.malformed_skipto")
with pytest.raises(RuntimeError):
pipeline(41)
16 changes: 13 additions & 3 deletions tests/test_result_semantics.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import pytest
from modpipe.modpipe_impl import Result, Done, SkipTo
from modpipe import Result, Done, SkipTo


def test_single_arg_result():
@@ -8,10 +8,10 @@ def test_single_arg_result():

def f(x):
assert x == 1
return 1
return 2

res_1 = res_0.apply_to(f, 1)
assert res_1.args == 1
assert res_1.args == 2


def test_tuple_arg_result():
@@ -75,3 +75,13 @@ def g(a, b):
assert res_0.args == (1, 2)
assert res_0.apply_to(f, 2) is res_0
assert res_0.apply_to(g, 2).args == 3


def test_result_repr():
def g(): pass

assert repr(Result(1)) == 'Result(1)'
assert repr(Done(2)) == 'Done(2)'
assert repr(SkipTo(g, 91)) == 'SkipTo(91)'


4 changes: 2 additions & 2 deletions tox.ini
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[tox]
envlist = py27,py35,py36,pypy,flake8
envlist = py35,py36,py37,flake8

[testenv]
deps = -rrequirements_dev.txt
@@ -8,5 +8,5 @@ commands = pytest {posargs:tests}
[testenv:flake8]
basepython = python
deps = flake8
commands = flake8 modpipe
commands = flake8 --ignore E501 modpipe