Skip to content

Commit

Permalink
MAINT: tools/refguide_check.py: misc fixes and cleanups + friendlier …
Browse files Browse the repository at this point in the history 
…output
  • Loading branch information
@pv @ev-br
pv authored and ev-br committed Jul 5, 2015
1 parent 787576f commit 9fcb5a8
Show file tree
Hide file tree
Showing 3 changed files with 166 additions and 79 deletions.
20 changes: 1 addition & 19 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,25 +23,7 @@ matrix:
- travis_retry pip install $NUMPYSPEC
- travis_retry pip install nose argparse
- travis_retry pip install matplotlib
- python runtests.py --doctests-only -s linalg
- python runtests.py --doctests-only -s cluster
- python runtests.py --doctests-only -s cluster.vq
- python runtests.py --doctests-only -s cluster.hierarchy
- python runtests.py --doctests-only -s fftpack
- python runtests.py --doctests-only -s interpolate
- python runtests.py --doctests-only -s integrate
- python runtests.py --doctests-only -s io
- python runtests.py --doctests-only -s misc
- python runtests.py --doctests-only -s ndimage
- python runtests.py --doctests-only -s odr
- python runtests.py --doctests-only -s optimize
- python runtests.py --doctests-only -s signal
- python runtests.py --doctests-only -s spatial
- python runtests.py --doctests-only -s sparse
- python runtests.py --doctests-only -s sparse.csgraph
- python runtests.py --doctests-only -s sparse.linalg
- python runtests.py --doctests-only -s stats
- python runtests.py --doctests-only -s stats.mstats
- python runtests.py --refguide-check
- python: 3.4
env:
- TESTMODE=fast
Expand Down
15 changes: 6 additions & 9 deletions runtests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@ def main(argv):
help="just build, do not run any tests")
parser.add_argument("--doctests", action="store_true", default=False,
help="Run doctests in module")
parser.add_argument("--doctests-only", action="store_true", default=False,
help="Only run doctests in submodule. (Do not run regular tests.)")
parser.add_argument("--refguide-check", action="store_true", default=False,
help="Run refguide check (do not run regular tests.)")
parser.add_argument("--coverage", action="store_true", default=False,
help=("report coverage of project code. HTML output goes "
"under build/coverage"))
Expand Down Expand Up @@ -176,13 +176,10 @@ def main(argv):
extra_argv += ['--cover-html',
'--cover-html-dir='+dst_dir]

if args.doctests_only:
if not args.submodule:
print("*** No submodule specified: Use '-s optimize', '-s stats.mstats' etc.")
sys.exit(-1)
cmd = [os.path.join('tools', 'refguide_check.py'),
'--check_docs',
args.submodule,]
if args.refguide_check:
cmd = [os.path.join('tools', 'refguide_check.py'), '--doctests']
if args.submodule:
cmd += [args.submodule]
os.execv(sys.executable, [sys.executable] + cmd)
sys.exit(0)

Expand Down
210 changes: 159 additions & 51 deletions tools/refguide_check.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,23 +25,67 @@
from __future__ import print_function

import sys
import os
import re
import copy
import inspect
import warnings
import doctest
import tempfile
import shutil
from doctest import NORMALIZE_WHITESPACE, ELLIPSIS, IGNORE_EXCEPTION_DETAIL

from argparse import ArgumentParser, REMAINDER

import numpy as np

import scipy
from scipy import (cluster, constants, fftpack, integrate, interpolate, io,
linalg, misc, ndimage, odr, optimize, signal, sparse,
spatial, special, stats)
# TODO: sparse.csgraph, sparse.linalg, stats.mstats, cluster.vq,
# cluster.hierarchy

BASE_MODULE = "scipy"

PUBLIC_SUBMODULES = [
'linalg',
'cluster',
'cluster.vq',
'cluster.hierarchy',
'fftpack',
'interpolate',
'integrate',
'io',
'misc',
'ndimage',
'odr',
'optimize',
'signal',
'spatial',
'sparse',
'sparse.csgraph',
'sparse.linalg',
'stats',
'stats.mstats',
]

# these names are known to fail doctesting and we like to keep it that way
# e.g. sometimes pseudocode is acceptable etc
DOCTEST_SKIPLIST = set([
'scipy.integrate.quad',
'scipy.interpolate.UnivariateSpline',
'scipy.stats.levy_stable'
])


def short_path(path, cwd=None):
"""
Return relative or absolute path name, whichever is shortest.
"""
if not isinstance(path, str):
return path
if cwd is None:
cwd = os.getcwd()
abspath = os.path.abspath(path)
relpath = os.path.relpath(path, cwd)
if len(abspath) <= len(relpath):
return abspath
return relpath


def find_funcnames(module):
Expand Down Expand Up @@ -77,12 +121,12 @@ def get_all_dict(module):
# Modules are almost always private; real submodules need a separate
# run of refguide_check.
all_dict = [name for name in all_dict
if not inspect.ismodule(getattr(module, name))]
if not inspect.ismodule(getattr(module, name, None))]

deprecated = []
not_deprecated = []
for name in all_dict:
f = getattr(module, name)
f = getattr(module, name, None)
if callable(f) and is_deprecated(f):
deprecated.append(name)
else:
Expand Down Expand Up @@ -118,46 +162,44 @@ def is_deprecated(f):

def report(all_dict, funcnames, deprecated, module_name):
"""Print out a report for the module"""

print("\n\n" + "=" * len(module_name))
print(module_name)
print("=" * len(module_name) + "\n")

num_all = len(all_dict)
num_ref = len(funcnames)
print("Number of non-deprecated functions in __all__: %i" % num_all)
print("Number of functions in refguide: %i" % num_ref)
print("Non-deprecated objects in __all__: %i" % num_all)
print("Objects in refguide: %i" % num_ref)

only_all, only_ref = compare(all_dict, funcnames)
dep_in_ref = set(only_ref).intersection(deprecated)
only_ref = set(only_ref).difference(deprecated)
if len(only_all) == len(only_ref) == 0:
print("\nAll good!")
print("\nNo missing or extraneous items!")
else:
if len(only_all) > 0:
print("")
print("Functions in %s.__all__ but not in refguide:" % module_name)
print("------------------------------------------")
print("Objects in %s.__all__ but not in refguide::\n" % module_name)
for name in only_all:
print(name)
print(" " + name)

if len(only_ref) > 0:
print("")
print("Objects in refguide but not functions in %s.__all__:" % module_name)
print("------------------------------------------")
print("Objects in refguide but not in %s.__all__::\n" % module_name)
for name in only_ref:
print(name)
print(" " + name)

if len(dep_in_ref) > 0:
print("")
print("Deprecated objects in refguide:")
print("------------------------------------------")
print("Deprecated objects in refguide::\n")
for name in deprecated:
print(name)
print(" " + name)


def check_docstrings(module, verbose):
"""Check code in docstrings of the module's public symbols.
"""
# these names are known to fail doctesting and we like to keep it that way
# e.g. sometimes pseudocode is acceptable etc
skiplist = set(['quad', 'UnivariateSpline', 'levy_stable'])

# the namespace to run examples in
ns = {'np': np,
'assert_allclose': np.testing.assert_allclose,
Expand All @@ -170,31 +212,61 @@ def check_docstrings(module, verbose):
'int32': np.int32,
'float64': np.float64,
'dtype': np.dtype,
'nan': np.nan, 'NaN': np.nan,
'inf': np.inf, 'Inf': np.inf,}
'nan': np.nan,
'NaN': np.nan,
'inf': np.inf,
'Inf': np.inf,}

# if MPL is available, use display-less backend
try:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
have_MPL = True
have_matplotlib = True
except ImportError:
have_MPL = False
have_matplotlib = False


def format_item_header(name):
return "\n\n" + name + "\n" + "-" * len(name)


class DTRunner(doctest.DocTestRunner):
stopwords = {'plt.', '.hist', '.show', '.ylim', '.subplot(',
'set_title', 'imshow', 'plt.show', 'ax.axis', 'plt.plot(',
'.title', '.ylabel', '.xlabel', 'set_ylim', 'set_xlim'}
rndm_markers = {'# random', '# Random', '#random', '#Random'}
DIVIDER = "\n"

def __init__(self, item_name, checker=None, verbose=None, optionflags=0):
self._item_name = item_name
doctest.DocTestRunner.__init__(self, checker=checker, verbose=verbose,
optionflags=optionflags)

def _report_item_name(self, out, new_line=False):
if self._item_name is not None:
out(format_item_header(self._item_name))
if new_line:
out("\n")
self._item_name = None

def report_success(self, out, test, example, got):
if self._verbose:
self._report_item_name(out, new_line=True)
return doctest.DocTestRunner.report_success(self, out, test, example, got)

def report_unexpected_exception(self, out, test, example, exc_info):
self._report_item_name(out)
return doctest.DocTestRunner.report_unexpected_exception(
self, out, test, example, exc_info)

def report_failure(self, out, test, example, got):
if (any(word in example.source for word in self.stopwords) or
any(word in example.want for word in self.rndm_markers)):
# do not complain if output does not match
pass
else:
self._report_item_name(out)
return doctest.DocTestRunner.report_failure(self, out, test,
example, got)

Expand Down Expand Up @@ -229,8 +301,8 @@ def check_output(self, want, got, optionflags):

# OK then, convert strings to objects
try:
a_want = eval(want, ns)
a_got = eval(got, ns)
a_want = eval(want, dict(ns))
a_got = eval(got, dict(ns))
except:
if not self.parse_namedtuples:
return False
Expand Down Expand Up @@ -272,48 +344,84 @@ def _do_check(self, want, got):
pass
return np.allclose(want, got, atol=self.atol, rtol=self.rtol)

# loop over non-deprecated items
# Loop over non-deprecated items
all_success = True
for name in get_all_dict(module)[0]:
obj = getattr(module, name)
full_name = module.__name__ + '.' + name

if name in skiplist:
if full_name in DOCTEST_SKIPLIST:
continue

if verbose:
print(name)
try:
obj = getattr(module, name)
except AttributeError:
import traceback
print(format_item_header(full_name))
print("Missing item!")
print(traceback.format_exc())
continue

finder = doctest.DocTestFinder()
tests = finder.find(obj, name, globs=ns)
try:
tests = finder.find(obj, name, globs=dict(ns))
except:
import traceback
print(format_item_header(full_name))
print("Failed to get doctests:")
print(traceback.format_exc())
continue

flags = NORMALIZE_WHITESPACE | ELLIPSIS | IGNORE_EXCEPTION_DETAIL
runner = DTRunner(full_name, checker=Checker(), optionflags=flags,
verbose=verbose)

# Run tests, trying to restore global state afterward
old_printoptions = np.get_printoptions()
old_errstate = np.seterr()
cwd = os.getcwd()
tmpdir = tempfile.mkdtemp()
try:
os.chdir(tmpdir)

for t in tests:
t.filename = short_path(t.filename, cwd)
fails, successes = runner.run(t)
if fails > 0:
all_success = False

runner = DTRunner(checker=Checker(), optionflags=flags)
for t in tests:
runner.run(t)
if have_matplotlib:
plt.close('all')
finally:
os.chdir(cwd)
shutil.rmtree(tmpdir)
np.set_printoptions(**old_printoptions)
np.seterr(**old_errstate)

if have_MPL:
plt.close('all')
if not verbose and all_success:
# Print at least a success message if no other output was produced
print("\nAll doctests pass!")


def main(argv):
parser = ArgumentParser(usage=__doc__.lstrip())
parser.add_argument("module_name", metavar="ARGS", default=[],
nargs=REMAINDER, help="Valid Scipy submodule name")
parser.add_argument("--check_docs", action="store_true")
parser.add_argument("module_names", metavar="SUBMODULES", default=list(PUBLIC_SUBMODULES),
nargs='*', help="Submodules to check (default: all public)")
parser.add_argument("--doctests", action="store_true", help="Run also doctests")
parser.add_argument("--verbose", action="store_true")
args = parser.parse_args(argv)

module_name = args.module_name[0].split(".")
module = scipy
for n in module_name:
module = getattr(module, n)
for submodule_name in args.module_names:
module_name = BASE_MODULE + '.' + submodule_name
__import__(module_name)
module = sys.modules[module_name]

if args.check_docs:
check_docstrings(module, args.verbose)
else:
funcnames = find_funcnames(module)
all_dict, deprecated = get_all_dict(module)
report(all_dict, funcnames, deprecated, module_name)

if args.doctests:
check_docstrings(module, args.verbose)


if __name__ == '__main__':
main(argv=sys.argv[1:])

0 comments on commit 9fcb5a8

Please sign in to comment.