r"""Test tools:
These test tools are based on the nose testing interface, and using
some of the conventions from SciPy.
http://projects.scipy.org/numpy/wiki/TestingGuidelines
- `test()`: Run the tests for the module.
- `bench()`: Run benchmarks for the module.
.. note:: The following functionality is obsolete. Use nosetests
instead.
This module facilitates this by providing an initialization of these
functions: Use the following template at the end of your module
definition files::
# Testing
from mmf.utils.mmf_test import run
run(__name__, __file__, locals())
This will modify the local dictionary defining :func:`test` and
:func:`test_suites` as well as providing runtime code to run the tests
if the module is executed.
These default functions allow for two types of tests:
1) Doctest code in the docstrings of the class. See the doctest
module for details about how these work.
2) Define a subclasses of the TestCase class provided
here. This may be defined in either the module file, or
(preferably) in a separate testing module.
The name of the testing module is the same as the module but with
`test_` prepended. This may be in either the same directory as the
module, or in a subdirectory called `tests`. (These two paths are
automatically searched. Of course, the test-module may also occur
anywhere on sys.path).
In addition to these two function, this module also provides a class
called Test that provides better control over testing (for example,
this will provide code coverage analysis.)
"""
import sys
import os
from optparse import OptionParser
import unittest
import textwrap
import warnings
import filecmp
import fnmatch
import nose
# This contains some useful test functions.
from nose.tools import *
# These decorators can be used to specify that certain functions
# should be excluded from tests or that they are slow. See dec.slow
# and dec.isastest
import decorators as dec
from decorators import *
import numpy as np
__all__ = ['run', 'TestCase', 'dec']
__all__.extend([_k for _k in nose.tools.__dict__.keys()
if not _k.startswith('_')])
__all__.extend(dec.__all__)
'''
# Some profiling code.
from test import pystone
import time
# TOLERANCE in Pystones
kPS = 1000
TOLERANCE = 0.5*kPS
class DurationError(AssertionError): pass
def local_pystone():
return pystone.pystones(loops=pystone.LOOPS)
def timedtest(max_num_pystones, current_pystone=local_pystone()):
""" decorator timedtest """
if not isinstance(max_num_pystones, float):
max_num_pystones = float(max_num_pystones)
def _timedtest(function):
def wrapper(*args, **kw):
start_time = time.time()
try:
return function(*args, **kw)
finally:
total_time = time.time() - start_time
if total_time == 0:
pystone_total_time = 0
else:
pystone_rate = current_pystone[0] / current_pystone[1]
pystone_total_time = total_time / pystone_rate
if pystone_total_time > (max_num_pystones + TOLERANCE):
raise DurationError((('Test too long (%.2f Ps, '
'need at most %.2f Ps)')
% (pystone_total_time,
max_num_pystones)))
return wrapper
return _timedtest
This decorator is not to use in production code, and would rather
fit in functional or unit tests. This make performance tests portable to any box and fits performance regression tests you would want to run in unit tests.
For example, in this test we want to be sure test_critical() does not last more than 2kPS:
>>> import unittest
>>> class MesTests(unittest.TestCase):
... @timedtest(2*kPS)
... def test_critical(self):
... a =''
... for i in range(50000):
... a = a + 'x' * 200
>>> suite = unittest.makeSuite(MesTests)
>>> unittest.TextTestRunner().run(suite)
<unittest._TextTestResult run=1 errors=0 failures=0>
'''
###################################
## Private variables
_abs_tol = np.finfo(float).eps*64
_rel_tol = np.finfo(float).eps*64
[docs]def skipknownfailure(f):
r"""Decorator to raise SkipTest for test known to fail
"""
def skipper(*args, **kwargs):
raise nose.SkipTest, 'This test is known to fail'
return nose.tools.make_decorator(f)(skipper)
## Here are some test functions.
def check_derivative(x, f, df=None, ddf=None, dddf=None, ddddf=None,
eps=np.finfo(float).eps, tol_factor=2.0):
r"""Check the derivatives `df(x)` and `ddf(x)` of `f(x)` using finite
difference formula. If the function can compute higher order
derivatives, then these will be used to estimate the optimal step
size and the acceptable error."""
fx = f(x)
if dddf:
dddfx = dddf(x)
else:
# Estimate dddfx
h3 = max(x, 1.0)*(18.0*eps)**(1./5.)
h3 = (x + h) - x
dddfx = (f(x+2.0*h)-2.0*(f(x+h)-f(x-h))-f(x-2.0*h))/\
h**3/2.0
if ddddf:
ddddfx = ddddf(x)
else:
# Estimate ddddfx
h4 = max(x, 1.0)*(18.0*eps)**(1./5.)
h4 = (x + h) - x
dddfx = (f(x+2.0*h)-2.0*(f(x+h)-f(x-h))-f(x-2.0*h))/\
h**3/2.0
h1 = (3.0*eps*fx/dddfx)**(1./3.)
h1 = (x + h1) - x
tol1 = h1**2*dddfx/2.0
def diff(dir1, dir2, recursive=True, print_report=True, skip=[],**kw):
r"""Return `True` if two directories are identical. See
:class:`filecmp.dircmp` for details.
Parameters
----------
dir1, dir2 : str
Directories to compare.
recursive : bool
Recursively check all subdirectories if `True`.
print_report : bool
If there are differences, print a message.
skip : [str]
Filenames matching these patterns will only be checked for existence: the
detailed comparison will be skipped. (Actually, it will be done, but the
call will not fail if there is a difference.)
"""
_d = filecmp.dircmp(dir1, dir2, **kw)
res = _diff_get_diffs(_d, recursive=recursive, skip=skip)
if not res and print_report:
if recursive:
_d.report_full_closure()
else:
_d.report()
return res
def _diff_get_diffs(d, recursive, skip=set()):
r"""Recursive helper for :func:`diff`."""
diffs = [_f for _f in d.diff_files
if not any([fnmatch.fnmatch(_f,_p) for _p in skip])]
res = not d.left_only and not d.right_only and not diffs
if recursive and res:
res = all([_diff_get_diffs(_d, skip=skip) for _d in d.subdirs])
return res
## These are for testing. They simulate user input.
def simulate_input(input):
r"""Return a function that simulates raw_input but returns the
values in the input list.
Examples
--------
>>> raw_input = simulate_input(["1", "2", "3"])
>>> raw_input("> ")
> 1
'1'
>>> raw_input("> ")
> 2
'2'
>>> raw_input("> ")
> 3
'3'
>>> raw_input("> ")
''
"""
if isinstance(input, str):
input = [input]
input.reverse()
def raw_input(msg, input=input):
"Simulate user entering input"
while 0 < len(input):
response = input.pop()
print msg + response
return response
return ""
return raw_input
def simulate_cancel(msg):
"Simulate user entering ctrl-c"
print msg + "^C"
raise KeyboardInterrupt
[docs]class TestCase(object):
r"""Test Case Class Stub with extra testing functions.
"""
[docs] def ok_(self, res, msg=None):
return ok_(res, msg)
[docs] def assertAlmostEqual(self, exact, approx,
abs_tol=_abs_tol, rel_tol=_rel_tol,
msg=None, relax=None):
r"""Fail if the two objects are unequal as determined by their
difference compared with the specified absolute and relative
tolerances.
Parameters
----------
exact, approx : float or array
Exact reference value and test value.
abs_err, rel_tol : float or (float, float)
Success if `abs_err < abs_tol or rel_err < rel_tol` where
`abs_err = abs(exact - approx)` and `rel_err =
abs_err/abs(exact)`. If pairs are provided, then the test
fails only if the maximum tolerances are not met, but a
warning is generated if the minimum tolerances are not met.
msg : str
Optional message.
Examples
--------
Here is a simple example of usage:
>>> tc = TestCase()
There is no problem if one of the tolerances (the absolute
tolerance here) is met:
>>> tc.assertAlmostEqual(0.5, 0.5 + _abs_tol/1.01)
but there is a failure if neither tolerance is
met:
>>> tc.assertAlmostEqual(0.5, 0.5 + _abs_tol*1.01)
Traceback (most recent call last):
...
AssertionError: Numbers not almost equal.
Max abs_err: 1.43219e-14 > 1.42109e-14
Max rel_err: 2.86438e-14 > 1.42109e-14
Again, no problem if at least one of the tolerances (the
relative tolerance here) is met:
>>> tc.assertAlmostEqual(2, 2*(1.0 + _abs_tol/1.01))
but there is a again failure if neither tolerance is
met:
>>> tc.assertAlmostEqual(2, 2*(1.0 + _abs_tol*1.01))
Traceback (most recent call last):
...
AssertionError: Numbers not almost equal.
Max abs_err: 2.88658e-14 > 1.42109e-14
Max rel_err: 1.44329e-14 > 1.42109e-14
Here is an example of providing two different errors. If the
tolerances are met, then there is no problem as before:
>>> tc.assertAlmostEqual(2, 2*(1.0 + _abs_tol/1.01),
... abs_tol=(_abs_tol, _abs_tol*2),
... rel_tol=(_rel_tol, _rel_tol*2))
The looser tolerances suppress errors, but the tighter
tolerances will generate a warning. (First we turn warnings
into errors so we can see the behaviour.)
>>> warnings.simplefilter('error', UserWarning)
>>> tc.assertAlmostEqual(2, 2*(1.0 + _abs_tol*1.01),
... abs_tol=(_abs_tol, _abs_tol*2),
... rel_tol=(_rel_tol, _rel_tol*2))
Traceback (most recent call last):
...
UserWarning: Numbers not almost equal.
Max abs_err: 2.88658e-14 > 1.42109e-14
Max rel_err: 1.44329e-14 > 1.42109e-14
`NaN`'s trigger a different type of error message than unmet
tolerances:
>>> tc.assertAlmostEqual(np.nan, 0)
Traceback (most recent call last):
...
AssertionError: NaN encountered.
These tests also work with arrays and lists, using
:mod:`numpy` to provide proper vectorization:
>>> tc.assertAlmostEqual([0, 1, 2], [0, 1, 2])
>>> tc.assertAlmostEqual([np.nan, 1, 2], [0, np.nan, 2])
Traceback (most recent call last):
...
AssertionError: 2 of 2*3 components are NaN.
>>> tc.assertAlmostEqual([0.1, 1, 2], [0, 1, 2])
Traceback (most recent call last):
...
AssertionError: 2 of 3 components of arrays not almost equal.
Max abs_err: 0.1 > 1.42109e-14
Max rel_err: 1 > 1.42109e-14
>>> tc.assertAlmostEqual([0.1, 1, 2], [0, 1, 2],
... abs_tol=(_abs_tol, 0.2))
Traceback (most recent call last):
...
UserWarning: 2 of 3 components of arrays not almost equal.
Max abs_err: 0.1 > 1.42109e-14
Max rel_err: 1 > 1.42109e-14
>>> tc.assertAlmostEqual([np.nan, 1, 2], [0, np.nan, 2])
Traceback (most recent call last):
...
AssertionError: 2 of 2*3 components are NaN.
"""
exact = np.asarray(exact)
approx = np.asarray(approx)
abs_err = np.abs(exact - approx)
rel_err = np.divide(abs_err, np.abs(exact))
abs_satisfied = (abs_err <= np.max(abs_tol))
rel_satisfied = (rel_err <= np.max(rel_tol))
satisfied = np.logical_or(abs_satisfied, rel_satisfied)
if not np.all(satisfied):
count = np.prod(abs_err.shape)
n_fail = np.sum(np.isnan(exact) + np.isnan(approx))
if 0 < n_fail:
if count > 1:
_msg = ("%i of 2*%i components are NaN."
%(n_fail, count))
else:
_msg = "NaN encountered."
else:
n_fail = np.sum(satisfied)
max_abs_err = np.max(np.where(satisfied, 0, abs_err))
max_rel_err = np.max(np.where(satisfied, 0, rel_err))
if count > 1:
msg1 = ("%i of %i components of arrays not almost equal."
%(n_fail, count))
else:
msg1 = "Numbers not almost equal."
_msg = textwrap.dedent("""\
%s
Max abs_err: %g > %g
Max rel_err: %g > %g"""%(msg1,
max_abs_err, np.max(abs_tol),
max_rel_err, np.max(rel_tol)))
if msg:
msg = "\n".join([msg, _msg])
else:
msg = _msg
err = AssertionError(msg)
err.exact = exact
err.approx = approx
raise(err)
# Check for warning cases
abs_satisfied = (abs_err <= np.min(abs_tol))
rel_satisfied = (rel_err <= np.min(rel_tol))
satisfied = np.logical_or(abs_satisfied, rel_satisfied)
if not np.all(satisfied):
count = np.prod(abs_err.shape)
n_fail = np.sum(satisfied)
max_abs_err = np.max(np.where(satisfied, 0, abs_err))
max_rel_err = np.max(np.where(satisfied, 0, rel_err))
if count > 1:
msg1 = ("%i of %i components of arrays not almost equal."
%(n_fail, count))
else:
msg1 = "Numbers not almost equal."
msg = textwrap.dedent("""\
%s
Max abs_err: %g > %g
Max rel_err: %g > %g"""%(msg1,
max_abs_err, np.min(abs_tol),
max_rel_err, np.min(rel_tol)))
err = UserWarning(msg)
err.exact = exact
err.approx = approx
warnings.warn(err)
# Here we form a parse to parse the arguments when a program is run
# with python for testing.
_usage = """%prog [options]
Run the test suite on %prog.
"""
_parser = OptionParser(usage=_usage)
_parser.add_option("-c", "--coverage",
action="store_true",
dest="coverage",
default=False,
help="Perform coverage analysis")
_parser.add_option("-p", "--profile",
action="store_true",
dest="profile",
default=False,
help="Perform hotshot profile analysis")
_parser.add_option("--full",
action="store_false",
dest="fast",
default=False,
help="Run all tests")
_parser.add_option("-f", "--fast",
action="store_true",
dest="fast",
default=True,
help="Run only 'quick' tests")
_parser.add_option("-b", "--benchmark",
action="store_true",
dest="bench",
default=False,
help="Run only benchmarks")
@dec.setastest(False)
def defineModuleTests(name, fileName, locals, verbose=False):
r"""Define tests for module and run if `name == __main__`. This is
designed to be called at the end of the main module file as
follows::
# Testing
from mmf.utils.mmf_test import run
run(__name__, __file__, locals())
.. note:: This functionality is obsolete. Use nosetests instead.
All tests may be run by calling the function test() and the test
suite may be accessed through the function :func:`test_suite`. These
are added to the dictionary locals for use in the module to be
tested.
Tests are searched for in the file fileName as well as in the file
`test_` + `moduleName` in either the current directory, or in the
directory tests/
Note that this is desined to allow a single module (file) to be
tested. To test an entire package you should use the standard
nose testing framework by running nosetests.
"""
# Extract module name.
if ".pyc" == fileName[-4:]:
# Get module from .py file if it exists.
newName = fileName[:-1]
if os.access(newName, os.R_OK):
fileName = newName
fileName = os.path.abspath(fileName)
(modulePath, moduleFile) = os.path.split(fileName)
(moduleName, ext) = os.path.splitext(moduleFile)
paths = [modulePath,
os.sep.join([modulePath, "test"]),
os.sep.join([modulePath, "tests"])]
names = [moduleName+".py",
"test_"+moduleName+".py"]
files = []
for p in paths:
for n in names:
f = os.sep.join([p, n])
if os.access(f, os.R_OK):
files.append(f)
nose_argv = [''] + files + ["--with-doctest"]
@dec.setastest(False)
def test(extra_args=[]):
print nose_argv+extra_args
nose.run(argv=nose_argv+extra_args)
def bench(extra_args=[]):
argv = nose_argv
argv += ["--match", r'(?:^|[\\b_\\.%s-])[Bb]ench'%os.sep]
nose.run(argv=argv+extra_args)
locals['test'] = test
locals['bench'] = bench
if name == "__main__":
(options, args) = _parser.parse_args()
if options.coverage:
args += ["--with-coverage", "--cover-package", moduleName]
if options.profile:
args.append("--with-profile")
if options.fast:
args += ["-A", "not slow"]
if options.bench:
bench(args)
else:
test(args)
[docs]def run(name, file, locals):
"""Run all the tests in the module
This is to be used by including the following code in your
source::
# Testing
from mmf.utils.mmf_test import run
run(__name__, __file__, locals())
.. note:: This functionality is obsolete. Use nosetests instead.
Parameters
----------
name : The name of the current module. Typically this is set to
the global variable `__name__`.
file : The name of the current file. Typically this is set to
the global variable `__file__`.
locals : Dictionary of local variables for the module.
Typically this is set to the result of :func:`locals()`.
"""
if name is '__main__':
import mmf.utils.mmf_test
mmf.utils.mmf_test.defineModuleTests(__name__,
__file__,
locals)