[PATCH] Add an option to run tests in random order

[Vinzent Steinberg]

Just run

>>> sympy.test(sort=False)

or

$ bin/test --random

This is to make sure that the test are independent of each other. The motivation
to do this is that sometimes tests in sympy are failing if not executed in a
certain order, see [1].

[1] http://groups.google.com/group/sympy/browse_thread/thread/a0900b89e7b5000f
---
bin/test | 13 ++++++++++---
sympy/utilities/runtests.py | 17 +++++++++++++----
2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/bin/test b/bin/test
index 5444667..d581e57 100755
--- a/bin/test
+++ b/bin/test
@@ -25,13 +25,20 @@ parser.add_option("--pdb", action="store_true", dest="pdb",
default=False, help="Run post mortem pdb on each failure")
parser.add_option("--no-colors", action="store_false", dest="colors",
default=True, help="Do not report colored [OK] and [FAIL]")
-parser.add_option("-k", dest="kw", help="only run tests matching the given keyword expression", metavar="KEYWORD", default="")
-parser.add_option("--tb", dest="tb", help="traceback verboseness (short/no) [default: %default]", metavar="TBSTYLE", default="short")
+parser.add_option("-k", dest="kw",
+ help="only run tests matching the given keyword expression",
+ metavar="KEYWORD", default="")
+parser.add_option("--tb", dest="tb",
+ help="traceback verboseness (short/no) [default: %default]",
+ metavar="TBSTYLE", default="short")
+parser.add_option("--random", action="store_false", dest="sort", default=True,
+ help="Run tests in random order instead of sorting them")

options, args = parser.parse_args()

ok = sympy.test(*args, **{"verbose": options.verbose, "kw": options.kw,
- "tb": options.tb, "pdb": options.pdb, "colors": options.colors})
+ "tb": options.tb, "pdb": options.pdb, "colors": options.colors,
+ "sort": options.sort})
if ok:
sys.exit(0)
else:
diff --git a/sympy/utilities/runtests.py b/sympy/utilities/runtests.py
index 92e1eeb..6ec7541 100644
--- a/sympy/utilities/runtests.py
+++ b/sympy/utilities/runtests.py
@@ -57,6 +57,8 @@ def test(*args, **kwargs):
"""
Run all tests containing any of the given strings in their path.

+ If sort=False, run them in random order (not default).
+
Warning: Tests in *very* deeply nested directories are not found.

Examples:
@@ -81,6 +83,7 @@ def test(*args, **kwargs):
kw = kwargs.get("kw", "")
post_mortem = kwargs.get("pdb", False)
colors = kwargs.get("colors", True)
+ sort = kwargs.get("sort", True)
r = PyTestReporter(verbose, tb, colors)
t = SymPyTests(r, kw, post_mortem)
if len(args) == 0:
@@ -89,10 +92,9 @@ def test(*args, **kwargs):
mypaths = []
for p in t.get_paths(dir='sympy'):
mypaths.extend(glob(p))
- mypaths = list(set(mypaths))
- mypaths.sort()
+ mypaths = set(mypaths)
t.add_paths([p for p in mypaths if any(a in p for a in args)])
- return t.test()
+ return t.test(sort=sort)

def doctest(*paths, **kwargs):
"""
@@ -194,12 +196,19 @@ def add_paths(self, paths):
else:
self._tests.extend(self.get_tests(path2))

- def test(self):
+ def test(self, sort=False):
"""
Runs the tests.

+ If sort=False run tests in random order.
+
Returns True if all tests pass, otherwise False.
"""
+ if sort:
+ self._tests.sort()
+ else:
+ from random import shuffle
+ shuffle(self._tests)
self._reporter.start()
for f in self._tests:
try:
--
1.6.4

[Ondrej Certik]

The patch looks ok to me, please push it in. Let's see what happens.

Ondrej

[Vinzent Steinberg]

Thanks, it's in.

Vinzent

2009/8/10 Ondrej Certik <ond...@certik.cz>

[Vinzent Steinberg]

BTW: It makes no difference for me if I run the tests in random order (on a 32 bit platform).

Vinzent

[Vinzent Steinberg]

On 10 Aug., 20:57, Vinzent Steinberg

<vinzent.steinb...@googlemail.com> wrote:
> BTW: It makes no difference for me if I run the tests in random order (on a
> 32 bit platform).

This is not correct: It just failed:

______________ sympy/series/tests/test_nseries.py:test_issue1016
_______________
File "sympy/series/tests/test_nseries.py", line 343, in
test_issue1016
assert ( sin(x)/(1 - cos(x)) ).nseries(x, 0, 2) == O(1/x)
File "sympy/core/basic.py", line 2207, in nseries
return self._eval_nseries(x, x0, n)
File "sympy/core/mul.py", line 712, in _eval_nseries
return powsimp(Mul(*terms).expand(), combine='exp', deep=True)
File "sympy/core/cache.py", line 85, in wrapper
func_cache_it_cache[k] = r = func(*args, **kw_args)
File "sympy/core/operations.py", line 42, in __new__
obj = C.Order(obj, *order_symbols)
File "sympy/core/cache.py", line 85, in wrapper
func_cache_it_cache[k] = r = func(*args, **kw_args)
File "sympy/series/order.py", line 131, in __new__
lst = expr.extract_leading_order(*symbols)
File "sympy/core/cache.py", line 85, in wrapper
func_cache_it_cache[k] = r = func(*args, **kw_args)
File "sympy/core/add.py", line 325, in extract_leading_order
if o.contains(of) and o != of:
File "sympy/core/cache.py", line 85, in wrapper
func_cache_it_cache[k] = r = func(*args, **kw_args)
File "sympy/series/order.py", line 257, in contains
combine='exp'), s) != 0
File "sympy/series/order.py", line 206, in find_limit
return limit(f, x, 0, dir="+")
File "sympy/series/limits.py", line 91, in limit
r = gruntz(e, z, z0, dir)
File "sympy/series/gruntz.py", line 504, in gruntz
return limitinf(e0, x)
File "sympy/series/gruntz.py", line 339, in limitinf
c0, e0 = mrv_leadterm(e,x)
File "sympy/series/gruntz.py", line 391, in mrv_leadterm
Omega = mrv(e,x)
File "sympy/series/gruntz.py", line 228, in mrv
raise NotImplementedError("Don't know how to calculate the mrv of
'%s'" % e)
NotImplementedError: Don't know how to calculate the mrv of 'O(_x)'

Vinzent