Commit f99a3a48 authored by Remi Perenon's avatar Remi Perenon

Modification of test files, adding of reference data files, modification of...

Modification of test files, adding of reference data files, modification of GMFT-minor changes-, RBT-suppress multiprocessor mode- and graph-ensure repeatability-
parent 850a86b0
Pipeline #3419 failed with stages
in 5 minutes and 29 seconds
...@@ -13,17 +13,18 @@ nosetests --verbosity=3 -P . ...@@ -13,17 +13,18 @@ nosetests --verbosity=3 -P .
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
status = $? status = $?
echo -e "$ROUGE""One or several unit tests failed" echo -e "$ROUGE""One or several unit tests failed"
exit status exit $status
fi fi
cd ../.. cd ../..
# Performs the functional tests # Performs the functional tests
cd Tests/FunctionalTests/Jobs cd Tests/FunctionalTests/Jobs
rm -rf Test_*
python BuildJobTests.py python BuildJobTests.py
nosetests --verbosity=3 --exe -P . nosetests --verbosity=3 --exe -P .
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
status=$? status=$?
echo -e "$ROUGE""One or several functional tests failed" echo -e "$ROUGE""One or several functional tests failed"
exit status exit $status
fi fi
...@@ -80,7 +80,10 @@ class GlobalMotionFilteredTrajectory(IJob): ...@@ -80,7 +80,10 @@ class GlobalMotionFilteredTrajectory(IJob):
""" """
self.numberOfSteps = self.configuration['frames']['number'] self.numberOfSteps = self.configuration['frames']['number']
# Store universe name for further restoration
self.old_universe_name = self.configuration['trajectory']['instance'].universe.__class__.__name__
self.configuration['trajectory']['instance'].universe.__class__.__name__ = 'InfiniteUniverse' self.configuration['trajectory']['instance'].universe.__class__.__name__ = 'InfiniteUniverse'
self.configuration['trajectory']['instance'].universe._descriptionArguments = lambda: '()' self.configuration['trajectory']['instance'].universe._descriptionArguments = lambda: '()'
...@@ -197,4 +200,7 @@ class GlobalMotionFilteredTrajectory(IJob): ...@@ -197,4 +200,7 @@ class GlobalMotionFilteredTrajectory(IJob):
# The output trajectory is closed. # The output trajectory is closed.
self._gmft.close() self._gmft.close()
# Restore universe name
self.configuration['trajectory']['instance'].universe.__class__.__name__ = self.old_universe_name
REGISTRY['gmft'] = GlobalMotionFilteredTrajectory REGISTRY['gmft'] = GlobalMotionFilteredTrajectory
...@@ -139,72 +139,7 @@ class IJob(Configurable): ...@@ -139,72 +139,7 @@ class IJob(Configurable):
self._status = None self._status = None
@classmethod
def build_parallelization_test(cls, testFile, parameters=None):
"""
Produce a file like object for a given job.\n
:Parameters:
#. parameters (dict): optional. If not None, the parameters with which the job file will be built.
"""
f = open(testFile, 'w')
# The first line contains the call to the python executable. This is necessary for the file to
# be autostartable.
f.write('#!%s\n\n' % sys.executable)
f.write('import os\n')
f.write('import unittest\n')
f.write('import numpy\n')
f.write('from Scientific.IO.NetCDF import NetCDFFile\n')
f.write('from Tests.UnitTest import UnitTest\n')
f.write('from MDANSE import REGISTRY\n\n')
f.write('class Test%sParallel(UnitTest):\n\n' % cls._type.upper())
f.write(' def test(self):\n')
# Writes the line that will initialize the |parameters| dictionary.
f.write(' parameters = {}\n')
for k, v in sorted(parameters.items()):
f.write(' parameters[%r] = %r\n' % (k, v))
f.write('\n job = REGISTRY[%r](%r)\n\n' % ('job',cls._type))
f.write(' parameters["running_mode"] = ("monoprocessor",1)\n')
f.write(' self.assertNotRaises(job.run,parameters,False)\n\n')
f.write(' f = NetCDFFile(job.configuration["output_files"]["files"][0],"r")\n')
f.write(' resMono = {}\n')
f.write(' for k,v in f.variables.items():\n')
f.write(' resMono[k] = v.getValue()\n')
f.write(' f.close()\n\n')
f.write(' parameters["running_mode"] = ("multiprocessor",2)\n')
f.write(' self.assertNotRaises(job.run,parameters,False)\n\n')
f.write(' f = NetCDFFile(job.configuration["output_files"]["files"][0],"r")\n')
f.write(' resMulti = {}\n')
f.write(' for k,v in f.variables.items():\n')
f.write(' resMulti[k] = v.getValue()\n')
f.write(' f.close()\n\n')
f.write(' for k in resMono.keys():\n')
f.write(' self.assertTrue(numpy.allclose(resMono[k],resMulti[k]))\n\n')
f.write('def suite():\n')
f.write(' loader = unittest.TestLoader()\n')
f.write(' s = unittest.TestSuite()\n')
f.write(' s.addTest(loader.loadTestsFromTestCase(Test%sParallel))\n' % cls._type.upper())
f.write(' return s\n\n')
f.write('if __name__ == "__main__":\n')
f.write(' unittest.main(verbosity=2)\n')
f.close()
os.chmod(testFile,stat.S_IRWXU)
@staticmethod @staticmethod
def set_pyro_server(): def set_pyro_server():
...@@ -298,53 +233,6 @@ class IJob(Configurable): ...@@ -298,53 +233,6 @@ class IJob(Configurable):
f.close() f.close()
os.chmod(jobFile,stat.S_IRWXU) os.chmod(jobFile,stat.S_IRWXU)
@classmethod
def build_test(cls, testFile, parameters=None):
"""
Produce a file like object for a given job.\n
:Parameters:
#. parameters (dict): optional. If not None, the parameters with which the job file will be built.
"""
f = open(testFile, 'w')
# The first line contains the call to the python executable. This is necessary for the file to
# be autostartable.
f.write('#!%s\n\n' % sys.executable)
f.write('import unittest\n')
f.write('from Tests.UnitTests.UnitTest import UnitTest\n')
f.write('from MDANSE import REGISTRY\n\n')
f.write('class Test%s(UnitTest):\n\n' % cls._type.upper())
f.write(' def test(self):\n')
# Writes the line that will initialize the |parameters| dictionary.
f.write(' parameters = {}\n')
if parameters is None:
parameters = cls.get_default_parameters()
for k, v in sorted(parameters.items()):
f.write(' parameters[%r] = %r\n' % (k, v))
# Sets |analysis| variable to an instance analysis to save.
f.write(' job = REGISTRY[%r][%r]()\n' % ('job',cls._type))
f.write(' self.assertNotRaises(job.run, parameters, status=False)\n\n')
f.write('def suite():\n')
f.write(' loader = unittest.TestLoader()\n')
f.write(' s = unittest.TestSuite()\n')
f.write(' s.addTest(loader.loadTestsFromTestCase(Test%s))\n' % cls._type.upper())
f.write(' return s\n\n')
f.write("if __name__ == '__main__':\n")
f.write(' unittest.main(verbosity=2)\n')
f.close()
os.chmod(testFile,stat.S_IRWXU)
def _run_monoprocessor(self): def _run_monoprocessor(self):
...@@ -380,7 +268,8 @@ class IJob(Configurable): ...@@ -380,7 +268,8 @@ class IJob(Configurable):
if self._status is not None: if self._status is not None:
if self._status.is_stopped(): if self._status.is_stopped():
self._status.cleanup() self._status.cleanup()
return # Break to ensure the master will be shutdowned
break
else: else:
self._status.update() self._status.update()
......
...@@ -65,7 +65,6 @@ class RigidBodyTrajectory(IJob): ...@@ -65,7 +65,6 @@ class RigidBodyTrajectory(IJob):
settings['reference']=('integer',{"mini":0}) settings['reference']=('integer',{"mini":0})
settings['remove_translation']=('boolean',{'default':False}) settings['remove_translation']=('boolean',{'default':False})
settings['output_files']=('output_files', {"formats":["netcdf"]}) settings['output_files']=('output_files', {"formats":["netcdf"]})
settings['running_mode']=('running_mode',{})
def initialize(self): def initialize(self):
""" """
......
...@@ -30,6 +30,8 @@ Created on Apr 13, 2015 ...@@ -30,6 +30,8 @@ Created on Apr 13, 2015
:author: Eric C. Pellegrini :author: Eric C. Pellegrini
''' '''
import collections
class Node(object): class Node(object):
def __init__(self, name, **kwargs): def __init__(self, name, **kwargs):
...@@ -44,7 +46,7 @@ class Node(object): ...@@ -44,7 +46,7 @@ class Node(object):
@property @property
def links(self): def links(self):
return set(self._links) return self._links
def add_link(self, other): def add_link(self, other):
self._links.add(other) self._links.add(other)
...@@ -54,18 +56,16 @@ class Graph(object): ...@@ -54,18 +56,16 @@ class Graph(object):
def __init__(self): def __init__(self):
self._nodes = {} self._nodes = collections.OrderedDict()
@property @property
def nodes(self): def nodes(self):
return self._nodes return self._nodes
def add_node(self, name, **kwargs): def add_node(self, name, **kwargs):
self._nodes[name] = Node(name, **kwargs) self._nodes[name] = Node(name, **kwargs)
def add_link(self, source, target): def add_link(self, source, target):
self._nodes[source].add_link(self._nodes[target]) self._nodes[source].add_link(self._nodes[target])
def build_connected_components(self): def build_connected_components(self):
...@@ -74,7 +74,7 @@ class Graph(object): ...@@ -74,7 +74,7 @@ class Graph(object):
result = [] result = []
# Make a copy of the set, so we can modify it. # Make a copy of the set, so we can modify it.
nodes = set(self._nodes.values()) nodes = [self._nodes[k] for k in sorted(self._nodes.keys())]
# Iterate while we still have nodes to process. # Iterate while we still have nodes to process.
while nodes: while nodes:
...@@ -102,7 +102,9 @@ class Graph(object): ...@@ -102,7 +102,9 @@ class Graph(object):
neighbors.difference_update(group) neighbors.difference_update(group)
# Remove the remaining nodes from the global set. # Remove the remaining nodes from the global set.
nodes.difference_update(neighbors) for neigh in neighbors:
if neigh in nodes:
nodes.remove(neigh)
# Add them to the group of connected nodes. # Add them to the group of connected nodes.
group.update(neighbors) group.update(neighbors)
...@@ -110,6 +112,10 @@ class Graph(object): ...@@ -110,6 +112,10 @@ class Graph(object):
# Add them to the queue, so we visit them in the next iterations. # Add them to the queue, so we visit them in the next iterations.
queue.extend(neighbors) queue.extend(neighbors)
# Sort the group
group = list(group)
group.sort(key = lambda n : n.name)
# Add the group to the list of groups. # Add the group to the list of groups.
result.append(group) result.append(group)
......
...@@ -35,16 +35,18 @@ import os ...@@ -35,16 +35,18 @@ import os
import glob import glob
def suite(): def suite():
files = glob.glob('Test*.py') files = glob.glob("Test*.py")
#os.chdir("Jobs")
modules = [__import__(os.path.splitext(f)[0],globals(),locals(),[],-1) for f in files] modules = [__import__(os.path.splitext(f)[0],globals(),locals(),[],-1) for f in files]
test_suite = unittest.TestSuite() test_suite = unittest.TestSuite()
for m in modules: for m in modules:
print m.__file__
test_suite.addTests(m.suite()) test_suite.addTests(m.suite())
return test_suite return test_suite
def run_test(): def run_test():
unittest.TextTestRunner(verbosity=2).run(suite()) unittest.TextTestRunner(verbosity=2).run(suite())
if __name__ == '__main__': if __name__ == '__main__':
#os.chdir("Jobs")
run_test() run_test()
This diff is collapsed.
import numpy
import collections
class Comparator():
def __init__(self):
self.s1 = ""
self.s2 = ""
pass
def compare(self, res1, res2):
description1 = res1.pop("description", None)
description2 = res2.pop("description", None)
ret = self.__compareDictionnaries(res1, res2)
if not (description1 is None) or not (description2 is None):
ret = ret and self.__compareDescriptions(description1, description2)
return ret
def __compareDescriptions(self, descr1, descr2):
temp = collections.Counter(descr1)
return temp == collections.Counter(descr2)
def __compareDictionnaries(self, res1, res2):
ret = True
# Dictionnary Testing
if isinstance(res1, dict) and isinstance(res2, dict):
# Dictionnary case
if len(res1) == len(res2):
for key in res1.keys():
if key in res2.keys():
ret = ret and self.__compareDictionnaries(res1[key], res2[key])
else:
ret = False
else:
ret = False
else:
# Can be anything, probe array case first
if hasattr(res1, "__len__") and hasattr(res2, "__len__") and (not isinstance(res1, str)) and (not isinstance(res2, str)) and (not isinstance(res1, unicode)) and (not isinstance(res2, unicode)):
# Array case:
try:
ret = ret and numpy.allclose(res1,res2)
except TypeError:
# Python list case
if len(res1) == len(res2):
for index in range(len(res1)):
ret = ret and self.__compareDictionnaries(res1[index], res2[index])
else:
ret = False
else:
# Single Values case
ret = ret and (res1==res2)
return ret
\ No newline at end of file
import unittest
import os
from BuildJobTests import JobFileGenerator
class JobForTest():
settings = {}
configuration = {"output_files":{"files":["./File.nc"]}}
_type = 'Test'
def set_multi_processor(self):
self.settings = {'running_mode':True}
def set_mono_processor(self):
self.settings = {}
def get_default_parameters(self):
return {}
class TestBuildJobTests(unittest.TestCase):
def test(self):
temp = os.path.join(os.path.split(__file__)[0], "Test_Test.py")
self.job = JobForTest()
self.object = JobFileGenerator(self.job)
self.assertTrue(os.path.isfile(temp))
os.remove(temp)
self.job.set_multi_processor()
self.object = JobFileGenerator(self.job)
self.assertTrue(os.path.isfile(temp))
os.remove(temp)
if __name__ == '__main__':
unittest.main(verbosity=2)
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment