Modification of test files, adding of reference data files, modification of...

Modification of test files, adding of reference data files, modification of GMFT-minor changes-, RBT-suppress multiprocessor mode- and graph-ensure repeatability-

BuildServer/Debian/run_tests.sh

@@ -13,17 +13,18 @@ nosetests --verbosity=3 -P .
 if [ $? -ne 0 ]; then
 	status = $?
 	echo -e "$ROUGE""One or several unit tests failed"
-	exit status
+	exit $status
 fi
 cd ../..
 
 # Performs the functional tests
 cd Tests/FunctionalTests/Jobs
+rm -rf Test_*
 python BuildJobTests.py
 nosetests --verbosity=3 --exe -P .
 if [ $? -ne 0 ]; then
 	status=$?
 	echo -e "$ROUGE""One or several functional tests failed"
-	exit status
+	exit $status
 fi
 

MDANSE/Framework/Jobs/GlobalMotionFilteredTrajectory.py

@@ -81,6 +81,9 @@ class GlobalMotionFilteredTrajectory(IJob):
 
         self.numberOfSteps = self.configuration['frames']['number']
         
+        # Store universe name for further restoration
+        self.old_universe_name = self.configuration['trajectory']['instance'].universe.__class__.__name__
+
         self.configuration['trajectory']['instance'].universe.__class__.__name__ = 'InfiniteUniverse'
         
         self.configuration['trajectory']['instance'].universe._descriptionArguments = lambda: '()'
@@ -197,4 +200,7 @@ class GlobalMotionFilteredTrajectory(IJob):
         # The output trajectory is closed.
         self._gmft.close()
         
+        # Restore universe name
+        self.configuration['trajectory']['instance'].universe.__class__.__name__ = self.old_universe_name
+        
 REGISTRY['gmft'] = GlobalMotionFilteredTrajectory

MDANSE/Framework/Jobs/IJob.py

@@ -139,71 +139,6 @@ class IJob(Configurable):
         
         self._status = None
                                             
-    @classmethod
-    def build_parallelization_test(cls, testFile, parameters=None):
-        """
-        Produce a file like object for a given job.\n
-        :Parameters:
-            #. parameters (dict): optional. If not None, the parameters with which the job file will be built.
-        """
-         
-        f = open(testFile, 'w')
-           
-        # The first line contains the call to the python executable. This is necessary for the file to
-        # be autostartable.
-        f.write('#!%s\n\n' % sys.executable)
-
-        f.write('import os\n')
-        f.write('import unittest\n')
-        f.write('import numpy\n')
-        f.write('from Scientific.IO.NetCDF import NetCDFFile\n')
-        f.write('from Tests.UnitTest import UnitTest\n')
-        f.write('from MDANSE import REGISTRY\n\n')
-                
-        f.write('class Test%sParallel(UnitTest):\n\n' % cls._type.upper())
-                    
-        f.write('    def test(self):\n')      
-
-        # Writes the line that will initialize the |parameters| dictionary.
-        f.write('        parameters = {}\n')
-        
-        for k, v in sorted(parameters.items()):
-            f.write('        parameters[%r] = %r\n' % (k, v))
-                
-        f.write('\n        job = REGISTRY[%r](%r)\n\n' % ('job',cls._type))
-
-        f.write('        parameters["running_mode"] = ("monoprocessor",1)\n')
-        f.write('        self.assertNotRaises(job.run,parameters,False)\n\n')
-
-        f.write('        f = NetCDFFile(job.configuration["output_files"]["files"][0],"r")\n')
-        f.write('        resMono = {}\n')
-        f.write('        for k,v in f.variables.items():\n')
-        f.write('            resMono[k] = v.getValue()\n')
-        f.write('        f.close()\n\n')
-
-        f.write('        parameters["running_mode"] = ("multiprocessor",2)\n')
-        f.write('        self.assertNotRaises(job.run,parameters,False)\n\n')
-
-        f.write('        f = NetCDFFile(job.configuration["output_files"]["files"][0],"r")\n')
-        f.write('        resMulti = {}\n')
-        f.write('        for k,v in f.variables.items():\n')
-        f.write('            resMulti[k] = v.getValue()\n')
-        f.write('        f.close()\n\n')
-
-        f.write('        for k in resMono.keys():\n')
-        f.write('            self.assertTrue(numpy.allclose(resMono[k],resMulti[k]))\n\n')
-            
-        f.write('def suite():\n')
-        f.write('    loader = unittest.TestLoader()\n')
-        f.write('    s = unittest.TestSuite()\n')
-        f.write('    s.addTest(loader.loadTestsFromTestCase(Test%sParallel))\n' % cls._type.upper())
-        f.write('    return s\n\n')
-        f.write('if __name__ == "__main__":\n')
-        f.write('    unittest.main(verbosity=2)\n')
-        
-        f.close()
-        
-        os.chmod(testFile,stat.S_IRWXU)
   
     @staticmethod
     def set_pyro_server():
@@ -299,53 +234,6 @@ class IJob(Configurable):
         
         os.chmod(jobFile,stat.S_IRWXU)
         
-    @classmethod
-    def build_test(cls, testFile, parameters=None):
-        """
-        Produce a file like object for a given job.\n
-        :Parameters:
-            #. parameters (dict): optional. If not None, the parameters with which the job file will be built.
-        """
-                
-        f = open(testFile, 'w')
-           
-        # The first line contains the call to the python executable. This is necessary for the file to
-        # be autostartable.
-        f.write('#!%s\n\n' % sys.executable)
-
-        f.write('import unittest\n')
-        f.write('from Tests.UnitTests.UnitTest import UnitTest\n')
-        f.write('from MDANSE import REGISTRY\n\n')
-                
-        f.write('class Test%s(UnitTest):\n\n' % cls._type.upper())
-        
-        f.write('    def test(self):\n')      
-
-        # Writes the line that will initialize the |parameters| dictionary.
-        f.write('        parameters = {}\n')
-        
-        if parameters is None:
-            parameters = cls.get_default_parameters()
-        
-        for k, v in sorted(parameters.items()):
-            f.write('        parameters[%r] = %r\n' % (k, v))
-    
-        # Sets |analysis| variable to an instance analysis to save. 
-        f.write('        job = REGISTRY[%r][%r]()\n' % ('job',cls._type))
-        f.write('        self.assertNotRaises(job.run, parameters, status=False)\n\n')
-        
-        f.write('def suite():\n')
-        f.write('    loader = unittest.TestLoader()\n')
-        f.write('    s = unittest.TestSuite()\n')
-        f.write('    s.addTest(loader.loadTestsFromTestCase(Test%s))\n' % cls._type.upper())
-        f.write('    return s\n\n')
-        f.write("if __name__ == '__main__':\n")
-        f.write('    unittest.main(verbosity=2)\n')
-        
-        f.close()
-        
-        os.chmod(testFile,stat.S_IRWXU)
-        
     def _run_monoprocessor(self):
 
         for index in range(self.numberOfSteps):
@@ -380,7 +268,8 @@ class IJob(Configurable):
             if self._status is not None:
                 if self._status.is_stopped():
                     self._status.cleanup()
-                    return
+                    # Break to ensure the master will be shutdowned
+                    break
                 else:
                     self._status.update()
             

MDANSE/Framework/Jobs/RigidBodyTrajectory.py

@@ -65,7 +65,6 @@ class RigidBodyTrajectory(IJob):
     settings['reference']=('integer',{"mini":0})
     settings['remove_translation']=('boolean',{'default':False})
     settings['output_files']=('output_files', {"formats":["netcdf"]})
-    settings['running_mode']=('running_mode',{})
     
     def initialize(self):
         """

MDANSE/Mathematics/Graph.py

@@ -30,6 +30,8 @@ Created on Apr 13, 2015
 :author: Eric C. Pellegrini
 '''
 
+import collections
+
 class Node(object):
     
     def __init__(self, name, **kwargs):
@@ -44,7 +46,7 @@ class Node(object):
 
     @property
     def links(self):
-        return set(self._links)
+        return self._links
     
     def add_link(self, other):
         self._links.add(other)
@@ -54,18 +56,16 @@ class Graph(object):
     
     def __init__(self):
         
-        self._nodes = {}
+        self._nodes = collections.OrderedDict()
         
     @property
     def nodes(self):
         return self._nodes
     
     def add_node(self, name, **kwargs):
-        
         self._nodes[name] = Node(name, **kwargs)
         
     def add_link(self, source, target):
-        
         self._nodes[source].add_link(self._nodes[target])
         
     def build_connected_components(self):
@@ -74,7 +74,7 @@ class Graph(object):
         result = []
 
         # Make a copy of the set, so we can modify it.
-        nodes = set(self._nodes.values())
+        nodes = [self._nodes[k] for k in sorted(self._nodes.keys())]
 
         # Iterate while we still have nodes to process.
         while nodes:
@@ -102,7 +102,9 @@ class Graph(object):
                 neighbors.difference_update(group)
     
                 # Remove the remaining nodes from the global set.
-                nodes.difference_update(neighbors)
+                for neigh in neighbors:
+                    if neigh in nodes:
+                        nodes.remove(neigh)
     
                 # Add them to the group of connected nodes.
                 group.update(neighbors)
@@ -110,6 +112,10 @@ class Graph(object):
                 # Add them to the queue, so we visit them in the next iterations.
                 queue.extend(neighbors)
     
+            # Sort the group
+            group = list(group)
+            group.sort(key = lambda n : n.name)
+
             # Add the group to the list of groups.
             result.append(group)
     

Tests/FunctionalTests/Jobs/AllTests.py

@@ -35,16 +35,18 @@ import os
 import glob
 
 def suite():
-    files = glob.glob('Test*.py')
+    files = glob.glob("Test*.py")
+    #os.chdir("Jobs")
     modules = [__import__(os.path.splitext(f)[0],globals(),locals(),[],-1) for f in files]
     test_suite = unittest.TestSuite()
     for m in modules:
+        print m.__file__
         test_suite.addTests(m.suite())
-
     return test_suite
 
 def run_test():
     unittest.TextTestRunner(verbosity=2).run(suite())
 
 if __name__ == '__main__':
+    #os.chdir("Jobs")
     run_test()

Tests/FunctionalTests/Jobs/Comparator.py

+import numpy
+import collections
+
+class Comparator():
+    def __init__(self):
+        self.s1 = ""
+        self.s2 = ""
+        pass
+
+    def compare(self, res1, res2):
+        description1 = res1.pop("description", None)
+        description2 = res2.pop("description", None)
+        
+        ret = self.__compareDictionnaries(res1, res2)
+        if not (description1 is None) or not (description2 is None):
+            ret = ret and self.__compareDescriptions(description1, description2)
+        return ret
+
+    def __compareDescriptions(self, descr1, descr2):
+        temp = collections.Counter(descr1)
+        return temp == collections.Counter(descr2)
+    
+    def __compareDictionnaries(self, res1, res2):
+        ret = True
+        # Dictionnary Testing
+        if isinstance(res1, dict) and isinstance(res2, dict):
+            # Dictionnary case
+            if len(res1) == len(res2):
+                for key in res1.keys():
+                    if key in res2.keys():       
+                        ret = ret and self.__compareDictionnaries(res1[key], res2[key])
+                    else:
+                        ret = False
+            else:
+                ret = False
+        else:
+            # Can be anything, probe array case first
+            if hasattr(res1, "__len__") and hasattr(res2, "__len__") and (not isinstance(res1, str)) and (not isinstance(res2, str)) and (not isinstance(res1, unicode)) and (not isinstance(res2, unicode)):
+                # Array case:
+                try:
+                    ret = ret and numpy.allclose(res1,res2)
+                except TypeError:
+                    # Python list case    
+                    if len(res1) == len(res2):
+                        for index in range(len(res1)):
+                            ret = ret and self.__compareDictionnaries(res1[index], res2[index])
+                    else:
+                        ret = False
+            else:
+                # Single Values case
+                ret = ret and (res1==res2)
+        return ret
\ No newline at end of file

Tests/FunctionalTests/Jobs/test_BuildJobTests.py

+import unittest
+import os
+from BuildJobTests import JobFileGenerator
+
+class JobForTest():
+    settings = {}
+    configuration = {"output_files":{"files":["./File.nc"]}}
+    _type = 'Test'
+    
+    def set_multi_processor(self):
+        self.settings = {'running_mode':True}
+    
+    def set_mono_processor(self):
+        self.settings = {}
+        
+    def get_default_parameters(self):
+        return {}
+
+class TestBuildJobTests(unittest.TestCase):
+    def test(self):
+        temp = os.path.join(os.path.split(__file__)[0], "Test_Test.py")
+        self.job = JobForTest()
+        self.object = JobFileGenerator(self.job)
+        self.assertTrue(os.path.isfile(temp))
+        os.remove(temp)
+        self.job.set_multi_processor()
+        self.object = JobFileGenerator(self.job)
+        self.assertTrue(os.path.isfile(temp))
+        os.remove(temp)
+        
+if __name__ == '__main__':
+    unittest.main(verbosity=2)
\ No newline at end of file