Add fix for Pyro smp run

Bug fix in FloatConfigurator
Added unit tests
Any error is now caught at the Job level when running a job
Removed unused imports

MDANSE/DistributedComputing/Slave.py

@@ -30,6 +30,16 @@ Created on Mar 30, 2015
 @author: pellegrini
 '''
 
+import os
+from MDANSE import PLATFORM
+
+import Pyro
+Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
+Pyro.config.PYRO_NS_URIFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+Pyro.config.PYRO_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+Pyro.config.PYRO_USER_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+Pyro.config.PYROSSL_CERTDIR = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+
 # Define (or import) all the task handlers.
 def do_run_step(job, step):
     '''

MDANSE/Framework/Configurable.py

@@ -30,7 +30,6 @@ Created on Mar 30, 2015
 @author: pellegrini
 '''
 
-import _abcoll
 import collections
 
 from MDANSE.Core.Error import Error
@@ -55,20 +54,20 @@ class Configurable(object):
             #.. 2-value is the dictionary of the keywords used when initializing the configurator.  
     '''
     
+    settings = collections.OrderedDict()
+    
     def __init__(self):
         '''
         Constructor
         '''
                
         self._configuration = {}
-
-        settings = getattr(self,"settings",{})
         
-        if not isinstance(settings,_abcoll.Mapping):
+        if not isinstance(self.settings,dict):
             raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
 
         self._configurators = {}
-        for name,(typ,kwds) in settings.items():
+        for name,(typ,kwds) in self.settings.items():
 
             try:
                 self._configurators[name] = REGISTRY["configurator"][typ](name, **kwds)
@@ -89,7 +88,15 @@ class Configurable(object):
         """
                 
         return self._configuration.setdefault(name,{})
+    
+    @classmethod
+    def set_settings(cls, settings):
+        
+        cls.settings.clear()
         
+        if isinstance(settings,dict):
+            cls.settings.update(settings)        
+    
     def setup(self,parameters):
         '''
         Setup the configuration according to a set of input parameters.
@@ -102,7 +109,7 @@ class Configurable(object):
         self._configuration.clear()
 
         self._configured=False
-                
+                        
         # If no configurator has to be configured, just return
         if not self._configurators:
             self._configured=True
@@ -121,7 +128,7 @@ class Configurable(object):
         configured = set()
                 
         while toBeConfigured != configured:
-            
+                        
             progress = False
 
             for name,conf in self._configurators.items():
@@ -177,7 +184,7 @@ class Configurable(object):
               
         settings = getattr(cls,"settings",{})
         
-        if not isinstance(settings,_abcoll.Mapping):
+        if not isinstance(settings,dict):
             raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
                                     
         doclist = []
@@ -243,7 +250,7 @@ class Configurable(object):
         
         settings = getattr(cls,"settings",{})
         
-        if not isinstance(settings,_abcoll.Mapping):
+        if not isinstance(settings,dict):
             raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
                                     
         params = collections.OrderedDict()

MDANSE/Framework/Configurators/AtomSelectionConfigurator.py

@@ -35,7 +35,7 @@ import operator
 
 import numpy
 
-from MDANSE.Framework.UserDefinitions.IUserDefinition import UD_STORE
+from MDANSE.Framework.UserDefinitions.IUserDefinition import UD_STORE, UserDefinitionError
 from MDANSE.Framework.Configurators.IConfigurator import IConfigurator, ConfiguratorError
 from MDANSE.Framework.AtomSelectionParser import AtomSelectionParser
 
@@ -80,21 +80,23 @@ class AtomSelectionConfigurator(IConfigurator):
         '''
                           
         trajConfig = configuration[self._dependencies['trajectory']]
-                
-        if not isinstance(value,basestring):
+        
+        if value is None:
+            value = 'all'
+        elif not isinstance(value,basestring):
             raise ConfiguratorError("invalid type for atom selection. Must be a string", self)
         
         self["value"] = value
         
-        ud = UD_STORE[trajConfig["basename"],"atom_selection",value]        
-        # The input value is a user definition: get it and update the configuration
-        if ud is not None:
-            self.update(ud)
+        try:
+            ud = UD_STORE[trajConfig["basename"],"atom_selection",value]        
         # The input value is an atom selection string: parse it and update the configuration
-        else:
+        except UserDefinitionError:
             parser = AtomSelectionParser(trajConfig["instance"])
             self["indexes"] = parser.parse(value)
             self["expression"] = value
+        else:
+            self.update(ud)
 
         self["n_selected_atoms"] = len(self["indexes"])
         atoms = sorted(trajConfig["universe"].atomList(), key = operator.attrgetter('index'))

MDANSE/Framework/Configurators/FloatConfigurator.py

@@ -64,7 +64,7 @@ class FloatConfigurator(IConfigurator):
         
         self._choices = choices if choices is not None else []
 
-    def configure(self, value):
+    def configure(self, configuration, value):
         '''
         Configure an input value.
                 

MDANSE/Framework/Configurators/RunningModeConfigurator.py

@@ -30,6 +30,8 @@ Created on May 22, 2015
 @author: Eric C. Pellegrini
 '''
 
+import os
+
 from MDANSE import PLATFORM
 from MDANSE.Framework.Configurators.IConfigurator import IConfigurator, ConfiguratorError
                         
@@ -72,7 +74,13 @@ class RunningModeConfigurator(IConfigurator):
         else:
 
             import Pyro
-            Pyro.config.PYRO_STORAGE=PLATFORM.home_directory()
+
+            Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
+            Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
+            Pyro.config.PYRO_NS_URIFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+            Pyro.config.PYRO_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+            Pyro.config.PYRO_USER_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+            Pyro.config.PYROSSL_CERTDIR = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
 
             slots = int(value[1])
                         

MDANSE/Framework/Jobs/IJob.py

@@ -46,7 +46,31 @@ from MDANSE.Framework.Jobs.JobStatus import JobStatus
 from MDANSE.Framework.OutputVariables.IOutputVariable import OutputData
 
 class JobError(Error):
-    pass
+    '''
+    This class handles any exception related to IJob-derived objects
+    '''
+    
+    def __init__(self,job,message=None):
+        '''
+        Initializes the the object.
+        
+        :param job: the configurator in which the exception was raised
+        :type job: IJob derived object
+        '''
+
+        if message is None:
+            message = sys.exc_info()[1]
+            
+        self._message = str(message)
+        
+        if job._status is not None:
+            job._status._state["state"] = "crashed"
+            job._status._state["info"] += "ERROR: %s" % self._message
+            job._status.update(force=True)
+            
+    def __str__(self):
+        
+        return self._message
 
 def key_generator(size=4, chars=None, prefix=""):
     
@@ -130,12 +154,7 @@ class IJob(Configurable):
             #. parameters (dict): optional. If not None, the parameters with which the job file will be built.
         """
          
-        # Try to open the output test file to see whether it is allowed.
-        try:
-            f = open(testFile, 'w')
-        # Case where for whatever reason, the file could not be opened for writing. Return.        
-        except IOError as e:
-            raise JobError(["Error when saving python batch file.",e])
+        f = open(testFile, 'w')
            
         # The first line contains the call to the python executable. This is necessary for the file to
         # be autostartable.
@@ -249,14 +268,8 @@ class IJob(Configurable):
             #. parameters (dict): optional. If not None, the parameters with which the job file will be built.
         """
                          
-        # Try to open the output job file to see whether it is allowed.
-        try:
-            f = open(jobFile, 'w')
-        
-        # Case where for whatever reason, the file could not be opened for writing. Return.        
-        except IOError as e:
-            raise JobError(["Error when saving python batch file.",e])
-           
+        f = open(jobFile, 'w')
+                   
         # The first line contains the call to the python executable. This is necessary for the file to
         # be autostartable.
         f.write('#!%s\n\n' % sys.executable)
@@ -290,8 +303,7 @@ class IJob(Configurable):
     
         # Sets |analysis| variable to an instance analysis to save. 
         f.write('job = REGISTRY[%r][%r](status=False)\n' % ('job',cls.type))
-        f.write('job.setup(parameters)\n')
-        f.write('job.run()')
+        f.write('job.run(parameters)')
          
         f.close()
         
@@ -305,12 +317,7 @@ class IJob(Configurable):
             #. parameters (dict): optional. If not None, the parameters with which the job file will be built.
         """
                 
-        # Try to open the output test file to see whether it is allowed.
-        try:
-            f = open(testFile, 'w')
-        # Case where for whatever reason, the file could not be opened for writing. Return.        
-        except IOError as e:
-            raise JobError(["Error when saving python batch file.",e])
+        f = open(testFile, 'w')
            
         # The first line contains the call to the python executable. This is necessary for the file to
         # be autostartable.
@@ -427,35 +434,34 @@ class IJob(Configurable):
             
     _runner = {"monoprocessor" : _run_monoprocessor, "multiprocessor" : _run_multiprocessor, "remote" : _run_remote}
 
-    def run(self, parameters=None):
+    def run(self, parameters):
         """
         Run the job.
         """
         
-        if parameters is not None:
-            self.setup(parameters)
-        
-        self.initialize()
-
-        self._info = 'Information about %s job.\n' % self._name
-        self._info += str(self)
-                                                  
-        LOGGER(self._info)
-                  
-        if getattr(self,'numberOfSteps', 0) <= 0:
-            raise JobError("Invalid number of steps for job %s" % self)
-
         try:
+            self.setup(parameters)
+            
+            self.initialize()
+    
+            self._info = 'Information about %s job.\n' % self._name
+            self._info += str(self)
+                                                      
+            LOGGER(self._info)
+                      
+            if getattr(self,'numberOfSteps', 0) <= 0:
+                raise JobError(self,"Invalid number of steps for job %s" % self._name)
+    
             mode = self.configuration['running_mode']['mode']
-        except:
-            raise JobError("Invalid running mode")
-        else:                        
+    
             IJob._runner[mode](self)
-
-        self.finalize()
-
-        if self._status is not None:
-            self._status.finish()
+    
+            self.finalize()
+    
+            if self._status is not None:
+                self._status.finish()
+        except:
+            raise JobError(self)
 
     @property
     def info(self):

MDANSE/Framework/OutputVariables/IOutputVariable.py

@@ -42,12 +42,12 @@ class OutputVariableError(Error):
 
 class OutputData(collections.OrderedDict):
     
-    def __setitem__(self,item):
+    def __setitem__(self,item,value):
         pass
     
     def add(self, dataName, dataType, data, **kwargs):
-        
-        self[dataName] = REGISTRY["output_variable"][dataType](data, dataName, **kwargs)
+                
+        collections.OrderedDict.__setitem__(self,dataName,REGISTRY["output_variable"][dataType](data, dataName, **kwargs))
     
     def write(self, basename, formats, header=None):
         

MDANSE/Framework/Status.py

@@ -135,7 +135,7 @@ class Status(object):
         self._stopped = True
         Publisher.sendMessage("status_stop",message=self)        
                 
-    def update(self):
+    def update(self,force=False):
         
         if self._updateStep == 0:
             return
@@ -145,8 +145,8 @@ class Status(object):
         lastUpdate = datetime.datetime.today()
 
         self._deltas.append(lastUpdate)
-                        
-        if (not self._currentStep % self._updateStep) or (total_seconds(lastUpdate-self._lastRefresh) > 10):
+                                
+        if force or (not self._currentStep % self._updateStep) or (total_seconds(lastUpdate-self._lastRefresh) > 10):
             
             self._lastRefresh = lastUpdate
             

MDANSE/Framework/UserDefinitions/IUserDefinition.py

@@ -209,7 +209,7 @@ class UserDefinitionStore(UnicodeDict):
             raise UserDefinitionError("Invalid key value: must be a 3-tuple")
             
         try:
-            ud = self[name]
+            ud = UnicodeDict.__getitem__(name)
             
         except (KeyError,TypeError):
             raise UserDefinitionError('The item %r could not be found' % str(name))

MDANSE/Mathematics/Signal.py

-import math
-
 import numpy
 
 from MDANSE.Core.Error import Error

Tests/UnitTests/TestConfigurator.py

+#MDANSE : Molecular Dynamics Analysis for Neutron Scattering Experiments
+#------------------------------------------------------------------------------------------
+#Copyright (C)
+#2015- Eric C. Pellegrini Institut Laue-Langevin
+#BP 156
+#6, rue Jules Horowitz
+#38042 Grenoble Cedex 9
+#France
+#pellegrini[at]ill.fr
+#goret[at]ill.fr
+#aoun[at]ill.fr
+#
+#This library is free software; you can redistribute it and/or
+#modify it under the terms of the GNU Lesser General Public
+#License as published by the Free Software Foundation; either
+#version 2.1 of the License, or (at your option) any later version.
+#
+#This library is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#Lesser General Public License for more details.
+#
+#You should have received a copy of the GNU Lesser General Public
+#License along with this library; if not, write to the Free Software
+#Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+''' 
+Created on May 29, 2015
+
+@author: Eric C. Pellegrini
+'''
+
+import os
+import unittest
+
+import numpy
+
+from MMTK.Trajectory import Trajectory
+
+from MDANSE.Framework.Configurable import Configurable
+from MDANSE.Framework.Configurators.IConfigurator import ConfiguratorError
+from MDANSE.Framework.Projectors.IProjector import ProjectorError
+from MDANSE.Framework.AtomSelectionParser import AtomSelectionParserError
+from UnitTest import UnitTest
+
+TRAJECTORIES_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),"UserData","Trajectories")
+
+class TestConfigurator(UnitTest):
+    '''
+    Unittest for the configurators used to setup an analysis in nMolDyn
+    '''
+
+    def setUp(self):
+                
+        self._configurableClass = Configurable
+        self._validTrajectory = Trajectory(None, os.path.join('..','..','Data','Trajectories','MMTK','protein_in_periodic_universe.nc'), "r")
+        self._parameters = {}
+
+    def tearDown(self):
+        
+        self._configurableClass.settings.clear()
+        self._parameters.clear()
+        
+    def test_integer(self):
+        '''
+        Test the integer configurator
+        '''
+ 
+        self._configurableClass.set_settings({"test_integer":('integer',{})})
+        
+        configurable = self._configurableClass()
+  
+        # Case of a valid integer
+        self._parameters["test_integer"] = 20
+        self.assertNotRaises(configurable.setup,self._parameters)
+                  
+        # Case of a float that will casted to an integer
+        self._parameters["test_integer"] = 20.2
+        self.assertNotRaises(configurable.setup,self._parameters)
+        self.assertEqual(configurable["test_integer"]["value"], 20)
+ 
+        # Case of a string that can be casted to an integer
+        self._parameters["test_integer"] = "30"
+        self.assertNotRaises(configurable.setup,self._parameters)
+        self.assertEqual(configurable["test_integer"]["value"], 30)
+         
+        # Case of a string that cannot be casted to an integer
+        self._parameters["test_integer"] = "xxxx"
+        self.assertRaises(ConfiguratorError, configurable.setup,self._parameters)
+ 
+        # Case of an object that cannot be casted to an integer
+        self._parameters["test_integer"] = [1,2]
+        self.assertRaises(ConfiguratorError, configurable.setup,self._parameters)
+ 
+    def test_float(self):
+        '''
+        Test the float configurator
+        '''
+ 
+        self._configurableClass.set_settings({"test_float":('float',{})})
+ 
+        configurable = self._configurableClass()
+ 
+        # Case of an integer that will be casted to a float
+        self._parameters["test_float"] = 20
+        self.assertNotRaises(configurable.setup,self._parameters)
+        self.assertEqual(configurable["test_float"]["value"], 20.0)
+         
+        # Case of a float
+        self._parameters["test_float"] = 20.2
+        self.assertNotRaises(configurable.setup,self._parameters)
+        self.assertEqual(configurable["test_float"]["value"], 20.2)
+ 
+        # Case of a string that can be casted to a float
+        self._parameters["test_float"] = "30.2"
+        self.assertNotRaises(configurable.setup,self._parameters)
+        self.assertEqual(configurable["test_float"]["value"], 30.2)
+         
+        # Case of a string that cannot be casted to a float
+        self._parameters["test_float"] = "xxxx"
+        self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
+ 
+        # Case of an object that cannot be casted to a float
+        self._parameters["test_float"] = [1,2]
+        self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
+ 
+    def test_mmtk_trajectory(self):
+        '''
+        Test the mmtk_trajectory configurator
+        '''
+ 
+        self._configurableClass.set_settings({"trajectory":('mmtk_trajectory',{})})
+ 
+        configurable = self._configurableClass()
+          
+        # Case of a valid trajectory
+        self._parameters["trajectory"] = self._validTrajectory.filename
+        self.assertNotRaises(configurable.setup,self._parameters)
+         
+        # Case of an unknown trajectory
+        self._parameters["trajectory"] = 'fsfsdjkfjkfjs'
+        self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
+         
+        # Case of an invalid type for input trajectory
+        self._parameters["trajectory"] = 1
+        self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
+ 
+        # Case of an invalid type for input trajectory
+        self._parameters["trajectory"] = [1,2,3]
+        self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
+                  
+    def test_projection(self):
+ 
+        self._configurableClass.set_settings({"projection":('projection',{})})
+ 
+        configurable = self._configurableClass()
+ 
+        data = numpy.random.uniform(0,1,(10,3))
+ 
+        # Wrong parameters
+        self._parameters["projection"] = 10
+        self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
+ 
+        self._parameters["projection"] = [1,2,3,4]
+        self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
+ 
+        # No projection - wrong projection type
+        self._parameters["projection"] = (30,None)
+        self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
+         
+        # No projection
+        self._parameters["projection"] = None
+        self.assertNotRaises(configurable.setup,self._parameters)
+        proj = configurable["projection"]['projector'](data)
+        self.assertTrue(numpy.array_equal(data,proj))
+ 
+        self._parameters["projection"] = ('null',None)
+        self.assertNotRaises(configurable.setup,self._parameters)
+        proj = configurable["projection"]['projector'](data)
+        self.assertTrue(numpy.array_equal(data,proj))
+         
+        # Axial projection - wrong parameters
+        self._parameters["projection"] = ('tutu',(1,0,0))
+        self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
+        self._parameters["projection"] = ('axial',(1,0,0,2,12,25))
+        self.assertRaises(ProjectorError, configurable.setup, self._parameters)
+        self._parameters["projection"] = ('axial',30)
+        self.assertRaises(ProjectorError, configurable.setup, self._parameters)
+ 
+        # Axial projection - null vector
+        self._parameters["projection"] = ('axial',(0,0,0))
+        self.assertRaises(ProjectorError, configurable.setup, self._parameters)
+ 
+        # Axial projection
+        self._parameters["projection"] = ('axial',(1,0,0))
+        self.assertNotRaises(configurable.setup,self._parameters)
+        proj = configurable["projection"]['projector'](data)
+        self.assertTrue(numpy.array_equal(data[:,0],proj[:,0]))
+ 
+        # Axial projection - wrong data
+        self._parameters["projection"] = ('axial',(1,0,0))
+        self.assertNotRaises(configurable.setup,self._parameters)
+        self.assertRaises(ProjectorError, configurable["projection"]['projector'].__call__,None)
+        self.assertRaises(ProjectorError, configurable["projection"]['projector'].__call__,[1])
+ 
+        # Planar projection - wrong parameters
+        self._parameters["projection"] = ('tutu',(1,0,0))
+        self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
+        self._parameters["projection"] = ('planar',(1,0,0,2,99,123))
+        self.assertRaises(ProjectorError, configurable.setup, self._parameters)
+        self._parameters["projection"] = ('planar',30)
+        self.assertRaises(ProjectorError, configurable.setup, self._parameters)
+ 
+        # Planar projection - null vector
+        self._parameters["projection"] = ('planar',(0,0,0))
+        self.assertRaises(ProjectorError,configurable.setup, self._parameters)
+ 
+        # Planar projection
+        self._parameters["projection"] = ('planar',(1,0,0))
+        self.assertNotRaises(configurable.setup,self._parameters)
+        proj = configurable["projection"]['projector'](data)
+        self.assertTrue(numpy.array_equal(numpy.zeros((data.shape[0],), dtype=numpy.float64),proj[:,0]))
+        self.assertTrue(numpy.array_equal(data[:,1],proj[:,1]))
+        self.assertTrue(numpy.array_equal(data[:,2],proj[:,2]))
+ 
+        # Planar projection - wrong data
+        self._parameters["projection"] = ('planar',(1,0,0))
+        self.assertNotRaises(configurable.setup,self._parameters)
+        self.assertRaises(ProjectorError, configurable["projection"]['projector'].__call__,None)