Add fix for Pyro smp run

Bug fix in FloatConfigurator
Added unit tests
Any error is now caught at the Job level when running a job
Removed unused imports

MDANSE/DistributedComputing/Slave.py

@@ -30,6 +30,16 @@ Created on Mar 30, 2015
 @author: pellegrini
 '''
 
+import os
+from MDANSE import PLATFORM
+
+import Pyro
+Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
+Pyro.config.PYRO_NS_URIFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+Pyro.config.PYRO_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+Pyro.config.PYRO_USER_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+Pyro.config.PYROSSL_CERTDIR = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+
 # Define (or import) all the task handlers.
 def do_run_step(job, step):
     '''

MDANSE/Framework/Configurable.py

@@ -30,7 +30,6 @@ Created on Mar 30, 2015
 @author: pellegrini
 '''
 
-import _abcoll
 import collections
 
 from MDANSE.Core.Error import Error
@@ -55,20 +54,20 @@ class Configurable(object):
             #.. 2-value is the dictionary of the keywords used when initializing the configurator.  
     '''
     
+    settings = collections.OrderedDict()
+    
     def __init__(self):
         '''
         Constructor
         '''
                
         self._configuration = {}
-
-        settings = getattr(self,"settings",{})
         
-        if not isinstance(settings,_abcoll.Mapping):
+        if not isinstance(self.settings,dict):
             raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
 
         self._configurators = {}
-        for name,(typ,kwds) in settings.items():
+        for name,(typ,kwds) in self.settings.items():
 
             try:
                 self._configurators[name] = REGISTRY["configurator"][typ](name, **kwds)
@@ -89,7 +88,15 @@ class Configurable(object):
         """
                 
         return self._configuration.setdefault(name,{})
+    
+    @classmethod
+    def set_settings(cls, settings):
+        
+        cls.settings.clear()
         
+        if isinstance(settings,dict):
+            cls.settings.update(settings)        
+    
     def setup(self,parameters):
         '''
         Setup the configuration according to a set of input parameters.
@@ -102,7 +109,7 @@ class Configurable(object):
         self._configuration.clear()
 
         self._configured=False
-                
+                        
         # If no configurator has to be configured, just return
         if not self._configurators:
             self._configured=True
@@ -121,7 +128,7 @@ class Configurable(object):
         configured = set()
                 
         while toBeConfigured != configured:
-            
+                        
             progress = False
 
             for name,conf in self._configurators.items():
@@ -177,7 +184,7 @@ class Configurable(object):
               
         settings = getattr(cls,"settings",{})
         
-        if not isinstance(settings,_abcoll.Mapping):
+        if not isinstance(settings,dict):
             raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
                                     
         doclist = []
@@ -243,7 +250,7 @@ class Configurable(object):
         
         settings = getattr(cls,"settings",{})
         
-        if not isinstance(settings,_abcoll.Mapping):
+        if not isinstance(settings,dict):
             raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
                                     
         params = collections.OrderedDict()

MDANSE/Framework/Configurators/AtomSelectionConfigurator.py

@@ -35,7 +35,7 @@ import operator
 
 import numpy
 
-from MDANSE.Framework.UserDefinitions.IUserDefinition import UD_STORE
+from MDANSE.Framework.UserDefinitions.IUserDefinition import UD_STORE, UserDefinitionError
 from MDANSE.Framework.Configurators.IConfigurator import IConfigurator, ConfiguratorError
 from MDANSE.Framework.AtomSelectionParser import AtomSelectionParser
 
@@ -80,21 +80,23 @@ class AtomSelectionConfigurator(IConfigurator):
         '''
                           
         trajConfig = configuration[self._dependencies['trajectory']]
-                
-        if not isinstance(value,basestring):
+        
+        if value is None:
+            value = 'all'
+        elif not isinstance(value,basestring):
             raise ConfiguratorError("invalid type for atom selection. Must be a string", self)
         
         self["value"] = value
         
-        ud = UD_STORE[trajConfig["basename"],"atom_selection",value]        
-        # The input value is a user definition: get it and update the configuration
-        if ud is not None:
-            self.update(ud)
+        try:
+            ud = UD_STORE[trajConfig["basename"],"atom_selection",value]        
         # The input value is an atom selection string: parse it and update the configuration
-        else:
+        except UserDefinitionError:
             parser = AtomSelectionParser(trajConfig["instance"])
             self["indexes"] = parser.parse(value)
             self["expression"] = value
+        else:
+            self.update(ud)
 
         self["n_selected_atoms"] = len(self["indexes"])
         atoms = sorted(trajConfig["universe"].atomList(), key = operator.attrgetter('index'))

MDANSE/Framework/Configurators/FloatConfigurator.py

@@ -64,7 +64,7 @@ class FloatConfigurator(IConfigurator):
         
         self._choices = choices if choices is not None else []
 
-    def configure(self, value):
+    def configure(self, configuration, value):
         '''
         Configure an input value.
                 

MDANSE/Framework/Configurators/RunningModeConfigurator.py

@@ -30,6 +30,8 @@ Created on May 22, 2015
 @author: Eric C. Pellegrini
 '''
 
+import os
+
 from MDANSE import PLATFORM
 from MDANSE.Framework.Configurators.IConfigurator import IConfigurator, ConfiguratorError
                         
@@ -72,7 +74,13 @@ class RunningModeConfigurator(IConfigurator):
         else:
 
             import Pyro
-            Pyro.config.PYRO_STORAGE=PLATFORM.home_directory()
+
+            Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
+            Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
+            Pyro.config.PYRO_NS_URIFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+            Pyro.config.PYRO_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+            Pyro.config.PYRO_USER_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
+            Pyro.config.PYROSSL_CERTDIR = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
 
             slots = int(value[1])
                         

MDANSE/Framework/Jobs/IJob.py

@@ -46,7 +46,31 @@ from MDANSE.Framework.Jobs.JobStatus import JobStatus
 from MDANSE.Framework.OutputVariables.IOutputVariable import OutputData
 
 class JobError(Error):
-    pass
+    '''
+    This class handles any exception related to IJob-derived objects
+    '''
+    
+    def __init__(self,job,message=None):
+        '''
+        Initializes the the object.
+        
+        :param job: the configurator in which the exception was raised
+        :type job: IJob derived object
+        '''
+
+        if message is None:
+            message = sys.exc_info()[1]
+            
+        self._message = str(message)
+        
+        if job._status is not None:
+            job._status._state["state"] = "crashed"
+            job._status._state["info"] += "ERROR: %s" % self._message
+            job._status.update(force=True)
+            
+    def __str__(self):
+        
+        return self._message
 
 def key_generator(size=4, chars=None, prefix=""):
     
@@ -130,12 +154,7 @@ class IJob(Configurable):
             #. parameters (dict): optional. If not None, the parameters with which the job file will be built.
         """
          
-        # Try to open the output test file to see whether it is allowed.
-        try:
-            f = open(testFile, 'w')
-        # Case where for whatever reason, the file could not be opened for writing. Return.        
-        except IOError as e:
-            raise JobError(["Error when saving python batch file.",e])
+        f = open(testFile, 'w')
            
         # The first line contains the call to the python executable. This is necessary for the file to
         # be autostartable.
@@ -249,14 +268,8 @@ class IJob(Configurable):
             #. parameters (dict): optional. If not None, the parameters with which the job file will be built.
         """
                          
-        # Try to open the output job file to see whether it is allowed.
-        try:
-            f = open(jobFile, 'w')
-        
-        # Case where for whatever reason, the file could not be opened for writing. Return.        
-        except IOError as e:
-            raise JobError(["Error when saving python batch file.",e])
-           
+        f = open(jobFile, 'w')
+                   
         # The first line contains the call to the python executable. This is necessary for the file to
         # be autostartable.
         f.write('#!%s\n\n' % sys.executable)
@@ -290,8 +303,7 @@ class IJob(Configurable):
     
         # Sets |analysis| variable to an instance analysis to save. 
         f.write('job = REGISTRY[%r][%r](status=False)\n' % ('job',cls.type))
-        f.write('job.setup(parameters)\n')
-        f.write('job.run()')
+        f.write('job.run(parameters)')
          
         f.close()
         
@@ -305,12 +317,7 @@ class IJob(Configurable):
             #. parameters (dict): optional. If not None, the parameters with which the job file will be built.
         """
                 
-        # Try to open the output test file to see whether it is allowed.
-        try:
-            f = open(testFile, 'w')
-        # Case where for whatever reason, the file could not be opened for writing. Return.        
-        except IOError as e:
-            raise JobError(["Error when saving python batch file.",e])
+        f = open(testFile, 'w')
            
         # The first line contains the call to the python executable. This is necessary for the file to
         # be autostartable.
@@ -427,35 +434,34 @@ class IJob(Configurable):
             
     _runner = {"monoprocessor" : _run_monoprocessor, "multiprocessor" : _run_multiprocessor, "remote" : _run_remote}
 
-    def run(self, parameters=None):
+    def run(self, parameters):
         """
         Run the job.
         """
         
-        if parameters is not None:
-            self.setup(parameters)
-        
-        self.initialize()
-
-        self._info = 'Information about %s job.\n' % self._name
-        self._info += str(self)
-                                                  
-        LOGGER(self._info)
-                  
-        if getattr(self,'numberOfSteps', 0) <= 0:
-            raise JobError("Invalid number of steps for job %s" % self)
-
         try:
+            self.setup(parameters)
+            
+            self.initialize()
+    
+            self._info = 'Information about %s job.\n' % self._name
+            self._info += str(self)
+                                                      
+            LOGGER(self._info)
+                      
+            if getattr(self,'numberOfSteps', 0) <= 0:
+                raise JobError(self,"Invalid number of steps for job %s" % self._name)
+    
             mode = self.configuration['running_mode']['mode']
-        except:
-            raise JobError("Invalid running mode")
-        else:                        
+    
             IJob._runner[mode](self)
-
-        self.finalize()
-
-        if self._status is not None:
-            self._status.finish()
+    
+            self.finalize()
+    
+            if self._status is not None:
+                self._status.finish()
+        except:
+            raise JobError(self)
 
     @property
     def info(self):

MDANSE/Framework/OutputVariables/IOutputVariable.py

@@ -42,12 +42,12 @@ class OutputVariableError(Error):
 
 class OutputData(collections.OrderedDict):
     
-    def __setitem__(self,item):
+    def __setitem__(self,item,value):
         pass
     
     def add(self, dataName, dataType, data, **kwargs):
-        
-        self[dataName] = REGISTRY["output_variable"][dataType](data, dataName, **kwargs)
+                
+        collections.OrderedDict.__setitem__(self,dataName,REGISTRY["output_variable"][dataType](data, dataName, **kwargs))
     
     def write(self, basename, formats, header=None):
         

MDANSE/Framework/Status.py

@@ -135,7 +135,7 @@ class Status(object):
         self._stopped = True
         Publisher.sendMessage("status_stop",message=self)        
                 
-    def update(self):
+    def update(self,force=False):
         
         if self._updateStep == 0:
             return
@@ -145,8 +145,8 @@ class Status(object):
         lastUpdate = datetime.datetime.today()
 
         self._deltas.append(lastUpdate)
-                        
-        if (not self._currentStep % self._updateStep) or (total_seconds(lastUpdate-self._lastRefresh) > 10):
+                                
+        if force or (not self._currentStep % self._updateStep) or (total_seconds(lastUpdate-self._lastRefresh) > 10):
             
             self._lastRefresh = lastUpdate
             

MDANSE/Framework/UserDefinitions/IUserDefinition.py

@@ -209,7 +209,7 @@ class UserDefinitionStore(UnicodeDict):
             raise UserDefinitionError("Invalid key value: must be a 3-tuple")
             
         try:
-            ud = self[name]
+            ud = UnicodeDict.__getitem__(name)
             
         except (KeyError,TypeError):
             raise UserDefinitionError('The item %r could not be found' % str(name))

MDANSE/Mathematics/Signal.py

-import math
-
 import numpy
 
 from MDANSE.Core.Error import Error