Commit a5ef8cf8 authored by eric pellegrini's avatar eric pellegrini

Add fix for Pyro smp run

Bug fix in FloatConfigurator
Added unit tests
Any error is now caught at the Job level when running a job
Removed unused imports
parent d2f3271c
...@@ -30,6 +30,16 @@ Created on Mar 30, 2015 ...@@ -30,6 +30,16 @@ Created on Mar 30, 2015
@author: pellegrini @author: pellegrini
''' '''
import os
from MDANSE import PLATFORM
import Pyro
Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
Pyro.config.PYRO_NS_URIFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYRO_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYRO_USER_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYROSSL_CERTDIR = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
# Define (or import) all the task handlers. # Define (or import) all the task handlers.
def do_run_step(job, step): def do_run_step(job, step):
''' '''
......
...@@ -30,7 +30,6 @@ Created on Mar 30, 2015 ...@@ -30,7 +30,6 @@ Created on Mar 30, 2015
@author: pellegrini @author: pellegrini
''' '''
import _abcoll
import collections import collections
from MDANSE.Core.Error import Error from MDANSE.Core.Error import Error
...@@ -55,20 +54,20 @@ class Configurable(object): ...@@ -55,20 +54,20 @@ class Configurable(object):
#.. 2-value is the dictionary of the keywords used when initializing the configurator. #.. 2-value is the dictionary of the keywords used when initializing the configurator.
''' '''
settings = collections.OrderedDict()
def __init__(self): def __init__(self):
''' '''
Constructor Constructor
''' '''
self._configuration = {} self._configuration = {}
settings = getattr(self,"settings",{})
if not isinstance(settings,_abcoll.Mapping): if not isinstance(self.settings,dict):
raise ConfigurationError("Invalid type for settings: must be a mapping-like object") raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
self._configurators = {} self._configurators = {}
for name,(typ,kwds) in settings.items(): for name,(typ,kwds) in self.settings.items():
try: try:
self._configurators[name] = REGISTRY["configurator"][typ](name, **kwds) self._configurators[name] = REGISTRY["configurator"][typ](name, **kwds)
...@@ -89,7 +88,15 @@ class Configurable(object): ...@@ -89,7 +88,15 @@ class Configurable(object):
""" """
return self._configuration.setdefault(name,{}) return self._configuration.setdefault(name,{})
@classmethod
def set_settings(cls, settings):
cls.settings.clear()
if isinstance(settings,dict):
cls.settings.update(settings)
def setup(self,parameters): def setup(self,parameters):
''' '''
Setup the configuration according to a set of input parameters. Setup the configuration according to a set of input parameters.
...@@ -102,7 +109,7 @@ class Configurable(object): ...@@ -102,7 +109,7 @@ class Configurable(object):
self._configuration.clear() self._configuration.clear()
self._configured=False self._configured=False
# If no configurator has to be configured, just return # If no configurator has to be configured, just return
if not self._configurators: if not self._configurators:
self._configured=True self._configured=True
...@@ -121,7 +128,7 @@ class Configurable(object): ...@@ -121,7 +128,7 @@ class Configurable(object):
configured = set() configured = set()
while toBeConfigured != configured: while toBeConfigured != configured:
progress = False progress = False
for name,conf in self._configurators.items(): for name,conf in self._configurators.items():
...@@ -177,7 +184,7 @@ class Configurable(object): ...@@ -177,7 +184,7 @@ class Configurable(object):
settings = getattr(cls,"settings",{}) settings = getattr(cls,"settings",{})
if not isinstance(settings,_abcoll.Mapping): if not isinstance(settings,dict):
raise ConfigurationError("Invalid type for settings: must be a mapping-like object") raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
doclist = [] doclist = []
...@@ -243,7 +250,7 @@ class Configurable(object): ...@@ -243,7 +250,7 @@ class Configurable(object):
settings = getattr(cls,"settings",{}) settings = getattr(cls,"settings",{})
if not isinstance(settings,_abcoll.Mapping): if not isinstance(settings,dict):
raise ConfigurationError("Invalid type for settings: must be a mapping-like object") raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
params = collections.OrderedDict() params = collections.OrderedDict()
......
...@@ -35,7 +35,7 @@ import operator ...@@ -35,7 +35,7 @@ import operator
import numpy import numpy
from MDANSE.Framework.UserDefinitions.IUserDefinition import UD_STORE from MDANSE.Framework.UserDefinitions.IUserDefinition import UD_STORE, UserDefinitionError
from MDANSE.Framework.Configurators.IConfigurator import IConfigurator, ConfiguratorError from MDANSE.Framework.Configurators.IConfigurator import IConfigurator, ConfiguratorError
from MDANSE.Framework.AtomSelectionParser import AtomSelectionParser from MDANSE.Framework.AtomSelectionParser import AtomSelectionParser
...@@ -80,21 +80,23 @@ class AtomSelectionConfigurator(IConfigurator): ...@@ -80,21 +80,23 @@ class AtomSelectionConfigurator(IConfigurator):
''' '''
trajConfig = configuration[self._dependencies['trajectory']] trajConfig = configuration[self._dependencies['trajectory']]
if not isinstance(value,basestring): if value is None:
value = 'all'
elif not isinstance(value,basestring):
raise ConfiguratorError("invalid type for atom selection. Must be a string", self) raise ConfiguratorError("invalid type for atom selection. Must be a string", self)
self["value"] = value self["value"] = value
ud = UD_STORE[trajConfig["basename"],"atom_selection",value] try:
# The input value is a user definition: get it and update the configuration ud = UD_STORE[trajConfig["basename"],"atom_selection",value]
if ud is not None:
self.update(ud)
# The input value is an atom selection string: parse it and update the configuration # The input value is an atom selection string: parse it and update the configuration
else: except UserDefinitionError:
parser = AtomSelectionParser(trajConfig["instance"]) parser = AtomSelectionParser(trajConfig["instance"])
self["indexes"] = parser.parse(value) self["indexes"] = parser.parse(value)
self["expression"] = value self["expression"] = value
else:
self.update(ud)
self["n_selected_atoms"] = len(self["indexes"]) self["n_selected_atoms"] = len(self["indexes"])
atoms = sorted(trajConfig["universe"].atomList(), key = operator.attrgetter('index')) atoms = sorted(trajConfig["universe"].atomList(), key = operator.attrgetter('index'))
......
...@@ -64,7 +64,7 @@ class FloatConfigurator(IConfigurator): ...@@ -64,7 +64,7 @@ class FloatConfigurator(IConfigurator):
self._choices = choices if choices is not None else [] self._choices = choices if choices is not None else []
def configure(self, value): def configure(self, configuration, value):
''' '''
Configure an input value. Configure an input value.
......
...@@ -30,6 +30,8 @@ Created on May 22, 2015 ...@@ -30,6 +30,8 @@ Created on May 22, 2015
@author: Eric C. Pellegrini @author: Eric C. Pellegrini
''' '''
import os
from MDANSE import PLATFORM from MDANSE import PLATFORM
from MDANSE.Framework.Configurators.IConfigurator import IConfigurator, ConfiguratorError from MDANSE.Framework.Configurators.IConfigurator import IConfigurator, ConfiguratorError
...@@ -72,7 +74,13 @@ class RunningModeConfigurator(IConfigurator): ...@@ -72,7 +74,13 @@ class RunningModeConfigurator(IConfigurator):
else: else:
import Pyro import Pyro
Pyro.config.PYRO_STORAGE=PLATFORM.home_directory()
Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
Pyro.config.PYRO_NS_URIFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYRO_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYRO_USER_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYROSSL_CERTDIR = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
slots = int(value[1]) slots = int(value[1])
......
...@@ -46,7 +46,31 @@ from MDANSE.Framework.Jobs.JobStatus import JobStatus ...@@ -46,7 +46,31 @@ from MDANSE.Framework.Jobs.JobStatus import JobStatus
from MDANSE.Framework.OutputVariables.IOutputVariable import OutputData from MDANSE.Framework.OutputVariables.IOutputVariable import OutputData
class JobError(Error): class JobError(Error):
pass '''
This class handles any exception related to IJob-derived objects
'''
def __init__(self,job,message=None):
'''
Initializes the the object.
:param job: the configurator in which the exception was raised
:type job: IJob derived object
'''
if message is None:
message = sys.exc_info()[1]
self._message = str(message)
if job._status is not None:
job._status._state["state"] = "crashed"
job._status._state["info"] += "ERROR: %s" % self._message
job._status.update(force=True)
def __str__(self):
return self._message
def key_generator(size=4, chars=None, prefix=""): def key_generator(size=4, chars=None, prefix=""):
...@@ -130,12 +154,7 @@ class IJob(Configurable): ...@@ -130,12 +154,7 @@ class IJob(Configurable):
#. parameters (dict): optional. If not None, the parameters with which the job file will be built. #. parameters (dict): optional. If not None, the parameters with which the job file will be built.
""" """
# Try to open the output test file to see whether it is allowed. f = open(testFile, 'w')
try:
f = open(testFile, 'w')
# Case where for whatever reason, the file could not be opened for writing. Return.
except IOError as e:
raise JobError(["Error when saving python batch file.",e])
# The first line contains the call to the python executable. This is necessary for the file to # The first line contains the call to the python executable. This is necessary for the file to
# be autostartable. # be autostartable.
...@@ -249,14 +268,8 @@ class IJob(Configurable): ...@@ -249,14 +268,8 @@ class IJob(Configurable):
#. parameters (dict): optional. If not None, the parameters with which the job file will be built. #. parameters (dict): optional. If not None, the parameters with which the job file will be built.
""" """
# Try to open the output job file to see whether it is allowed. f = open(jobFile, 'w')
try:
f = open(jobFile, 'w')
# Case where for whatever reason, the file could not be opened for writing. Return.
except IOError as e:
raise JobError(["Error when saving python batch file.",e])
# The first line contains the call to the python executable. This is necessary for the file to # The first line contains the call to the python executable. This is necessary for the file to
# be autostartable. # be autostartable.
f.write('#!%s\n\n' % sys.executable) f.write('#!%s\n\n' % sys.executable)
...@@ -290,8 +303,7 @@ class IJob(Configurable): ...@@ -290,8 +303,7 @@ class IJob(Configurable):
# Sets |analysis| variable to an instance analysis to save. # Sets |analysis| variable to an instance analysis to save.
f.write('job = REGISTRY[%r][%r](status=False)\n' % ('job',cls.type)) f.write('job = REGISTRY[%r][%r](status=False)\n' % ('job',cls.type))
f.write('job.setup(parameters)\n') f.write('job.run(parameters)')
f.write('job.run()')
f.close() f.close()
...@@ -305,12 +317,7 @@ class IJob(Configurable): ...@@ -305,12 +317,7 @@ class IJob(Configurable):
#. parameters (dict): optional. If not None, the parameters with which the job file will be built. #. parameters (dict): optional. If not None, the parameters with which the job file will be built.
""" """
# Try to open the output test file to see whether it is allowed. f = open(testFile, 'w')
try:
f = open(testFile, 'w')
# Case where for whatever reason, the file could not be opened for writing. Return.
except IOError as e:
raise JobError(["Error when saving python batch file.",e])
# The first line contains the call to the python executable. This is necessary for the file to # The first line contains the call to the python executable. This is necessary for the file to
# be autostartable. # be autostartable.
...@@ -427,35 +434,34 @@ class IJob(Configurable): ...@@ -427,35 +434,34 @@ class IJob(Configurable):
_runner = {"monoprocessor" : _run_monoprocessor, "multiprocessor" : _run_multiprocessor, "remote" : _run_remote} _runner = {"monoprocessor" : _run_monoprocessor, "multiprocessor" : _run_multiprocessor, "remote" : _run_remote}
def run(self, parameters=None): def run(self, parameters):
""" """
Run the job. Run the job.
""" """
if parameters is not None:
self.setup(parameters)
self.initialize()
self._info = 'Information about %s job.\n' % self._name
self._info += str(self)
LOGGER(self._info)
if getattr(self,'numberOfSteps', 0) <= 0:
raise JobError("Invalid number of steps for job %s" % self)
try: try:
self.setup(parameters)
self.initialize()
self._info = 'Information about %s job.\n' % self._name
self._info += str(self)
LOGGER(self._info)
if getattr(self,'numberOfSteps', 0) <= 0:
raise JobError(self,"Invalid number of steps for job %s" % self._name)
mode = self.configuration['running_mode']['mode'] mode = self.configuration['running_mode']['mode']
except:
raise JobError("Invalid running mode")
else:
IJob._runner[mode](self) IJob._runner[mode](self)
self.finalize() self.finalize()
if self._status is not None: if self._status is not None:
self._status.finish() self._status.finish()
except:
raise JobError(self)
@property @property
def info(self): def info(self):
......
...@@ -42,12 +42,12 @@ class OutputVariableError(Error): ...@@ -42,12 +42,12 @@ class OutputVariableError(Error):
class OutputData(collections.OrderedDict): class OutputData(collections.OrderedDict):
def __setitem__(self,item): def __setitem__(self,item,value):
pass pass
def add(self, dataName, dataType, data, **kwargs): def add(self, dataName, dataType, data, **kwargs):
self[dataName] = REGISTRY["output_variable"][dataType](data, dataName, **kwargs) collections.OrderedDict.__setitem__(self,dataName,REGISTRY["output_variable"][dataType](data, dataName, **kwargs))
def write(self, basename, formats, header=None): def write(self, basename, formats, header=None):
......
...@@ -135,7 +135,7 @@ class Status(object): ...@@ -135,7 +135,7 @@ class Status(object):
self._stopped = True self._stopped = True
Publisher.sendMessage("status_stop",message=self) Publisher.sendMessage("status_stop",message=self)
def update(self): def update(self,force=False):
if self._updateStep == 0: if self._updateStep == 0:
return return
...@@ -145,8 +145,8 @@ class Status(object): ...@@ -145,8 +145,8 @@ class Status(object):
lastUpdate = datetime.datetime.today() lastUpdate = datetime.datetime.today()
self._deltas.append(lastUpdate) self._deltas.append(lastUpdate)
if (not self._currentStep % self._updateStep) or (total_seconds(lastUpdate-self._lastRefresh) > 10): if force or (not self._currentStep % self._updateStep) or (total_seconds(lastUpdate-self._lastRefresh) > 10):
self._lastRefresh = lastUpdate self._lastRefresh = lastUpdate
......
...@@ -209,7 +209,7 @@ class UserDefinitionStore(UnicodeDict): ...@@ -209,7 +209,7 @@ class UserDefinitionStore(UnicodeDict):
raise UserDefinitionError("Invalid key value: must be a 3-tuple") raise UserDefinitionError("Invalid key value: must be a 3-tuple")
try: try:
ud = self[name] ud = UnicodeDict.__getitem__(name)
except (KeyError,TypeError): except (KeyError,TypeError):
raise UserDefinitionError('The item %r could not be found' % str(name)) raise UserDefinitionError('The item %r could not be found' % str(name))
......
import math
import numpy import numpy
from MDANSE.Core.Error import Error from MDANSE.Core.Error import Error
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment