Commit a5ef8cf8 authored by eric pellegrini's avatar eric pellegrini

Add fix for Pyro smp run

Bug fix in FloatConfigurator
Added unit tests
Any error is now caught at the Job level when running a job
Removed unused imports
parent d2f3271c
......@@ -30,6 +30,16 @@ Created on Mar 30, 2015
@author: pellegrini
'''
import os
from MDANSE import PLATFORM
import Pyro
Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
Pyro.config.PYRO_NS_URIFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYRO_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYRO_USER_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYROSSL_CERTDIR = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
# Define (or import) all the task handlers.
def do_run_step(job, step):
'''
......
......@@ -30,7 +30,6 @@ Created on Mar 30, 2015
@author: pellegrini
'''
import _abcoll
import collections
from MDANSE.Core.Error import Error
......@@ -55,20 +54,20 @@ class Configurable(object):
#.. 2-value is the dictionary of the keywords used when initializing the configurator.
'''
settings = collections.OrderedDict()
def __init__(self):
'''
Constructor
'''
self._configuration = {}
settings = getattr(self,"settings",{})
if not isinstance(settings,_abcoll.Mapping):
if not isinstance(self.settings,dict):
raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
self._configurators = {}
for name,(typ,kwds) in settings.items():
for name,(typ,kwds) in self.settings.items():
try:
self._configurators[name] = REGISTRY["configurator"][typ](name, **kwds)
......@@ -89,7 +88,15 @@ class Configurable(object):
"""
return self._configuration.setdefault(name,{})
@classmethod
def set_settings(cls, settings):
cls.settings.clear()
if isinstance(settings,dict):
cls.settings.update(settings)
def setup(self,parameters):
'''
Setup the configuration according to a set of input parameters.
......@@ -102,7 +109,7 @@ class Configurable(object):
self._configuration.clear()
self._configured=False
# If no configurator has to be configured, just return
if not self._configurators:
self._configured=True
......@@ -121,7 +128,7 @@ class Configurable(object):
configured = set()
while toBeConfigured != configured:
progress = False
for name,conf in self._configurators.items():
......@@ -177,7 +184,7 @@ class Configurable(object):
settings = getattr(cls,"settings",{})
if not isinstance(settings,_abcoll.Mapping):
if not isinstance(settings,dict):
raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
doclist = []
......@@ -243,7 +250,7 @@ class Configurable(object):
settings = getattr(cls,"settings",{})
if not isinstance(settings,_abcoll.Mapping):
if not isinstance(settings,dict):
raise ConfigurationError("Invalid type for settings: must be a mapping-like object")
params = collections.OrderedDict()
......
......@@ -35,7 +35,7 @@ import operator
import numpy
from MDANSE.Framework.UserDefinitions.IUserDefinition import UD_STORE
from MDANSE.Framework.UserDefinitions.IUserDefinition import UD_STORE, UserDefinitionError
from MDANSE.Framework.Configurators.IConfigurator import IConfigurator, ConfiguratorError
from MDANSE.Framework.AtomSelectionParser import AtomSelectionParser
......@@ -80,21 +80,23 @@ class AtomSelectionConfigurator(IConfigurator):
'''
trajConfig = configuration[self._dependencies['trajectory']]
if not isinstance(value,basestring):
if value is None:
value = 'all'
elif not isinstance(value,basestring):
raise ConfiguratorError("invalid type for atom selection. Must be a string", self)
self["value"] = value
ud = UD_STORE[trajConfig["basename"],"atom_selection",value]
# The input value is a user definition: get it and update the configuration
if ud is not None:
self.update(ud)
try:
ud = UD_STORE[trajConfig["basename"],"atom_selection",value]
# The input value is an atom selection string: parse it and update the configuration
else:
except UserDefinitionError:
parser = AtomSelectionParser(trajConfig["instance"])
self["indexes"] = parser.parse(value)
self["expression"] = value
else:
self.update(ud)
self["n_selected_atoms"] = len(self["indexes"])
atoms = sorted(trajConfig["universe"].atomList(), key = operator.attrgetter('index'))
......
......@@ -64,7 +64,7 @@ class FloatConfigurator(IConfigurator):
self._choices = choices if choices is not None else []
def configure(self, value):
def configure(self, configuration, value):
'''
Configure an input value.
......
......@@ -30,6 +30,8 @@ Created on May 22, 2015
@author: Eric C. Pellegrini
'''
import os
from MDANSE import PLATFORM
from MDANSE.Framework.Configurators.IConfigurator import IConfigurator, ConfiguratorError
......@@ -72,7 +74,13 @@ class RunningModeConfigurator(IConfigurator):
else:
import Pyro
Pyro.config.PYRO_STORAGE=PLATFORM.home_directory()
Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
Pyro.config.PYRO_STORAGE = PLATFORM.home_directory()
Pyro.config.PYRO_NS_URIFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYRO_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYRO_USER_LOGFILE = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
Pyro.config.PYROSSL_CERTDIR = os.path.join(Pyro.config.PYRO_STORAGE,'Pyro_NS_URI')
slots = int(value[1])
......
......@@ -46,7 +46,31 @@ from MDANSE.Framework.Jobs.JobStatus import JobStatus
from MDANSE.Framework.OutputVariables.IOutputVariable import OutputData
class JobError(Error):
pass
'''
This class handles any exception related to IJob-derived objects
'''
def __init__(self,job,message=None):
'''
Initializes the the object.
:param job: the configurator in which the exception was raised
:type job: IJob derived object
'''
if message is None:
message = sys.exc_info()[1]
self._message = str(message)
if job._status is not None:
job._status._state["state"] = "crashed"
job._status._state["info"] += "ERROR: %s" % self._message
job._status.update(force=True)
def __str__(self):
return self._message
def key_generator(size=4, chars=None, prefix=""):
......@@ -130,12 +154,7 @@ class IJob(Configurable):
#. parameters (dict): optional. If not None, the parameters with which the job file will be built.
"""
# Try to open the output test file to see whether it is allowed.
try:
f = open(testFile, 'w')
# Case where for whatever reason, the file could not be opened for writing. Return.
except IOError as e:
raise JobError(["Error when saving python batch file.",e])
f = open(testFile, 'w')
# The first line contains the call to the python executable. This is necessary for the file to
# be autostartable.
......@@ -249,14 +268,8 @@ class IJob(Configurable):
#. parameters (dict): optional. If not None, the parameters with which the job file will be built.
"""
# Try to open the output job file to see whether it is allowed.
try:
f = open(jobFile, 'w')
# Case where for whatever reason, the file could not be opened for writing. Return.
except IOError as e:
raise JobError(["Error when saving python batch file.",e])
f = open(jobFile, 'w')
# The first line contains the call to the python executable. This is necessary for the file to
# be autostartable.
f.write('#!%s\n\n' % sys.executable)
......@@ -290,8 +303,7 @@ class IJob(Configurable):
# Sets |analysis| variable to an instance analysis to save.
f.write('job = REGISTRY[%r][%r](status=False)\n' % ('job',cls.type))
f.write('job.setup(parameters)\n')
f.write('job.run()')
f.write('job.run(parameters)')
f.close()
......@@ -305,12 +317,7 @@ class IJob(Configurable):
#. parameters (dict): optional. If not None, the parameters with which the job file will be built.
"""
# Try to open the output test file to see whether it is allowed.
try:
f = open(testFile, 'w')
# Case where for whatever reason, the file could not be opened for writing. Return.
except IOError as e:
raise JobError(["Error when saving python batch file.",e])
f = open(testFile, 'w')
# The first line contains the call to the python executable. This is necessary for the file to
# be autostartable.
......@@ -427,35 +434,34 @@ class IJob(Configurable):
_runner = {"monoprocessor" : _run_monoprocessor, "multiprocessor" : _run_multiprocessor, "remote" : _run_remote}
def run(self, parameters=None):
def run(self, parameters):
"""
Run the job.
"""
if parameters is not None:
self.setup(parameters)
self.initialize()
self._info = 'Information about %s job.\n' % self._name
self._info += str(self)
LOGGER(self._info)
if getattr(self,'numberOfSteps', 0) <= 0:
raise JobError("Invalid number of steps for job %s" % self)
try:
self.setup(parameters)
self.initialize()
self._info = 'Information about %s job.\n' % self._name
self._info += str(self)
LOGGER(self._info)
if getattr(self,'numberOfSteps', 0) <= 0:
raise JobError(self,"Invalid number of steps for job %s" % self._name)
mode = self.configuration['running_mode']['mode']
except:
raise JobError("Invalid running mode")
else:
IJob._runner[mode](self)
self.finalize()
if self._status is not None:
self._status.finish()
self.finalize()
if self._status is not None:
self._status.finish()
except:
raise JobError(self)
@property
def info(self):
......
......@@ -42,12 +42,12 @@ class OutputVariableError(Error):
class OutputData(collections.OrderedDict):
def __setitem__(self,item):
def __setitem__(self,item,value):
pass
def add(self, dataName, dataType, data, **kwargs):
self[dataName] = REGISTRY["output_variable"][dataType](data, dataName, **kwargs)
collections.OrderedDict.__setitem__(self,dataName,REGISTRY["output_variable"][dataType](data, dataName, **kwargs))
def write(self, basename, formats, header=None):
......
......@@ -135,7 +135,7 @@ class Status(object):
self._stopped = True
Publisher.sendMessage("status_stop",message=self)
def update(self):
def update(self,force=False):
if self._updateStep == 0:
return
......@@ -145,8 +145,8 @@ class Status(object):
lastUpdate = datetime.datetime.today()
self._deltas.append(lastUpdate)
if (not self._currentStep % self._updateStep) or (total_seconds(lastUpdate-self._lastRefresh) > 10):
if force or (not self._currentStep % self._updateStep) or (total_seconds(lastUpdate-self._lastRefresh) > 10):
self._lastRefresh = lastUpdate
......
......@@ -209,7 +209,7 @@ class UserDefinitionStore(UnicodeDict):
raise UserDefinitionError("Invalid key value: must be a 3-tuple")
try:
ud = self[name]
ud = UnicodeDict.__getitem__(name)
except (KeyError,TypeError):
raise UserDefinitionError('The item %r could not be found' % str(name))
......
import math
import numpy
from MDANSE.Core.Error import Error
......
#MDANSE : Molecular Dynamics Analysis for Neutron Scattering Experiments
#------------------------------------------------------------------------------------------
#Copyright (C)
#2015- Eric C. Pellegrini Institut Laue-Langevin
#BP 156
#6, rue Jules Horowitz
#38042 Grenoble Cedex 9
#France
#pellegrini[at]ill.fr
#goret[at]ill.fr
#aoun[at]ill.fr
#
#This library is free software; you can redistribute it and/or
#modify it under the terms of the GNU Lesser General Public
#License as published by the Free Software Foundation; either
#version 2.1 of the License, or (at your option) any later version.
#
#This library is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
#Lesser General Public License for more details.
#
#You should have received a copy of the GNU Lesser General Public
#License along with this library; if not, write to the Free Software
#Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
'''
Created on May 29, 2015
@author: Eric C. Pellegrini
'''
import os
import unittest
import numpy
from MMTK.Trajectory import Trajectory
from MDANSE.Framework.Configurable import Configurable
from MDANSE.Framework.Configurators.IConfigurator import ConfiguratorError
from MDANSE.Framework.Projectors.IProjector import ProjectorError
from MDANSE.Framework.AtomSelectionParser import AtomSelectionParserError
from UnitTest import UnitTest
TRAJECTORIES_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),"UserData","Trajectories")
class TestConfigurator(UnitTest):
'''
Unittest for the configurators used to setup an analysis in nMolDyn
'''
def setUp(self):
self._configurableClass = Configurable
self._validTrajectory = Trajectory(None, os.path.join('..','..','Data','Trajectories','MMTK','protein_in_periodic_universe.nc'), "r")
self._parameters = {}
def tearDown(self):
self._configurableClass.settings.clear()
self._parameters.clear()
def test_integer(self):
'''
Test the integer configurator
'''
self._configurableClass.set_settings({"test_integer":('integer',{})})
configurable = self._configurableClass()
# Case of a valid integer
self._parameters["test_integer"] = 20
self.assertNotRaises(configurable.setup,self._parameters)
# Case of a float that will casted to an integer
self._parameters["test_integer"] = 20.2
self.assertNotRaises(configurable.setup,self._parameters)
self.assertEqual(configurable["test_integer"]["value"], 20)
# Case of a string that can be casted to an integer
self._parameters["test_integer"] = "30"
self.assertNotRaises(configurable.setup,self._parameters)
self.assertEqual(configurable["test_integer"]["value"], 30)
# Case of a string that cannot be casted to an integer
self._parameters["test_integer"] = "xxxx"
self.assertRaises(ConfiguratorError, configurable.setup,self._parameters)
# Case of an object that cannot be casted to an integer
self._parameters["test_integer"] = [1,2]
self.assertRaises(ConfiguratorError, configurable.setup,self._parameters)
def test_float(self):
'''
Test the float configurator
'''
self._configurableClass.set_settings({"test_float":('float',{})})
configurable = self._configurableClass()
# Case of an integer that will be casted to a float
self._parameters["test_float"] = 20
self.assertNotRaises(configurable.setup,self._parameters)
self.assertEqual(configurable["test_float"]["value"], 20.0)
# Case of a float
self._parameters["test_float"] = 20.2
self.assertNotRaises(configurable.setup,self._parameters)
self.assertEqual(configurable["test_float"]["value"], 20.2)
# Case of a string that can be casted to a float
self._parameters["test_float"] = "30.2"
self.assertNotRaises(configurable.setup,self._parameters)
self.assertEqual(configurable["test_float"]["value"], 30.2)
# Case of a string that cannot be casted to a float
self._parameters["test_float"] = "xxxx"
self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
# Case of an object that cannot be casted to a float
self._parameters["test_float"] = [1,2]
self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
def test_mmtk_trajectory(self):
'''
Test the mmtk_trajectory configurator
'''
self._configurableClass.set_settings({"trajectory":('mmtk_trajectory',{})})
configurable = self._configurableClass()
# Case of a valid trajectory
self._parameters["trajectory"] = self._validTrajectory.filename
self.assertNotRaises(configurable.setup,self._parameters)
# Case of an unknown trajectory
self._parameters["trajectory"] = 'fsfsdjkfjkfjs'
self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
# Case of an invalid type for input trajectory
self._parameters["trajectory"] = 1
self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
# Case of an invalid type for input trajectory
self._parameters["trajectory"] = [1,2,3]
self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
def test_projection(self):
self._configurableClass.set_settings({"projection":('projection',{})})
configurable = self._configurableClass()
data = numpy.random.uniform(0,1,(10,3))
# Wrong parameters
self._parameters["projection"] = 10
self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
self._parameters["projection"] = [1,2,3,4]
self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
# No projection - wrong projection type
self._parameters["projection"] = (30,None)
self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
# No projection
self._parameters["projection"] = None
self.assertNotRaises(configurable.setup,self._parameters)
proj = configurable["projection"]['projector'](data)
self.assertTrue(numpy.array_equal(data,proj))
self._parameters["projection"] = ('null',None)
self.assertNotRaises(configurable.setup,self._parameters)
proj = configurable["projection"]['projector'](data)
self.assertTrue(numpy.array_equal(data,proj))
# Axial projection - wrong parameters
self._parameters["projection"] = ('tutu',(1,0,0))
self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
self._parameters["projection"] = ('axial',(1,0,0,2,12,25))
self.assertRaises(ProjectorError, configurable.setup, self._parameters)
self._parameters["projection"] = ('axial',30)
self.assertRaises(ProjectorError, configurable.setup, self._parameters)
# Axial projection - null vector
self._parameters["projection"] = ('axial',(0,0,0))
self.assertRaises(ProjectorError, configurable.setup, self._parameters)
# Axial projection
self._parameters["projection"] = ('axial',(1,0,0))
self.assertNotRaises(configurable.setup,self._parameters)
proj = configurable["projection"]['projector'](data)
self.assertTrue(numpy.array_equal(data[:,0],proj[:,0]))
# Axial projection - wrong data
self._parameters["projection"] = ('axial',(1,0,0))
self.assertNotRaises(configurable.setup,self._parameters)
self.assertRaises(ProjectorError, configurable["projection"]['projector'].__call__,None)
self.assertRaises(ProjectorError, configurable["projection"]['projector'].__call__,[1])
# Planar projection - wrong parameters
self._parameters["projection"] = ('tutu',(1,0,0))
self.assertRaises(ConfiguratorError, configurable.setup, self._parameters)
self._parameters["projection"] = ('planar',(1,0,0,2,99,123))
self.assertRaises(ProjectorError, configurable.setup, self._parameters)
self._parameters["projection"] = ('planar',30)
self.assertRaises(ProjectorError, configurable.setup, self._parameters)
# Planar projection - null vector
self._parameters["projection"] = ('planar',(0,0,0))
self.assertRaises(ProjectorError,configurable.setup, self._parameters)
# Planar projection
self._parameters["projection"] = ('planar',(1,0,0))
self.assertNotRaises(configurable.setup,self._parameters)
proj = configurable["projection"]['projector'](data)
self.assertTrue(numpy.array_equal(numpy.zeros((data.shape[0],), dtype=numpy.float64),proj[:,0]))
self.assertTrue(numpy.array_equal(data[:,1],proj[:,1]))
self.assertTrue(numpy.array_equal(data[:,2],proj[:,2]))
# Planar projection - wrong data
self._parameters["projection"] = ('planar',(1,0,0))
self.assertNotRaises(configurable.setup,self._parameters)
self.assertRaises(ProjectorError, configurable["projection"]['projector'].__call__,None)
self.assertRaises(ProjectorError, configurable["projection"]['projector'].__call__,[1])
def test_atom_selection(self):
self._configurableClass.set_settings({"trajectory":('mmtk_trajectory',{}),
"atom_selection":('atom_selection',{'dependencies':{'trajectory':'trajectory'}})})
configurable = self._configurableClass()
# Test wrong parameter for atom selection
self._parameters["trajectory"] = self._validTrajectory.filename
self._parameters["atom_selection"] = 10
self.assertRaises(ConfiguratorError,configurable.setup,self._parameters)
# Test wrong parameter for atom selection
self._parameters["trajectory"] = self._validTrajectory.filename
self._parameters["atom_selection"] = 'dadsada'
self.assertRaises(AtomSelectionParserError,configurable.setup,self._parameters)
# Test that an empty selection selection raises a SelectionParserError
self._parameters["trajectory"] = self._validTrajectory.filename
self._parameters["atom_selection"] = 'atomelement fsfsd)'
self.assertRaises(AtomSelectionParserError,configurable.setup,self._parameters)
# Test that None parameters selects everything
self._parameters["trajectory"] = self._validTrajectory.filename