Skip to content

GitLab

Commit 7fa7ff83 authored by eric pellegrini's avatar eric pellegrini
Browse files

Many modifications to make the first job work

parent 98f7f78f
Expand all Hide whitespace changes
Inline Side-by-side
MDANSE/Core/ClassRegistry.py deleted 100644 → 0
View file @ 98f7f78f
import abc
class _Meta(type):
'''
Metaclass that allows to use the __getitem__ method at a class level for the class that has been built.
The class that uses this metaclass must define a class attribute named _registry that will be used
by the __getitem__ method.
'''
def __getitem__(self, item):
"""
Returns a given item stored in the class registry
"""
return self._registry[item]
class ClassRegistry(abc.ABCMeta):
'''
Metaclass that registers the subclasses of bases classes.
The internal registry is defined as a nested dictionary whose keys
are the |type| class attribute of the base classes and values another dictionary
whose keys are the |type| class attribute of the subclasses and values are the corresponding
class instances.
Hence any base or child class that does not define |type| class attribute will not be resgistered.
'''
__metaclass__ = _Meta
__interfaces = []
_registry = {}
def __init__(self, name, bases, namespace):
'''
Constructor of a class metaclassed by ClassFactory
:param name: the name of the class to be built by this metaclass
:param bases: the base classes of the class to be built by this metaclass
:param namespace: the attributes and methods of the class to be built by this metaclass
'''
super(ClassRegistry, self).__init__(name, bases, namespace)
# Get the typ of the class
typ = getattr(self, 'type', None)
if typ is None:
return
metaClass = namespace.get("__metaclass__", None)
if metaClass is ClassRegistry:
ClassRegistry.__interfaces.append(self)
ClassRegistry._registry[typ] = {}
else:
for interface in ClassRegistry.__interfaces:
if issubclass(self, interface):
ClassRegistry._registry[interface.type][typ] = self
break
@classmethod
def info(cls, interface):
'''
Returns informations about the subclasses of a given base class stored in the registry.
:param cls: the ClassRegsitry instance
:param interface: the name of base class of whom information about its subclasses is requested
'''
if not cls._registry.has_key(interface):
return "The interface " + interface + " is not registered"
import inspect
import os
# Dictionnay whose keys are the package names and values and list of (job name, job path) stored in the corresponding package.
packages = {}
# Loop over the registry items.
for k, v in cls._registry[interface].items():
# Get the module corresponding to the job class.
mod = inspect.getmodule(v)
# The package hosting the module.
modPackage = mod.__package__
# The module file.
modFilename = mod.__file__
# If no package could be found, guess a name using the directory name of the module file.
if modPackage is None:
modPackage = os.path.split(os.path.dirname(modFilename))[1]
# Update the packages dictionary.
if packages.has_key(modPackage):
packages[modPackage].append([k, v.__name__])
else:
packages[modPackage] = [[k, v.__name__]]
contents = []
# Print the contents of the packages dictionary.
contents.append("="*130)
contents.append("%-50s %-40s %-s" % ("Package", "Name", "Class"))
contents.append("="*130)
for k, v in sorted(packages.items()):
for vv in sorted(v):
contents.append("%-50s %-40s %-s" % (k, vv[0], vv[1]))
contents.append('-' * 130)
contents = "\n".join(contents)
return contents
MDANSE/Core/Logger.py deleted 100644 → 0
View file @ 98f7f78f
import logging
import sys
class Logger(object):
levels = {"debug" : logging.DEBUG,
"info" : logging.INFO,
"warning" : logging.WARNING,
"error" : logging.ERROR,
"fatal" : logging.CRITICAL,
"critical" : logging.FATAL
}
def __call__(self, message, level="info", loggers=None):
lvl = Logger.levels.get(level, None)
# If the logging level is unkwnown, skip that log
if lvl is None:
return
if loggers is None:
loggers = logging.Logger.manager.loggerDict.keys()
else:
loggers = [n for n in loggers if logging.Logger.manager.loggerDict.has_key(n)]
for n in loggers:
logging.getLogger(n).log(lvl, message)
def start(self, logger=None):
from nMOLDYN import _nmoldyn_excepthook
sys.excepthook = _nmoldyn_excepthook
if loggers is None:
loggers = logging.Logger.manager.loggerDict.keys()
else:
loggers = [n for n in loggers if logging.Logger.manager.loggerDict.has_key(n)]
for n in loggers:
logging.getLogger(n).disabled = False
def stop(self, loggers=None):
sys.excepthook = _sys_excepthook
if loggers is None:
loggers = logging.Logger.manager.loggerDict.keys()
else:
loggers = [n for n in loggers if logging.Logger.manager.loggerDict.has_key(n)]
for n in loggers:
logging.getLogger(n).disabled = True
def set_level(self, level, loggers=None):
lvl = Logger.levels.get(level, None)
if lvl is None:
return
if loggers is None:
loggers = logging.Logger.manager.loggerDict.keys()
else:
loggers = [n for n in loggers if logging.Logger.manager.loggerDict.has_key(n)]
for loggerName in loggers:
logging.getLogger(loggerName).setLevel(lvl)
def add_logger(self, name, handler, level="error"):
if logging.Logger.manager.loggerDict.has_key(name):
return
logging.getLogger(name).addHandler(handler)
self.set_level(level, loggers=[name])
MDANSE/DistributedComputing/MasterSlave.py
View file @ 7fa7ff83
......@@ -524,15 +524,15 @@ def initializeMasterProcess(label, slave_script=None, slave_module=None,
source = file(slave_script).read()
else:
source = """
import nMOLDYN.DistributedComputing.MasterSlave
import MDANSE.DistributedComputing.MasterSlave
from %s import *
""" % slave_module
if debug:
source += "print 'Slave definitions:'\n"
source += "print dir()\n"
source += "nMOLDYN.DistributedComputing.MasterSlave.debug=True\n"
source += "MDANSE.DistributedComputing.MasterSlave.debug=True\n"
source += """
nMOLDYN.DistributedComputing.MasterSlave.startSlaveProcess()
MDANSE.DistributedComputing.MasterSlave.startSlaveProcess()
"""
process.task_manager.storeData(slave_code = source,
cwd = os.getcwd())
......
MDANSE/DistributedComputing/Slave.py
View file @ 7fa7ff83
......@@ -4,12 +4,12 @@ Functions:
* do_run_step: performs the analysis step by step.
"""
# Define (or import) all the task handlers.
# Define (or import) all the task handlers.
def do_run_step(job, step):
return job.run_step(step)
from nMOLDYN.DistributedComputing.MasterSlave import startSlaveProcess
from MDANSE.DistributedComputing.MasterSlave import startSlaveProcess
startSlaveProcess(master_host="localhost:%d")
MDANSE/Framework/Configurables/Configurable.py
View file @ 7fa7ff83
......@@ -51,7 +51,7 @@ class Configurable(object):
If not found raise a ConfigurationError.
"""
if self._configurators.has_key(name):
if self.configurators.has_key(name):
return self._configuration.setdefault(name,{})
raise ConfigurationError("The item %r is not valid for this configuration." % name)
......@@ -110,7 +110,7 @@ class Configurable(object):
info = []
for configurator in self._configuration:
for configurator in self._configuration.values():
info.append(configurator.get_information())
......
MDANSE/Framework/Configurables/Configurators.py
View file @ 7fa7ff83
......@@ -44,7 +44,7 @@ from Scientific.Geometry import Vector
from MDANSE import ELEMENTS, PLATFORM, REGISTRY, USER_DEFINITIONS
from MDANSE.Core.Error import Error
from MDANSE.Framework.Configurables.Jobs.Selectors import SelectionParser
from MDANSE.Framework.Selectors import SelectionParser
from MDANSE.Mathematics.Arithmetic import ComplexNumber
from MDANSE.Mathematics.Signal import INTERPOLATION_ORDER
from MDANSE.MolecularDynamics.Trajectory import find_atoms_in_molecule
......@@ -66,7 +66,7 @@ class ConfiguratorError(Error):
def __str__(self):
if self._configurator is not None:
self._message = "%r --> %s" % (self._configurator.name,self._message)
self._message = "Configurator: %r --> %s" % (self._configurator.name,self._message)
return self._message
......@@ -81,7 +81,7 @@ class ConfiguratorsDict(collections.OrderedDict):
try:
self[name] = REGISTRY["configurator"][typ](name, *args, **kwargs)
except KeyError:
raise ConfiguratorError("Invalid type for %r configurator" % name)
raise ConfiguratorError("invalid type for %r configurator" % name)
def build_doc(self):
......@@ -130,6 +130,15 @@ class ConfiguratorsDict(collections.OrderedDict):
table += '+%s+%s+%s+\n'%(s[0]*'-',s[1]*'-',s[2]*'-')
return table
def get_default_parameters(self):
params = collections.OrderedDict()
for k,v in self.items():
params[k] = v.default
return params
class Configurator(dict):
......@@ -185,7 +194,7 @@ class Configurator(dict):
def add_dependency(self, name, conf):
if self._dependencies.has_key(name):
raise ConfiguratorError("The configurator %s already has %s dependency" % (self._name,name))
raise ConfiguratorError("duplicate dependendy for configurator %s" % name, self)
def check_dependencies(self, configured):
......@@ -213,7 +222,7 @@ class InputDirectoryConfigurator(Configurator):
value = PLATFORM.get_path(value)
if not os.path.exists(value):
raise ConfiguratorError('Invalid type for input value', self)
raise ConfiguratorError('invalid type for input value', self)
self['value'] = value
......@@ -242,7 +251,7 @@ class OutputDirectoryConfigurator(Configurator):
if self._new:
if os.path.exists(value):
raise ConfiguratorError("The output directory must not exist", self)
raise ConfiguratorError("the output directory must not exist", self)
self['value'] = value
......@@ -297,12 +306,12 @@ class StringConfigurator(Configurator):
if not self._acceptNullString:
if not value:
raise ConfiguratorError("Null string not accepted", self)
raise ConfiguratorError("invalid null string", self)
if self._evalType is not None:
value = ast.literal_eval(value)
if not isinstance(value,self._evalType):
raise ConfiguratorError("Invalid type for the evaluated string", self)
raise ConfiguratorError("the string can not be eval to %r type" % self._evalType.__name__, self)
self['value'] = value
......@@ -339,7 +348,7 @@ class BooleanConfigurator(Configurator):
value = value.lower()
if not self._shortCuts.has_key(value):
raise ConfiguratorError('Invalid boolean string', self)
raise ConfiguratorError('invalid boolean string', self)
value = bool(value)
......@@ -403,7 +412,7 @@ class MultipleChoicesConfigurator(Configurator):
if self._nChoices is not None:
if len(value) != self._nChoices:
raise ConfiguratorError("Invalid number of choices.", self)
raise ConfiguratorError("invalid number of choices.", self)
indexes = []
for v in value:
......@@ -457,15 +466,15 @@ class ComplexConfigurator(Configurator):
if self._choices:
if not value in self._choices:
raise ConfiguratorError('The input value is not a valid choice.', self)
raise ConfiguratorError('the input value is not a valid choice.', self)
if self._mini is not None:
if value.modulus() < self._mini.modulus():
raise ConfiguratorError("The input value is lower than %r." % self._mini, self)
raise ConfiguratorError("the input value is lower than %r." % self._mini, self)
if self._maxi is not None:
if value.modulus() > self._maxi.modulus():
raise ConfiguratorError("The input value is higher than %r." % self._maxi, self)
raise ConfiguratorError("the input value is higher than %r." % self._maxi, self)
self['value'] = value
......@@ -517,15 +526,15 @@ class FloatConfigurator(Configurator):
if self._choices:
if not value in self._choices:
raise ConfiguratorError('The input value is not a valid choice.', self)
raise ConfiguratorError('the input value is not a valid choice.', self)
if self._mini is not None:
if value < self._mini:
raise ConfiguratorError("The input value is lower than %r." % self._mini, self)
raise ConfiguratorError("the input value is lower than %r." % self._mini, self)
if self._maxi is not None:
if value > self._maxi:
raise ConfiguratorError("The input value is higher than %r." % self._maxi, self)
raise ConfiguratorError("the input value is higher than %r." % self._maxi, self)
self['value'] = value
......@@ -576,7 +585,7 @@ class QVectorsConfigurator(Configurator):
data = generator.run()
if not data:
raise ConfiguratorError("No Q vectors could be generated", self)
raise ConfiguratorError("no Q vectors could be generated", self)
self["parameters"] = parameters
self["type"] = generator.type
......@@ -671,7 +680,7 @@ class InputFileConfigurator(Configurator):
value = PLATFORM.get_path(value)
if not os.path.exists(value):
raise ConfiguratorError("The input file %r does not exist." % value, self)
raise ConfiguratorError("the input file %r does not exist." % value, self)
self["value"] = value
self["filename"] = value
......@@ -719,15 +728,15 @@ class IntegerConfigurator(Configurator):
if self._choices:
if not value in self._choices:
raise ConfiguratorError('The input value is not a valid choice.', self)
raise ConfiguratorError('the input value is not a valid choice.', self)
if self._mini is not None:
if value < self._mini:
raise ConfiguratorError("The input value is lower than %r." % self._mini, self)
raise ConfiguratorError("the input value is lower than %r." % self._mini, self)
if self._maxi is not None:
if value > self._maxi:
raise ConfiguratorError("The input value is higher than %r." % self._maxi, self)
raise ConfiguratorError("the input value is higher than %r." % self._maxi, self)
self['value'] = value
......@@ -775,13 +784,13 @@ class NetCDFInputFileConfigurator(InputFileConfigurator):
self['instance'] = NetCDFFile(self['value'], 'r')
except IOError:
raise ConfiguratorError("Can not open %r NetCDF file for reading" % self['value'])
raise ConfiguratorError("can not open %r NetCDF file for reading" % self['value'])
for v in self._variables:
try:
self[v] = self['instance'].variables[v]
except KeyError:
raise ConfiguratorError("The variable %r was not found in %r NetCDF file" % (v,self["value"]))
raise ConfiguratorError("the variable %r was not found in %r NetCDF file" % (v,self["value"]))
@property
def variables(self):
......@@ -806,7 +815,7 @@ class MMTKNetCDFTrajectoryConfigurator(InputFileConfigurator):
InputFileConfigurator.configure(self, configuration, value)
inputTraj = REGISTRY["inputdata"]["mmtk_trajectory"](self['value'])
inputTraj = REGISTRY["input_data"]["mmtk_trajectory"](self['value'])
self['instance'] = inputTraj.trajectory
......@@ -826,9 +835,9 @@ class MMTKNetCDFTrajectoryConfigurator(InputFileConfigurator):
self['has_velocities'] = 'velocities' in self['instance'].variables()
def get_information(self):
info = ["MMTK input trajectory: %r\n" % self["filename"]]
info.append("Number of steps: %d\n") % self["length"]
info.append("Number of steps: %d\n" % self["length"])
info.append("Size of the universe: %d\n" % self["universe"].numberOfAtoms())
if (self['has_velocities']):
info.append("The trajectory contains atomic velocities\n")
......@@ -859,25 +868,25 @@ class OutputFilesConfigurator(Configurator):
dirname = os.getcwd()
if not basename:
raise ConfiguratorError("Empty basename for the output file.", self)
raise ConfiguratorError("empty basename for the output file.", self)
root = os.path.join(dirname, basename)
try:
PLATFORM.create_directory(dirname)
except:
raise ConfiguratorError("The directory %r is not writable" % dirname)
raise ConfiguratorError("the directory %r is not writable" % dirname)
if not formats:
raise ConfiguratorError("No output formats specified", self)
raise ConfiguratorError("no output formats specified", self)
for fmt in formats:
if not fmt in self._formats:
raise ConfiguratorError("The output file format %r is not a valid output format" % fmt, self)
raise ConfiguratorError("the output file format %r is not a valid output format" % fmt, self)
if not REGISTRY["format"].has_key(fmt):
raise ConfiguratorError("The output file format %r is not registered as a valid file format." % fmt, self)
raise ConfiguratorError("the output file format %r is not registered as a valid file format." % fmt, self)
self["root"] = root
self["formats"] = formats
......@@ -916,14 +925,14 @@ class ProjectionConfigurator(Configurator):
raise ConfiguratorError(e)
if not isinstance(mode,basestring):
raise ConfiguratorError("Invalid type for projection mode: must be a string")
raise ConfiguratorError("invalid type for projection mode: must be a string")
mode = mode.lower()
try:
self["projector"] = REGISTRY['projector'][mode]()
except KeyError:
raise ConfiguratorError("The projector %r is unknow" % mode)
raise ConfiguratorError("the projector %r is unknown" % mode)
else:
self["projector"].set_axis(axis)
self["axis"] = self["projector"].axis
......@@ -956,7 +965,7 @@ class InterpolationOrderConfigurator(IntegerConfigurator):
trajConfig = configuration[self._dependencies['trajectory']]
if not "velocities" in trajConfig['instance'].variables():
raise ConfiguratorError("The trajectory does not contain any velocities. Use an interpolation order higher than 0", self)
raise ConfiguratorError("the trajectory does not contain any velocities. Use an interpolation order higher than 0", self)
self["variable"] = "velocities"
......@@ -1013,7 +1022,7 @@ class RangeConfigurator(Configurator):
value = value[value <= self._maxi]
if value.size == 0:
raise ConfiguratorError("The input range is empty." , self)
raise ConfiguratorError("the input range is empty." , self)
if self._sort:
value = numpy.sort(value)
......@@ -1115,7 +1124,7 @@ class FramesConfigurator(RangeConfigurator):
def get_information(self):
return "%d frames selected from %s to %s (last value excluded) with a time step of %s" % \
return "%d frames selected (first=%.3f ; last = %.3f ; time step = %.3f)" % \
(self["n_frames"],self["time"][0],self["time"][-1],self["time_step"])
class RunningModeConfigurator(Configurator):
......@@ -1154,10 +1163,10 @@ class RunningModeConfigurator(Configurator):
maxSlots = multiprocessing.cpu_count()
del multiprocessing
if slots > maxSlots:
raise ConfiguratorError("Invalid number of allocated slots.", self)
raise ConfiguratorError("invalid number of allocated slots.", self)
if slots <= 0:
raise ConfiguratorError("Invalid number of allocated slots.", self)
raise ConfiguratorError("invalid number of allocated slots.", self)
self['mode'] = mode
......@@ -1206,11 +1215,11 @@ class InstrumentResolutionConfigurator(Configurator):
dmax = resolution.timeWindow.max()-1
if dmax > 0.1:
raise ConfiguratorError('''The resolution function is too sharp for the available frequency step.
raise ConfiguratorError('''the resolution function is too sharp for the available frequency step.
You can change your resolution function settings to make it broader or use "ideal" kernel if you do not want to smooth your signal.
For a gaussian resolution function, this would correspond to a sigma at least equal to the frequency step (%s)''' % df,self)
elif dmax < -0.1:
raise ConfiguratorError('''The resolution function is too broad.
raise ConfiguratorError('''the resolution function is too broad.
You should change your resolution function settings to make it sharper.''',self)
self["frequency_window"] = resolution.frequencyWindow
......@@ -1269,7 +1278,7 @@ class AtomSelectionConfigurator(Configurator):
value = "all"
if not isinstance(value,basestring):
raise ConfiguratorError("Invalid type for atom selection. Must be a string", self)
raise ConfiguratorError("invalid type for atom selection. Must be a string", self)
self["value"] = value
......@@ -1288,11 +1297,11 @@ class AtomSelectionConfigurator(Configurator):
self["elements"] = [[at.symbol] for at in selectedAtoms]
if self._dependencies.has_key("grouping_level"):
self.group(configuration, selectedAtoms, configuration[self._dependencies['grouping_level']]['value'])
self.group(selectedAtoms, configuration[self._dependencies['grouping_level']]['value'])
else:
self.group(selectedAtoms)
self.set_contents(configuration)
self.set_contents()
@staticmethod
def find_parent(atom, level):
......@@ -1374,7 +1383,7 @@ class AtomTransmutationConfigurator(Configurator):
self["atom_selection"] = configuration[self._dependencies['atom_selection']]
if self["atom_selection"]["level"] != "atom":
raise ConfiguratorError("The atom transmutation can only be set with a grouping level set to %r" % 'atom', self)
raise ConfiguratorError("the atom transmutation can only be set with a grouping level set to %r" % 'atom', self)
trajConfig = configuration[self._dependencies['trajectory']]
......@@ -1535,7 +1544,7 @@ class GroupingLevelConfigurator(SingleChoiceConfigurator):
def get_information(self):
return "Grouping level: %d" % self["value"]
return "Grouping level: %r\n" % self["value"]
class WeightsConfigurator(SingleChoiceConfigurator):
"""
......@@ -1559,7 +1568,7 @@ class WeightsConfigurator(SingleChoiceConfigurator):
value = value.lower()
if not value in ELEMENTS.numericProperties: