reorganise Base and Option
[tiramisu.git] / tiramisu / option / optiondescription.py
index 991274b..9780828 100644 (file)
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2014 Team tiramisu (see AUTHORS for all contributors)
+# Copyright (C) 2014-2017 Team tiramisu (see AUTHORS for all contributors)
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU Lesser General Public License as published by the
@@ -24,78 +24,22 @@ import re
 
 from ..i18n import _
 from ..setting import groups, undefined, owners  # , log
-from .baseoption import BaseOption, SymLinkOption, Option, allowed_const_list
+from .baseoption import BaseOption, SymLinkOption, Option, ALLOWED_CONST_LIST
 from . import MasterSlaves
 from ..error import ConfigError, ConflictError
-from ..storage import get_storages_option
 from ..autolib import carry_out_calculation
 
 
-StorageOptionDescription = get_storages_option('optiondescription')
-
 name_regexp = re.compile(r'^[a-zA-Z\d\-_]*$')
 
 import sys
-if sys.version_info[0] >= 3:  # pragma: optional cover
+if sys.version_info[0] >= 3:  # pragma: no cover
     xrange = range
 del(sys)
 
 
-class OptionDescription(BaseOption, StorageOptionDescription):
-    """Config's schema (organisation, group) and container of Options
-    The `OptionsDescription` objects lives in the `tiramisu.config.Config`.
-    """
-    __slots__ = tuple()
-
-    def __init__(self, name, doc, children, requires=None, properties=None):
-        """
-        :param children: a list of options (including optiondescriptions)
-
-        """
-        super(OptionDescription, self).__init__(name, doc=doc,
-                                                requires=requires,
-                                                properties=properties,
-                                                callback=False)
-        child_names = []
-        dynopt_names = []
-        for child in children:
-            name = child.impl_getname()
-            child_names.append(name)
-            if isinstance(child, DynOptionDescription):
-                dynopt_names.append(name)
-
-        #better performance like this
-        valid_child = copy(child_names)
-        valid_child.sort()
-        old = None
-        for child in valid_child:
-            if child == old:  # pragma: optional cover
-                raise ConflictError(_('duplicate option name: '
-                                      '{0}').format(child))
-            if dynopt_names:
-                for dynopt in dynopt_names:
-                    if child != dynopt and child.startswith(dynopt):
-                        raise ConflictError(_('option must not start as '
-                                              'dynoptiondescription'))
-            old = child
-        self._add_children(child_names, children)
-        _setattr = object.__setattr__
-        _setattr(self, '_cache_consistencies', None)
-        # the group_type is useful for filtering OptionDescriptions in a config
-        _setattr(self, '_group_type', groups.default)
-
-    def impl_getdoc(self):
-        return self.impl_get_information('doc')
-
-    def impl_validate(self, *args, **kwargs):
-        """usefull for OptionDescription"""
-        pass
-
-    def impl_getpaths(self, include_groups=False, _currpath=None):
-        """returns a list of all paths in self, recursively
-           _currpath should not be provided (helps with recursion)
-        """
-        return _impl_getpaths(self, include_groups, _currpath)
+class CacheOptionDescription(BaseOption):
+    __slots__ = ('_cache_paths', '_cache_consistencies', '_cache_force_store_values')
 
     def impl_build_cache(self, config, path='', _consistencies=None,
                          cache_option=None, force_store_values=None):
@@ -112,9 +56,7 @@ class OptionDescription(BaseOption, StorageOptionDescription):
         else:
             init = False
         for option in self._impl_getchildren(dyn=False):
-            #FIXME specifique id for sqlalchemy?
-            #FIXME avec sqlalchemy ca marche le multi parent ? (dans des configs diffĂ©rentes)
-            cache_option.append(option._get_id())
+            cache_option.append(option)
             if path == '':
                 subpath = option.impl_getname()
             else:
@@ -125,27 +67,34 @@ class OptionDescription(BaseOption, StorageOptionDescription):
                                         cache_option, force_store_values)
                 #cannot set multi option as OptionDescription requires
             else:
+                if option.impl_is_master_slaves('master'):
+                    if not getattr(option, '_dependencies', None):
+                        options = set()
+                    else:
+                        options = set(option._dependencies)
+                    options.add(option.impl_get_master_slaves())
+                    option._dependencies = tuple(options)
                 option._set_readonly(True)
                 is_multi = option.impl_is_multi()
                 if not isinstance(option, SymLinkOption) and 'force_store_value' in option.impl_getproperties():
                     force_store_values.append((subpath, option))
                 for func, all_cons_opts, params in option._get_consistencies():
                     option._valid_consistencies(all_cons_opts[1:], init=False)
-                    if func not in allowed_const_list and is_multi:
-                        is_slave = option.impl_is_master_slaves()
-                        if not is_slave:
-                            raise ValueError(_('malformed consistency option "{0}" '
+                    if func not in ALLOWED_CONST_LIST and is_multi:
+                        is_masterslaves = option.impl_is_master_slaves()
+                        if not is_masterslaves:
+                            raise ConfigError(_('malformed consistency option "{0}" '
                                                'must be a master/slaves').format(
                                                    option.impl_getname()))
                         masterslaves = option.impl_get_master_slaves()
                     for opt in all_cons_opts:
-                        if func not in allowed_const_list and is_multi:
+                        if func not in ALLOWED_CONST_LIST and is_multi:
                             if not opt.impl_is_master_slaves():
-                                raise ValueError(_('malformed consistency option "{0}" '
+                                raise ConfigError(_('malformed consistency option "{0}" '
                                                    'must not be a multi for "{1}"').format(
                                                        option.impl_getname(), opt.impl_getname()))
                             elif masterslaves != opt.impl_get_master_slaves():
-                                raise ValueError(_('malformed consistency option "{0}" '
+                                raise ConfigError(_('malformed consistency option "{0}" '
                                                    'must be in same master/slaves for "{1}"').format(
                                                        option.impl_getname(), opt.impl_getname()))
                         _consistencies.setdefault(opt,
@@ -178,7 +127,6 @@ class OptionDescription(BaseOption, StorageOptionDescription):
                                                                'must not be a multi for {1}').format(
                                                                    require_opt.impl_getname(), option.impl_getname()))
         if init:
-            session = config._impl_values._p_.getsession()
             if len(cache_option) != len(set(cache_option)):
                 for idx in xrange(1, len(cache_option) + 1):
                     opt = cache_option.pop(0)
@@ -187,24 +135,21 @@ class OptionDescription(BaseOption, StorageOptionDescription):
             if _consistencies != {}:
                 self._cache_consistencies = {}
                 for opt, cons in _consistencies.items():
-                    if opt._get_id() not in cache_option:  # pragma: optional cover
+                    if opt not in cache_option:  # pragma: optional cover
                         raise ConfigError(_('consistency with option {0} '
                                             'which is not in Config').format(
                                                 opt.impl_getname()))
                     self._cache_consistencies[opt] = tuple(cons)
             self._cache_force_store_values = force_store_values
             self._set_readonly(False)
-            del(session)
 
+    def impl_already_build_caches(self):
+        return getattr(self, '_cache_paths', None) is not None
 
-    def impl_build_force_store_values(self, config):
+    def impl_build_force_store_values(self, config, force_store_values):
         session = config._impl_values._p_.getsession()
+        value_set = False
         for subpath, option in self._cache_force_store_values:
-            value = config.cfgimpl_get_values()._get_cached_value(option,
-                                                                  path=subpath,
-                                                                  validate=False,
-                                                                  trusted_cached_properties=False,
-                                                                  validate_properties=True)
             if option.impl_is_master_slaves('slave'):
                 # problem with index
                 raise ConfigError(_('a slave ({0}) cannot have '
@@ -212,8 +157,290 @@ class OptionDescription(BaseOption, StorageOptionDescription):
             if option._is_subdyn():
                 raise ConfigError(_('a dynoption ({0}) cannot have '
                                     'force_store_value property').format(subpath))
-            config._impl_values._p_.setvalue(subpath, value,
-                                             owners.forced, None, session)
+            if force_store_values and not config._impl_values._p_.hasvalue(subpath, session):
+                value = config.cfgimpl_get_values()._get_cached_value(option,
+                                                                      path=subpath,
+                                                                      validate=False,
+                                                                      trusted_cached_properties=False,
+                                                                      validate_properties=True)
+                value_set = True
+                config._impl_values._p_.setvalue(subpath, value,
+                                                 owners.forced, None, session, False)
+
+        if value_set:
+            config._impl_values._p_.commit()
+
+    def impl_build_cache_option(self, _currpath=None, cache_path=None,
+                                cache_option=None):
+
+        if self.impl_is_readonly() or (_currpath is None and getattr(self, '_cache_paths', None) is not None):
+            # cache already set
+            return
+        if _currpath is None:
+            save = True
+            _currpath = []
+        else:
+            save = False
+        if cache_path is None:
+            cache_path = []
+            cache_option = []
+        for option in self._impl_getchildren(dyn=False):
+            attr = option.impl_getname()
+            path = str('.'.join(_currpath + [attr]))
+            cache_option.append(option)
+            cache_path.append(path)
+            if option.impl_is_optiondescription():
+                _currpath.append(attr)
+                option.impl_build_cache_option(_currpath, cache_path,
+                                               cache_option)
+                _currpath.pop()
+        if save:
+            _setattr = object.__setattr__
+            _setattr(self, '_cache_paths', (tuple(cache_option), tuple(cache_path)))
+
+
+class OptionDescriptionWalk(CacheOptionDescription):
+    __slots__ = ('_children',)
+
+    def impl_get_options_paths(self, bytype, byname, _subpath, only_first, context):
+        find_results = []
+
+        def _rebuild_dynpath(path, suffix, dynopt):
+            found = False
+            spath = path.split('.')
+            for length in xrange(1, len(spath)):
+                subpath = '.'.join(spath[0:length])
+                subopt = self.impl_get_opt_by_path(subpath)
+                if dynopt == subopt:
+                    found = True
+                    break
+            if not found:  # pragma: no cover
+                raise ConfigError(_('cannot find dynpath'))
+            subpath = subpath + suffix
+            for slength in xrange(length, len(spath)):
+                subpath = subpath + '.' + spath[slength] + suffix
+            return subpath
+
+        def _filter_by_name(path, option):
+            name = option.impl_getname()
+            if option._is_subdyn():
+                if byname.startswith(name):
+                    found = False
+                    for suffix in option._subdyn._impl_get_suffixes(
+                            context):
+                        if byname == name + suffix:
+                            found = True
+                            path = _rebuild_dynpath(path, suffix,
+                                                    option._subdyn)
+                            option = option._impl_to_dyn(
+                                name + suffix, path)
+                            break
+                    if not found:
+                        return False
+            else:
+                if not byname == name:
+                    return False
+            find_results.append((path, option))
+            return True
+
+        def _filter_by_type(path, option):
+            if isinstance(option, bytype):
+                #if byname is not None, check option byname in _filter_by_name
+                #not here
+                if byname is None:
+                    if option._is_subdyn():
+                        name = option.impl_getname()
+                        for suffix in option._subdyn._impl_get_suffixes(
+                                context):
+                            spath = _rebuild_dynpath(path, suffix,
+                                                     option._subdyn)
+                            find_results.append((spath, option._impl_to_dyn(
+                                name + suffix, spath)))
+                    else:
+                        find_results.append((path, option))
+                return True
+            return False
+
+        def _filter(path, option):
+            if bytype is not None:
+                retval = _filter_by_type(path, option)
+                if byname is None:
+                    return retval
+            if byname is not None:
+                return _filter_by_name(path, option)
+
+        opts, paths = self._cache_paths
+        for index in xrange(0, len(paths)):
+            option = opts[index]
+            if option.impl_is_optiondescription():
+                continue
+            path = paths[index]
+            if _subpath is not None and not path.startswith(_subpath + '.'):
+                continue
+            if bytype == byname is None:
+                if option._is_subdyn():
+                    name = option.impl_getname()
+                    for suffix in option._subdyn._impl_get_suffixes(
+                            context):
+                        spath = _rebuild_dynpath(path, suffix,
+                                                 option._subdyn)
+                        find_results.append((spath, option._impl_to_dyn(
+                            name + suffix, spath)))
+                else:
+                    find_results.append((path, option))
+            else:
+                if _filter(path, option) is False:
+                    continue
+            if only_first:
+                return find_results
+        return find_results
+
+    def _impl_st_getchildren(self, context, only_dyn=False):
+        for child in self._children[1]:
+            if only_dyn is False or child.impl_is_dynoptiondescription():
+                yield(child)
+
+    def _getattr(self, name, suffix=undefined, context=undefined, dyn=True):
+        error = False
+        if suffix is not undefined:
+            if undefined in [suffix, context]:  # pragma: no cover
+                raise ConfigError(_("suffix and context needed if "
+                                    "it's a dyn option"))
+            if name.endswith(suffix):
+                oname = name[:-len(suffix)]
+                child = self._children[1][self._children[0].index(oname)]
+                return self._impl_get_dynchild(child, suffix)
+            else:
+                error = True
+        else:
+            if name in self._children[0]:
+                child = self._children[1][self._children[0].index(name)]
+                if dyn and child.impl_is_dynoptiondescription():
+                    error = True
+                else:
+                    return child
+            else:
+                child = self._impl_search_dynchild(name, context=context)
+                if child != []:
+                    return child
+                error = True
+        if error:
+            raise AttributeError(_('unknown Option {0} '
+                                   'in OptionDescription {1}'
+                                   '').format(name, self.impl_getname()))
+
+    def impl_getpaths(self, include_groups=False, _currpath=None):
+        """returns a list of all paths in self, recursively
+           _currpath should not be provided (helps with recursion)
+        """
+        return _impl_getpaths(self, include_groups, _currpath)
+
+    def impl_get_opt_by_path(self, path):
+        if getattr(self, '_cache_paths', None) is None:
+            raise ConfigError(_('use impl_get_opt_by_path only with root OptionDescription'))
+        if path not in self._cache_paths[1]:
+            raise AttributeError(_('no option for path {0}').format(path))
+        return self._cache_paths[0][self._cache_paths[1].index(path)]
+
+    def impl_get_path_by_opt(self, opt):
+        if getattr(self, '_cache_paths', None) is None:
+            raise ConfigError(_('use impl_get_path_by_opt only with root OptionDescription'))
+        if opt not in self._cache_paths[0]:
+            raise AttributeError(_('no option {0} found').format(opt))
+        return self._cache_paths[1][self._cache_paths[0].index(opt)]
+
+    def _impl_getchildren(self, dyn=True, context=undefined):
+        for child in self._impl_st_getchildren(context):
+            cname = child.impl_getname()
+            if dyn and child.impl_is_dynoptiondescription():
+                path = cname
+                for value in child._impl_get_suffixes(context):
+                    yield SynDynOptionDescription(child,
+                                                  cname + value,
+                                                  path + value, value)
+            else:
+                yield child
+
+    def impl_getchildren(self):
+        return list(self._impl_getchildren())
+
+    def __getattr__(self, name, context=undefined):
+        if name.startswith('_'):  # or name.startswith('impl_'):
+            return object.__getattribute__(self, name)
+        if '.' in name:
+            path = name.split('.')[0]
+            subpath = '.'.join(name.split('.')[1:])
+            return self.__getattr__(path, context=context).__getattr__(subpath, context=context)
+        return self._getattr(name, context=context)
+
+    def _impl_search_dynchild(self, name, context):
+        ret = []
+        for child in self._impl_st_getchildren(context, only_dyn=True):
+            cname = child.impl_getname()
+            if name.startswith(cname):
+                path = cname
+                for value in child._impl_get_suffixes(context):
+                    if name == cname + value:
+                        return SynDynOptionDescription(child, name, path + value, value)
+        return ret
+
+    def _impl_get_dynchild(self, child, suffix):
+        name = child.impl_getname() + suffix
+        path = self.impl_getname() + suffix + '.' + name
+        if isinstance(child, OptionDescription):
+            return SynDynOptionDescription(child, name, path, suffix)
+        else:
+            return child._impl_to_dyn(name, path)
+
+
+class OptionDescription(OptionDescriptionWalk):
+    """Config's schema (organisation, group) and container of Options
+    The `OptionsDescription` objects lives in the `tiramisu.config.Config`.
+    """
+    __slots__ = ('_group_type',)
+
+    def __init__(self, name, doc, children, requires=None, properties=None):
+        """
+        :param children: a list of options (including optiondescriptions)
+
+        """
+        super(OptionDescription, self).__init__(name, doc=doc,
+                                                requires=requires,
+                                                properties=properties)
+        child_names = []
+        dynopt_names = []
+        for child in children:
+            name = child.impl_getname()
+            child_names.append(name)
+            if isinstance(child, DynOptionDescription):
+                dynopt_names.append(name)
+
+        #better performance like this
+        valid_child = copy(child_names)
+        valid_child.sort()
+        old = None
+        for child in valid_child:
+            if child == old:  # pragma: optional cover
+                raise ConflictError(_('duplicate option name: '
+                                      '{0}').format(child))
+            if dynopt_names:
+                for dynopt in dynopt_names:
+                    if child != dynopt and child.startswith(dynopt):
+                        raise ConflictError(_('option must not start as '
+                                              'dynoptiondescription'))
+            old = child
+        _setattr = object.__setattr__
+        _setattr(self, '_children', (tuple(child_names), tuple(children)))
+        _setattr(self, '_cache_consistencies', None)
+        # the group_type is useful for filtering OptionDescriptions in a config
+        _setattr(self, '_group_type', groups.default)
+
+    def impl_getdoc(self):
+        return self.impl_get_information('doc')
+
+    def impl_validate(self, *args, **kwargs):
+        """usefull for OptionDescription"""
+        pass
 
     # ____________________________________________________________
     def impl_set_group_type(self, group_type):
@@ -249,66 +476,17 @@ class OptionDescription(BaseOption, StorageOptionDescription):
             raise ValueError(_('group_type: {0}'
                                ' not allowed').format(group_type))
 
-    def _impl_getstate(self, descr=None):
-        """enables us to export into a dict
-        :param descr: parent :class:`tiramisu.option.OptionDescription`
-        """
-        if descr is None:
-            self.impl_build_cache_option()
-            descr = self
-        super(OptionDescription, self)._impl_getstate(descr)
-        self._state_group_type = str(self._group_type)
-        for option in self._impl_getchildren():
-            option._impl_getstate(descr)
+    def impl_get_group_type(self):
+        return self._group_type
 
     def __getstate__(self):
-        """special method to enable the serialization with pickle
-        """
-        stated = True
-        try:
-            # the `_state` attribute is a flag that which tells us if
-            # the serialization can be performed
-            self._stated
-        except AttributeError:
-            # if cannot delete, _impl_getstate never launch
-            # launch it recursivement
-            # _stated prevent __getstate__ launch more than one time
-            # _stated is delete, if re-serialize, re-lauch _impl_getstate
-            self._impl_getstate()
-            stated = False
-        return super(OptionDescription, self).__getstate__(stated)
-
-    def _impl_setstate(self, descr=None):
-        """enables us to import from a dict
-        :param descr: parent :class:`tiramisu.option.OptionDescription`
-        """
-        if descr is None:
-            self._cache_consistencies = None
-            self.impl_build_cache_option()
-            descr = self
-        self._group_type = getattr(groups, self._state_group_type)
-        if isinstance(self._group_type, groups.MasterGroupType):
-            MasterSlaves(self.impl_getname(), self.impl_getchildren(),
-                         validate=False)
-        del(self._state_group_type)
-        super(OptionDescription, self)._impl_setstate(descr)
-        for option in self._impl_getchildren(dyn=False):
-            option._impl_setstate(descr)
-
-    def __setstate__(self, state):
-        super(OptionDescription, self).__setstate__(state)
-        try:
-            self._stated
-        except AttributeError:
-            self._impl_setstate()
+        raise NotImplementedError()
 
     def _impl_get_suffixes(self, context):
         callback, callback_params = self.impl_get_callback()
         values = carry_out_calculation(self, context=context,
                                        callback=callback,
                                        callback_params=callback_params)
-        if isinstance(values, Exception):
-            raise values
         if len(values) > len(set(values)):
             raise ConfigError(_('DynOptionDescription callback return not unique value'))
         for val in values:
@@ -316,51 +494,6 @@ class OptionDescription(BaseOption, StorageOptionDescription):
                 raise ValueError(_("invalid suffix: {0} for option").format(val))
         return values
 
-    def _impl_search_dynchild(self, name=undefined, context=undefined):
-        ret = []
-        for child in self._impl_st_getchildren(context, only_dyn=True):
-            cname = child.impl_getname()
-            if name is undefined or name.startswith(cname):
-                path = cname
-                for value in child._impl_get_suffixes(context):
-                    if name is undefined:
-                        ret.append(SynDynOptionDescription(child, cname + value, path + value, value))
-                    elif name == cname + value:
-                        return SynDynOptionDescription(child, name, path + value, value)
-        return ret
-
-    def _impl_get_dynchild(self, child, suffix):
-        name = child.impl_getname() + suffix
-        path = self.impl_getname() + suffix + '.' + name
-        if isinstance(child, OptionDescription):
-            return SynDynOptionDescription(child, name, path, suffix)
-        else:
-            return child._impl_to_dyn(name, path)
-
-    def _impl_getchildren(self, dyn=True, context=undefined):
-        for child in self._impl_st_getchildren(context):
-            cname = child.impl_getname()
-            if dyn and child.impl_is_dynoptiondescription():
-                path = cname
-                for value in child._impl_get_suffixes(context):
-                    yield SynDynOptionDescription(child,
-                                                  cname + value,
-                                                  path + value, value)
-            else:
-                yield child
-
-    def impl_getchildren(self):
-        return list(self._impl_getchildren())
-
-    def __getattr__(self, name, context=undefined):
-        if name.startswith('_'):  # or name.startswith('impl_'):
-            return object.__getattribute__(self, name)
-        if '.' in name:
-            path = name.split('.')[0]
-            subpath = '.'.join(name.split('.')[1:])
-            return self.__getattr__(path, context=context).__getattr__(subpath, context=context)
-        return self._getattr(name, context=context)
-
 
 class DynOptionDescription(OptionDescription):
     def __init__(self, name, doc, children, requires=None, properties=None,
@@ -377,9 +510,6 @@ class DynOptionDescription(OptionDescription):
             if isinstance(child, SymLinkOption):
                 raise ConfigError(_('cannot set symlinkoption in a '
                                     'dynoptiondescription'))
-            if isinstance(child, SymLinkOption):
-                raise ConfigError(_('cannot set symlinkoption in a '
-                                    'dynoptiondescription'))
             child._impl_setsubdyn(self)
         self.impl_set_callback(callback, callback_params)
 
@@ -408,8 +538,7 @@ class SynDynOptionDescription(object):
     def _impl_getchildren(self, dyn=True, context=undefined):
         children = []
         for child in self._opt._impl_getchildren():
-            children.append(self._opt._impl_get_dynchild(child, self._suffix))
-        return children
+            yield(self._opt._impl_get_dynchild(child, self._suffix))
 
     def impl_getchildren(self):
         return self._impl_getchildren()
@@ -425,19 +554,19 @@ class SynDynOptionDescription(object):
 
 
 def _impl_getpaths(klass, include_groups, _currpath):
-        """returns a list of all paths in klass, recursively
-           _currpath should not be provided (helps with recursion)
-        """
-        if _currpath is None:
-            _currpath = []
-        paths = []
-        for option in klass._impl_getchildren():
-            attr = option.impl_getname()
-            if option.impl_is_optiondescription():
-                if include_groups:
-                    paths.append('.'.join(_currpath + [attr]))
-                paths += option.impl_getpaths(include_groups=include_groups,
-                                              _currpath=_currpath + [attr])
-            else:
+    """returns a list of all paths in klass, recursively
+       _currpath should not be provided (helps with recursion)
+    """
+    if _currpath is None:
+        _currpath = []
+    paths = []
+    for option in klass._impl_getchildren():
+        attr = option.impl_getname()
+        if option.impl_is_optiondescription():
+            if include_groups:
                 paths.append('.'.join(_currpath + [attr]))
-        return paths
+            paths += option.impl_getpaths(include_groups=include_groups,
+                                          _currpath=_currpath + [attr])
+        else:
+            paths.append('.'.join(_currpath + [attr]))
+    return paths