tiramisu/value.py

   1 # -*- coding: utf-8 -*-
   2 "takes care of the option's values and multi values"
   3 # Copyright (C) 2013-2017 Team tiramisu (see AUTHORS for all contributors)
   4 #
   5 # This program is free software: you can redistribute it and/or modify it
   6 # under the terms of the GNU Lesser General Public License as published by the
   7 # Free Software Foundation, either version 3 of the License, or (at your
   8 # option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  12 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
  13 # details.
  14 #
  15 # You should have received a copy of the GNU Lesser General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 # ____________________________________________________________
  18 from time import time
  19 import sys
  20 import weakref
  21 from .error import ConfigError, SlaveError, PropertiesOptionError
  22 from .setting import owners, expires_time, undefined
  23 from .autolib import carry_out_calculation
  24 from .i18n import _
  25 from .option import SymLinkOption, DynSymLinkOption, Option
  26 i_i = 0
  27
  28
  29 class Values(object):
  30     """The `Config`'s root is indeed  in charge of the `Option()`'s values,
  31     but the values are physicaly located here, in `Values`, wich is also
  32     responsible of a caching utility.
  33     """
  34     __slots__ = ('context', '_p_', '__weakref__')
  35
  36     def __init__(self, context, storage):
  37         """
  38         Initializes the values's dict.
  39
  40         :param context: the context is the home config's values
  41
  42         """
  43         self.context = weakref.ref(context)
  44         # the storage type is dictionary or sqlite3
  45         self._p_ = storage
  46
  47     def _getcontext(self):
  48         """context could be None, we need to test it
  49         context is None only if all reference to `Config` object is deleted
  50         (for example we delete a `Config` and we manipulate a reference to
  51         old `SubConfig`, `Values`, `Multi` or `Settings`)
  52         """
  53         context = self.context()
  54         if context is None:
  55             raise ConfigError(_('the context does not exist anymore'))
  56         return context
  57
  58     def _get_multi(self, opt, path):
  59         return Multi([], self.context, opt, path)
  60
  61     def _getdefaultvalue(self, opt, path, with_meta, index, submulti_index, validate):
  62         # if value has callback and is not set
  63         if opt.impl_has_callback():
  64             callback, callback_params = opt.impl_get_callback()
  65             value = carry_out_calculation(opt, context=self._getcontext(),
  66                                           callback=callback,
  67                                           callback_params=callback_params,
  68                                           index=index, validate=validate)
  69             if isinstance(value, list) and index is not None:
  70                 #if return a list and index is set, return value only if
  71                 #it's a submulti without submulti_index and without list of list
  72                 if opt.impl_is_submulti() and submulti_index is undefined and \
  73                         (len(value) == 0 or not isinstance(value[0], list)):
  74                     return value
  75                 if not opt.impl_is_submulti() and len(value) > index:
  76                     return value[index]
  77             else:
  78                 return value
  79         if with_meta:
  80             meta = self._getcontext().cfgimpl_get_meta()
  81             if meta is not None:
  82                 value = meta.cfgimpl_get_values(
  83                 )._get_cached_value(opt, path, index=index, submulti_index=submulti_index,
  84                                     from_masterslave=True)
  85                 if isinstance(value, Exception):
  86                     if not isinstance(value, PropertiesOptionError):  # pragma: no cover
  87                         raise value
  88                 else:
  89                     if isinstance(value, Multi):
  90                         if index is not None:
  91                             value = value[index]
  92                             if isinstance(value, SubMulti):
  93                                 if submulti_index is not undefined:
  94                                     value = value[submulti_index]
  95                                 else:
  96                                     value = list(value)
  97                         else:
  98                             new_value = []
  99                             for val in value:
 100                                 if isinstance(val, SubMulti):
 101                                     val = list(val)
 102                                 new_value.append(val)
 103                             value = new_value
 104                             del new_value
 105                     return value
 106         # now try to get default value
 107         value = opt.impl_getdefault()
 108         if opt.impl_is_multi() and index is not None:
 109             if value == []:
 110                 value = opt.impl_getdefault_multi()
 111             else:
 112                 if len(value) > index:
 113                     value = value[index]
 114                 else:
 115                     value = opt.impl_getdefault_multi()
 116         return value
 117
 118     def _getvalue(self, opt, path, self_properties, index, submulti_index,
 119                   with_meta, masterlen, session, validate):
 120         """actually retrieves the value
 121
 122         :param opt: the `option.Option()` object
 123         :returns: the option's value (or the default value if not set)
 124         """
 125         force_default = 'frozen' in self_properties and \
 126             'force_default_on_freeze' in self_properties
 127         # not default value
 128         if index is None or not opt.impl_is_master_slaves('slave'):
 129             _index = None
 130         else:
 131             _index = index
 132         is_default = self._p_.getowner(path, owners.default, session, only_default=True, index=_index) == owners.default
 133         if not is_default and not force_default:
 134             if opt.impl_is_master_slaves('slave'):
 135                 return self._p_.getvalue(path, session, index)
 136             else:
 137                 value = self._p_.getvalue(path, session)
 138                 if index is not None:
 139                     if len(value) > index:
 140                         return value[index]
 141                     #value is smaller than expected
 142                     #so return default value
 143                 else:
 144                     return value
 145         return self._getdefaultvalue(opt, path, with_meta, index,
 146                                      submulti_index, validate)
 147
 148     def get_modified_values(self):
 149         return self._p_.get_modified_values()
 150
 151     def __contains__(self, opt):
 152         """
 153         implements the 'in' keyword syntax in order provide a pythonic way
 154         to kow if an option have a value
 155
 156         :param opt: the `option.Option()` object
 157         """
 158         path = opt.impl_getpath(self._getcontext())
 159         return self._contains(path)
 160
 161     def _contains(self, path, session=None):
 162         if session is None:
 163             session = self._p_.getsession()
 164         return self._p_.hasvalue(path, session)
 165
 166     def __delitem__(self, opt):
 167         """overrides the builtins `del()` instructions"""
 168         self.reset(opt)
 169
 170     def reset(self, opt, path=None, validate=True, _setting_properties=None):
 171         context = self._getcontext()
 172         setting = context.cfgimpl_get_settings()
 173         if path is None:
 174             path = opt.impl_getpath(context)
 175         if _setting_properties is None:
 176             _setting_properties = setting._getproperties(read_write=False)
 177         session = self._p_.getsession()
 178         hasvalue = self._contains(path, session)
 179
 180         if validate and hasvalue and 'validator' in _setting_properties:
 181             session = context.cfgimpl_get_values()._p_.getsession()
 182             fake_context = context._gen_fake_values(session)
 183             fake_value = fake_context.cfgimpl_get_values()
 184             fake_value.reset(opt, path, validate=False)
 185             ret = fake_value._get_cached_value(opt, path,
 186                                                setting_properties=_setting_properties,
 187                                                check_frozen=True)
 188             if isinstance(ret, Exception):
 189                 raise ret
 190         if opt.impl_is_master_slaves('master'):
 191             opt.impl_get_master_slaves().reset(opt, self, _setting_properties)
 192         if hasvalue:
 193             if 'force_store_value' in setting._getproperties(opt=opt,
 194                                                              path=path,
 195                                                              setting_properties=_setting_properties,
 196                                                              read_write=False,
 197                                                              apply_requires=False):
 198                 value = self._getdefaultvalue(opt, path, True, undefined, undefined, validate)
 199                 if isinstance(value, Exception):  # pragma: no cover
 200                     raise value
 201                 self._setvalue(opt, path, value, force_owner=owners.forced)
 202             else:
 203                 self._p_.resetvalue(path, session)
 204             context.cfgimpl_reset_cache(opt=opt, path=path)
 205
 206     def _isempty(self, opt, value, force_allow_empty_list=False, index=None):
 207         "convenience method to know if an option is empty"
 208         if value is undefined:
 209             return False
 210         else:
 211             empty = opt._empty
 212             if index in [None, undefined] and opt.impl_is_multi():
 213                 if force_allow_empty_list:
 214                     allow_empty_list = True
 215                 else:
 216                     allow_empty_list = opt.impl_allow_empty_list()
 217                     if allow_empty_list is undefined:
 218                         if opt.impl_is_master_slaves('slave'):
 219                             allow_empty_list = True
 220                         else:
 221                             allow_empty_list = False
 222                 isempty = value is None or (not allow_empty_list and value == []) or \
 223                     None in value or empty in value
 224             else:
 225                 isempty = value is None or value == empty
 226         return isempty
 227
 228     def __getitem__(self, opt):
 229         "enables us to use the pythonic dictionary-like access to values"
 230         return self._get_cached_value(opt)
 231
 232     def getitem(self, opt, validate=True, force_permissive=False):
 233         """
 234         """
 235         return self._get_cached_value(opt, validate=validate,
 236                                       force_permissive=force_permissive)
 237
 238     def _get_cached_value(self, opt, path=None, validate=True,
 239                           force_permissive=False, trusted_cached_properties=True,
 240                           validate_properties=True,
 241                           setting_properties=undefined, self_properties=undefined,
 242                           index=None, submulti_index=undefined, from_masterslave=False,
 243                           with_meta=True, masterlen=undefined, check_frozen=False,
 244                           session=None, display_warnings=True):
 245         context = self._getcontext()
 246         settings = context.cfgimpl_get_settings()
 247         if path is None:
 248             path = opt.impl_getpath(context)
 249         ntime = None
 250         if setting_properties is undefined:
 251             setting_properties = settings._getproperties(read_write=False)
 252         if self_properties is undefined:
 253             self_properties = settings._getproperties(opt, path,
 254                                                       read_write=False,
 255                                                       setting_properties=setting_properties,
 256                                                       index=index)
 257         if 'cache' in setting_properties and self._p_.hascache(path, index):
 258             if 'expire' in setting_properties:
 259                 ntime = int(time())
 260             is_cached, value = self._p_.getcache(path, ntime, index)
 261             if is_cached:
 262                 if opt.impl_is_multi() and not isinstance(value, Multi) and index is None:
 263                     value = Multi(value, self.context, opt, path)
 264                 if not trusted_cached_properties:
 265                     # revalidate properties (because of not default properties)
 266                     props = settings.validate_properties(opt, False, False, value=value,
 267                                                          path=path,
 268                                                          force_permissive=force_permissive,
 269                                                          setting_properties=setting_properties,
 270                                                          self_properties=self_properties,
 271                                                          index=index)
 272                     if props:
 273                         return props
 274                 return value
 275         if session is None:
 276             session = self._p_.getsession()
 277         if not from_masterslave and opt.impl_is_master_slaves():
 278             val = opt.impl_get_master_slaves().getitem(self, opt, path,
 279                                                        validate,
 280                                                        force_permissive,
 281                                                        trusted_cached_properties,
 282                                                        validate_properties,
 283                                                        session,
 284                                                        setting_properties=setting_properties,
 285                                                        index=index,
 286                                                        self_properties=self_properties,
 287                                                        check_frozen=check_frozen)
 288         else:
 289             val = self._get_validated_value(opt, path, validate,
 290                                             force_permissive,
 291                                             validate_properties,
 292                                             setting_properties,
 293                                             self_properties,
 294                                             with_meta=with_meta,
 295                                             masterlen=masterlen,
 296                                             index=index,
 297                                             submulti_index=submulti_index,
 298                                             check_frozen=check_frozen,
 299                                             session=session,
 300                                             display_warnings=display_warnings)
 301         if isinstance(val, Exception):
 302             return val
 303         # cache doesn't work with SubMulti yet
 304         if not isinstance(val, SubMulti) and 'cache' in setting_properties and \
 305                 validate and validate_properties and force_permissive is False \
 306                 and trusted_cached_properties is True:
 307             if 'expire' in setting_properties:
 308                 if ntime is None:
 309                     ntime = int(time())
 310                 ntime = ntime + expires_time
 311             self._p_.setcache(path, val, ntime, index)
 312         return val
 313
 314     def _get_validated_value(self, opt, path, validate, force_permissive,
 315                              validate_properties, setting_properties,
 316                              self_properties,
 317                              index=None, submulti_index=undefined,
 318                              with_meta=True,
 319                              masterlen=undefined,
 320                              check_frozen=False,
 321                              session=None, display_warnings=True):
 322         """same has getitem but don't touch the cache
 323         index is None for slave value, if value returned is not a list, just return []
 324         """
 325         context = self._getcontext()
 326         setting = context.cfgimpl_get_settings()
 327         config_error = None
 328         if session is None:
 329             session = self._p_.getsession()
 330         value = self._getvalue(opt, path, self_properties, index, submulti_index,
 331                                with_meta, masterlen, session, validate)
 332         if isinstance(value, Exception):
 333             value_error = True
 334             if isinstance(value, ConfigError):
 335                 # For calculating properties, we need value (ie for mandatory
 336                 # value).
 337                 # If value is calculating with a PropertiesOptionError's option
 338                 # _getvalue raise a ConfigError.
 339                 # We can not raise ConfigError if this option should raise
 340                 # PropertiesOptionError too. So we get config_error and raise
 341                 # ConfigError if properties did not raise.
 342                 config_error = value
 343                 # value is not set, for 'undefined' (cannot set None because of
 344                 # mandatory property)
 345                 value = undefined
 346             else:  # pragma: no cover
 347                 raise value
 348         else:
 349             value_error = False
 350             if opt.impl_is_multi():
 351                 if index is None:
 352                     value = Multi(value, self.context, opt, path)
 353                 elif opt.impl_is_submulti() and submulti_index is undefined:
 354                     value = SubMulti(value, self.context, opt, path,
 355                                      index)
 356
 357             if validate:
 358                 if submulti_index is undefined:
 359                     force_submulti_index = None
 360                 else:
 361                     force_submulti_index = submulti_index
 362                 err = opt.impl_validate(value, context,
 363                                         'validator' in setting_properties,
 364                                         force_index=index,
 365                                         force_submulti_index=force_submulti_index,
 366                                         display_error=True,
 367                                         display_warnings=False)
 368                 if err:
 369                     config_error = err
 370                     value = None
 371
 372         if validate_properties:
 373             if config_error is not None:
 374                 # should not raise PropertiesOptionError if option is
 375                 # mandatory
 376                 val_props = undefined
 377             else:
 378                 val_props = value
 379             props = setting.validate_properties(opt, False, check_frozen, value=val_props,
 380                                                 path=path,
 381                                                 force_permissive=force_permissive,
 382                                                 setting_properties=setting_properties,
 383                                                 self_properties=self_properties,
 384                                                 index=index)
 385             if props:
 386                 return props
 387         if not value_error and validate and display_warnings:
 388             opt.impl_validate(value, context,
 389                               'validator' in setting_properties,
 390                               force_index=index,
 391                               force_submulti_index=force_submulti_index,
 392                               display_error=False,
 393                               display_warnings=display_warnings,
 394                               setting_properties=setting_properties,)
 395         if config_error is not None:
 396             return config_error
 397         return value
 398
 399     def __setitem__(self, opt, value):
 400         raise ConfigError(_('you should only set value with config'))
 401
 402     def setitem(self, opt, value, path, force_permissive=False,
 403                 check_frozen=True, not_raises=False, index=None):
 404         # check_frozen is, for example, used with "force_store_value"
 405         # user didn't change value, so not write
 406         # valid opt
 407         context = self._getcontext()
 408         setting_properties = context.cfgimpl_get_settings()._getproperties(read_write=False)
 409         if 'validator' in setting_properties:
 410             session = context.cfgimpl_get_values()._p_.getsession()
 411             fake_context = context._gen_fake_values(session)
 412             fake_values = fake_context.cfgimpl_get_values()
 413             fake_values._setvalue(opt, path, value, index=index)
 414             props = fake_values.validate(opt, value, path,
 415                                          check_frozen=check_frozen,
 416                                          force_permissive=force_permissive,
 417                                          setting_properties=setting_properties,
 418                                          session=session, not_raises=not_raises,
 419                                          index=index)
 420             if props and not_raises:
 421                 return
 422             err = opt.impl_validate(value, fake_context, display_warnings=False, force_index=index)
 423             if err:
 424                 raise err
 425             opt.impl_validate(value, fake_context, display_error=False)
 426         self._setvalue(opt, path, value, index=index)
 427
 428     def _setvalue(self, opt, path, value, force_owner=undefined, index=None):
 429         context = self._getcontext()
 430         context.cfgimpl_reset_cache(opt=opt, path=path)
 431         if force_owner is undefined:
 432             owner = context.cfgimpl_get_settings().getowner()
 433         else:
 434             owner = force_owner
 435         # in storage, value must not be a multi
 436         if isinstance(value, Multi):
 437             if not opt.impl_is_master_slaves('slave') or index is None:
 438                 value = list(value)
 439                 if opt.impl_is_submulti():
 440                     for idx, val in enumerate(value):
 441                         if isinstance(val, SubMulti):
 442                             value[idx] = list(val)
 443             else:
 444                 value = value[index]
 445         session = self._p_.getsession()
 446         #FIXME pourquoi là et pas dans masterslaves ??
 447         if opt.impl_is_master_slaves('slave'):
 448             if index is not None:
 449                 self._p_.setvalue(path, value, owner, index, session)
 450             else:
 451                 self._p_.resetvalue(path, session)
 452                 for idx, val in enumerate(value):
 453                     self._p_.setvalue(path, val, owner, idx, session)
 454         else:
 455             self._p_.setvalue(path, value, owner, None, session)
 456         del(session)
 457
 458     def validate(self, opt, value, path, check_frozen=True, force_permissive=False,
 459                  setting_properties=undefined, valid_masterslave=True,
 460                  not_raises=False, session=None, index=None):
 461         if valid_masterslave and opt.impl_is_master_slaves():
 462             if session is None:
 463                 session = self._p_.getsession()
 464             if index is not None:
 465                 len_value = index
 466                 setitem = False
 467             else:
 468                 len_value = len(value)
 469                 setitem = True
 470             val = opt.impl_get_master_slaves().validate(self, opt, len_value, path, session, setitem=setitem)
 471             if isinstance(val, Exception):
 472                 return val
 473         props = self._getcontext().cfgimpl_get_settings().validate_properties(opt,
 474                                                                               False,
 475                                                                               check_frozen,
 476                                                                               value=value,
 477                                                                               path=path,
 478                                                                               force_permissive=force_permissive,
 479                                                                               setting_properties=setting_properties,
 480                                                                               index=index)
 481         if props:
 482             if not_raises:
 483                 return props
 484             raise props
 485
 486     def _is_meta(self, opt, path, session):
 487         context = self._getcontext()
 488         setting = context.cfgimpl_get_settings()
 489         self_properties = setting._getproperties(opt, path, read_write=False)
 490         if 'frozen' in self_properties and 'force_default_on_freeze' in self_properties:
 491             return False
 492         if self._p_.getowner(path, owners.default, session, only_default=True) is not owners.default:
 493             return False
 494         if context.cfgimpl_get_meta() is not None:
 495             return True
 496         return False
 497
 498     def getowner(self, opt, index=None, force_permissive=False, session=None):
 499         """
 500         retrieves the option's owner
 501
 502         :param opt: the `option.Option` object
 503         :param force_permissive: behaves as if the permissive property
 504                                  was present
 505         :returns: a `setting.owners.Owner` object
 506         """
 507         if isinstance(opt, SymLinkOption) and \
 508                 not isinstance(opt, DynSymLinkOption):
 509             opt = opt._impl_getopt()
 510         path = opt.impl_getpath(self._getcontext())
 511         return self._getowner(opt, path, session, index=index, force_permissive=force_permissive)
 512
 513     def _getowner(self, opt, path, session, validate_properties=True,
 514                   force_permissive=False, validate_meta=undefined,
 515                   self_properties=undefined, only_default=False,
 516                   index=None):
 517         """get owner of an option
 518         """
 519         if session is None:
 520             session = self._p_.getsession()
 521         if not isinstance(opt, Option) and not isinstance(opt,
 522                                                           DynSymLinkOption):
 523             raise ConfigError(_('owner only avalaible for an option'))
 524         context = self._getcontext()
 525         if self_properties is undefined:
 526             self_properties = context.cfgimpl_get_settings()._getproperties(
 527                 opt, path, read_write=False)
 528         if 'frozen' in self_properties and 'force_default_on_freeze' in self_properties:
 529             return owners.default
 530         if validate_properties:
 531             value = self._get_cached_value(opt, path, True, force_permissive, None, True,
 532                                            self_properties=self_properties, session=session,
 533                                            index=index)
 534             if isinstance(value, Exception):
 535                 raise value
 536
 537         owner = self._p_.getowner(path, owners.default, session, only_default=only_default, index=index)
 538         if validate_meta is undefined:
 539             if opt.impl_is_master_slaves('slave'):
 540                 master = opt.impl_get_master_slaves().getmaster(opt)
 541                 masterp = master.impl_getpath(context)
 542                 validate_meta = self._is_meta(opt, masterp, session)
 543             else:
 544                 validate_meta = True
 545         if validate_meta:
 546             meta = context.cfgimpl_get_meta()
 547             if owner is owners.default and meta is not None:
 548                 owner = meta.cfgimpl_get_values()._getowner(opt, path, session,
 549                                                             validate_properties=validate_properties,
 550                                                             force_permissive=force_permissive,
 551                                                             self_properties=self_properties,
 552                                                             only_default=only_default, index=index)
 553         return owner
 554
 555     def setowner(self, opt, owner, index=None):
 556         """
 557         sets a owner to an option
 558
 559         :param opt: the `option.Option` object
 560         :param owner: a valid owner, that is a `setting.owners.Owner` object
 561         """
 562         if not isinstance(owner, owners.Owner):
 563             raise TypeError(_("invalid generic owner {0}").format(str(owner)))
 564
 565         path = opt.impl_getpath(self._getcontext())
 566         session = self._p_.getsession()
 567         if not self._p_.hasvalue(path, session):
 568             raise ConfigError(_('no value for {0} cannot change owner to {1}'
 569                                 '').format(path, owner))
 570         props = self._getcontext().cfgimpl_get_settings().validate_properties(opt,
 571                                                                               False,
 572                                                                               True,
 573                                                                               path,
 574                                                                               index=index)
 575         if props:
 576             raise props
 577         self._p_.setowner(path, owner, session, index=index)
 578
 579     def is_default_owner(self, opt, validate_properties=True,
 580                          validate_meta=True, index=None,
 581                          force_permissive=False):
 582         """
 583         :param config: *must* be only the **parent** config
 584                        (not the toplevel config)
 585         :return: boolean
 586         """
 587         path = opt.impl_getpath(self._getcontext())
 588         return self._is_default_owner(opt, path, session=None,
 589                                       validate_properties=validate_properties,
 590                                       validate_meta=validate_meta, index=index,
 591                                       force_permissive=force_permissive)
 592
 593     def _is_default_owner(self, opt, path, session, validate_properties=True,
 594                           validate_meta=True, self_properties=undefined,
 595                           index=None, force_permissive=False):
 596         d = self._getowner(opt, path, session, validate_properties=validate_properties,
 597                            validate_meta=validate_meta,
 598                            self_properties=self_properties, only_default=True,
 599                            index=index, force_permissive=force_permissive)
 600         return d == owners.default
 601
 602     # information
 603     def set_information(self, key, value):
 604         """updates the information's attribute
 605
 606         :param key: information's key (ex: "help", "doc"
 607         :param value: information's value (ex: "the help string")
 608         """
 609         self._p_.set_information(key, value)
 610
 611     def get_information(self, key, default=undefined):
 612         """retrieves one information's item
 613
 614         :param key: the item string (ex: "help")
 615         """
 616         return self._p_.get_information(key, default)
 617
 618     def del_information(self, key, raises=True):
 619         self._p_.del_information(key, raises)
 620
 621     def mandatory_warnings(self, force_permissive=True):
 622         """convenience function to trace Options that are mandatory and
 623         where no value has been set
 624
 625         :returns: generator of mandatory Option's path
 626         """
 627         context = self._getcontext()
 628         settings = context.cfgimpl_get_settings()
 629         setting_properties = context.cfgimpl_get_settings()._getproperties()
 630         setting_properties.update(['mandatory', 'empty'])
 631         def _is_properties_option(err, path):
 632             if not isinstance(err, Exception):
 633                 pass
 634             elif isinstance(err, PropertiesOptionError):
 635                 if err.proptype == ['mandatory']:
 636                     return path
 637             elif isinstance(err, ConfigError):
 638                 #assume that uncalculated value is an empty value
 639                 return path
 640             else:
 641                 raise err
 642
 643         def _mandatory_warnings(description, currpath=None):
 644             if currpath is None:
 645                 currpath = []
 646             for opt in description._impl_getchildren(context=context):
 647                 name = opt.impl_getname()
 648                 path = '.'.join(currpath + [name])
 649
 650                 if opt.impl_is_optiondescription():
 651                     if not settings.validate_properties(opt, True, False, path=path,
 652                                                         force_permissive=True,
 653                                                         setting_properties=setting_properties):
 654                         for path in _mandatory_warnings(opt, currpath + [name]):
 655                             yield path
 656                 else:
 657                     if isinstance(opt, SymLinkOption) and \
 658                             not isinstance(opt, DynSymLinkOption):
 659                         continue
 660                     self_properties = settings._getproperties(opt, path,
 661                                                               read_write=False,
 662                                                               setting_properties=setting_properties)
 663                     if 'mandatory' in self_properties or 'empty' in self_properties:
 664                         err = self._get_cached_value(opt, path=path,
 665                                                      trusted_cached_properties=False,
 666                                                      force_permissive=True,
 667                                                      setting_properties=setting_properties,
 668                                                      self_properties=self_properties,
 669                                                      validate=True,
 670                                                      display_warnings=False)
 671                         if opt.impl_is_master_slaves('slave') and isinstance(err, list):
 672                             for val in err:
 673                                 ret = _is_properties_option(val, path)
 674                                 if ret is not None:
 675                                     yield ret
 676                                     break
 677                         else:
 678                             ret = _is_properties_option(err, path)
 679                             if ret is not None:
 680                                 yield ret
 681
 682         descr = context.cfgimpl_get_description()
 683         for path in _mandatory_warnings(descr):
 684             yield path
 685
 686     def force_cache(self):
 687         """parse all option to force data in cache
 688         """
 689         context = self.context()
 690         if not 'cache' in context.cfgimpl_get_settings():
 691             raise ConfigError(_('can force cache only if cache '
 692                                 'is actived in config'))
 693         #FIXME properties and value should update "expired" time
 694         for path in context.cfgimpl_get_description().impl_getpaths(
 695                 include_groups=True):
 696             err = context.getattr(path, returns_raise=True)
 697             if isinstance(err, Exception) and not isinstance(err, PropertiesOptionError):  # pragma: no cover
 698                 raise err
 699
 700     def __getstate__(self):
 701         return {'_p_': self._p_}
 702
 703     def _impl_setstate(self, storage):
 704         self._p_._storage = storage
 705
 706     def __setstate__(self, states):
 707         self._p_ = states['_p_']
 708
 709
 710 # ____________________________________________________________
 711 # multi types
 712 class Multi(list):
 713     """multi options values container
 714     that support item notation for the values of multi options"""
 715     __slots__ = ('opt', 'path', 'context', '__weakref__')
 716
 717     def __init__(self, value, context, opt, path):
 718         """
 719         :param value: the Multi wraps a list value
 720         :param context: the home config that has the values
 721         :param opt: the option object that have this Multi value
 722         :param path: path of the option
 723         """
 724         if value is None:
 725             value = []
 726         if not opt.impl_is_submulti() and isinstance(value, Multi):
 727             raise ValueError(_('{0} is already a Multi ').format(
 728                 opt.impl_getname()))
 729         self.opt = opt
 730         self.path = path
 731         if not isinstance(context, weakref.ReferenceType):
 732             raise ValueError('context must be a Weakref')
 733         self.context = context
 734         if not isinstance(value, list):
 735             if not '_index' in self.__slots__ and opt.impl_is_submulti():
 736                 value = [[value]]
 737             else:
 738                 value = [value]
 739         elif value != [] and not '_index' in self.__slots__ and \
 740                 opt.impl_is_submulti() and not isinstance(value[0], list):
 741             value = [value]
 742         super(Multi, self).__init__(value)
 743         if opt.impl_is_submulti():
 744             if not '_index' in self.__slots__:
 745                 for idx, val in enumerate(self):
 746                     if not isinstance(val, SubMulti):
 747                         super(Multi, self).__setitem__(idx, SubMulti(val,
 748                                                                      context,
 749                                                                      opt, path,
 750                                                                      idx))
 751                     self[idx].submulti = weakref.ref(self)
 752
 753     def _getcontext(self):
 754         """context could be None, we need to test it
 755         context is None only if all reference to `Config` object is deleted
 756         (for example we delete a `Config` and we manipulate a reference to
 757         old `SubConfig`, `Values`, `Multi` or `Settings`)
 758         """
 759         context = self.context()
 760         if context is None:
 761             raise ConfigError(_('the context does not exist anymore'))
 762         return context
 763
 764     def __setitem__(self, index, value):
 765         self._setitem(index, value)
 766
 767     def _setitem(self, index, value, validate=True):
 768         context = self._getcontext()
 769         setting = context.cfgimpl_get_settings()
 770         setting_properties = setting._getproperties(read_write=False)
 771         if index < 0:
 772             index = self.__len__() + index
 773         if 'validator' in setting_properties and validate:
 774             session = context.cfgimpl_get_values()._p_.getsession()
 775             fake_context = context._gen_fake_values(session)
 776             fake_multi = Multi(list(self), weakref.ref(fake_context), self.opt, self.path)
 777             fake_multi._setitem(index, value, validate=False)
 778             self._validate(value, fake_context, index, True)
 779         #assume not checking mandatory property
 780         super(Multi, self).__setitem__(index, value)
 781         self._store(index=index)
 782
 783     #def __repr__(self, *args, **kwargs):
 784     #    return super(Multi, self).__repr__(*args, **kwargs)
 785
 786     def __getitem__(self, index):
 787         value = super(Multi, self).__getitem__(index)
 788         if isinstance(value, PropertiesOptionError):
 789             raise value
 790         return value
 791
 792     def __delitem__(self, index):
 793         return self.pop(index)
 794
 795     def _getdefaultvalue(self, index):
 796         values = self._getcontext().cfgimpl_get_values()
 797         value = values._getdefaultvalue(self.opt, self.path, True, index,
 798                                         undefined, True)
 799         if self.opt.impl_is_submulti():
 800             value = SubMulti(value, self.context, self.opt, self.path, index)
 801         return value
 802
 803     def append(self, value=undefined, force=False, setitem=True, validate=True,
 804                force_permissive=False):
 805         """the list value can be updated (appened)
 806         only if the option is a master
 807         """
 808         if not force and self.opt.impl_is_master_slaves('slave'):
 809             raise SlaveError(_("cannot append a value on a multi option {0}"
 810                                " which is a slave").format(self.opt.impl_getname()))
 811         index = self.__len__()
 812         if value is undefined:
 813             value = self._getdefaultvalue(index)
 814         if validate and value not in [None, undefined]:
 815             context = self._getcontext()
 816             setting = context.cfgimpl_get_settings()
 817             setting_properties = setting._getproperties(read_write=False)
 818             if 'validator' in setting_properties:
 819                 session = context.cfgimpl_get_values()._p_.getsession()
 820                 fake_context = context._gen_fake_values(session)
 821                 fake_multi = Multi(list(self), weakref.ref(fake_context), self.opt, self.path)
 822                 if isinstance(fake_multi, Exception):
 823                     raise fake_multi
 824                 fake_multi.append(value, validate=False, force=True,
 825                                   setitem=setitem)
 826                 self._validate(value, fake_context, index, True)
 827         if not '_index' in self.__slots__ and self.opt.impl_is_submulti():
 828             if not isinstance(value, SubMulti):
 829                 value = SubMulti(value, self.context, self.opt, self.path, index)
 830             value.submulti = weakref.ref(self)
 831         super(Multi, self).append(value)
 832         if setitem:
 833             self._store(force=force)
 834
 835     def append_properties_error(self, err):
 836         super(Multi, self).append(err)
 837
 838     def sort(self, cmp=None, key=None, reverse=False):
 839         if self.opt.impl_is_master_slaves():
 840             raise SlaveError(_("cannot sort multi option {0} if master or slave"
 841                                "").format(self.opt.impl_getname()))
 842         if sys.version_info[0] >= 3:  # pragma: no cover
 843             if cmp is not None:
 844                 raise ValueError(_('cmp is not permitted in python v3 or '
 845                                    'greater'))
 846             super(Multi, self).sort(key=key, reverse=reverse)
 847         else:
 848             super(Multi, self).sort(cmp=cmp, key=key, reverse=reverse)
 849         self._store()
 850
 851     def reverse(self):
 852         if self.opt.impl_is_master_slaves():
 853             raise SlaveError(_("cannot reverse multi option {0} if master or "
 854                                "slave").format(self.opt.impl_getname()))
 855         super(Multi, self).reverse()
 856         self._store()
 857
 858     def insert(self, index, value, validate=True):
 859         if self.opt.impl_is_master_slaves():
 860             raise SlaveError(_("cannot insert multi option {0} if master or "
 861                                "slave").format(self.opt.impl_getname()))
 862         context = self._getcontext()
 863         setting = setting = context.cfgimpl_get_settings()
 864         setting_properties = setting._getproperties(read_write=False)
 865         if 'validator' in setting_properties and validate and value is not None:
 866             session = context.cfgimpl_get_values()._p_.getsession()
 867             fake_context = context._gen_fake_values(session)
 868             fake_multi = Multi(list(self), weakref.ref(fake_context), self.opt, self.path)
 869             fake_multi.insert(index, value, validate=False)
 870             self._validate(value, fake_context, index, True)
 871         super(Multi, self).insert(index, value)
 872         self._store()
 873
 874     def extend(self, iterable, validate=True):
 875         if self.opt.impl_is_master_slaves():
 876             raise SlaveError(_("cannot extend multi option {0} if master or "
 877                                "slave").format(self.opt.impl_getname()))
 878         index = getattr(self, '_index', None)
 879         context = self._getcontext()
 880         setting = context.cfgimpl_get_settings()
 881         setting_properties = setting._getproperties(read_write=False)
 882         if 'validator' in setting_properties and validate:
 883             session = context.cfgimpl_get_values()._p_.getsession()
 884             fake_context = context._gen_fake_values(session)
 885             fake_multi = Multi(list(self), weakref.ref(fake_context), self.opt, self.path)
 886             if index is None:
 887                 fake_multi.extend(iterable, validate=False)
 888                 self._validate(fake_multi, fake_context, index)
 889             else:
 890                 fake_multi[index].extend(iterable, validate=False)
 891                 self._validate(fake_multi[index], fake_context, index)
 892         super(Multi, self).extend(iterable)
 893         self._store()
 894
 895     def _validate(self, value, fake_context, force_index, submulti=False):
 896         err = self.opt.impl_validate(value, context=fake_context,
 897                                      force_index=force_index,
 898                                      multi=self)
 899         if err:
 900             raise err
 901
 902     def pop(self, index, force=False):
 903         """the list value can be updated (poped)
 904         only if the option is a master
 905
 906         :param index: remove item a index
 907         :type index: int
 908         :param force: force pop item (withoud check master/slave)
 909         :type force: boolean
 910         :returns: item at index
 911         """
 912         context = self._getcontext()
 913         if not force:
 914             if self.opt.impl_is_master_slaves('slave'):
 915                 raise SlaveError(_("cannot pop a value on a multi option {0}"
 916                                    " which is a slave").format(self.opt.impl_getname()))
 917             if self.opt.impl_is_master_slaves('master'):
 918                 self.opt.impl_get_master_slaves().pop(self.opt,
 919                                                       context.cfgimpl_get_values(), index)
 920         #set value without valid properties
 921         ret = super(Multi, self).pop(index)
 922         self._store(force=force)
 923         return ret
 924
 925     def remove(self, value):
 926         idx = self.index(value)
 927         return self.pop(idx)
 928
 929     def _store(self, force=False, index=None):
 930         values = self._getcontext().cfgimpl_get_values()
 931         if not force:
 932             #FIXME could get properties an pass it
 933             values.validate(self.opt, self, self.path,
 934                             valid_masterslave=False)
 935         values._setvalue(self.opt, self.path, self, index=index)
 936
 937
 938 class SubMulti(Multi):
 939     __slots__ = ('_index', 'submulti')
 940
 941     def __init__(self, value, context, opt, path, index):
 942         """
 943         :param index: index (only for slave with submulti)
 944         :type index: `int`
 945         """
 946         self._index = index
 947         super(SubMulti, self).__init__(value, context, opt, path)
 948
 949     def append(self, value=undefined):
 950         super(SubMulti, self).append(value, force=True)
 951
 952     def pop(self, index):
 953         return super(SubMulti, self).pop(index, force=True)
 954
 955     def __setitem__(self, index, value):
 956         self._setitem(index, value)
 957
 958     def _store(self, force=False, index=None):
 959         #force is unused here
 960         values = self._getcontext().cfgimpl_get_values()
 961         values.validate(self.opt, self, self.path, valid_masterslave=False)
 962         values._setvalue(self.opt, self.path, self.submulti())
 963
 964     def _validate(self, value, fake_context, force_index, submulti=False):
 965         if value is not None:
 966             if submulti is False:
 967                 super(SubMulti, self)._validate(value, fake_context,
 968                                                 force_index, submulti)
 969             else:
 970                 err = self.opt.impl_validate(value, context=fake_context,
 971                                              force_index=self._index,
 972                                              force_submulti_index=force_index,
 973                                              multi=self)
 974                 if err:
 975                     raise err
 976
 977     def _getdefaultvalue(self, index):
 978         values = self._getcontext().cfgimpl_get_values()
 979         return values._getdefaultvalue(self.opt, self.path, True, index,
 980                                        self._index, True)