many improvment
[tiramisu.git] / tiramisu / value.py
1 # -*- coding: utf-8 -*-
2 "takes care of the option's values and multi values"
3 # Copyright (C) 2013-2017 Team tiramisu (see AUTHORS for all contributors)
4 #
5 # This program is free software: you can redistribute it and/or modify it
6 # under the terms of the GNU Lesser General Public License as published by the
7 # Free Software Foundation, either version 3 of the License, or (at your
8 # option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful, but WITHOUT
11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
12 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
13 # details.
14 #
15 # You should have received a copy of the GNU Lesser General Public License
16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 # ____________________________________________________________
18 from time import time
19 import sys
20 import weakref
21 from .error import ConfigError, SlaveError, PropertiesOptionError
22 from .setting import owners, expires_time, undefined
23 from .autolib import carry_out_calculation
24 from .i18n import _
25 from .option import SymLinkOption, DynSymLinkOption, Option
26 i_i = 0
27
28
29 class Values(object):
30     """The `Config`'s root is indeed  in charge of the `Option()`'s values,
31     but the values are physicaly located here, in `Values`, wich is also
32     responsible of a caching utility.
33     """
34     __slots__ = ('context', '_p_', '__weakref__')
35
36     def __init__(self, context, storage):
37         """
38         Initializes the values's dict.
39
40         :param context: the context is the home config's values
41
42         """
43         self.context = weakref.ref(context)
44         # the storage type is dictionary or sqlite3
45         self._p_ = storage
46
47     def _getcontext(self):
48         """context could be None, we need to test it
49         context is None only if all reference to `Config` object is deleted
50         (for example we delete a `Config` and we manipulate a reference to
51         old `SubConfig`, `Values`, `Multi` or `Settings`)
52         """
53         context = self.context()
54         if context is None:
55             raise ConfigError(_('the context does not exist anymore'))
56         return context
57
58     def _get_multi(self, opt, path):
59         return Multi([], self.context, opt, path)
60
61     def _getdefaultvalue(self, opt, path, with_meta, index, submulti_index, validate):
62         # if value has callback and is not set
63         if opt.impl_has_callback():
64             callback, callback_params = opt.impl_get_callback()
65             value = carry_out_calculation(opt, context=self._getcontext(),
66                                           callback=callback,
67                                           callback_params=callback_params,
68                                           index=index, validate=validate)
69             if isinstance(value, list) and index is not None:
70                 #if return a list and index is set, return value only if
71                 #it's a submulti without submulti_index and without list of list
72                 if opt.impl_is_submulti() and submulti_index is undefined and \
73                         (len(value) == 0 or not isinstance(value[0], list)):
74                     return value
75                 if not opt.impl_is_submulti() and len(value) > index:
76                     return value[index]
77             else:
78                 return value
79         if with_meta:
80             meta = self._getcontext().cfgimpl_get_meta()
81             if meta is not None:
82                 value = meta.cfgimpl_get_values(
83                 )._get_cached_value(opt, path, index=index, submulti_index=submulti_index,
84                                     from_masterslave=True)
85                 if isinstance(value, Exception):
86                     if not isinstance(value, PropertiesOptionError):  # pragma: no cover
87                         raise value
88                 else:
89                     if isinstance(value, Multi):
90                         if index is not None:
91                             value = value[index]
92                             if isinstance(value, SubMulti):
93                                 if submulti_index is not undefined:
94                                     value = value[submulti_index]
95                                 else:
96                                     value = list(value)
97                         else:
98                             new_value = []
99                             for val in value:
100                                 if isinstance(val, SubMulti):
101                                     val = list(val)
102                                 new_value.append(val)
103                             value = new_value
104                             del new_value
105                     return value
106         # now try to get default value
107         value = opt.impl_getdefault()
108         if opt.impl_is_multi() and index is not None:
109             if value == []:
110                 value = opt.impl_getdefault_multi()
111             else:
112                 if len(value) > index:
113                     value = value[index]
114                 else:
115                     value = opt.impl_getdefault_multi()
116         return value
117
118     def _getvalue(self, opt, path, self_properties, index, submulti_index,
119                   with_meta, masterlen, session, validate):
120         """actually retrieves the value
121
122         :param opt: the `option.Option()` object
123         :returns: the option's value (or the default value if not set)
124         """
125         force_default = 'frozen' in self_properties and \
126             'force_default_on_freeze' in self_properties
127         # not default value
128         if index is None or not opt.impl_is_master_slaves('slave'):
129             _index = None
130         else:
131             _index = index
132         owner, value = self._p_.getowner(path, owners.default, session, only_default=True,
133                                            index=_index, with_value=True)
134         is_default = owner == owners.default
135         if not is_default and not force_default:
136             if index is not None and not opt.impl_is_master_slaves('slave'):
137                 if len(value) > index:
138                     return value[index]
139                 #value is smaller than expected
140                 #so return default value
141             else:
142                 return value
143         return self._getdefaultvalue(opt, path, with_meta, index,
144                                      submulti_index, validate)
145
146     def get_modified_values(self):
147         return self._p_.get_modified_values()
148
149     def __contains__(self, opt):
150         """
151         implements the 'in' keyword syntax in order provide a pythonic way
152         to kow if an option have a value
153
154         :param opt: the `option.Option()` object
155         """
156         path = opt.impl_getpath(self._getcontext())
157         return self._contains(path)
158
159     def _contains(self, path, session=None):
160         if session is None:
161             session = self._p_.getsession()
162         return self._p_.hasvalue(path, session)
163
164     def __delitem__(self, opt):
165         """overrides the builtins `del()` instructions"""
166         self.reset(opt)
167
168     def reset(self, opt, path=None, validate=True, _setting_properties=None):
169         context = self._getcontext()
170         setting = context.cfgimpl_get_settings()
171         if path is None:
172             path = opt.impl_getpath(context)
173         if _setting_properties is None:
174             _setting_properties = setting._getproperties(read_write=False)
175         session = self._p_.getsession()
176         hasvalue = self._contains(path, session)
177
178         if validate and hasvalue and 'validator' in _setting_properties:
179             session = context.cfgimpl_get_values()._p_.getsession()
180             fake_context = context._gen_fake_values(session)
181             fake_value = fake_context.cfgimpl_get_values()
182             fake_value.reset(opt, path, validate=False)
183             ret = fake_value._get_cached_value(opt, path,
184                                                setting_properties=_setting_properties,
185                                                check_frozen=True)
186             if isinstance(ret, Exception):
187                 raise ret
188         if opt.impl_is_master_slaves('master'):
189             opt.impl_get_master_slaves().reset(opt, self, _setting_properties)
190         if hasvalue:
191             if 'force_store_value' in setting._getproperties(opt=opt,
192                                                              path=path,
193                                                              setting_properties=_setting_properties,
194                                                              read_write=False,
195                                                              apply_requires=False):
196                 value = self._getdefaultvalue(opt, path, True, undefined, undefined, validate)
197                 if isinstance(value, Exception):  # pragma: no cover
198                     raise value
199                 self._setvalue(opt, path, value, force_owner=owners.forced)
200             else:
201                 self._p_.resetvalue(path, session)
202             context.cfgimpl_reset_cache(opt=opt, path=path)
203
204     def _isempty(self, opt, value, force_allow_empty_list=False, index=None):
205         "convenience method to know if an option is empty"
206         if value is undefined:
207             return False
208         else:
209             empty = opt._empty
210             if index in [None, undefined] and opt.impl_is_multi():
211                 if force_allow_empty_list:
212                     allow_empty_list = True
213                 else:
214                     allow_empty_list = opt.impl_allow_empty_list()
215                     if allow_empty_list is undefined:
216                         if opt.impl_is_master_slaves('slave'):
217                             allow_empty_list = True
218                         else:
219                             allow_empty_list = False
220                 isempty = value is None or (not allow_empty_list and value == []) or \
221                     None in value or empty in value
222             else:
223                 isempty = value is None or value == empty
224         return isempty
225
226     def __getitem__(self, opt):
227         "enables us to use the pythonic dictionary-like access to values"
228         return self._get_cached_value(opt)
229
230     def getitem(self, opt, validate=True, force_permissive=False):
231         """
232         """
233         return self._get_cached_value(opt, validate=validate,
234                                       force_permissive=force_permissive)
235
236     def _get_cached_value(self, opt, path=None, validate=True,
237                           force_permissive=False, trusted_cached_properties=True,
238                           validate_properties=True,
239                           setting_properties=undefined, self_properties=undefined,
240                           index=None, submulti_index=undefined, from_masterslave=False,
241                           with_meta=True, masterlen=undefined, check_frozen=False,
242                           session=None, display_warnings=True):
243         context = self._getcontext()
244         settings = context.cfgimpl_get_settings()
245         if path is None:
246             path = opt.impl_getpath(context)
247         ntime = None
248         if setting_properties is undefined:
249             setting_properties = settings._getproperties(read_write=False)
250         if self_properties is undefined:
251             self_properties = settings._getproperties(opt, path,
252                                                       read_write=False,
253                                                       setting_properties=setting_properties,
254                                                       index=index)
255         if 'cache' in setting_properties and self._p_.hascache(path, index):
256             if 'expire' in setting_properties:
257                 ntime = int(time())
258             is_cached, value = self._p_.getcache(path, ntime, index)
259             if is_cached:
260                 if opt.impl_is_multi() and not isinstance(value, Multi) and index is None:
261                     value = Multi(value, self.context, opt, path)
262                 if not trusted_cached_properties:
263                     # revalidate properties (because of not default properties)
264                     props = settings.validate_properties(opt, False, False, value=value,
265                                                          path=path,
266                                                          force_permissive=force_permissive,
267                                                          setting_properties=setting_properties,
268                                                          self_properties=self_properties,
269                                                          index=index)
270                     if props:
271                         return props
272                 return value
273         if session is None:
274             session = self._p_.getsession()
275         if not from_masterslave and opt.impl_is_master_slaves():
276             val = opt.impl_get_master_slaves().getitem(self, opt, path,
277                                                        validate,
278                                                        force_permissive,
279                                                        trusted_cached_properties,
280                                                        validate_properties,
281                                                        session,
282                                                        setting_properties=setting_properties,
283                                                        index=index,
284                                                        self_properties=self_properties,
285                                                        check_frozen=check_frozen)
286         else:
287             val = self._get_validated_value(opt, path, validate,
288                                             force_permissive,
289                                             validate_properties,
290                                             setting_properties,
291                                             self_properties,
292                                             with_meta=with_meta,
293                                             masterlen=masterlen,
294                                             index=index,
295                                             submulti_index=submulti_index,
296                                             check_frozen=check_frozen,
297                                             session=session,
298                                             display_warnings=display_warnings)
299         if isinstance(val, Exception):
300             return val
301         # cache doesn't work with SubMulti yet
302         if not isinstance(val, SubMulti) and 'cache' in setting_properties and \
303                 validate and validate_properties and force_permissive is False \
304                 and trusted_cached_properties is True:
305             if 'expire' in setting_properties:
306                 if ntime is None:
307                     ntime = int(time())
308                 ntime = ntime + expires_time
309             self._p_.setcache(path, val, ntime, index)
310         return val
311
312     def _get_validated_value(self, opt, path, validate, force_permissive,
313                              validate_properties, setting_properties,
314                              self_properties,
315                              index=None, submulti_index=undefined,
316                              with_meta=True,
317                              masterlen=undefined,
318                              check_frozen=False,
319                              session=None, display_warnings=True):
320         """same has getitem but don't touch the cache
321         index is None for slave value, if value returned is not a list, just return []
322         """
323         context = self._getcontext()
324         setting = context.cfgimpl_get_settings()
325         config_error = None
326         if session is None:
327             session = self._p_.getsession()
328         value = self._getvalue(opt, path, self_properties, index, submulti_index,
329                                with_meta, masterlen, session, validate)
330         if isinstance(value, Exception):
331             value_error = True
332             if isinstance(value, ConfigError):
333                 # For calculating properties, we need value (ie for mandatory
334                 # value).
335                 # If value is calculating with a PropertiesOptionError's option
336                 # _getvalue raise a ConfigError.
337                 # We can not raise ConfigError if this option should raise
338                 # PropertiesOptionError too. So we get config_error and raise
339                 # ConfigError if properties did not raise.
340                 config_error = value
341                 # value is not set, for 'undefined' (cannot set None because of
342                 # mandatory property)
343                 value = undefined
344             else:  # pragma: no cover
345                 raise value
346         else:
347             value_error = False
348             if opt.impl_is_multi():
349                 if index is None:
350                     value = Multi(value, self.context, opt, path)
351                 elif opt.impl_is_submulti() and submulti_index is undefined:
352                     value = SubMulti(value, self.context, opt, path,
353                                      index)
354
355             if validate:
356                 if submulti_index is undefined:
357                     force_submulti_index = None
358                 else:
359                     force_submulti_index = submulti_index
360                 err = opt.impl_validate(value, context,
361                                         'validator' in setting_properties,
362                                         force_index=index,
363                                         force_submulti_index=force_submulti_index,
364                                         display_error=True,
365                                         display_warnings=False,
366                                         setting_properties=setting_properties)
367                 if err:
368                     config_error = err
369                     value = None
370
371         if validate_properties:
372             if config_error is not None:
373                 # should not raise PropertiesOptionError if option is
374                 # mandatory
375                 val_props = undefined
376             else:
377                 val_props = value
378             props = setting.validate_properties(opt, False, check_frozen, value=val_props,
379                                                 path=path,
380                                                 force_permissive=force_permissive,
381                                                 setting_properties=setting_properties,
382                                                 self_properties=self_properties,
383                                                 index=index)
384             if props:
385                 return props
386         if not value_error and validate and display_warnings:
387             opt.impl_validate(value, context,
388                               'validator' in setting_properties,
389                               force_index=index,
390                               force_submulti_index=force_submulti_index,
391                               display_error=False,
392                               display_warnings=display_warnings,
393                               setting_properties=setting_properties)
394         if config_error is not None:
395             return config_error
396         return value
397
398     def __setitem__(self, opt, value):
399         raise ConfigError(_('you should only set value with config'))
400
401     def setitem(self, opt, value, path, force_permissive=False,
402                 check_frozen=True, not_raises=False, index=None,
403                 _setting_properties=undefined):
404         # check_frozen is, for example, used with "force_store_value"
405         # user didn't change value, so not write
406         # valid opt
407         context = self._getcontext()
408         if _setting_properties is undefined:
409             _setting_properties = context.cfgimpl_get_settings()._getproperties(read_write=False)
410         if 'validator' in _setting_properties:
411             session = context.cfgimpl_get_values()._p_.getsession()
412             fake_context = context._gen_fake_values(session)
413             fake_values = fake_context.cfgimpl_get_values()
414             fake_values._setvalue(opt, path, value, index=index)
415             props = fake_values.validate(opt, value, path,
416                                          check_frozen=check_frozen,
417                                          force_permissive=force_permissive,
418                                          setting_properties=_setting_properties,
419                                          session=session, not_raises=not_raises,
420                                          index=index)
421             if props and not_raises:
422                 return
423             err = opt.impl_validate(value, fake_context, display_warnings=False, force_index=index,
424                                     setting_properties=_setting_properties)
425             if err:
426                 raise err
427             opt.impl_validate(value, fake_context, display_error=False,
428                               setting_properties=_setting_properties)
429         self._setvalue(opt, path, value, index=index)
430
431     def _setvalue(self, opt, path, value, force_owner=undefined, index=None):
432         context = self._getcontext()
433         context.cfgimpl_reset_cache(opt=opt, path=path)
434         if force_owner is undefined:
435             owner = context.cfgimpl_get_settings().getowner()
436         else:
437             owner = force_owner
438         # in storage, value must not be a multi
439         if isinstance(value, Multi):
440             if not opt.impl_is_master_slaves('slave') or index is None:
441                 value = list(value)
442                 if opt.impl_is_submulti():
443                     for idx, val in enumerate(value):
444                         if isinstance(val, SubMulti):
445                             value[idx] = list(val)
446             else:
447                 value = value[index]
448         session = self._p_.getsession()
449         #FIXME pourquoi là et pas dans masterslaves ??
450         if opt.impl_is_master_slaves('slave'):
451             if index is not None:
452                 self._p_.setvalue(path, value, owner, index, session)
453             else:
454                 self._p_.resetvalue(path, session)
455                 for idx, val in enumerate(value):
456                     self._p_.setvalue(path, val, owner, idx, session)
457         else:
458             self._p_.setvalue(path, value, owner, None, session)
459         del(session)
460
461     def validate(self, opt, value, path, check_frozen=True, force_permissive=False,
462                  setting_properties=undefined, valid_masterslave=True,
463                  not_raises=False, session=None, index=None):
464         if valid_masterslave and opt.impl_is_master_slaves():
465             if session is None:
466                 session = self._p_.getsession()
467             if index is not None:
468                 len_value = index
469                 setitem = False
470             else:
471                 len_value = len(value)
472                 setitem = True
473             val = opt.impl_get_master_slaves().validate(self, opt, len_value, path, session, setitem=setitem)
474             if isinstance(val, Exception):
475                 return val
476         props = self._getcontext().cfgimpl_get_settings().validate_properties(opt,
477                                                                               False,
478                                                                               check_frozen,
479                                                                               value=value,
480                                                                               path=path,
481                                                                               force_permissive=force_permissive,
482                                                                               setting_properties=setting_properties,
483                                                                               index=index)
484         if props:
485             if not_raises:
486                 return props
487             raise props
488
489     def _is_meta(self, opt, path, session):
490         context = self._getcontext()
491         setting = context.cfgimpl_get_settings()
492         self_properties = setting._getproperties(opt, path, read_write=False)
493         if 'frozen' in self_properties and 'force_default_on_freeze' in self_properties:
494             return False
495         if self._p_.getowner(path, owners.default, session, only_default=True) is not owners.default:
496             return False
497         if context.cfgimpl_get_meta() is not None:
498             return True
499         return False
500
501     def getowner(self, opt, index=None, force_permissive=False, session=None):
502         """
503         retrieves the option's owner
504
505         :param opt: the `option.Option` object
506         :param force_permissive: behaves as if the permissive property
507                                  was present
508         :returns: a `setting.owners.Owner` object
509         """
510         if isinstance(opt, SymLinkOption) and \
511                 not isinstance(opt, DynSymLinkOption):
512             opt = opt._impl_getopt()
513         path = opt.impl_getpath(self._getcontext())
514         return self._getowner(opt, path, session, index=index, force_permissive=force_permissive)
515
516     def _getowner(self, opt, path, session, validate_properties=True,
517                   force_permissive=False, validate_meta=undefined,
518                   self_properties=undefined, only_default=False,
519                   index=None):
520         """get owner of an option
521         """
522         if session is None:
523             session = self._p_.getsession()
524         if not isinstance(opt, Option) and not isinstance(opt,
525                                                           DynSymLinkOption):
526             raise ConfigError(_('owner only avalaible for an option'))
527         context = self._getcontext()
528         if self_properties is undefined:
529             self_properties = context.cfgimpl_get_settings()._getproperties(
530                 opt, path, read_write=False)
531         if 'frozen' in self_properties and 'force_default_on_freeze' in self_properties:
532             return owners.default
533         if validate_properties:
534             value = self._get_cached_value(opt, path, True, force_permissive, None, True,
535                                            self_properties=self_properties, session=session,
536                                            index=index)
537             if isinstance(value, Exception):
538                 raise value
539
540         owner = self._p_.getowner(path, owners.default, session, only_default=only_default, index=index)
541         if validate_meta is undefined:
542             if opt.impl_is_master_slaves('slave'):
543                 master = opt.impl_get_master_slaves().getmaster(opt)
544                 masterp = master.impl_getpath(context)
545                 validate_meta = self._is_meta(opt, masterp, session)
546             else:
547                 validate_meta = True
548         if validate_meta:
549             meta = context.cfgimpl_get_meta()
550             if owner is owners.default and meta is not None:
551                 owner = meta.cfgimpl_get_values()._getowner(opt, path, session,
552                                                             validate_properties=validate_properties,
553                                                             force_permissive=force_permissive,
554                                                             self_properties=self_properties,
555                                                             only_default=only_default, index=index)
556         return owner
557
558     def setowner(self, opt, owner, index=None):
559         """
560         sets a owner to an option
561
562         :param opt: the `option.Option` object
563         :param owner: a valid owner, that is a `setting.owners.Owner` object
564         """
565         if not isinstance(owner, owners.Owner):
566             raise TypeError(_("invalid generic owner {0}").format(str(owner)))
567
568         path = opt.impl_getpath(self._getcontext())
569         session = self._p_.getsession()
570         if not self._p_.hasvalue(path, session):
571             raise ConfigError(_('no value for {0} cannot change owner to {1}'
572                                 '').format(path, owner))
573         props = self._getcontext().cfgimpl_get_settings().validate_properties(opt,
574                                                                               False,
575                                                                               True,
576                                                                               path,
577                                                                               index=index)
578         if props:
579             raise props
580         self._p_.setowner(path, owner, session, index=index)
581
582     def is_default_owner(self, opt, validate_properties=True,
583                          validate_meta=True, index=None,
584                          force_permissive=False):
585         """
586         :param config: *must* be only the **parent** config
587                        (not the toplevel config)
588         :return: boolean
589         """
590         path = opt.impl_getpath(self._getcontext())
591         return self._is_default_owner(opt, path, session=None,
592                                       validate_properties=validate_properties,
593                                       validate_meta=validate_meta, index=index,
594                                       force_permissive=force_permissive)
595
596     def _is_default_owner(self, opt, path, session, validate_properties=True,
597                           validate_meta=True, self_properties=undefined,
598                           index=None, force_permissive=False):
599         d = self._getowner(opt, path, session, validate_properties=validate_properties,
600                            validate_meta=validate_meta,
601                            self_properties=self_properties, only_default=True,
602                            index=index, force_permissive=force_permissive)
603         return d == owners.default
604
605     # information
606     def set_information(self, key, value):
607         """updates the information's attribute
608
609         :param key: information's key (ex: "help", "doc"
610         :param value: information's value (ex: "the help string")
611         """
612         self._p_.set_information(key, value)
613
614     def get_information(self, key, default=undefined):
615         """retrieves one information's item
616
617         :param key: the item string (ex: "help")
618         """
619         return self._p_.get_information(key, default)
620
621     def del_information(self, key, raises=True):
622         self._p_.del_information(key, raises)
623
624     def mandatory_warnings(self, force_permissive=True):
625         """convenience function to trace Options that are mandatory and
626         where no value has been set
627
628         :returns: generator of mandatory Option's path
629         """
630         context = self._getcontext()
631         settings = context.cfgimpl_get_settings()
632         setting_properties = context.cfgimpl_get_settings()._getproperties()
633         setting_properties.update(['mandatory', 'empty'])
634         def _is_properties_option(err, path):
635             if not isinstance(err, Exception):
636                 pass
637             elif isinstance(err, PropertiesOptionError):
638                 if err.proptype == ['mandatory']:
639                     return path
640             elif isinstance(err, ConfigError):
641                 #assume that uncalculated value is an empty value
642                 return path
643             else:
644                 raise err
645
646         def _mandatory_warnings(description, currpath=None):
647             if currpath is None:
648                 currpath = []
649             for opt in description._impl_getchildren(context=context):
650                 name = opt.impl_getname()
651                 path = '.'.join(currpath + [name])
652
653                 if opt.impl_is_optiondescription():
654                     if not settings.validate_properties(opt, True, False, path=path,
655                                                         force_permissive=True,
656                                                         setting_properties=setting_properties):
657                         for path in _mandatory_warnings(opt, currpath + [name]):
658                             yield path
659                 else:
660                     if isinstance(opt, SymLinkOption) and \
661                             not isinstance(opt, DynSymLinkOption):
662                         continue
663                     self_properties = settings._getproperties(opt, path,
664                                                               read_write=False,
665                                                               setting_properties=setting_properties)
666                     if 'mandatory' in self_properties or 'empty' in self_properties:
667                         err = self._get_cached_value(opt, path=path,
668                                                      trusted_cached_properties=False,
669                                                      force_permissive=True,
670                                                      setting_properties=setting_properties,
671                                                      self_properties=self_properties,
672                                                      validate=True,
673                                                      display_warnings=False)
674                         if opt.impl_is_master_slaves('slave') and isinstance(err, list):
675                             for val in err:
676                                 ret = _is_properties_option(val, path)
677                                 if ret is not None:
678                                     yield ret
679                                     break
680                         else:
681                             ret = _is_properties_option(err, path)
682                             if ret is not None:
683                                 yield ret
684
685         descr = context.cfgimpl_get_description()
686         for path in _mandatory_warnings(descr):
687             yield path
688
689     def force_cache(self):
690         """parse all option to force data in cache
691         """
692         context = self.context()
693         if not 'cache' in context.cfgimpl_get_settings():
694             raise ConfigError(_('can force cache only if cache '
695                                 'is actived in config'))
696         #FIXME properties and value should update "expired" time
697         for path in context.cfgimpl_get_description().impl_getpaths(
698                 include_groups=True):
699             err = context.getattr(path, returns_raise=True)
700             if isinstance(err, Exception) and not isinstance(err, PropertiesOptionError):  # pragma: no cover
701                 raise err
702
703     def __getstate__(self):
704         return {'_p_': self._p_}
705
706     def _impl_setstate(self, storage):
707         self._p_._storage = storage
708
709     def __setstate__(self, states):
710         self._p_ = states['_p_']
711
712
713 # ____________________________________________________________
714 # multi types
715 class Multi(list):
716     """multi options values container
717     that support item notation for the values of multi options"""
718     __slots__ = ('opt', 'path', 'context', '__weakref__')
719
720     def __init__(self, value, context, opt, path):
721         """
722         :param value: the Multi wraps a list value
723         :param context: the home config that has the values
724         :param opt: the option object that have this Multi value
725         :param path: path of the option
726         """
727         if value is None:
728             value = []
729         if not opt.impl_is_submulti() and isinstance(value, Multi):
730             raise ValueError(_('{0} is already a Multi ').format(
731                 opt.impl_getname()))
732         self.opt = opt
733         self.path = path
734         if not isinstance(context, weakref.ReferenceType):
735             raise ValueError('context must be a Weakref')
736         self.context = context
737         if not isinstance(value, list):
738             if not '_index' in self.__slots__ and opt.impl_is_submulti():
739                 value = [[value]]
740             else:
741                 value = [value]
742         elif value != [] and not '_index' in self.__slots__ and \
743                 opt.impl_is_submulti() and not isinstance(value[0], list):
744             value = [value]
745         super(Multi, self).__init__(value)
746         if opt.impl_is_submulti():
747             if not '_index' in self.__slots__:
748                 for idx, val in enumerate(self):
749                     if not isinstance(val, SubMulti):
750                         super(Multi, self).__setitem__(idx, SubMulti(val,
751                                                                      context,
752                                                                      opt, path,
753                                                                      idx))
754                     self[idx].submulti = weakref.ref(self)
755
756     def _getcontext(self):
757         """context could be None, we need to test it
758         context is None only if all reference to `Config` object is deleted
759         (for example we delete a `Config` and we manipulate a reference to
760         old `SubConfig`, `Values`, `Multi` or `Settings`)
761         """
762         context = self.context()
763         if context is None:
764             raise ConfigError(_('the context does not exist anymore'))
765         return context
766
767     def __setitem__(self, index, value):
768         self._setitem(index, value)
769
770     def _setitem(self, index, value, validate=True):
771         context = self._getcontext()
772         setting = context.cfgimpl_get_settings()
773         setting_properties = setting._getproperties(read_write=False)
774         if index < 0:
775             index = self.__len__() + index
776         if 'validator' in setting_properties and validate:
777             session = context.cfgimpl_get_values()._p_.getsession()
778             fake_context = context._gen_fake_values(session)
779             fake_multi = Multi(list(self), weakref.ref(fake_context), self.opt, self.path)
780             fake_multi._setitem(index, value, validate=False)
781             self._validate(value, fake_context, index, True)
782         #assume not checking mandatory property
783         super(Multi, self).__setitem__(index, value)
784         self._store(index=index)
785
786     #def __repr__(self, *args, **kwargs):
787     #    return super(Multi, self).__repr__(*args, **kwargs)
788
789     def __getitem__(self, index):
790         value = super(Multi, self).__getitem__(index)
791         if isinstance(value, PropertiesOptionError):
792             raise value
793         return value
794
795     def __delitem__(self, index):
796         return self.pop(index)
797
798     def _getdefaultvalue(self, index):
799         values = self._getcontext().cfgimpl_get_values()
800         value = values._getdefaultvalue(self.opt, self.path, True, index,
801                                         undefined, True)
802         if self.opt.impl_is_submulti():
803             value = SubMulti(value, self.context, self.opt, self.path, index)
804         return value
805
806     def append(self, value=undefined, force=False, setitem=True, validate=True,
807                force_permissive=False):
808         """the list value can be updated (appened)
809         only if the option is a master
810         """
811         if not force and self.opt.impl_is_master_slaves('slave'):
812             raise SlaveError(_("cannot append a value on a multi option {0}"
813                                " which is a slave").format(self.opt.impl_getname()))
814         index = self.__len__()
815         if value is undefined:
816             value = self._getdefaultvalue(index)
817         if validate and value not in [None, undefined]:
818             context = self._getcontext()
819             setting = context.cfgimpl_get_settings()
820             setting_properties = setting._getproperties(read_write=False)
821             if 'validator' in setting_properties:
822                 session = context.cfgimpl_get_values()._p_.getsession()
823                 fake_context = context._gen_fake_values(session)
824                 fake_multi = Multi(list(self), weakref.ref(fake_context), self.opt, self.path)
825                 if isinstance(fake_multi, Exception):
826                     raise fake_multi
827                 fake_multi.append(value, validate=False, force=True,
828                                   setitem=setitem)
829                 self._validate(value, fake_context, index, True)
830         if not '_index' in self.__slots__ and self.opt.impl_is_submulti():
831             if not isinstance(value, SubMulti):
832                 value = SubMulti(value, self.context, self.opt, self.path, index)
833             value.submulti = weakref.ref(self)
834         super(Multi, self).append(value)
835         if setitem:
836             self._store(force=force)
837
838     def append_properties_error(self, err):
839         super(Multi, self).append(err)
840
841     def sort(self, cmp=None, key=None, reverse=False):
842         if self.opt.impl_is_master_slaves():
843             raise SlaveError(_("cannot sort multi option {0} if master or slave"
844                                "").format(self.opt.impl_getname()))
845         if sys.version_info[0] >= 3:  # pragma: no cover
846             if cmp is not None:
847                 raise ValueError(_('cmp is not permitted in python v3 or '
848                                    'greater'))
849             super(Multi, self).sort(key=key, reverse=reverse)
850         else:
851             super(Multi, self).sort(cmp=cmp, key=key, reverse=reverse)
852         self._store()
853
854     def reverse(self):
855         if self.opt.impl_is_master_slaves():
856             raise SlaveError(_("cannot reverse multi option {0} if master or "
857                                "slave").format(self.opt.impl_getname()))
858         super(Multi, self).reverse()
859         self._store()
860
861     def insert(self, index, value, validate=True):
862         if self.opt.impl_is_master_slaves():
863             raise SlaveError(_("cannot insert multi option {0} if master or "
864                                "slave").format(self.opt.impl_getname()))
865         context = self._getcontext()
866         setting = setting = context.cfgimpl_get_settings()
867         setting_properties = setting._getproperties(read_write=False)
868         if 'validator' in setting_properties and validate and value is not None:
869             session = context.cfgimpl_get_values()._p_.getsession()
870             fake_context = context._gen_fake_values(session)
871             fake_multi = Multi(list(self), weakref.ref(fake_context), self.opt, self.path)
872             fake_multi.insert(index, value, validate=False)
873             self._validate(value, fake_context, index, True)
874         super(Multi, self).insert(index, value)
875         self._store()
876
877     def extend(self, iterable, validate=True):
878         if self.opt.impl_is_master_slaves():
879             raise SlaveError(_("cannot extend multi option {0} if master or "
880                                "slave").format(self.opt.impl_getname()))
881         index = getattr(self, '_index', None)
882         context = self._getcontext()
883         setting = context.cfgimpl_get_settings()
884         setting_properties = setting._getproperties(read_write=False)
885         if 'validator' in setting_properties and validate:
886             session = context.cfgimpl_get_values()._p_.getsession()
887             fake_context = context._gen_fake_values(session)
888             fake_multi = Multi(list(self), weakref.ref(fake_context), self.opt, self.path)
889             if index is None:
890                 fake_multi.extend(iterable, validate=False)
891                 self._validate(fake_multi, fake_context, index)
892             else:
893                 fake_multi[index].extend(iterable, validate=False)
894                 self._validate(fake_multi[index], fake_context, index)
895         super(Multi, self).extend(iterable)
896         self._store()
897
898     def _validate(self, value, fake_context, force_index, submulti=False):
899         err = self.opt.impl_validate(value, context=fake_context,
900                                      force_index=force_index,
901                                      multi=self)
902         if err:
903             raise err
904
905     def pop(self, index, force=False):
906         """the list value can be updated (poped)
907         only if the option is a master
908
909         :param index: remove item a index
910         :type index: int
911         :param force: force pop item (withoud check master/slave)
912         :type force: boolean
913         :returns: item at index
914         """
915         context = self._getcontext()
916         if not force:
917             if self.opt.impl_is_master_slaves('slave'):
918                 raise SlaveError(_("cannot pop a value on a multi option {0}"
919                                    " which is a slave").format(self.opt.impl_getname()))
920             if self.opt.impl_is_master_slaves('master'):
921                 self.opt.impl_get_master_slaves().pop(self.opt,
922                                                       context.cfgimpl_get_values(), index)
923         #set value without valid properties
924         ret = super(Multi, self).pop(index)
925         self._store(force=force)
926         return ret
927
928     def remove(self, value):
929         idx = self.index(value)
930         return self.pop(idx)
931
932     def _store(self, force=False, index=None):
933         values = self._getcontext().cfgimpl_get_values()
934         if not force:
935             #FIXME could get properties an pass it
936             values.validate(self.opt, self, self.path,
937                             valid_masterslave=False)
938         values._setvalue(self.opt, self.path, self, index=index)
939
940
941 class SubMulti(Multi):
942     __slots__ = ('_index', 'submulti')
943
944     def __init__(self, value, context, opt, path, index):
945         """
946         :param index: index (only for slave with submulti)
947         :type index: `int`
948         """
949         self._index = index
950         super(SubMulti, self).__init__(value, context, opt, path)
951
952     def append(self, value=undefined):
953         super(SubMulti, self).append(value, force=True)
954
955     def pop(self, index):
956         return super(SubMulti, self).pop(index, force=True)
957
958     def __setitem__(self, index, value):
959         self._setitem(index, value)
960
961     def _store(self, force=False, index=None):
962         #force is unused here
963         values = self._getcontext().cfgimpl_get_values()
964         values.validate(self.opt, self, self.path, valid_masterslave=False)
965         values._setvalue(self.opt, self.path, self.submulti())
966
967     def _validate(self, value, fake_context, force_index, submulti=False):
968         if value is not None:
969             if submulti is False:
970                 super(SubMulti, self)._validate(value, fake_context,
971                                                 force_index, submulti)
972             else:
973                 err = self.opt.impl_validate(value, context=fake_context,
974                                              force_index=self._index,
975                                              force_submulti_index=force_index,
976                                              multi=self)
977                 if err:
978                     raise err
979
980     def _getdefaultvalue(self, index):
981         values = self._getcontext().cfgimpl_get_values()
982         return values._getdefaultvalue(self.opt, self.path, True, index,
983                                        self._index, True)