diff options
| -rw-r--r-- | Doc/library/configparser.rst | 46 | ||||
| -rw-r--r-- | Lib/configparser.py | 170 | ||||
| -rw-r--r-- | Lib/test/test_configparser.py | 223 |
3 files changed, 386 insertions, 53 deletions
diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index ae93504917..c9187a3441 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -162,10 +162,8 @@ Boolean values from ``'yes'``/``'no'``, ``'on'``/``'off'``, True Apart from :meth:`getboolean`, config parsers also provide equivalent -:meth:`getint` and :meth:`getfloat` methods, but these are far less -useful since conversion using :func:`int` and :func:`float` is -sufficient for these types. - +:meth:`getint` and :meth:`getfloat` methods. You can register your own +converters and customize the provided ones. [1]_ Fallback Values --------------- @@ -555,10 +553,10 @@ the :meth:`__init__` options: Comment prefixes are strings that indicate the start of a valid comment within a config file. *comment_prefixes* are used only on otherwise empty lines - (optionally indented) whereas *inline_comment_prefixes* can be used - after every valid value (e.g. section names, options and empty lines - as well). By default inline comments are disabled and ``'#'`` and - ``';'`` are used as prefixes for whole line comments. + (optionally indented) whereas *inline_comment_prefixes* can be used after + every valid value (e.g. section names, options and empty lines as well). By + default inline comments are disabled and ``'#'`` and ``';'`` are used as + prefixes for whole line comments. .. versionchanged:: 3.2 In previous versions of :mod:`configparser` behaviour matched @@ -667,10 +665,26 @@ the :meth:`__init__` options: `dedicated documentation section <#interpolation-of-values>`_. :class:`RawConfigParser` has a default value of ``None``. +* *converters*, default value: not set + + Config parsers provide option value getters that perform type conversion. By + default :meth:`getint`, :meth:`getfloat`, and :meth:`getboolean` are + implemented. Should other getters be desirable, users may define them in + a subclass or pass a dictionary where each key is a name of the converter and + each value is a callable implementing said conversion. For instance, passing + ``{'decimal': decimal.Decimal}`` would add :meth:`getdecimal` on both the + parser object and all section proxies. In other words, it will be possible + to write both ``parser_instance.getdecimal('section', 'key', fallback=0)`` + and ``parser_instance['section'].getdecimal('key', 0)``. + + If the converter needs to access the state of the parser, it can be + implemented as a method on a config parser subclass. If the name of this + method starts with ``get``, it will be available on all section proxies, in + the dict-compatible form (see the ``getdecimal()`` example above). More advanced customization may be achieved by overriding default values of -these parser attributes. The defaults are defined on the classes, so they -may be overridden by subclasses or by attribute assignment. +these parser attributes. The defaults are defined on the classes, so they may +be overridden by subclasses or by attribute assignment. .. attribute:: BOOLEAN_STATES @@ -863,7 +877,7 @@ interpolation if an option used is not defined elsewhere. :: ConfigParser Objects -------------------- -.. class:: ConfigParser(defaults=None, dict_type=collections.OrderedDict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation()) +.. class:: ConfigParser(defaults=None, dict_type=collections.OrderedDict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={}) The main configuration parser. When *defaults* is given, it is initialized into the dictionary of intrinsic defaults. When *dict_type* is given, it @@ -903,6 +917,12 @@ ConfigParser Objects converts option names to lower case), the values ``foo %(bar)s`` and ``foo %(BAR)s`` are equivalent. + When *converters* is given, it should be a dictionary where each key + represents the name of a type converter and each value is a callable + implementing the conversion from string to the desired datatype. Every + converter gets its own corresponding :meth:`get*()` method on the parser + object and section proxies. + .. versionchanged:: 3.1 The default *dict_type* is :class:`collections.OrderedDict`. @@ -911,6 +931,9 @@ ConfigParser Objects *empty_lines_in_values*, *default_section* and *interpolation* were added. + .. versionchanged:: 3.5 + The *converters* argument was added. + .. method:: defaults() @@ -1286,3 +1309,4 @@ Exceptions .. [1] Config parsers allow for heavy customization. If you are interested in changing the behaviour outlined by the footnote reference, consult the `Customizing Parser Behaviour`_ section. + diff --git a/Lib/configparser.py b/Lib/configparser.py index 5843b0fc03..ecd06600b1 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -17,7 +17,8 @@ ConfigParser -- responsible for parsing a list of __init__(defaults=None, dict_type=_default_dict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, - empty_lines_in_values=True): + empty_lines_in_values=True, default_section='DEFAULT', + interpolation=<unset>, converters=<unset>): Create the parser. When `defaults' is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. @@ -47,6 +48,25 @@ ConfigParser -- responsible for parsing a list of When `allow_no_value' is True (default: False), options without values are accepted; the value presented for these is None. + When `default_section' is given, the name of the special section is + named accordingly. By default it is called ``"DEFAULT"`` but this can + be customized to point to any other valid section name. Its current + value can be retrieved using the ``parser_instance.default_section`` + attribute and may be modified at runtime. + + When `interpolation` is given, it should be an Interpolation subclass + instance. It will be used as the handler for option value + pre-processing when using getters. RawConfigParser object s don't do + any sort of interpolation, whereas ConfigParser uses an instance of + BasicInterpolation. The library also provides a ``zc.buildbot`` + inspired ExtendedInterpolation implementation. + + When `converters` is given, it should be a dictionary where each key + represents the name of a type converter and each value is a callable + implementing the conversion from string to the desired datatype. Every + converter gets its corresponding get*() method on the parser object and + section proxies. + sections() Return all the configuration section names, sans DEFAULT. @@ -129,9 +149,11 @@ import warnings __all__ = ["NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", - "InterpolationSyntaxError", "ParsingError", - "MissingSectionHeaderError", + "InterpolationMissingOptionError", "InterpolationSyntaxError", + "ParsingError", "MissingSectionHeaderError", "ConfigParser", "SafeConfigParser", "RawConfigParser", + "Interpolation", "BasicInterpolation", "ExtendedInterpolation", + "LegacyInterpolation", "SectionProxy", "ConverterMapping", "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH"] DEFAULTSECT = "DEFAULT" @@ -580,11 +602,12 @@ class RawConfigParser(MutableMapping): comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=DEFAULTSECT, - interpolation=_UNSET): + interpolation=_UNSET, converters=_UNSET): self._dict = dict_type self._sections = self._dict() self._defaults = self._dict() + self._converters = ConverterMapping(self) self._proxies = self._dict() self._proxies[default_section] = SectionProxy(self, default_section) if defaults: @@ -612,6 +635,8 @@ class RawConfigParser(MutableMapping): self._interpolation = self._DEFAULT_INTERPOLATION if self._interpolation is None: self._interpolation = Interpolation() + if converters is not _UNSET: + self._converters.update(converters) def defaults(self): return self._defaults @@ -775,36 +800,31 @@ class RawConfigParser(MutableMapping): def _get(self, section, conv, option, **kwargs): return conv(self.get(section, option, **kwargs)) - def getint(self, section, option, *, raw=False, vars=None, - fallback=_UNSET): + def _get_conv(self, section, option, conv, *, raw=False, vars=None, + fallback=_UNSET, **kwargs): try: - return self._get(section, int, option, raw=raw, vars=vars) + return self._get(section, conv, option, raw=raw, vars=vars, + **kwargs) except (NoSectionError, NoOptionError): if fallback is _UNSET: raise - else: - return fallback + return fallback + + # getint, getfloat and getboolean provided directly for backwards compat + def getint(self, section, option, *, raw=False, vars=None, + fallback=_UNSET, **kwargs): + return self._get_conv(section, option, int, raw=raw, vars=vars, + fallback=fallback, **kwargs) def getfloat(self, section, option, *, raw=False, vars=None, - fallback=_UNSET): - try: - return self._get(section, float, option, raw=raw, vars=vars) - except (NoSectionError, NoOptionError): - if fallback is _UNSET: - raise - else: - return fallback + fallback=_UNSET, **kwargs): + return self._get_conv(section, option, float, raw=raw, vars=vars, + fallback=fallback, **kwargs) def getboolean(self, section, option, *, raw=False, vars=None, - fallback=_UNSET): - try: - return self._get(section, self._convert_to_boolean, option, - raw=raw, vars=vars) - except (NoSectionError, NoOptionError): - if fallback is _UNSET: - raise - else: - return fallback + fallback=_UNSET, **kwargs): + return self._get_conv(section, option, self._convert_to_boolean, + raw=raw, vars=vars, fallback=fallback, **kwargs) def items(self, section=_UNSET, raw=False, vars=None): """Return a list of (name, value) tuples for each option in a section. @@ -1154,6 +1174,10 @@ class RawConfigParser(MutableMapping): if not isinstance(value, str): raise TypeError("option values must be strings") + @property + def converters(self): + return self._converters + class ConfigParser(RawConfigParser): """ConfigParser implementing interpolation.""" @@ -1194,6 +1218,10 @@ class SectionProxy(MutableMapping): """Creates a view on a section of the specified `name` in `parser`.""" self._parser = parser self._name = name + for conv in parser.converters: + key = 'get' + conv + getter = functools.partial(self.get, _impl=getattr(parser, key)) + setattr(self, key, getter) def __repr__(self): return '<Section: {}>'.format(self._name) @@ -1227,22 +1255,6 @@ class SectionProxy(MutableMapping): else: return self._parser.defaults() - def get(self, option, fallback=None, *, raw=False, vars=None): - return self._parser.get(self._name, option, raw=raw, vars=vars, - fallback=fallback) - - def getint(self, option, fallback=None, *, raw=False, vars=None): - return self._parser.getint(self._name, option, raw=raw, vars=vars, - fallback=fallback) - - def getfloat(self, option, fallback=None, *, raw=False, vars=None): - return self._parser.getfloat(self._name, option, raw=raw, vars=vars, - fallback=fallback) - - def getboolean(self, option, fallback=None, *, raw=False, vars=None): - return self._parser.getboolean(self._name, option, raw=raw, vars=vars, - fallback=fallback) - @property def parser(self): # The parser object of the proxy is read-only. @@ -1252,3 +1264,77 @@ class SectionProxy(MutableMapping): def name(self): # The name of the section on a proxy is read-only. return self._name + + def get(self, option, fallback=None, *, raw=False, vars=None, + _impl=None, **kwargs): + """Get an option value. + + Unless `fallback` is provided, `None` will be returned if the option + is not found. + + """ + # If `_impl` is provided, it should be a getter method on the parser + # object that provides the desired type conversion. + if not _impl: + _impl = self._parser.get + return _impl(self._name, option, raw=raw, vars=vars, + fallback=fallback, **kwargs) + + +class ConverterMapping(MutableMapping): + """Enables reuse of get*() methods between the parser and section proxies. + + If a parser class implements a getter directly, the value for the given + key will be ``None``. The presence of the converter name here enables + section proxies to find and use the implementation on the parser class. + """ + + GETTERCRE = re.compile(r"^get(?P<name>.+)$") + + def __init__(self, parser): + self._parser = parser + self._data = {} + for getter in dir(self._parser): + m = self.GETTERCRE.match(getter) + if not m or not callable(getattr(self._parser, getter)): + continue + self._data[m.group('name')] = None # See class docstring. + + def __getitem__(self, key): + return self._data[key] + + def __setitem__(self, key, value): + try: + k = 'get' + key + except TypeError: + raise ValueError('Incompatible key: {} (type: {})' + ''.format(key, type(key))) + if k == 'get': + raise ValueError('Incompatible key: cannot use "" as a name') + self._data[key] = value + func = functools.partial(self._parser._get_conv, conv=value) + func.converter = value + setattr(self._parser, k, func) + for proxy in self._parser.values(): + getter = functools.partial(proxy.get, _impl=func) + setattr(proxy, k, getter) + + def __delitem__(self, key): + try: + k = 'get' + (key or None) + except TypeError: + raise KeyError(key) + del self._data[key] + for inst in itertools.chain((self._parser,), self._parser.values()): + try: + delattr(inst, k) + except AttributeError: + # don't raise since the entry was present in _data, silently + # clean up + continue + + def __iter__(self): + return iter(self._data) + + def __len__(self): + return len(self._data) diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index b43950191c..a7c6127630 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -1584,6 +1584,34 @@ class CoverageOneHundredTestCase(unittest.TestCase): """) self.assertEqual(repr(parser['section']), '<Section: section>') + def test_inconsistent_converters_state(self): + parser = configparser.ConfigParser() + import decimal + parser.converters['decimal'] = decimal.Decimal + parser.read_string(""" + [s1] + one = 1 + [s2] + two = 2 + """) + self.assertIn('decimal', parser.converters) + self.assertEqual(parser.getdecimal('s1', 'one'), 1) + self.assertEqual(parser.getdecimal('s2', 'two'), 2) + self.assertEqual(parser['s1'].getdecimal('one'), 1) + self.assertEqual(parser['s2'].getdecimal('two'), 2) + del parser.getdecimal + with self.assertRaises(AttributeError): + parser.getdecimal('s1', 'one') + self.assertIn('decimal', parser.converters) + del parser.converters['decimal'] + self.assertNotIn('decimal', parser.converters) + with self.assertRaises(AttributeError): + parser.getdecimal('s1', 'one') + with self.assertRaises(AttributeError): + parser['s1'].getdecimal('one') + with self.assertRaises(AttributeError): + parser['s2'].getdecimal('two') + class ExceptionPicklingTestCase(unittest.TestCase): """Tests for issue #13760: ConfigParser exceptions are not picklable.""" @@ -1763,6 +1791,7 @@ class InlineCommentStrippingTestCase(unittest.TestCase): self.assertEqual(s['k2'], 'v2') self.assertEqual(s['k3'], 'v3;#//still v3# and still v3') + class ExceptionContextTestCase(unittest.TestCase): """ Test that implementation details doesn't leak through raising exceptions. """ @@ -1816,5 +1845,199 @@ class ExceptionContextTestCase(unittest.TestCase): config.remove_option('Section1', 'an_int') self.assertIs(cm.exception.__suppress_context__, True) + +class ConvertersTestCase(BasicTestCase, unittest.TestCase): + """Introduced in 3.5, issue #18159.""" + + config_class = configparser.ConfigParser + + def newconfig(self, defaults=None): + instance = super().newconfig(defaults=defaults) + instance.converters['list'] = lambda v: [e.strip() for e in v.split() + if e.strip()] + return instance + + def test_converters(self): + cfg = self.newconfig() + self.assertIn('boolean', cfg.converters) + self.assertIn('list', cfg.converters) + self.assertIsNone(cfg.converters['int']) + self.assertIsNone(cfg.converters['float']) + self.assertIsNone(cfg.converters['boolean']) + self.assertIsNotNone(cfg.converters['list']) + self.assertEqual(len(cfg.converters), 4) + with self.assertRaises(ValueError): + cfg.converters[''] = lambda v: v + with self.assertRaises(ValueError): + cfg.converters[None] = lambda v: v + cfg.read_string(""" + [s] + str = string + int = 1 + float = 0.5 + list = a b c d e f g + bool = yes + """) + s = cfg['s'] + self.assertEqual(s['str'], 'string') + self.assertEqual(s['int'], '1') + self.assertEqual(s['float'], '0.5') + self.assertEqual(s['list'], 'a b c d e f g') + self.assertEqual(s['bool'], 'yes') + self.assertEqual(cfg.get('s', 'str'), 'string') + self.assertEqual(cfg.get('s', 'int'), '1') + self.assertEqual(cfg.get('s', 'float'), '0.5') + self.assertEqual(cfg.get('s', 'list'), 'a b c d e f g') + self.assertEqual(cfg.get('s', 'bool'), 'yes') + self.assertEqual(cfg.get('s', 'str'), 'string') + self.assertEqual(cfg.getint('s', 'int'), 1) + self.assertEqual(cfg.getfloat('s', 'float'), 0.5) + self.assertEqual(cfg.getlist('s', 'list'), ['a', 'b', 'c', 'd', + 'e', 'f', 'g']) + self.assertEqual(cfg.getboolean('s', 'bool'), True) + self.assertEqual(s.get('str'), 'string') + self.assertEqual(s.getint('int'), 1) + self.assertEqual(s.getfloat('float'), 0.5) + self.assertEqual(s.getlist('list'), ['a', 'b', 'c', 'd', + 'e', 'f', 'g']) + self.assertEqual(s.getboolean('bool'), True) + with self.assertRaises(AttributeError): + cfg.getdecimal('s', 'float') + with self.assertRaises(AttributeError): + s.getdecimal('float') + import decimal + cfg.converters['decimal'] = decimal.Decimal + self.assertIn('decimal', cfg.converters) + self.assertIsNotNone(cfg.converters['decimal']) + self.assertEqual(len(cfg.converters), 5) + dec0_5 = decimal.Decimal('0.5') + self.assertEqual(cfg.getdecimal('s', 'float'), dec0_5) + self.assertEqual(s.getdecimal('float'), dec0_5) + del cfg.converters['decimal'] + self.assertNotIn('decimal', cfg.converters) + self.assertEqual(len(cfg.converters), 4) + with self.assertRaises(AttributeError): + cfg.getdecimal('s', 'float') + with self.assertRaises(AttributeError): + s.getdecimal('float') + with self.assertRaises(KeyError): + del cfg.converters['decimal'] + with self.assertRaises(KeyError): + del cfg.converters[''] + with self.assertRaises(KeyError): + del cfg.converters[None] + + +class BlatantOverrideConvertersTestCase(unittest.TestCase): + """What if somebody overrode a getboolean()? We want to make sure that in + this case the automatic converters do not kick in.""" + + config = """ + [one] + one = false + two = false + three = long story short + + [two] + one = false + two = false + three = four + """ + + def test_converters_at_init(self): + cfg = configparser.ConfigParser(converters={'len': len}) + cfg.read_string(self.config) + self._test_len(cfg) + self.assertIsNotNone(cfg.converters['len']) + + def test_inheritance(self): + class StrangeConfigParser(configparser.ConfigParser): + gettysburg = 'a historic borough in south central Pennsylvania' + + def getboolean(self, section, option, *, raw=False, vars=None, + fallback=configparser._UNSET): + if section == option: + return True + return super().getboolean(section, option, raw=raw, vars=vars, + fallback=fallback) + def getlen(self, section, option, *, raw=False, vars=None, + fallback=configparser._UNSET): + return self._get_conv(section, option, len, raw=raw, vars=vars, + fallback=fallback) + + cfg = StrangeConfigParser() + cfg.read_string(self.config) + self._test_len(cfg) + self.assertIsNone(cfg.converters['len']) + self.assertTrue(cfg.getboolean('one', 'one')) + self.assertTrue(cfg.getboolean('two', 'two')) + self.assertFalse(cfg.getboolean('one', 'two')) + self.assertFalse(cfg.getboolean('two', 'one')) + cfg.converters['boolean'] = cfg._convert_to_boolean + self.assertFalse(cfg.getboolean('one', 'one')) + self.assertFalse(cfg.getboolean('two', 'two')) + self.assertFalse(cfg.getboolean('one', 'two')) + self.assertFalse(cfg.getboolean('two', 'one')) + + def _test_len(self, cfg): + self.assertEqual(len(cfg.converters), 4) + self.assertIn('boolean', cfg.converters) + self.assertIn('len', cfg.converters) + self.assertNotIn('tysburg', cfg.converters) + self.assertIsNone(cfg.converters['int']) + self.assertIsNone(cfg.converters['float']) + self.assertIsNone(cfg.converters['boolean']) + self.assertEqual(cfg.getlen('one', 'one'), 5) + self.assertEqual(cfg.getlen('one', 'two'), 5) + self.assertEqual(cfg.getlen('one', 'three'), 16) + self.assertEqual(cfg.getlen('two', 'one'), 5) + self.assertEqual(cfg.getlen('two', 'two'), 5) + self.assertEqual(cfg.getlen('two', 'three'), 4) + self.assertEqual(cfg.getlen('two', 'four', fallback=0), 0) + with self.assertRaises(configparser.NoOptionError): + cfg.getlen('two', 'four') + self.assertEqual(cfg['one'].getlen('one'), 5) + self.assertEqual(cfg['one'].getlen('two'), 5) + self.assertEqual(cfg['one'].getlen('three'), 16) + self.assertEqual(cfg['two'].getlen('one'), 5) + self.assertEqual(cfg['two'].getlen('two'), 5) + self.assertEqual(cfg['two'].getlen('three'), 4) + self.assertEqual(cfg['two'].getlen('four', 0), 0) + self.assertEqual(cfg['two'].getlen('four'), None) + + def test_instance_assignment(self): + cfg = configparser.ConfigParser() + cfg.getboolean = lambda section, option: True + cfg.getlen = lambda section, option: len(cfg[section][option]) + cfg.read_string(self.config) + self.assertEqual(len(cfg.converters), 3) + self.assertIn('boolean', cfg.converters) + self.assertNotIn('len', cfg.converters) + self.assertIsNone(cfg.converters['int']) + self.assertIsNone(cfg.converters['float']) + self.assertIsNone(cfg.converters['boolean']) + self.assertTrue(cfg.getboolean('one', 'one')) + self.assertTrue(cfg.getboolean('two', 'two')) + self.assertTrue(cfg.getboolean('one', 'two')) + self.assertTrue(cfg.getboolean('two', 'one')) + cfg.converters['boolean'] = cfg._convert_to_boolean + self.assertFalse(cfg.getboolean('one', 'one')) + self.assertFalse(cfg.getboolean('two', 'two')) + self.assertFalse(cfg.getboolean('one', 'two')) + self.assertFalse(cfg.getboolean('two', 'one')) + self.assertEqual(cfg.getlen('one', 'one'), 5) + self.assertEqual(cfg.getlen('one', 'two'), 5) + self.assertEqual(cfg.getlen('one', 'three'), 16) + self.assertEqual(cfg.getlen('two', 'one'), 5) + self.assertEqual(cfg.getlen('two', 'two'), 5) + self.assertEqual(cfg.getlen('two', 'three'), 4) + # If a getter impl is assigned straight to the instance, it won't + # be available on the section proxies. + with self.assertRaises(AttributeError): + self.assertEqual(cfg['one'].getlen('one'), 5) + with self.assertRaises(AttributeError): + self.assertEqual(cfg['two'].getlen('one'), 5) + + if __name__ == '__main__': unittest.main() |