Prepare switch from optparse to argparse.

New interface function: frontend.get_default_settings()
easy access to default settings for programmatic use.

Add deprecation warnings.
Catch them when accessing the deprecated functions from Docutils.

Make docstrings conform to the Documentation Policies
(first line: summary, next line empty).

git-svn-id: https://svn.code.sf.net/p/docutils/code/trunk@9047 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

10 files changed, 171 insertions, 89 deletions

diff --git a/docutils/HISTORY.txt b/docutils/HISTORY.txt
index 469758a56..803546548 100644
--- a/docutils/HISTORY.txt
+++ b/docutils/HISTORY.txt
@@ -135,6 +135,7 @@ Release 0.18 (2021-10-26)
 * docutils/frontend.py
   - mark as provisional (will switch from using "optparse" to "argparse").
   - remove hack for the now obsolete "mod_python" Apache module.
+  - new function get_default_settings()
 
 * docutils/nodes.py
 
diff --git a/docutils/docutils/core.py b/docutils/docutils/core.py
index 80158eaa2..c06057876 100644
--- a/docutils/docutils/core.py
+++ b/docutils/docutils/core.py
@@ -18,6 +18,7 @@ __docformat__ = 'reStructuredText'
 import pprint
 import os
 import sys
+import warnings
 
 from docutils import (__version__, __version_details__, SettingsSpec,
                       io, utils, readers, writers)
@@ -102,6 +103,9 @@ class Publisher:
     def setup_option_parser(self, usage=None, description=None,
                             settings_spec=None, config_section=None,
                             **defaults):
+        warnings.warn('Publisher.setup_option_parser is deprecated, '
+                      'and will be removed in Docutils 0.21.',
+                      DeprecationWarning, stacklevel=2)
         if config_section:
             if not settings_spec:
                 settings_spec = SettingsSpec()
@@ -110,22 +114,32 @@ class Publisher:
             if len(parts) > 1 and parts[-1] == 'application':
                 settings_spec.config_section_dependencies = ['applications']
         # @@@ Add self.source & self.destination to components in future?
-        option_parser = OptionParser(
+        return OptionParser(
             components=(self.parser, self.reader, self.writer, settings_spec),
             defaults=defaults, read_config_files=True,
             usage=usage, description=description)
-        return option_parser
+
+    def _setup_settings_parser(self, *args, **kwargs):
+        # Provisional: will change (docutils.frontend.OptionParser will
+        # be replaced by a parser based on arparse.ArgumentParser)
+        # and may be removed later.
+        with warnings.catch_warnings():
+            warnings.filterwarnings('ignore', category=DeprecationWarning)
+            return self.setup_option_parser(*args, **kwargs)
 
     def get_settings(self, usage=None, description=None,
                      settings_spec=None, config_section=None, **defaults):
         """
-        Set and return default settings (overrides in `defaults` dict).
+        Return settings from components and config files.
 
-        Set components first (`self.set_reader` & `self.set_writer`).
-        Explicitly setting `self.settings` disables command line option
-        processing from `self.publish()`.
+        Please set components first (`self.set_reader` & `self.set_writer`).
+        Use keyword arguments to override component defaults
+        (before updating from configuration files).
+
+        Calling this function also sets `self.settings` which makes
+        `self.publish()` skip parsing command line options.
         """
-        option_parser = self.setup_option_parser(
+        option_parser = self._setup_settings_parser(
             usage, description, settings_spec, config_section, **defaults)
         self.settings = option_parser.get_default_values()
         return self.settings
@@ -145,14 +159,14 @@ class Publisher:
                              settings_spec=None, config_section=None,
                              **defaults):
         """
-        Set parse command line arguments and set ``self.settings``.
+        Parse command line arguments and set ``self.settings``.
 
         Pass an empty sequence to `argv` to avoid reading `sys.argv`
         (the default behaviour).
 
         Set components first (`self.set_reader` & `self.set_writer`).
         """
-        option_parser = self.setup_option_parser(
+        option_parser = self._setup_settings_parser(
             usage, description, settings_spec, config_section, **defaults)
         if argv is None:
             argv = sys.argv[1:]
diff --git a/docutils/docutils/frontend.py b/docutils/docutils/frontend.py
index c61e2b56c..549472ccf 100644
--- a/docutils/docutils/frontend.py
+++ b/docutils/docutils/frontend.py
@@ -15,23 +15,39 @@ See https://docutils.sourceforge.io/docs/api/runtime-settings.html.
 Exports the following classes:
 
 * `OptionParser`: Standard Docutils command-line processing.
+  Deprecated. Will be replaced by an ArgumentParser.
 * `Option`: Customized version of `optparse.Option`; validation support.
+  Deprecated. Will be removed.
 * `Values`: Runtime settings; objects are simple structs
   (``object.attribute``).  Supports cumulative list settings (attributes).
+  Deprecated. Will be removed.
 * `ConfigParser`: Standard Docutils config file processing.
+  Provisional. Details will change.
 
 Also exports the following functions:
 
-* Option callbacks: `store_multiple()`, `read_config_file()`.
-* Setting validators: `validate_encoding()`,
-  `validate_encoding_error_handler()`,
+Interface function:
+   `get_default_settings()`.  New in 0.19.
+
+Option callbacks:
+   `store_multiple()`, `read_config_file()`. Deprecated.
+
+Setting validators:
+  `validate_encoding()`, `validate_encoding_error_handler()`,
   `validate_encoding_and_error_handler()`,
-  `validate_boolean()`, `validate_ternary()`, `validate_threshold()`,
+  `validate_boolean()`, `validate_ternary()`,
+  `validate_nonnegative_int()`, `validate_threshold()`,
   `validate_colon_separated_string_list()`,
   `validate_comma_separated_list()`,
-  `validate_dependency_file()`.
-* `make_paths_absolute()`.
-* SettingSpec creation: `filter_settings_spec()`.
+  `validate_url_trailing_slash()`,
+  `validate_dependency_file()`,
+  `validate_strip_class()`
+  `validate_smartquotes_locales()`.
+
+  Provisional.
+
+Misc:
+  `make_paths_absolute()`, `filter_settings_spec()`. Provisional.
 """
 
 __docformat__ = 'reStructuredText'
@@ -47,7 +63,7 @@ import sys
 import warnings
 
 import docutils
-from docutils import io
+from docutils import io, utils
 
 
 def store_multiple(option, opt, value, parser, *args, **kwargs):
@@ -207,10 +223,10 @@ def validate_url_trailing_slash(
 def validate_dependency_file(setting, value, option_parser,
                              config_parser=None, config_section=None):
     try:
-        return docutils.utils.DependencyList(value)
+        return utils.DependencyList(value)
     except OSError:
         # TODO: warn/info?
-        return docutils.utils.DependencyList(None)
+        return utils.DependencyList(None)
 
 
 def validate_strip_class(setting, value, option_parser,
@@ -317,17 +333,22 @@ def filter_settings_spec(settings_spec, *exclude, **replace):
 
 
 class Values(optparse.Values):
+    """Storage for option values.
 
-    """
     Updates list attributes by extension rather than by replacement.
     Works in conjunction with the `OptionParser.lists` instance attribute.
+
+    Deprecated. Will be removed.
     """
 
     def __init__(self, *args, **kwargs):
-        optparse.Values.__init__(self, *args, **kwargs)
+        warnings.warn('frontend.Values class will be removed '
+                      'in Docutils 0.21 or later.',
+                      DeprecationWarning, stacklevel=2)
+        super().__init__(*args, **kwargs)
         if getattr(self, 'record_dependencies', None) is None:
             # Set up dummy dependency list.
-            self.record_dependencies = docutils.utils.DependencyList()
+            self.record_dependencies = utils.DependencyList()
 
     def update(self, other_dict, option_parser):
         if isinstance(other_dict, Values):
@@ -354,9 +375,19 @@ class Values(optparse.Values):
 
 
 class Option(optparse.Option):
+    """Add validation and override support to `optparse.Option`.
+
+    Deprecated. Will be removed.
+    """
 
     ATTRS = optparse.Option.ATTRS + ['validator', 'overrides']
 
+    def __init__(self, *args, **kwargs):
+        warnings.warn('The frontend.Option class will be removed '
+                      'in Docutils 0.21 or later.',
+                      DeprecationWarning, stacklevel=2)
+        super().__init__(*args, **kwargs)
+
     def process(self, opt, value, values, parser):
         """
         Call the validator function on applicable settings and
@@ -381,23 +412,29 @@ class Option(optparse.Option):
 
 
 class OptionParser(optparse.OptionParser, docutils.SettingsSpec):
-
     """
-    Parser for command-line and library use.  The `settings_spec`
-    specification here and in other Docutils components are merged to build
-    the set of command-line options and runtime settings for this process.
+    Settings parser for command-line and library use.
+
+    The `settings_spec` specification here and in other Docutils components
+    are merged to build the set of command-line options and runtime settings
+    for this process.
 
     Common settings (defined below) and component-specific settings must not
     conflict.  Short options are reserved for common settings, and components
     are restricted to using long options.
+
+    Deprecated.
+    Will be replaced by a subclass of `argparse.ArgumentParser`.
     """
 
     standard_config_files = [
         '/etc/docutils.conf',           # system-wide
         './docutils.conf',              # project-specific
         '~/.docutils']                  # user-specific
-    """Docutils configuration files, using ConfigParser syntax.  Filenames
-    will be tilde-expanded later.  Later files override earlier ones."""
+    """Docutils configuration files, using ConfigParser syntax.
+
+    Filenames will be tilde-expanded later. Later files override earlier ones.
+    """
 
     threshold_choices = 'info 1 warning 2 error 3 severe 4 none 5'.split()
     """Possible inputs for for --report and --halt threshold values."""
@@ -588,9 +625,10 @@ class OptionParser(optparse.OptionParser, docutils.SettingsSpec):
                          '_source': None,
                          '_destination': None,
                          '_config_files': None}
-    """Defaults for settings without command-line option equivalents."""
+    """Defaults for settings without command-line option equivalents.
 
-    relative_path_settings = ('warning_stream',)
+    See https://docutils.sourceforge.io/docs/user/config.html#internal-settings
+    """
 
     config_section = 'general'
 
@@ -601,12 +639,13 @@ class OptionParser(optparse.OptionParser, docutils.SettingsSpec):
                            sys.version.split()[0], sys.platform))
     """Default version message."""
 
-    def __init__(self, components=(), defaults=None, read_config_files=None,
+    def __init__(self, components=(), defaults=None, read_config_files=False,
                  *args, **kwargs):
-        """
+        """Set up OptionParser instance.
+
         `components` is a list of Docutils components each containing a
-        ``.settings_spec`` attribute.  `defaults` is a mapping of setting
-        default overrides.
+        ``.settings_spec`` attribute.
+        `defaults` is a mapping of setting default overrides.
         """
 
         self.lists = {}
@@ -615,14 +654,17 @@ class OptionParser(optparse.OptionParser, docutils.SettingsSpec):
         self.config_files = []
         """List of paths of applied configuration files."""
 
-        optparse.OptionParser.__init__(
-            self, option_class=Option, add_help_option=None,
-            formatter=optparse.TitledHelpFormatter(width=78),
-            *args, **kwargs)
+        self.relative_path_settings = ['warning_stream']  # will be modified
+
+        warnings.warn('The frontend.OptionParser class will be replaced '
+                      'by a subclass of argparse.ArgumentParser '
+                      'in Docutils 0.21 or later.',
+                      DeprecationWarning, stacklevel=2)
+        super().__init__(option_class=Option, add_help_option=None,
+                         formatter=optparse.TitledHelpFormatter(width=78),
+                         *args, **kwargs)
         if not self.version:
             self.version = self.version_template
-        # Make an instance copy (it will be modified):
-        self.relative_path_settings = list(self.relative_path_settings)
         self.components = (self, *components)
         self.populate_from_components(self.components)
         self.defaults.update(defaults or {})
@@ -668,14 +710,16 @@ class OptionParser(optparse.OptionParser, docutils.SettingsSpec):
     @classmethod
     def get_standard_config_files(cls):
         """Return list of config files, from environment or standard."""
-        try:
+        if 'DOCUTILSCONFIG' in os.environ:
             config_files = os.environ['DOCUTILSCONFIG'].split(os.pathsep)
-        except KeyError:
+        else:
             config_files = cls.standard_config_files
         return [os.path.expanduser(f) for f in config_files if f.strip()]
 
     def get_standard_config_settings(self):
-        settings = Values()
+        with warnings.catch_warnings():
+            warnings.filterwarnings('ignore', category=DeprecationWarning)
+            settings = Values()
         for filename in self.get_standard_config_files():
             settings.update(self.get_config_file_settings(filename), self)
         return settings
@@ -683,11 +727,12 @@ class OptionParser(optparse.OptionParser, docutils.SettingsSpec):
     def get_config_file_settings(self, config_file):
         """Returns a dictionary containing appropriate config file settings."""
         config_parser = ConfigParser()
-        # parse config file, add filename if found and successfull read.
-        self.config_files += config_parser.read(config_file, self)
-        base_path = os.path.dirname(config_file)
+        # parse config file, add filename if found and successfully read.
         applied = set()
-        settings = Values()
+        with warnings.catch_warnings():
+            warnings.filterwarnings('ignore', category=DeprecationWarning)
+            self.config_files += config_parser.read(config_file, self)
+            settings = Values()
         for component in self.components:
             if not component:
                 continue
@@ -696,9 +741,11 @@ class OptionParser(optparse.OptionParser, docutils.SettingsSpec):
                 if section in applied:
                     continue
                 applied.add(section)
-                settings.update(config_parser.get_section(section), self)
-        make_paths_absolute(
-            settings.__dict__, self.relative_path_settings, base_path)
+                if config_parser.has_section(section):
+                    settings.update(config_parser[section], self)
+        make_paths_absolute(settings.__dict__,
+                            self.relative_path_settings,
+                            os.path.dirname(config_file))
         return settings.__dict__
 
     def check_values(self, values, args):
@@ -731,7 +778,9 @@ class OptionParser(optparse.OptionParser, docutils.SettingsSpec):
 
     def get_default_values(self):
         """Needed to get custom `Values` instances."""
-        defaults = Values(self.defaults)
+        with warnings.catch_warnings():
+            warnings.filterwarnings('ignore', category=DeprecationWarning)
+            defaults = Values(self.defaults)
         defaults._config_files = self.config_files
         return defaults
 
@@ -792,7 +841,7 @@ Skipping "%s" configuration file.
             warnings.warn('frontend.ConfigParser.read(): parameter '
                           '"option_parser" will be removed '
                           'in Docutils 0.21 or later.',
-                          PendingDeprecationWarning, stacklevel=2)
+                          DeprecationWarning, stacklevel=2)
         read_ok = []
         if isinstance(filenames, str):
             filenames = [filenames]
@@ -845,12 +894,10 @@ Skipping "%s" configuration file.
                             setting, value, option_parser,
                             config_parser=self, config_section=section)
                     except Exception as err:
-                        raise ValueError(
-                            'Error in config file "%s", section "[%s]":\n'
-                            '    %s\n'
-                            '        %s = %s'
-                            % (filename, section, io.error_string(err),
-                               setting, value))
+                        raise ValueError(f'Error in config file "{filename}", '
+                                         f'section "[{section}]":\n'
+                                         f'    {io.error_string(err)}\n'
+                                         f'        {setting} = {value}')
                     self.set(section, setting, new_value)
                 if option.overrides:
                     self.set(section, option.overrides, None)
@@ -873,8 +920,8 @@ Skipping "%s" configuration file.
         catch KeyError.
         """
         warnings.warn('frontend.OptionParser.get_section() '
-                      'will be removed in Docutils 0.22 or later.',
-                      PendingDeprecationWarning, stacklevel=2)
+                      'will be removed in Docutils 0.21 or later.',
+                      DeprecationWarning, stacklevel=2)
         try:
             return dict(self[section])
         except KeyError:
@@ -883,3 +930,19 @@ Skipping "%s" configuration file.
 
 class ConfigDeprecationWarning(FutureWarning):
     """Warning for deprecated configuration file features."""
+
+
+def get_default_settings(*components):
+    """Return default runtime settings for `components`.
+
+    Return a `frontend.Values` instance with defaults for generic Docutils
+    settings and settings from the `components` (`SettingsSpec` instances).
+
+    This corresponds to steps 1 and 2 in the `runtime settings priority`__.
+
+    __ https://docutils.sourceforge.io/docs/api/runtime-settings.html
+       #settings-priority
+    """
+    with warnings.catch_warnings():
+        warnings.filterwarnings('ignore', category=DeprecationWarning)
+        return OptionParser(components).get_default_values()
diff --git a/docutils/docutils/utils/__init__.py b/docutils/docutils/utils/__init__.py
index 599f183a5..2819e3f4a 100644
--- a/docutils/docutils/utils/__init__.py
+++ b/docutils/docutils/utils/__init__.py
@@ -18,7 +18,8 @@ import unicodedata
 
 from docutils import ApplicationError, DataError, __version_info__
 from docutils import io, nodes
-from docutils.nodes import unescape  # noqa: F401  backwards compatibility
+# for backwards compatibility
+from docutils.nodes import unescape  # noqa: F401
 
 
 class SystemMessage(ApplicationError):
@@ -436,17 +437,26 @@ def new_document(source_path, settings=None):
         `settings` : optparse.Values object
             Runtime settings.  If none are provided, a default core set will
             be used.  If you will use the document object with any Docutils
-            components, you must provide their default settings as well.  For
-            example, if parsing rST, at least provide the rst-parser settings,
-            obtainable as follows::
+            components, you must provide their default settings as well.
+
+            For example, if parsing rST, at least provide the rst-parser
+            settings, obtainable as follows:
+
+            Defaults for parser component::
+
+                settings = docutils.frontend.get_default_settings(
+                               docutils.parsers.rst.Parser)
+
+            Defaults and configuration file customizations::
+
+                settings = docutils.core.Publisher(
+                    parser=docutils.parsers.rst.Parser).get_settings()
 
-                settings = docutils.frontend.OptionParser(
-                               components=(docutils.parsers.rst.Parser,)
-                               ).get_default_values()
     """
+    # Import at top of module would lead to circular dependency!
     from docutils import frontend
     if settings is None:
-        settings = frontend.OptionParser().get_default_values()
+        settings = frontend.get_default_settings()
     source_path = decode_path(source_path)
     reporter = new_reporter(source_path, settings)
     document = nodes.document(settings, reporter, source=source_path)
diff --git a/docutils/test/DocutilsTestSupport.py b/docutils/test/DocutilsTestSupport.py
index 69b589aeb..b677129b9 100644
--- a/docutils/test/DocutilsTestSupport.py
+++ b/docutils/test/DocutilsTestSupport.py
@@ -311,8 +311,7 @@ class TransformTestCase(CustomTestCase):
     cases that have nothing to do with the input and output of the transform.
     """
 
-    option_parser = frontend.OptionParser(components=(rst.Parser,))
-    settings = option_parser.get_default_values()
+    settings = frontend.get_default_settings(rst.Parser)
     settings.report_level = 1
     settings.halt_level = 5
     settings.debug = package_unittest.debug
@@ -433,8 +432,7 @@ class ParserTestCase(CustomTestCase):
     parser = rst.Parser()
     """Parser shared by all ParserTestCases."""
 
-    option_parser = frontend.OptionParser(components=(rst.Parser,))
-    settings = option_parser.get_default_values()
+    settings = frontend.get_default_settings(rst.Parser)
     settings.report_level = 5
     settings.halt_level = 5
     settings.debug = package_unittest.debug
@@ -495,8 +493,7 @@ class PEPParserTestCase(ParserTestCase):
     parser = rst.Parser(rfc2822=True, inliner=rst.states.Inliner())
     """Parser shared by all PEPParserTestCases."""
 
-    option_parser = frontend.OptionParser(components=(rst.Parser, pep.Reader))
-    settings = option_parser.get_default_values()
+    settings = frontend.get_default_settings(rst.Parser, pep.Reader)
     settings.report_level = 5
     settings.halt_level = 5
     settings.debug = package_unittest.debug
@@ -536,8 +533,7 @@ class RecommonmarkParserTestCase(ParserTestCase):
 
     if md_parser_class:
         parser = md_parser_class()
-        option_parser = frontend.OptionParser(components=(md_parser_class,))
-        settings = option_parser.get_default_values()
+        settings = frontend.get_default_settings(md_parser_class)
         settings.report_level = 5
         settings.halt_level = 5
         settings.debug = package_unittest.debug
diff --git a/docutils/test/test_error_reporting.py b/docutils/test/test_error_reporting.py
index 03ee54e9a..16f1082c5 100644
--- a/docutils/test/test_error_reporting.py
+++ b/docutils/test/test_error_reporting.py
@@ -32,10 +32,9 @@ import warnings
 
 from docutils import frontend, utils
 import docutils.parsers.rst
-from docutils.utils.error_reporting import SafeString, ErrorString, ErrorOutput
-
 warnings.filterwarnings('ignore', category=DeprecationWarning,
-                        message=r'.*utils\.error_reporting')
+                        module='.*error_reporting')
+from docutils.utils.error_reporting import SafeString, ErrorString, ErrorOutput  # noqa: E402, E501
 
 
 class SafeStringTests(unittest.TestCase):
@@ -241,8 +240,7 @@ class ErrorReportingTests(unittest.TestCase):
     parser = docutils.parsers.rst.Parser()
     """Parser shared by all ParserTestCases."""
 
-    option_parser = frontend.OptionParser(components=(parser,))
-    settings = option_parser.get_default_values()
+    settings = frontend.get_default_settings(parser)
     settings.report_level = 1
     settings.halt_level = 1
     settings.warning_stream = ''
diff --git a/docutils/test/test_language.py b/docutils/test/test_language.py
index 6fc92c9a3..ac47d46c0 100755
--- a/docutils/test/test_language.py
+++ b/docutils/test/test_language.py
@@ -23,7 +23,7 @@ from docutils.parsers.rst import directives, roles
 import docutils.utils
 import docutils.frontend
 
-_settings = docutils.frontend.OptionParser().get_default_values()
+_settings = docutils.frontend.get_default_settings()
 _reporter = docutils.utils.new_reporter('', _settings)
 
 reference_language = 'en'
diff --git a/docutils/test/test_parsers/test_parser.py b/docutils/test/test_parsers/test_parser.py
index 4157bb712..7d76eaf49 100644
--- a/docutils/test/test_parsers/test_parser.py
+++ b/docutils/test/test_parsers/test_parser.py
@@ -17,8 +17,8 @@ class RstParserTests(unittest.TestCase):
     def test_inputrestrictions(self):
         parser_class = parsers.get_parser_class('rst')
         parser = parser_class()
-        document = utils.new_document('test data', frontend.OptionParser(
-                    components=(parser, )).get_default_values())
+        document = utils.new_document('test data',
+                                      frontend.get_default_settings(parser))
         # input must be unicode at all times
         self.assertRaises(TypeError, parser.parse, b'hol', document)
 
diff --git a/docutils/test/test_settings.py b/docutils/test/test_settings.py
index aed6f770b..94c116564 100755
--- a/docutils/test/test_settings.py
+++ b/docutils/test/test_settings.py
@@ -108,10 +108,9 @@ class ConfigFileTests(unittest.TestCase):
     """Comparison method shared by all tests."""
 
     def setUp(self):
-        warnings.filterwarnings(action='ignore',
+        warnings.filterwarnings('ignore',
                                 category=frontend.ConfigDeprecationWarning)
-        warnings.filterwarnings(action='ignore', module='docutils.frontend',
-                                category=PendingDeprecationWarning)
+        warnings.filterwarnings('ignore', category=DeprecationWarning)
         self.option_parser = frontend.OptionParser(
             components=(pep_html.Writer, rst.Parser), read_config_files=None)
 
@@ -357,5 +356,6 @@ class HelperFunctionsTests(unittest.TestCase):
             reporter.set_conditions('foo', 1, 4)  # trigger warning
 
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/docutils/tools/quicktest.py b/docutils/tools/quicktest.py
index ba2fa8122..8093c0aba 100755
--- a/docutils/tools/quicktest.py
+++ b/docutils/tools/quicktest.py
@@ -15,7 +15,7 @@ import sys
 import os
 import getopt
 import docutils
-from docutils.frontend import OptionParser
+from docutils import frontend
 from docutils.utils import new_document
 from docutils.parsers.rst import Parser
 
@@ -206,7 +206,7 @@ Use the next dialog to build a command line:
 def main():
     # process cmdline arguments:
     inputFile, outputFile, outputFormat, optargs = getArgs()
-    settings = OptionParser(components=(Parser,)).get_default_values()
+    settings = frontend.get_default_settings(Parser)
     settings.debug = optargs['debug']
     parser = Parser()
     input = inputFile.read()