Merge "Switch from pkg_resources to importlib"

4 files changed, 116 insertions, 12 deletions

diff --git a/swift/__init__.py b/swift/__init__.py
index 9927a84c8..0ba983d2e 100644
--- a/swift/__init__.py
+++ b/swift/__init__.py
@@ -18,16 +18,30 @@ import sys
 import gettext
 import warnings
 
-import pkg_resources
+__version__ = None
 
+# First, try to get our version out of PKG-INFO. If we're installed,
+# this'll let us find our version without pulling in pbr. After all, if
+# we're installed on a system, we're not in a Git-managed source tree, so
+# pbr doesn't really buy us anything.
 try:
-    # First, try to get our version out of PKG-INFO. If we're installed,
-    # this'll let us find our version without pulling in pbr. After all, if
-    # we're installed on a system, we're not in a Git-managed source tree, so
-    # pbr doesn't really buy us anything.
-    __version__ = __canonical_version__ = pkg_resources.get_provider(
-        pkg_resources.Requirement.parse('swift')).version
-except pkg_resources.DistributionNotFound:
+    import importlib.metadata
+except ImportError:
+    # python < 3.8
+    import pkg_resources
+    try:
+        __version__ = __canonical_version__ = pkg_resources.get_provider(
+            pkg_resources.Requirement.parse('swift')).version
+    except pkg_resources.DistributionNotFound:
+        pass
+else:
+    try:
+        __version__ = __canonical_version__ = importlib.metadata.distribution(
+            'swift').version
+    except importlib.metadata.PackageNotFoundError:
+        pass
+
+if __version__ is None:
     # No PKG-INFO? We're probably running from a checkout, then. Let pbr do
     # its thing to figure out a version number.
     import pbr.version
@@ -35,6 +49,7 @@ except pkg_resources.DistributionNotFound:
     __version__ = _version_info.release_string()
     __canonical_version__ = _version_info.version_string()
 
+
 _localedir = os.environ.get('SWIFT_LOCALEDIR')
 _t = gettext.translation('swift', localedir=_localedir, fallback=True)
 
diff --git a/swift/common/middleware/s3api/etree.py b/swift/common/middleware/s3api/etree.py
index e16b75340..4c24f8427 100644
--- a/swift/common/middleware/s3api/etree.py
+++ b/swift/common/middleware/s3api/etree.py
@@ -15,7 +15,17 @@
 
 import lxml.etree
 from copy import deepcopy
-from pkg_resources import resource_stream  # pylint: disable-msg=E0611
+try:
+    # importlib.resources was introduced in py37, but couldn't handle
+    # resources in subdirectories (which we use); files() added support
+    from importlib.resources import files
+    del files
+except ImportError:
+    # python < 3.9
+    from pkg_resources import resource_stream  # pylint: disable-msg=E0611
+else:
+    import importlib.resources
+    resource_stream = None
 import six
 
 from swift.common.utils import get_logger
@@ -70,7 +80,13 @@ def fromstring(text, root_tag=None, logger=None):
         # validate XML
         try:
             path = 'schema/%s.rng' % camel_to_snake(root_tag)
-            with resource_stream(__name__, path) as rng:
+            if resource_stream:
+                # python < 3.9
+                stream = resource_stream(__name__, path)
+            else:
+                stream = importlib.resources.files(
+                    __name__.rsplit('.', 1)[0]).joinpath(path).open('rb')
+            with stream as rng:
                 lxml.etree.RelaxNG(file=rng).assertValid(elem)
         except IOError as e:
             # Probably, the schema file doesn't exist.
diff --git a/swift/common/utils.py b/swift/common/utils.py
index d90f38046..d6e9a515f 100644
--- a/swift/common/utils.py
+++ b/swift/common/utils.py
@@ -58,7 +58,12 @@ import eventlet.debug
 import eventlet.greenthread
 import eventlet.patcher
 import eventlet.semaphore
-import pkg_resources
+try:
+    import importlib.metadata
+    pkg_resources = None
+except ImportError:
+    # python < 3.8
+    import pkg_resources
 from eventlet import GreenPool, sleep, Timeout
 from eventlet.event import Event
 from eventlet.green import socket, threading
@@ -6415,7 +6420,19 @@ def load_pkg_resource(group, uri):
 
     if scheme != 'egg':
         raise TypeError('Unhandled URI scheme: %r' % scheme)
-    return pkg_resources.load_entry_point(dist, group, name)
+
+    if pkg_resources:
+        # python < 3.8
+        return pkg_resources.load_entry_point(dist, group, name)
+
+    # May raise importlib.metadata.PackageNotFoundError
+    meta = importlib.metadata.distribution(dist)
+
+    entry_points = [ep for ep in meta.entry_points
+                    if ep.group == group and ep.name == name]
+    if not entry_points:
+        raise ImportError("Entry point %r not found" % ((group, name),))
+    return entry_points[0].load()
 
 
 class PipeMutex(object):
diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py
index 80001ed1c..3c1ec8228 100644
--- a/test/unit/common/test_utils.py
+++ b/test/unit/common/test_utils.py
@@ -4801,6 +4801,9 @@ cluster_dfw1 = http://dfw1.host/v1/
                        'X-Backend-Redirect-Timestamp': '-1'})
         self.assertIn('Invalid timestamp', str(exc))
 
+    @unittest.skipIf(sys.version_info >= (3, 8),
+                     'pkg_resources loading is only available on python 3.7 '
+                     'and earlier')
     @mock.patch('pkg_resources.load_entry_point')
     def test_load_pkg_resource(self, mock_driver):
         tests = {
@@ -4824,6 +4827,59 @@ cluster_dfw1 = http://dfw1.host/v1/
             utils.load_pkg_resource(*args)
         self.assertEqual("Unhandled URI scheme: 'nog'", str(cm.exception))
 
+    @unittest.skipIf(sys.version_info < (3, 8),
+                     'importlib loading is only available on python 3.8 '
+                     'and later')
+    @mock.patch('importlib.metadata.distribution')
+    def test_load_pkg_resource_importlib(self, mock_driver):
+        import importlib.metadata
+        repl_obj = object()
+        ec_obj = object()
+        other_obj = object()
+        mock_driver.return_value.entry_points = [
+            importlib.metadata.EntryPoint(group='swift.diskfile',
+                                          name='replication.fs',
+                                          value=repl_obj),
+            importlib.metadata.EntryPoint(group='swift.diskfile',
+                                          name='erasure_coding.fs',
+                                          value=ec_obj),
+            importlib.metadata.EntryPoint(group='swift.section',
+                                          name='thing.other',
+                                          value=other_obj),
+        ]
+        for ep in mock_driver.return_value.entry_points:
+            ep.load = lambda ep=ep: ep.value
+        tests = {
+            ('swift.diskfile', 'egg:swift#replication.fs'): repl_obj,
+            ('swift.diskfile', 'egg:swift#erasure_coding.fs'): ec_obj,
+            ('swift.section', 'egg:swift#thing.other'): other_obj,
+            ('swift.section', 'swift#thing.other'): other_obj,
+            ('swift.section', 'thing.other'): other_obj,
+        }
+        for args, expected in tests.items():
+            self.assertIs(expected, utils.load_pkg_resource(*args))
+            self.assertEqual(mock_driver.mock_calls, [mock.call('swift')])
+            mock_driver.reset_mock()
+
+        with self.assertRaises(TypeError) as cm:
+            args = ('swift.diskfile', 'nog:swift#replication.fs')
+            utils.load_pkg_resource(*args)
+        self.assertEqual("Unhandled URI scheme: 'nog'", str(cm.exception))
+
+        with self.assertRaises(ImportError) as cm:
+            args = ('swift.diskfile', 'other.fs')
+            utils.load_pkg_resource(*args)
+        self.assertEqual(
+            "Entry point ('swift.diskfile', 'other.fs') not found",
+            str(cm.exception))
+
+        with self.assertRaises(ImportError) as cm:
+            args = ('swift.missing', 'thing.other')
+            utils.load_pkg_resource(*args)
+        self.assertEqual(
+            "Entry point ('swift.missing', 'thing.other') not found",
+            str(cm.exception))
+
     @with_tempdir
     def test_systemd_notify(self, tempdir):
         m_sock = mock.Mock(connect=mock.Mock(), sendall=mock.Mock())