Avoid using __file__ in places that matternext-release

Assuming __file__ exists breaks packaging netaddr with PyOxidirzer[1].

Initially I wanted to use pkgutil.get_data()[2] as it's been in the Python
standard library since Python 2.6 but it always reads and returns the
whole resource and I decided I don't like reading whole out.txt and
iab.txt in OUI and IAB constructors just to read few small bits of data.
importlib.resources provides an API[3] that should in the usual cases
cases[4] avoid loading whole resources into memory. Maybe IAB and OUI
constructors shouldn't read files and the API needs to be rethought but
that's an issue for another day.

Granted, this is a tradeoff, as we have a dependency now which means
slighly more network traffic and slightly more complexity, but all
things considered I think that's better than the alternative.

(Ironically this introduces a new piece of code using __file__ but this
should be benign as it's not in the code that'll be present in
a PyOxidizer-produced binary. It's necessary to have that setup.py hack
as netaddr can't be imported without importlib_resources or
importlib.resources present now so it can't be unconditionally imported
from setup.py where importlib_resources may not be installed yet).

Fixes GH-188.

[1] https://github.com/indygreg/PyOxidizer/issues/69
[2] https://docs.python.org/2/library/pkgutil.html#pkgutil.get_data
[3] https://docs.python.org/3.9/library/importlib.html#importlib.resources.open_binary
[4] https://gitlab.com/python-devs/importlib_resources/-/blob/2707fb7384e76cda715de14bea5956339969950f/importlib_resources/_py3.py#L24

7 files changed, 49 insertions, 43 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index be00d3b..76c8095 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -18,6 +18,7 @@ jobs:
     - name: Install dependencies
       run: |
         pip install --upgrade -r requirements.txt
+        pip install .
     - name: Run tests
       run: |
         make test_with_junitxml
diff --git a/netaddr/compat.py b/netaddr/compat.py
index 0fe69ee..ad80386 100644
--- a/netaddr/compat.py
+++ b/netaddr/compat.py
@@ -113,3 +113,7 @@ else:
     raise RuntimeError(
         'this module only supports Python 2.4.x or higher (including 3.x)!')
 
+try:
+    from importlib import resources as _importlib_resources
+except ImportError:
+    import importlib_resources as _importlib_resources
diff --git a/netaddr/eui/__init__.py b/netaddr/eui/__init__.py
index 91f1f39..c288ad3 100644
--- a/netaddr/eui/__init__.py
+++ b/netaddr/eui/__init__.py
@@ -13,7 +13,7 @@ from netaddr.strategy import eui48 as _eui48, eui64 as _eui64
 from netaddr.strategy.eui48 import mac_eui48
 from netaddr.strategy.eui64 import eui64_base
 from netaddr.ip import IPAddress
-from netaddr.compat import _is_int, _is_str
+from netaddr.compat import _importlib_resources, _is_int, _is_str
 
 
 class BaseIdentifier(object):
@@ -91,7 +91,7 @@ class OUI(BaseIdentifier):
 
         #   Discover offsets.
         if self._value in ieee.OUI_INDEX:
-            fh = open(ieee.OUI_REGISTRY_PATH, 'rb')
+            fh = _importlib_resources.open_binary(__package__, 'oui.txt')
             for (offset, size) in ieee.OUI_INDEX[self._value]:
                 fh.seek(offset)
                 data = fh.read(size).decode('UTF-8')
@@ -256,7 +256,7 @@ class IAB(BaseIdentifier):
 
         #   Discover offsets.
         if self._value in ieee.IAB_INDEX:
-            fh = open(ieee.IAB_REGISTRY_PATH, 'rb')
+            fh = _importlib_resources.open_binary(__package__, 'iab.txt')
             (offset, size) = ieee.IAB_INDEX[self._value][0]
             self.record['offset'] = offset
             self.record['size'] = size
diff --git a/netaddr/eui/ieee.py b/netaddr/eui/ieee.py
index 36380f6..3e66d55 100755
--- a/netaddr/eui/ieee.py
+++ b/netaddr/eui/ieee.py
@@ -35,24 +35,13 @@ More details can be found at the following URLs :-
 import os.path as _path
 import csv as _csv
 
-from netaddr.compat import _bytes_type
+from netaddr.compat import _bytes_type, _importlib_resources
 from netaddr.core import Subscriber, Publisher
 
 
-#: Path to local copy of IEEE OUI Registry data file.
-OUI_REGISTRY_PATH = _path.join(_path.dirname(__file__), 'oui.txt')
-#: Path to netaddr OUI index file.
-OUI_INDEX_PATH = _path.join(_path.dirname(__file__), 'oui.idx')
-
 #: OUI index lookup dictionary.
 OUI_INDEX = {}
 
-#: Path to local copy of IEEE IAB Registry data file.
-IAB_REGISTRY_PATH = _path.join(_path.dirname(__file__), 'iab.txt')
-
-#: Path to netaddr IAB index file.
-IAB_INDEX_PATH = _path.join(_path.dirname(__file__), 'iab.idx')
-
 #: IAB index lookup dictionary.
 IAB_INDEX = {}
 
@@ -258,22 +247,29 @@ class IABIndexParser(Publisher):
         self.notify(record)
 
 
-def create_index_from_registry(registry_path, index_path, parser):
+def create_index_from_registry(registry_fh, index_path, parser):
     """Generate an index files from the IEEE registry file."""
-    oui_parser = parser(registry_path)
+    oui_parser = parser(registry_fh)
     oui_parser.attach(FileIndexer(index_path))
     oui_parser.parse()
 
 
 def create_indices():
     """Create indices for OUI and IAB file based lookups"""
-    create_index_from_registry(OUI_REGISTRY_PATH, OUI_INDEX_PATH, OUIIndexParser)
-    create_index_from_registry(IAB_REGISTRY_PATH, IAB_INDEX_PATH, IABIndexParser)
-
-
-def load_index(index, index_path):
+    create_index_from_registry(
+        _path.join(_path.dirname(__file__), 'oui.txt'),
+        _path.join(_path.dirname(__file__), 'oui.idx'),
+        OUIIndexParser,
+    )
+    create_index_from_registry(
+        _path.join(_path.dirname(__file__), 'iab.txt'),
+        _path.join(_path.dirname(__file__), 'iab.idx'),
+        IABIndexParser,
+    )
+
+
+def load_index(index, fp):
     """Load index from file into index data structure."""
-    fp = open(index_path, 'rb')
     try:
         for row in _csv.reader([x.decode('UTF-8') for x in fp]):
             (key, offset, size) = [int(_) for _ in row]
@@ -285,8 +281,8 @@ def load_index(index, index_path):
 
 def load_indices():
     """Load OUI and IAB lookup indices into memory"""
-    load_index(OUI_INDEX, OUI_INDEX_PATH)
-    load_index(IAB_INDEX, IAB_INDEX_PATH)
+    load_index(OUI_INDEX, _importlib_resources.open_binary(__package__, 'oui.idx'))
+    load_index(IAB_INDEX, _importlib_resources.open_binary(__package__, 'iab.idx'))
 
 
 if __name__ == '__main__':
diff --git a/netaddr/ip/iana.py b/netaddr/ip/iana.py
index 48b547d..116c8ff 100755
--- a/netaddr/ip/iana.py
+++ b/netaddr/ip/iana.py
@@ -29,13 +29,12 @@ More details can be found at the following URLs :-
     - IEEE Protocols Information Home Page - http://www.iana.org/protocols/
 """
 
-import os.path as _path
 import sys as _sys
 from xml.sax import make_parser, handler
 
 from netaddr.core import Publisher, Subscriber
 from netaddr.ip import IPAddress, IPNetwork, IPRange, cidr_abbrev_to_verbose
-from netaddr.compat import _dict_items, _callable
+from netaddr.compat import _dict_items, _callable, _importlib_resources
 
 
 
@@ -362,21 +361,21 @@ def load_info():
     Parse and load internal IANA data lookups with the latest information from
     data files.
     """
-    PATH = _path.dirname(__file__)
-
-    ipv4 = IPv4Parser(open(_path.join(PATH, 'ipv4-address-space.xml')))
+    ipv4 = IPv4Parser(_importlib_resources.open_binary(__package__, 'ipv4-address-space.xml'))
     ipv4.attach(DictUpdater(IANA_INFO['IPv4'], 'IPv4', 'prefix'))
     ipv4.parse()
 
-    ipv6 = IPv6Parser(open(_path.join(PATH, 'ipv6-address-space.xml')))
+    ipv6 = IPv6Parser(_importlib_resources.open_binary(__package__, 'ipv6-address-space.xml'))
     ipv6.attach(DictUpdater(IANA_INFO['IPv6'], 'IPv6', 'prefix'))
     ipv6.parse()
 
-    ipv6ua = IPv6UnicastParser(open(_path.join(PATH, 'ipv6-unicast-address-assignments.xml')))
+    ipv6ua = IPv6UnicastParser(
+        _importlib_resources.open_binary(__package__, 'ipv6-unicast-address-assignments.xml'),
+    )
     ipv6ua.attach(DictUpdater(IANA_INFO['IPv6_unicast'], 'IPv6_unicast', 'prefix'))
     ipv6ua.parse()
 
-    mcast = MulticastParser(open(_path.join(PATH, 'multicast-addresses.xml')))
+    mcast = MulticastParser(_importlib_resources.open_binary(__package__, 'multicast-addresses.xml'))
     mcast.attach(DictUpdater(IANA_INFO['multicast'], 'multicast', 'address'))
     mcast.parse()
 
diff --git a/netaddr/tests/eui/test_ieee_parsers.py b/netaddr/tests/eui/test_ieee_parsers.py
index 81f1faa..f14edf4 100644
--- a/netaddr/tests/eui/test_ieee_parsers.py
+++ b/netaddr/tests/eui/test_ieee_parsers.py
@@ -1,18 +1,17 @@
-import os
+import contextlib
 import sys
 
 import pytest
 
+from netaddr.compat import _importlib_resources
 from netaddr.eui.ieee import OUIIndexParser, IABIndexParser, FileIndexer
 
 
-SAMPLE_DIR = os.path.dirname(__file__)
-
 @pytest.mark.skipif(sys.version_info > (3,), reason="requires python 2.x")
 def test_oui_parser_py2():
     from cStringIO import StringIO
     outfile = StringIO()
-    with open(os.path.join(SAMPLE_DIR, 'sample_oui.txt'), 'rb') as infile:
+    with contextlib.closing(_importlib_resources.open_binary(__package__, 'sample_oui.txt')) as infile:
         iab_parser = OUIIndexParser(infile)
         iab_parser.attach(FileIndexer(outfile))
         iab_parser.parse()
@@ -23,7 +22,7 @@ def test_oui_parser_py2():
 def test_iab_parser_py2():
     from cStringIO import StringIO
     outfile = StringIO()
-    with open(os.path.join(SAMPLE_DIR, 'sample_iab.txt'), 'rb') as infile:
+    with contextlib.closing(_importlib_resources.open_binary(__package__, 'sample_iab.txt')) as infile:
         iab_parser = IABIndexParser(infile)
         iab_parser.attach(FileIndexer(outfile))
         iab_parser.parse()
@@ -34,7 +33,7 @@ def test_iab_parser_py2():
 def test_oui_parser_py3():
     from io import StringIO
     outfile = StringIO()
-    with open(os.path.join(SAMPLE_DIR, 'sample_oui.txt'), 'rb') as infile:
+    with contextlib.closing(_importlib_resources.open_binary(__package__, 'sample_oui.txt')) as infile:
         iab_parser = OUIIndexParser(infile)
         iab_parser.attach(FileIndexer(outfile))
         iab_parser.parse()
@@ -45,7 +44,7 @@ def test_oui_parser_py3():
 def test_iab_parser_py3():
     from io import StringIO
     outfile = StringIO()
-    with open(os.path.join(SAMPLE_DIR, 'sample_iab.txt'), 'rb') as infile:
+    with contextlib.closing(_importlib_resources.open_binary(__package__, 'sample_iab.txt')) as infile:
         iab_parser = IABIndexParser(infile)
         iab_parser.attach(FileIndexer(outfile))
         iab_parser.parse()
diff --git a/setup.py b/setup.py
index 32620fe..ac403dc 100644
--- a/setup.py
+++ b/setup.py
@@ -13,8 +13,6 @@ from setuptools import setup
 if os.path.exists('MANIFEST'):
     os.remove('MANIFEST')
 
-import netaddr
-
 keywords = [
     'Networking', 'Systems Administration', 'IANA', 'IEEE', 'CIDR', 'IP',
     'IPv4', 'IPv6', 'CIDR', 'EUI', 'MAC', 'MAC-48', 'EUI-48', 'EUI-64'
@@ -171,7 +169,16 @@ def main():
         platforms=platforms,
         entry_points={'console_scripts': ['netaddr = netaddr.cli:main']},
         url='https://github.com/drkjam/netaddr/',
-        version=netaddr.__version__,
+        version=(
+            [
+                ln for ln in open(os.path.join(os.path.dirname(__file__), 'netaddr', '__init__.py'))
+                if '__version__' in ln
+            ][0]
+            .split('=')[-1]
+            .strip()
+            .strip('\'"')
+        ),
+        install_requires=['importlib-resources;python_version<"3.7"'],
     )
 
     setup(**setup_options)