Fix pytest --mypy segfault: "too many open files" (#428)

* attempt to solve #426 without breaking #413, needs tests

* reproduce bug #426

* increase number of empty-bodied tests

* default thread.join(0) and cleanup temp files during httpretty.disable()

* fix functional target

* actions: install dependencies in separate step

* stop running the bugfixes target within functional

* move cleanup into httpretty.reset()

closes #426 

17 files changed, 200 insertions, 13 deletions

diff --git a/.github/workflows/pyenv.yml b/.github/workflows/pyenv.yml
index e5e6bb1..4baf808 100644
--- a/.github/workflows/pyenv.yml
+++ b/.github/workflows/pyenv.yml
@@ -24,10 +24,14 @@ jobs:
       uses: gabrielfalcao/pyenv-action@v7
       with:
         default: "${{ matrix.python }}"
+        command: make setup
 
     - name: Unit Tests
       run: make unit
 
+    - name: Test Bugfixes
+      run: make bugfixes
+
     - name: PyOpenSSL tests
       run: make pyopenssl
 
diff --git a/Makefile b/Makefile
index 9a70f5f..5c5cae2 100644
--- a/Makefile
+++ b/Makefile
@@ -17,13 +17,13 @@ $(VENV):  # creates $(VENV) folder if does not exist
 	python3 -mvenv $(VENV)
 	$(VENV)/bin/pip install -U pip setuptools
 
-setup $(VENV)/bin/sphinx-build $(VENV)/bin/twine $(VENV)/bin/nosetests $(VENV)/bin/python $(VENV)/bin/pip: # installs latest pip
+setup $(VENV)/bin/sphinx-build $(VENV)/bin/twine $(VENV)/bin/nosetests $(VENV)/bin/pytest $(VENV)/bin/python $(VENV)/bin/pip: # installs latest pip
 	test -e $(VENV)/bin/pip || make $(VENV)
 	$(VENV)/bin/pip install -r development.txt
 	$(VENV)/bin/pip install -e .
 
 # Runs the unit and functional tests
-tests: unit functional pyopenssl
+tests: unit bugfixes functional pyopenssl
 
 
 tdd: $(VENV)/bin/nosetests  # runs all tests
@@ -40,10 +40,13 @@ unit: $(VENV)/bin/nosetests  # runs only unit tests
 pyopenssl: $(VENV)/bin/nosetests
 	$(VENV)/bin/nosetests --cover-erase tests/pyopenssl
 
+bugfixes: $(VENV)/bin/nosetests $(VENV)/bin/pytest   # runs tests for specific bugfixes
+	$(VENV)/bin/pytest -v --maxfail=1 --mypy tests/bugfixes/pytest
+	$(VENV)/bin/nosetests tests/bugfixes/nosetests
+
 
 # runs functional tests
 functional: $(VENV)/bin/nosetests  # runs functional tests
-	$(VENV)/bin/nosetests tests/functional/bugfixes
 	$(VENV)/bin/nosetests tests/functional
 
 
diff --git a/development.txt b/development.txt
index fb46b3e..aa6cab6 100644
--- a/development.txt
+++ b/development.txt
@@ -27,3 +27,4 @@ twine>=1.15.0
 urllib3>=1.25.8
 boto3>=1.17.72
 ndg-httpsclient>=0.5.1
+pytest-mypy==0.8.1
diff --git a/httpretty/__init__.py b/httpretty/__init__.py
index 50971b0..5e7d01f 100644
--- a/httpretty/__init__.py
+++ b/httpretty/__init__.py
@@ -29,6 +29,7 @@
 from . import core
 
 from .core import httpretty, httprettified, EmptyRequestHeaders
+from .core import set_default_thread_timeout, get_default_thread_timeout
 from .errors import HTTPrettyError, UnmockedError
 from .version import version
 
diff --git a/httpretty/core.py b/httpretty/core.py
index 20dd281..9d89203 100644
--- a/httpretty/core.py
+++ b/httpretty/core.py
@@ -73,6 +73,59 @@ from datetime import datetime
 from datetime import timedelta
 from errno import EAGAIN
 
+class __internals__:
+    thread_timeout = 0  # https://github.com/gabrielfalcao/HTTPretty/issues/426
+    temp_files = []
+    threads = []
+
+    @classmethod
+    def cleanup_sockets(cls):
+        cls.cleanup_temp_files()
+        cls.cleanup_threads()
+
+    @classmethod
+    def cleanup_threads(cls):
+        for t in cls.threads:
+            t.join(cls.thread_timeout)
+            if t.is_alive():
+                raise socket.timeout(cls.thread_timeout)
+
+    @classmethod
+    def create_thread(cls, *args, **kwargs):
+        return threading.Thread(*args, **kwargs)
+
+    @classmethod
+    def cleanup_temp_files(cls):
+        for fd in cls.temp_files[:]:
+            try:
+                fd.close()
+            except Exception as e:
+                logger.debug('error closing file {}: {}'.format(fd, e))
+            cls.temp_files.remove(fd)
+
+    @classmethod
+    def create_temp_file(cls):
+        fd = tempfile.TemporaryFile()
+        cls.temp_files.append(fd)
+        return fd
+
+def set_default_thread_timeout(timeout):
+    """sets the default thread timeout for HTTPretty threads
+
+    :param timeout: int
+    """
+    __internals__.thread_timeout = timeout
+
+def get_default_thread_timeout():
+    """sets the default thread timeout for HTTPretty threads
+
+    :returns: int
+    """
+
+    return __internals__.thread_timeout
+
+
+SOCKET_GLOBAL_DEFAULT_TIMEOUT = socket._GLOBAL_DEFAULT_TIMEOUT
 old_socket = socket.socket
 old_socketpair = getattr(socket, 'socketpair', None)
 old_SocketType = socket.SocketType
@@ -155,6 +208,7 @@ DEFAULT_HTTPS_PORTS = frozenset([443])
 POTENTIAL_HTTPS_PORTS = set(DEFAULT_HTTPS_PORTS)
 
 
+
 def FALLBACK_FUNCTION(x):
     return x
 
@@ -359,29 +413,65 @@ class HTTPrettyRequestEmpty(object):
     headers = EmptyRequestHeaders()
 
 
+
 class FakeSockFile(object):
     """Fake socket file descriptor. Under the hood all data is written in
     a temporary file, giving it a real file descriptor number.
 
     """
     def __init__(self):
-        self.file = tempfile.TemporaryFile()
+        self.file = None
+        self._fileno = None
+        self.__closed__ = None
+        self.reset()
+
+    def reset(self):
+        if self.file:
+            try:
+                self.file.close()
+            except Exception as e:
+                logger.debug('error closing file {}: {}'.format(self.file, e))
+            self.file = None
+
+        self.file = __internals__.create_temp_file()
         self._fileno = self.file.fileno()
+        self.__closed__ = False
 
     def getvalue(self):
         if hasattr(self.file, 'getvalue'):
-            return self.file.getvalue()
+            value = self.file.getvalue()
         else:
-            return self.file.read()
+            value = self.file.read()
+        self.file.seek(0)
+        return value
 
     def close(self):
-        self.socket.close()
+        if self.__closed__:
+            return
+        self.__closed__ = True
+        self.flush()
+
+    def flush(self):
+        try:
+            super().flush()
+        except Exception as e:
+            logger.debug('error closing file {}: {}'.format(self, e))
+
+        try:
+            self.file.flush()
+        except Exception as e:
+            logger.debug('error closing file {}: {}'.format(self.file, e))
+
+
 
     def fileno(self):
         return self._fileno
 
     def __getattr__(self, name):
-        return getattr(self.file, name)
+        try:
+            return getattr(self.file, name)
+        except AttributeError:
+            return super().__getattribute__(name)
 
     def __del__(self):
         try:
@@ -389,6 +479,11 @@ class FakeSockFile(object):
         except (ValueError, AttributeError):
             pass
 
+        # Adding the line below as a potential fix of github issue #426
+        # that seems to be a compatible the solution of #413
+        self.file.close()
+
+
 
 class FakeSSLSocket(object):
     """Shorthand for :py:class:`~httpretty.core.fakesock`
@@ -593,17 +688,18 @@ class fakesock(object):
             self._bufsize = bufsize
 
             if self._entry:
-                t = threading.Thread(
+                t = __internals__.create_thread(
                     target=self._entry.fill_filekind, args=(self.fd,)
                 )
+
                 t.start()
-                if self.timeout == socket._GLOBAL_DEFAULT_TIMEOUT:
-                    timeout = None
+                if self.timeout == SOCKET_GLOBAL_DEFAULT_TIMEOUT:
+                    timeout = get_default_thread_timeout()
                 else:
                     timeout = self.timeout
-                t.join(timeout)
+                t.join(None)
                 if t.is_alive():
-                    raise socket.timeout
+                    raise socket.timeout(timeout)
 
             return self.fd
 
@@ -1496,6 +1592,7 @@ class httpretty(HttpBaseClass):
         cls._entries.clear()
         cls.latest_requests = []
         cls.last_request = HTTPrettyRequestEmpty()
+        __internals__.cleanup_sockets()
 
     @classmethod
     def historify_request(cls, headers, body='', sock=None):
diff --git a/tests/functional/bugfixes/__init__.py b/tests/bugfixes/nosetests/__init__.py
index e69de29..e69de29 100644
--- a/tests/functional/bugfixes/__init__.py
+++ b/tests/bugfixes/nosetests/__init__.py
diff --git a/tests/functional/bugfixes/test_242_ssl_bad_handshake.py b/tests/bugfixes/nosetests/test_242_ssl_bad_handshake.py
index 0653b41..0653b41 100644
--- a/tests/functional/bugfixes/test_242_ssl_bad_handshake.py
+++ b/tests/bugfixes/nosetests/test_242_ssl_bad_handshake.py
diff --git a/tests/functional/bugfixes/test_387_regex_port.py b/tests/bugfixes/nosetests/test_387_regex_port.py
index c3f90cd..c3f90cd 100644
--- a/tests/functional/bugfixes/test_387_regex_port.py
+++ b/tests/bugfixes/nosetests/test_387_regex_port.py
diff --git a/tests/functional/bugfixes/test_388_unmocked_error_with_url.py b/tests/bugfixes/nosetests/test_388_unmocked_error_with_url.py
index 751d0ad..751d0ad 100644
--- a/tests/functional/bugfixes/test_388_unmocked_error_with_url.py
+++ b/tests/bugfixes/nosetests/test_388_unmocked_error_with_url.py
diff --git a/tests/functional/bugfixes/test_413_regex.py b/tests/bugfixes/nosetests/test_413_regex.py
index 2131f7f..2131f7f 100644
--- a/tests/functional/bugfixes/test_413_regex.py
+++ b/tests/bugfixes/nosetests/test_413_regex.py
diff --git a/tests/functional/bugfixes/test_414_httpx.py b/tests/bugfixes/nosetests/test_414_httpx.py
index 6360ddc..6360ddc 100644
--- a/tests/functional/bugfixes/test_414_httpx.py
+++ b/tests/bugfixes/nosetests/test_414_httpx.py
diff --git a/tests/functional/bugfixes/test_416_boto3.py b/tests/bugfixes/nosetests/test_416_boto3.py
index 9d11eaa..9d11eaa 100644
--- a/tests/functional/bugfixes/test_416_boto3.py
+++ b/tests/bugfixes/nosetests/test_416_boto3.py
diff --git a/tests/functional/bugfixes/test_417_openssl.py b/tests/bugfixes/nosetests/test_417_openssl.py
index 29d82da..29d82da 100644
--- a/tests/functional/bugfixes/test_417_openssl.py
+++ b/tests/bugfixes/nosetests/test_417_openssl.py
diff --git a/tests/functional/bugfixes/test_eventlet.py b/tests/bugfixes/nosetests/test_eventlet.py
index 25a58a9..25a58a9 100644
--- a/tests/functional/bugfixes/test_eventlet.py
+++ b/tests/bugfixes/nosetests/test_eventlet.py
diff --git a/tests/functional/bugfixes/test_redis.py b/tests/bugfixes/nosetests/test_redis.py
index e7328a4..e7328a4 100644
--- a/tests/functional/bugfixes/test_redis.py
+++ b/tests/bugfixes/nosetests/test_redis.py
diff --git a/tests/functional/bugfixes/test_tornado_bind_unused_port.py b/tests/bugfixes/nosetests/test_tornado_bind_unused_port.py
index 82cb2b9..82cb2b9 100644
--- a/tests/functional/bugfixes/test_tornado_bind_unused_port.py
+++ b/tests/bugfixes/nosetests/test_tornado_bind_unused_port.py
diff --git a/tests/bugfixes/pytest/test_426_mypy_segfault.py b/tests/bugfixes/pytest/test_426_mypy_segfault.py
new file mode 100644
index 0000000..3a60601
--- /dev/null
+++ b/tests/bugfixes/pytest/test_426_mypy_segfault.py
@@ -0,0 +1,81 @@
+import time
+import requests
+import json
+import unittest
+import re
+import httpretty
+
+
+class GenerateTests(type):
+    def __init__(cls, name, bases, attrs):
+        if name in ('GenerateTestMeta',): return
+
+        count = getattr(cls, '__generate_count__', attrs.get('__generate_count__'))
+        if not isinstance(count, int):
+            raise SyntaxError(f'Metaclass requires def `__generate_count__ = NUMBER_OF_TESTS` to be set to an integer')
+
+        generate_method = getattr(cls, '__generate_method__', attrs.get('__generate_method__'))
+        if not callable(generate_method):
+            raise SyntaxError(f'Metaclass requires def `__generate_method__(test_name):` to be implemented')
+
+
+        for x in range(count):
+            test_name = "test_{}".format(x)
+            def test_func(self, *args, **kwargs):
+                run_test = generate_method(test_name)
+                run_test(self, *args, **kwargs)
+
+            test_func.__name__ = test_name
+            attrs[test_name] = test_func
+            setattr(cls, test_name, test_func)
+
+
+class TestBug426MypySegfaultWithCallbackAndPayload(unittest.TestCase, metaclass=GenerateTests):
+    __generate_count__ = 1000
+
+    def __generate_method__(test_name):
+        @httpretty.httprettified(allow_net_connect=False)
+        def test_func(self):
+            httpretty.register_uri(httpretty.GET, 'http://github.com', body=self.json_response_callback({"kind": "insecure"}))
+            httpretty.register_uri(httpretty.GET, 'https://github.com', body=self.json_response_callback({"kind": "secure"}))
+            httpretty.register_uri(httpretty.POST, re.compile('github.com/.*'), body=self.json_response_callback({"kind": "regex"}) )
+
+            response = requests.post(
+                'https://github.com/foo',
+                headers={
+                    "Content-Type": "application/json"
+                },
+                data=json.dumps({test_name: time.time()}))
+
+            assert response.status_code == 200
+
+            try:
+                response = requests.get('https://gitlab.com')
+                assert response.status_code == 200
+            except Exception:
+                pass
+
+        return test_func
+
+    def json_response_callback(self, data):
+
+        payload = dict(data)
+        payload.update({
+            "time": time.time()
+        })
+
+        def request_callback(request, path, headers):
+            return [200, headers, json.dumps(payload)]
+
+        return request_callback
+
+
+class TestBug426MypySegfaultWithEmptyMethod(unittest.TestCase, metaclass=GenerateTests):
+    __generate_count__ = 10000
+
+    def __generate_method__(test_name):
+        @httpretty.httprettified(allow_net_connect=False)
+        def test_func(self):
+            pass
+
+        return test_func