Merge pull request #723 from rthalley/httpx_if_possible

For DoH, use httpx and with HTTP/2 if we can

8 files changed, 272 insertions, 23 deletions

diff --git a/dns/asyncquery.py b/dns/asyncquery.py
index deeff27..0018541 100644
--- a/dns/asyncquery.py
+++ b/dns/asyncquery.py
@@ -17,9 +17,11 @@
 
 """Talk to a DNS server."""
 
+import base64
 import socket
 import struct
 import time
+import urllib
 
 import dns.asyncbackend
 import dns.exception
@@ -31,8 +33,10 @@ import dns.rdataclass
 import dns.rdatatype
 
 from dns.query import _compute_times, _matches_destination, BadResponse, ssl, \
-    UDPMode
+    UDPMode, have_doh, _have_httpx, _have_http2, NoDOH
 
+if _have_httpx:
+    import httpx
 
 # for brevity
 _lltuple = dns.inet.low_level_address_tuple
@@ -354,6 +358,88 @@ async def tls(q, where, timeout=None, port=853, source=None, source_port=0,
         if not sock and s:
             await s.close()
 
+async def https(q, where, timeout=None, port=443, source=None, source_port=0,
+                one_rr_per_rrset=False, ignore_trailing=False, client=None,
+                path='/dns-query', post=True, verify=True):
+    """Return the response obtained after sending a query via DNS-over-HTTPS.
+
+    *client*, a ``httpx.AsyncClient``.  If provided, the client to use for
+    the query.
+
+    Unlike the other dnspython async functions, a backend cannot be provided
+    in this function because httpx always auto-detects the async backend.
+
+    See :py:func:`dns.query.https()` for the documentation of the other
+    parameters, exceptions, and return type of this method.
+    """
+
+    if not _have_httpx:
+        raise NoDOH('httpx is not available.')  # pragma: no cover
+
+    _httpx_ok = True
+
+    wire = q.to_wire()
+    try:
+        af = dns.inet.af_for_address(where)
+    except ValueError:
+        af = None
+    transport = None
+    headers = {
+        "accept": "application/dns-message"
+    }
+    if af is not None:
+        if af == socket.AF_INET:
+            url = 'https://{}:{}{}'.format(where, port, path)
+        elif af == socket.AF_INET6:
+            url = 'https://[{}]:{}{}'.format(where, port, path)
+    else:
+        url = where
+    if source is not None:
+        transport = httpx.AsyncHTTPTransport(local_address=source[0])
+
+    # After 3.6 is no longer supported, this can use an AsyncExitStack
+    client_to_close = None
+    try:
+        if not client:
+            client = httpx.AsyncClient(http1=True, http2=_have_http2,
+                                       verify=verify, transport=transport)
+            client_to_close = client
+
+        # see https://tools.ietf.org/html/rfc8484#section-4.1.1 for DoH
+        # GET and POST examples
+        if post:
+            headers.update({
+                "content-type": "application/dns-message",
+                "content-length": str(len(wire))
+            })
+            response = await client.post(url, headers=headers, content=wire,
+                                         timeout=timeout)
+        else:
+            wire = base64.urlsafe_b64encode(wire).rstrip(b"=")
+            wire = wire.decode()  # httpx does a repr() if we give it bytes
+            response = await client.get(url, headers=headers, timeout=timeout,
+                                        params={"dns": wire})
+    finally:
+        if client_to_close:
+            await client.aclose()
+
+    # see https://tools.ietf.org/html/rfc8484#section-4.2.1 for info about DoH
+    # status codes
+    if response.status_code < 200 or response.status_code > 299:
+        raise ValueError('{} responded with status code {}'
+                         '\nResponse body: {}'.format(where,
+                                                      response.status_code,
+                                                      response.content))
+    r = dns.message.from_wire(response.content,
+                              keyring=q.keyring,
+                              request_mac=q.request_mac,
+                              one_rr_per_rrset=one_rr_per_rrset,
+                              ignore_trailing=ignore_trailing)
+    r.time = response.elapsed
+    if not q.is_response(r):
+        raise BadResponse
+    return r
+
 async def inbound_xfr(where, txn_manager, query=None,
                       port=53, timeout=None, lifetime=None, source=None,
                       source_port=0, udp_mode=UDPMode.NEVER, backend=None):
diff --git a/dns/asyncresolver.py b/dns/asyncresolver.py
index e94c6fd..ed29dee 100644
--- a/dns/asyncresolver.py
+++ b/dns/asyncresolver.py
@@ -87,8 +87,9 @@ class Resolver(dns.resolver.BaseResolver):
                                                   raise_on_truncation=True,
                                                   backend=backend)
                     else:
-                        # We don't do DoH yet.
-                        raise NotImplementedError
+                        response = await dns.asyncquery.https(request,
+                                                              nameserver,
+                                                              timeout=timeout)
                 except Exception as ex:
                     (_, done) = resolution.query_result(None, ex)
                     continue
diff --git a/dns/query.py b/dns/query.py
index fee5d6a..fbf76d8 100644
--- a/dns/query.py
+++ b/dns/query.py
@@ -42,9 +42,25 @@ try:
     import requests
     from requests_toolbelt.adapters.source import SourceAddressAdapter
     from requests_toolbelt.adapters.host_header_ssl import HostHeaderSSLAdapter
-    have_doh = True
+    _have_requests = True
 except ImportError:  # pragma: no cover
-    have_doh = False
+    _have_requests = False
+
+_have_httpx = False
+_have_http2 = False
+try:
+    import httpx
+    _have_httpx = True
+    try:
+        # See if http2 support is available.
+        with httpx.Client(http2=True):
+            _have_http2 = True
+    except Exception:
+        pass
+except ImportError:  # pragma: no cover
+    pass
+
+have_doh = _have_requests or _have_httpx
 
 try:
     import ssl
@@ -258,8 +274,8 @@ def https(q, where, timeout=None, port=443, source=None, source_port=0,
     *ignore_trailing*, a ``bool``. If ``True``, ignore trailing
     junk at end of the received message.
 
-    *session*, a ``requests.session.Session``.  If provided, the session to use
-    to send the queries.
+    *session*, an ``httpx.Client`` or ``requests.session.Session``.  If
+    provided, the client/session to use to send the queries.
 
     *path*, a ``str``. If *where* is an IP address, then *path* will be used to
     construct the URL to send the DNS query to.
@@ -275,12 +291,15 @@ def https(q, where, timeout=None, port=443, source=None, source_port=0,
     """
 
     if not have_doh:
-        raise NoDOH  # pragma: no cover
+        raise NoDOH('Neither httpx nor requests is available.')  # pragma: no cover
+
+    _httpx_ok = _have_httpx
 
     wire = q.to_wire()
     (af, _, source) = _destination_and_source(where, port, source, source_port,
                                               False)
     transport_adapter = None
+    transport = None
     headers = {
         "accept": "application/dns-message"
     }
@@ -290,19 +309,48 @@ def https(q, where, timeout=None, port=443, source=None, source_port=0,
         elif af == socket.AF_INET6:
             url = 'https://[{}]:{}{}'.format(where, port, path)
     elif bootstrap_address is not None:
+        _httpx_ok = False
         split_url = urllib.parse.urlsplit(where)
         headers['Host'] = split_url.hostname
         url = where.replace(split_url.hostname, bootstrap_address)
-        transport_adapter = HostHeaderSSLAdapter()
+        if _have_requests:
+            transport_adapter = HostHeaderSSLAdapter()
     else:
         url = where
     if source is not None:
         # set source port and source address
-        transport_adapter = SourceAddressAdapter(source)
+        if _have_httpx:
+            if source_port == 0:
+                transport = httpx.HTTPTransport(local_address=source[0])
+            else:
+                _httpx_ok = False
+        if _have_requests:
+            transport_adapter = SourceAddressAdapter(source)
+
+    if session:
+        if _have_httpx:
+            _is_httpx = isinstance(session, httpx.Client)
+        else:
+            _is_httpx = False
+        if _is_httpx and not _httpx_ok:
+            raise NoDOH('Session is httpx, but httpx cannot be used for '
+                        'the requested operation.')
+    else:
+        _is_httpx = _httpx_ok
+
+    if not _httpx_ok and not _have_requests:
+        raise NoDOH('Cannot use httpx for this operation, and '
+                    'requests is not available.')
 
     with contextlib.ExitStack() as stack:
         if not session:
-            session = stack.enter_context(requests.sessions.Session())
+            if _is_httpx:
+                session = stack.enter_context(httpx.Client(http1=True,
+                                                           http2=_have_http2,
+                                                           verify=verify,
+                                                           transport=transport))
+            else:
+                session = stack.enter_context(requests.sessions.Session())
 
         if transport_adapter:
             session.mount(url, transport_adapter)
@@ -314,13 +362,23 @@ def https(q, where, timeout=None, port=443, source=None, source_port=0,
                 "content-type": "application/dns-message",
                 "content-length": str(len(wire))
             })
-            response = session.post(url, headers=headers, data=wire,
-                                    timeout=timeout, verify=verify)
+            if _is_httpx:
+                response = session.post(url, headers=headers, content=wire,
+                                        timeout=timeout)
+            else:
+                response = session.post(url, headers=headers, data=wire,
+                                        timeout=timeout, verify=verify)
         else:
             wire = base64.urlsafe_b64encode(wire).rstrip(b"=")
-            response = session.get(url, headers=headers,
-                                   timeout=timeout, verify=verify,
-                                   params={"dns": wire})
+            if _is_httpx:
+                wire = wire.decode()  # httpx does a repr() if we give it bytes
+                response = session.get(url, headers=headers,
+                                       timeout=timeout,
+                                       params={"dns": wire})
+            else:
+                response = session.get(url, headers=headers,
+                                       timeout=timeout, verify=verify,
+                                       params={"dns": wire})
 
     # see https://tools.ietf.org/html/rfc8484#section-4.2.1 for info about DoH
     # status codes
diff --git a/doc/async-query.rst b/doc/async-query.rst
index 7202bdf..dc22692 100644
--- a/doc/async-query.rst
+++ b/doc/async-query.rst
@@ -9,8 +9,10 @@ processing their responses.  If you want "stub resolver" behavior, then
 you should use the higher level ``dns.asyncresolver`` module; see
 :ref:`async_resolver`.
 
-There is currently no support for DNS-over-HTTPS using asynchronous
-I/O but we hope to offer this in the future.
+For UDP and TCP, the module provides a single "do everything" query
+function, and also provides the send and receive halves of this function
+individually for situations where more sophisticated I/O handling is
+being used by the application.
 
 UDP
 ---
@@ -32,6 +34,12 @@ TLS
 
 .. autofunction:: dns.asyncquery.tls
 
+HTTPS
+-----
+
+.. autofunction:: dns.asyncquery.https
+
+
 Zone Transfers
 --------------
 
diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst
index 32e34d9..f4c6e93 100644
--- a/doc/whatsnew.rst
+++ b/doc/whatsnew.rst
@@ -15,6 +15,8 @@ What's New in dnspython
   an error trace like the NoNameservers exception.  This class is a subclass of
   dns.exception.Timeout for backwards compatibility.
 
+* DNS-over-HTTPS is now supported for asynchronous queries and resolutions.
+
 2.1.0
 ----------------------
 
diff --git a/pyproject.toml b/pyproject.toml
index 51bfbae..06648a2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,6 +30,8 @@ exclude = [
 
 [tool.poetry.dependencies]
 python = "^3.6"
+httpx = {version=">=0.21.1", optional=true, python=">=3.6.2"}
+h2 = {version=">=4.1.0", optional=true, python=">=3.6.2"}
 requests-toolbelt = {version="^0.9.1", optional=true}
 requests = {version="^2.23.0", optional=true}
 idna = {version=">=2.1,<4.0", optional=true}
@@ -50,7 +52,7 @@ wheel = "^0.35.0"
 pylint = "^2.7.4"
 
 [tool.poetry.extras]
-doh = ['requests', 'requests-toolbelt']
+doh = ['httpx', 'h2', 'requests', 'requests-toolbelt']
 idna = ['idna']
 dnssec = ['cryptography']
 trio = ['trio']
diff --git a/setup.cfg b/setup.cfg
index be56502..4d88451 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -44,7 +44,7 @@ test_suite = tests
 setup_requires = setuptools>=44; wheel; setuptools_scm[toml]>=3.4.3
 
 [options.extras_require]
-DOH = requests; requests-toolbelt
+DOH = httpx>=0.21.1; h2>=4.1.0; requests; requests-toolbelt
 IDNA = idna>=2.1
 DNSSEC = cryptography>=2.6
 trio = trio>=0.14.0; sniffio>=1.1
diff --git a/tests/test_doh.py b/tests/test_doh.py
index 835e07d..9dc4cec 100644
--- a/tests/test_doh.py
+++ b/tests/test_doh.py
@@ -23,10 +23,13 @@ import dns.query
 import dns.rdatatype
 import dns.resolver
 
-if dns.query.have_doh:
+if dns.query._have_requests:
     import requests
     from requests.exceptions import SSLError
 
+if dns.query._have_httpx:
+    import httpx
+
 # Probe for IPv4 and IPv6
 resolver_v4_addresses = []
 resolver_v6_addresses = []
@@ -66,9 +69,10 @@ try:
 except socket.gaierror:
     _network_available = False
 
-@unittest.skipUnless(dns.query.have_doh and _network_available,
+
+@unittest.skipUnless(dns.query._have_requests and _network_available,
                      "Python requests cannot be imported; no DNS over HTTPS (DOH)")
-class DNSOverHTTPSTestCase(unittest.TestCase):
+class DNSOverHTTPSTestCaseRequests(unittest.TestCase):
     def setUp(self):
         self.session = requests.sessions.Session()
 
@@ -140,5 +144,93 @@ class DNSOverHTTPSTestCase(unittest.TestCase):
         self.assertTrue('8.8.4.4' in seen)
 
 
+@unittest.skipUnless(dns.query._have_httpx and _network_available,
+                     "Python httpx cannot be imported; no DNS over HTTPS (DOH)")
+class DNSOverHTTPSTestCaseHttpx(unittest.TestCase):
+    def setUp(self):
+        self.session = httpx.Client(http1=True, http2=True, verify=True)
+
+    def tearDown(self):
+        self.session.close()
+
+    def test_get_request(self):
+        nameserver_url = random.choice(KNOWN_ANYCAST_DOH_RESOLVER_URLS)
+        q = dns.message.make_query('example.com.', dns.rdatatype.A)
+        r = dns.query.https(q, nameserver_url, session=self.session, post=False,
+                            timeout=4)
+        self.assertTrue(q.is_response(r))
+
+    def test_get_request_http1(self):
+        saved_have_http2 = dns.query._have_http2
+        try:
+            dns.query._have_http2 = False
+            nameserver_url = random.choice(KNOWN_ANYCAST_DOH_RESOLVER_URLS)
+            q = dns.message.make_query('example.com.', dns.rdatatype.A)
+            r = dns.query.https(q, nameserver_url, session=self.session, post=False,
+                                timeout=4)
+            self.assertTrue(q.is_response(r))
+        finally:
+            dns.query._have_http2 = saved_have_http2
+
+    def test_post_request(self):
+        nameserver_url = random.choice(KNOWN_ANYCAST_DOH_RESOLVER_URLS)
+        q = dns.message.make_query('example.com.', dns.rdatatype.A)
+        r = dns.query.https(q, nameserver_url, session=self.session, post=True,
+                            timeout=4)
+        self.assertTrue(q.is_response(r))
+
+    def test_build_url_from_ip(self):
+        self.assertTrue(resolver_v4_addresses or resolver_v6_addresses)
+        if resolver_v4_addresses:
+            nameserver_ip = random.choice(resolver_v4_addresses)
+            q = dns.message.make_query('example.com.', dns.rdatatype.A)
+            # For some reason Google's DNS over HTTPS fails when you POST to
+            # https://8.8.8.8/dns-query
+            # So we're just going to do GET requests here
+            r = dns.query.https(q, nameserver_ip, session=self.session,
+                                post=False, timeout=4)
+
+            self.assertTrue(q.is_response(r))
+        if resolver_v6_addresses:
+            nameserver_ip = random.choice(resolver_v6_addresses)
+            q = dns.message.make_query('example.com.', dns.rdatatype.A)
+            r = dns.query.https(q, nameserver_ip, session=self.session,
+                                post=False, timeout=4)
+            self.assertTrue(q.is_response(r))
+
+    def test_bootstrap_address_fails(self):
+        # We test this to see if v4 is available
+        if resolver_v4_addresses:
+            ip = '185.228.168.168'
+            invalid_tls_url = 'https://{}/doh/family-filter/'.format(ip)
+            valid_tls_url = 'https://doh.cleanbrowsing.org/doh/family-filter/'
+            q = dns.message.make_query('example.com.', dns.rdatatype.A)
+            # make sure CleanBrowsing's IP address will fail TLS certificate
+            # check
+            with self.assertRaises(httpx.ConnectError):
+                dns.query.https(q, invalid_tls_url, session=self.session,
+                                timeout=4)
+            # We can't do the Host header and SNI magic with httpx, but
+            # we are demanding httpx be used by providing a session, so
+            # we should get a NoDOH exception.
+            with self.assertRaises(dns.query.NoDOH):
+                dns.query.https(q, valid_tls_url, session=self.session,
+                                bootstrap_address=ip, timeout=4)
+
+    def test_new_session(self):
+        nameserver_url = random.choice(KNOWN_ANYCAST_DOH_RESOLVER_URLS)
+        q = dns.message.make_query('example.com.', dns.rdatatype.A)
+        r = dns.query.https(q, nameserver_url, timeout=4)
+        self.assertTrue(q.is_response(r))
+
+    def test_resolver(self):
+        res = dns.resolver.Resolver(configure=False)
+        res.nameservers = ['https://dns.google/dns-query']
+        answer = res.resolve('dns.google', 'A')
+        seen = set([rdata.address for rdata in answer])
+        self.assertTrue('8.8.8.8' in seen)
+        self.assertTrue('8.8.4.4' in seen)
+
+
 if __name__ == '__main__':
     unittest.main()