a way of doing commentscomments

34 files changed, 189 insertions, 88 deletions

diff --git a/dns/rdata.py b/dns/rdata.py
index e114fe3..98ded79 100644
--- a/dns/rdata.py
+++ b/dns/rdata.py
@@ -111,7 +111,7 @@ def _constify(o):
 class Rdata:
     """Base class for all DNS rdata types."""
 
-    __slots__ = ['rdclass', 'rdtype']
+    __slots__ = ['rdclass', 'rdtype', 'rdcomment']
 
     def __init__(self, rdclass, rdtype):
         """Initialize an rdata.
@@ -123,6 +123,7 @@ class Rdata:
 
         object.__setattr__(self, 'rdclass', rdclass)
         object.__setattr__(self, 'rdtype', rdtype)
+        object.__setattr__(self, 'rdcomment', None)
 
     def __setattr__(self, name, value):
         # Rdatas are immutable
@@ -153,6 +154,10 @@ class Rdata:
     def __setstate__(self, state):
         for slot, val in state.items():
             object.__setattr__(self, slot, val)
+        if not hasattr(self, 'rdcomment'):
+            # Pickled rdata from 2.0.x might not have a rdcomment, so add
+            # it if needed.
+            object.__setattr__(self, 'rdcomment', None)
 
     def covers(self):
         """Return the type a Rdata covers.
@@ -319,6 +324,8 @@ class Rdata:
         # Ensure that all of the arguments correspond to valid fields.
         # Don't allow rdclass or rdtype to be changed, though.
         for key in kwargs:
+            if key == 'rdcomment':
+                continue
             if key not in parameters:
                 raise AttributeError("'{}' object has no attribute '{}'"
                                      .format(self.__class__.__name__, key))
@@ -336,6 +343,11 @@ class Rdata:
         # this validation can go away.
         rd = self.__class__(*args)
         dns.rdata.from_text(rd.rdclass, rd.rdtype, rd.to_text())
+        # The comment is not set in the constructor, so give it special
+        # handling.
+        rdcomment = kwargs.get('rdcomment', self.rdcomment)
+        if rdcomment is not None:
+            object.__setattr__(rd, 'rdcomment', rdcomment)
         return rd
 
 
@@ -364,13 +376,7 @@ class GenericRdata(Rdata):
             raise dns.exception.SyntaxError(
                 r'generic rdata does not start with \#')
         length = tok.get_int()
-        chunks = []
-        while 1:
-            token = tok.get()
-            if token.is_eol_or_eof():
-                break
-            chunks.append(token.value.encode())
-        hex = b''.join(chunks)
+        hex = tok.concatenate_remaining_identifiers().encode()
         data = binascii.unhexlify(hex)
         if len(data) != length:
             raise dns.exception.SyntaxError(
@@ -459,6 +465,7 @@ def from_text(rdclass, rdtype, tok, origin=None, relativize=True,
     rdclass = dns.rdataclass.RdataClass.make(rdclass)
     rdtype = dns.rdatatype.RdataType.make(rdtype)
     cls = get_rdata_class(rdclass, rdtype)
+    rdata = None
     if cls != GenericRdata:
         # peek at first token
         token = tok.get()
@@ -470,12 +477,17 @@ def from_text(rdclass, rdtype, tok, origin=None, relativize=True,
             # wire form from the generic syntax, and then run
             # from_wire on it.
             #
-            rdata = GenericRdata.from_text(rdclass, rdtype, tok, origin,
-                                           relativize, relativize_to)
-            return from_wire(rdclass, rdtype, rdata.data, 0, len(rdata.data),
-                             origin)
-    return cls.from_text(rdclass, rdtype, tok, origin, relativize,
-                         relativize_to)
+            grdata = GenericRdata.from_text(rdclass, rdtype, tok, origin,
+                                            relativize, relativize_to)
+            rdata = from_wire(rdclass, rdtype, grdata.data, 0, len(grdata.data),
+                              origin)
+    if rdata is None:
+        rdata = cls.from_text(rdclass, rdtype, tok, origin, relativize,
+                              relativize_to)
+    token = tok.get_eol_as_token()
+    if token.comment is not None:
+        object.__setattr__(rdata, 'rdcomment', token.comment)
+    return rdata
 
 
 def from_wire_parser(rdclass, rdtype, parser, origin=None):
diff --git a/dns/rdataset.py b/dns/rdataset.py
index 660415e..8e70a08 100644
--- a/dns/rdataset.py
+++ b/dns/rdataset.py
@@ -176,7 +176,7 @@ class Rdataset(dns.set.Set):
         return not self.__eq__(other)
 
     def to_text(self, name=None, origin=None, relativize=True,
-                override_rdclass=None, **kw):
+                override_rdclass=None, want_comments=False, **kw):
         """Convert the rdataset into DNS master file format.
 
         See ``dns.name.Name.choose_relativity`` for more information
@@ -194,6 +194,9 @@ class Rdataset(dns.set.Set):
 
         *relativize*, a ``bool``.  If ``True``, names will be relativized
         to *origin*.
+
+        *want_comments*, a ``bool``.  If ``True``, emit comments for rdata
+        which have them.  The default is ``False``.
         """
 
         if name is not None:
@@ -219,11 +222,16 @@ class Rdataset(dns.set.Set):
                                          dns.rdatatype.to_text(self.rdtype)))
         else:
             for rd in self:
-                s.write('%s%s%d %s %s %s\n' %
+                extra = ''
+                if want_comments:
+                    if rd.rdcomment:
+                        extra = f' ;{rd.rdcomment}'
+                s.write('%s%s%d %s %s %s%s\n' %
                         (ntext, pad, self.ttl, dns.rdataclass.to_text(rdclass),
                          dns.rdatatype.to_text(self.rdtype),
                          rd.to_text(origin=origin, relativize=relativize,
-                         **kw)))
+                                    **kw),
+                         extra))
         #
         # We strip off the final \n for the caller's convenience in printing
         #
diff --git a/dns/rdtypes/ANY/GPOS.py b/dns/rdtypes/ANY/GPOS.py
index 03677fd..8285b3f 100644
--- a/dns/rdtypes/ANY/GPOS.py
+++ b/dns/rdtypes/ANY/GPOS.py
@@ -93,7 +93,6 @@ class GPOS(dns.rdata.Rdata):
         latitude = tok.get_string()
         longitude = tok.get_string()
         altitude = tok.get_string()
-        tok.get_eol()
         return cls(rdclass, rdtype, latitude, longitude, altitude)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/ANY/HINFO.py b/dns/rdtypes/ANY/HINFO.py
index 587e0ad..6c1ccfa 100644
--- a/dns/rdtypes/ANY/HINFO.py
+++ b/dns/rdtypes/ANY/HINFO.py
@@ -50,7 +50,6 @@ class HINFO(dns.rdata.Rdata):
                   relativize_to=None):
         cpu = tok.get_string(max_length=255)
         os = tok.get_string(max_length=255)
-        tok.get_eol()
         return cls(rdclass, rdtype, cpu, os)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/ANY/HIP.py b/dns/rdtypes/ANY/HIP.py
index 1c774bb..437ee73 100644
--- a/dns/rdtypes/ANY/HIP.py
+++ b/dns/rdtypes/ANY/HIP.py
@@ -59,10 +59,7 @@ class HIP(dns.rdata.Rdata):
             raise dns.exception.SyntaxError("HIT too long")
         key = base64.b64decode(tok.get_string().encode())
         servers = []
-        while 1:
-            token = tok.get()
-            if token.is_eol_or_eof():
-                break
+        for token in tok.get_remaining():
             server = tok.as_name(token, origin, relativize, relativize_to)
             servers.append(server)
         return cls(rdclass, rdtype, hit, algorithm, key, servers)
diff --git a/dns/rdtypes/ANY/ISDN.py b/dns/rdtypes/ANY/ISDN.py
index 6834b3c..b07594f 100644
--- a/dns/rdtypes/ANY/ISDN.py
+++ b/dns/rdtypes/ANY/ISDN.py
@@ -52,14 +52,11 @@ class ISDN(dns.rdata.Rdata):
     def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True,
                   relativize_to=None):
         address = tok.get_string()
-        t = tok.get()
-        if not t.is_eol_or_eof():
-            tok.unget(t)
-            subaddress = tok.get_string()
+        tokens = tok.get_remaining(max_tokens=1)
+        if len(tokens) >= 1:
+            subaddress = tokens[0].unescape().value
         else:
-            tok.unget(t)
             subaddress = ''
-        tok.get_eol()
         return cls(rdclass, rdtype, address, subaddress)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/ANY/LOC.py b/dns/rdtypes/ANY/LOC.py
index eb00a1c..0602734 100644
--- a/dns/rdtypes/ANY/LOC.py
+++ b/dns/rdtypes/ANY/LOC.py
@@ -245,25 +245,22 @@ class LOC(dns.rdata.Rdata):
             t = t[0: -1]
         altitude = float(t) * 100.0        # m -> cm
 
-        token = tok.get().unescape()
-        if not token.is_eol_or_eof():
-            value = token.value
+        tokens = tok.get_remaining(max_tokens=3)
+        if len(tokens) >= 1:
+            value = tokens[0].unescape().value
             if value[-1] == 'm':
                 value = value[0: -1]
             size = float(value) * 100.0        # m -> cm
-            token = tok.get().unescape()
-            if not token.is_eol_or_eof():
-                value = token.value
+            if len(tokens) >= 2:
+                value = tokens[1].unescape().value
                 if value[-1] == 'm':
                     value = value[0: -1]
                 hprec = float(value) * 100.0        # m -> cm
-                token = tok.get().unescape()
-                if not token.is_eol_or_eof():
-                    value = token.value
+                if len(tokens) >= 3:
+                    value = tokens[2].unescape().value
                     if value[-1] == 'm':
                         value = value[0: -1]
                     vprec = float(value) * 100.0        # m -> cm
-                    tok.get_eol()
 
         # Try encoding these now so we raise if they are bad
         _encode_size(size, "size")
diff --git a/dns/rdtypes/ANY/NSEC3PARAM.py b/dns/rdtypes/ANY/NSEC3PARAM.py
index 8ac7627..31ab8b7 100644
--- a/dns/rdtypes/ANY/NSEC3PARAM.py
+++ b/dns/rdtypes/ANY/NSEC3PARAM.py
@@ -57,7 +57,6 @@ class NSEC3PARAM(dns.rdata.Rdata):
             salt = ''
         else:
             salt = binascii.unhexlify(salt.encode())
-        tok.get_eol()
         return cls(rdclass, rdtype, algorithm, flags, iterations, salt)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/ANY/RP.py b/dns/rdtypes/ANY/RP.py
index 7446de6..a6054da 100644
--- a/dns/rdtypes/ANY/RP.py
+++ b/dns/rdtypes/ANY/RP.py
@@ -43,7 +43,6 @@ class RP(dns.rdata.Rdata):
                   relativize_to=None):
         mbox = tok.get_name(origin, relativize, relativize_to)
         txt = tok.get_name(origin, relativize, relativize_to)
-        tok.get_eol()
         return cls(rdclass, rdtype, mbox, txt)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/ANY/SOA.py b/dns/rdtypes/ANY/SOA.py
index e93274e..32b0a86 100644
--- a/dns/rdtypes/ANY/SOA.py
+++ b/dns/rdtypes/ANY/SOA.py
@@ -59,7 +59,6 @@ class SOA(dns.rdata.Rdata):
         retry = tok.get_ttl()
         expire = tok.get_ttl()
         minimum = tok.get_ttl()
-        tok.get_eol()
         return cls(rdclass, rdtype, mname, rname, serial, refresh, retry,
                    expire, minimum)
 
diff --git a/dns/rdtypes/ANY/URI.py b/dns/rdtypes/ANY/URI.py
index 84296f5..7d6d068 100644
--- a/dns/rdtypes/ANY/URI.py
+++ b/dns/rdtypes/ANY/URI.py
@@ -54,7 +54,6 @@ class URI(dns.rdata.Rdata):
         target = tok.get().unescape()
         if not (target.is_quoted_string() or target.is_identifier()):
             raise dns.exception.SyntaxError("URI target must be a string")
-        tok.get_eol()
         return cls(rdclass, rdtype, priority, weight, target.value)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/ANY/X25.py b/dns/rdtypes/ANY/X25.py
index 214f1dc..29b9c4d 100644
--- a/dns/rdtypes/ANY/X25.py
+++ b/dns/rdtypes/ANY/X25.py
@@ -44,7 +44,6 @@ class X25(dns.rdata.Rdata):
     def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True,
                   relativize_to=None):
         address = tok.get_string()
-        tok.get_eol()
         return cls(rdclass, rdtype, address)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/CH/A.py b/dns/rdtypes/CH/A.py
index b738ac6..330fcae 100644
--- a/dns/rdtypes/CH/A.py
+++ b/dns/rdtypes/CH/A.py
@@ -41,7 +41,6 @@ class A(dns.rdata.Rdata):
                   relativize_to=None):
         domain = tok.get_name(origin, relativize, relativize_to)
         address = tok.get_uint16(base=8)
-        tok.get_eol()
         return cls(rdclass, rdtype, domain, address)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/IN/A.py b/dns/rdtypes/IN/A.py
index 8b71e32..35ec46f 100644
--- a/dns/rdtypes/IN/A.py
+++ b/dns/rdtypes/IN/A.py
@@ -40,7 +40,6 @@ class A(dns.rdata.Rdata):
     def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True,
                   relativize_to=None):
         address = tok.get_identifier()
-        tok.get_eol()
         return cls(rdclass, rdtype, address)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/IN/AAAA.py b/dns/rdtypes/IN/AAAA.py
index 08f9d67..c37b82a 100644
--- a/dns/rdtypes/IN/AAAA.py
+++ b/dns/rdtypes/IN/AAAA.py
@@ -40,7 +40,6 @@ class AAAA(dns.rdata.Rdata):
     def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True,
                   relativize_to=None):
         address = tok.get_identifier()
-        tok.get_eol()
         return cls(rdclass, rdtype, address)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/IN/APL.py b/dns/rdtypes/IN/APL.py
index ab7fe4b..3b1b8d1 100644
--- a/dns/rdtypes/IN/APL.py
+++ b/dns/rdtypes/IN/APL.py
@@ -87,11 +87,8 @@ class APL(dns.rdata.Rdata):
     def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True,
                   relativize_to=None):
         items = []
-        while True:
-            token = tok.get().unescape()
-            if token.is_eol_or_eof():
-                break
-            item = token.value
+        for token in tok.get_remaining():
+            item = token.unescape().value
             if item[0] == '!':
                 negation = True
                 item = item[1:]
diff --git a/dns/rdtypes/IN/NAPTR.py b/dns/rdtypes/IN/NAPTR.py
index 48d4356..a861b67 100644
--- a/dns/rdtypes/IN/NAPTR.py
+++ b/dns/rdtypes/IN/NAPTR.py
@@ -72,7 +72,6 @@ class NAPTR(dns.rdata.Rdata):
         service = tok.get_string()
         regexp = tok.get_string()
         replacement = tok.get_name(origin, relativize, relativize_to)
-        tok.get_eol()
         return cls(rdclass, rdtype, order, preference, flags, service,
                    regexp, replacement)
 
diff --git a/dns/rdtypes/IN/NSAP.py b/dns/rdtypes/IN/NSAP.py
index 227465f..78730a1 100644
--- a/dns/rdtypes/IN/NSAP.py
+++ b/dns/rdtypes/IN/NSAP.py
@@ -41,7 +41,6 @@ class NSAP(dns.rdata.Rdata):
     def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True,
                   relativize_to=None):
         address = tok.get_string()
-        tok.get_eol()
         if address[0:2] != '0x':
             raise dns.exception.SyntaxError('string does not start with 0x')
         address = address[2:].replace('.', '')
diff --git a/dns/rdtypes/IN/PX.py b/dns/rdtypes/IN/PX.py
index 946d79f..288bb12 100644
--- a/dns/rdtypes/IN/PX.py
+++ b/dns/rdtypes/IN/PX.py
@@ -47,7 +47,6 @@ class PX(dns.rdata.Rdata):
         preference = tok.get_uint16()
         map822 = tok.get_name(origin, relativize, relativize_to)
         mapx400 = tok.get_name(origin, relativize, relativize_to)
-        tok.get_eol()
         return cls(rdclass, rdtype, preference, map822, mapx400)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/IN/SRV.py b/dns/rdtypes/IN/SRV.py
index 485153f..a3debab 100644
--- a/dns/rdtypes/IN/SRV.py
+++ b/dns/rdtypes/IN/SRV.py
@@ -49,7 +49,6 @@ class SRV(dns.rdata.Rdata):
         weight = tok.get_uint16()
         port = tok.get_uint16()
         target = tok.get_name(origin, relativize, relativize_to)
-        tok.get_eol()
         return cls(rdclass, rdtype, priority, weight, port, target)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/IN/WKS.py b/dns/rdtypes/IN/WKS.py
index d66d858..9b5e87e 100644
--- a/dns/rdtypes/IN/WKS.py
+++ b/dns/rdtypes/IN/WKS.py
@@ -59,12 +59,10 @@ class WKS(dns.rdata.Rdata):
         else:
             protocol = socket.getprotobyname(protocol)
         bitmap = bytearray()
-        while 1:
-            token = tok.get().unescape()
-            if token.is_eol_or_eof():
-                break
-            if token.value.isdigit():
-                serv = int(token.value)
+        for token in tok.get_remaining():
+            value = token.unescape().value
+            if value.isdigit():
+                serv = int(value)
             else:
                 if protocol != _proto_udp and protocol != _proto_tcp:
                     raise NotImplementedError("protocol must be TCP or UDP")
@@ -72,7 +70,7 @@ class WKS(dns.rdata.Rdata):
                     protocol_text = "udp"
                 else:
                     protocol_text = "tcp"
-                serv = socket.getservbyname(token.value, protocol_text)
+                serv = socket.getservbyname(value, protocol_text)
             i = serv // 8
             l = len(bitmap)
             if l < i + 1:
diff --git a/dns/rdtypes/euibase.py b/dns/rdtypes/euibase.py
index c1677a8..ba44571 100644
--- a/dns/rdtypes/euibase.py
+++ b/dns/rdtypes/euibase.py
@@ -44,7 +44,6 @@ class EUIBase(dns.rdata.Rdata):
     def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True,
                   relativize_to=None):
         text = tok.get_string()
-        tok.get_eol()
         if len(text) != cls.text_len:
             raise dns.exception.SyntaxError(
                 'Input text must have %s characters' % cls.text_len)
diff --git a/dns/rdtypes/mxbase.py b/dns/rdtypes/mxbase.py
index d6a6efe..723b762 100644
--- a/dns/rdtypes/mxbase.py
+++ b/dns/rdtypes/mxbase.py
@@ -44,7 +44,6 @@ class MXBase(dns.rdata.Rdata):
                   relativize_to=None):
         preference = tok.get_uint16()
         exchange = tok.get_name(origin, relativize, relativize_to)
-        tok.get_eol()
         return cls(rdclass, rdtype, preference, exchange)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/nsbase.py b/dns/rdtypes/nsbase.py
index 93d3ee5..212f8c0 100644
--- a/dns/rdtypes/nsbase.py
+++ b/dns/rdtypes/nsbase.py
@@ -40,7 +40,6 @@ class NSBase(dns.rdata.Rdata):
     def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True,
                   relativize_to=None):
         target = tok.get_name(origin, relativize, relativize_to)
-        tok.get_eol()
         return cls(rdclass, rdtype, target)
 
     def _to_wire(self, file, compress=None, origin=None, canonicalize=False):
diff --git a/dns/rdtypes/txtbase.py b/dns/rdtypes/txtbase.py
index ad0093d..38c5601 100644
--- a/dns/rdtypes/txtbase.py
+++ b/dns/rdtypes/txtbase.py
@@ -63,10 +63,8 @@ class TXTBase(dns.rdata.Rdata):
     def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True,
                   relativize_to=None):
         strings = []
-        while 1:
-            token = tok.get().unescape_to_bytes()
-            if token.is_eol_or_eof():
-                break
+        for token in tok.get_remaining():
+            token = token.unescape_to_bytes()
             if not (token.is_quoted_string() or token.is_identifier()):
                 raise dns.exception.SyntaxError("expected a string")
             if len(token.value) > 255:
diff --git a/dns/rdtypes/util.py b/dns/rdtypes/util.py
index a63d1a0..3a089ed 100644
--- a/dns/rdtypes/util.py
+++ b/dns/rdtypes/util.py
@@ -113,11 +113,8 @@ class Bitmap:
 
     def from_text(self, tok):
         rdtypes = []
-        while True:
-            token = tok.get().unescape()
-            if token.is_eol_or_eof():
-                break
-            rdtype = dns.rdatatype.from_text(token.value)
+        for token in tok.get_remaining():
+            rdtype = dns.rdatatype.from_text(token.unescape().value)
             if rdtype == 0:
                 raise dns.exception.SyntaxError(f"{self.type_name} with bit 0")
             rdtypes.append(rdtype)
diff --git a/dns/tokenizer.py b/dns/tokenizer.py
index 3e5d2ba..0c117ab 100644
--- a/dns/tokenizer.py
+++ b/dns/tokenizer.py
@@ -48,12 +48,13 @@ class Token:
     has_escape: Does the token value contain escapes?
     """
 
-    def __init__(self, ttype, value='', has_escape=False):
+    def __init__(self, ttype, value='', has_escape=False, comment=None):
         """Initialize a token instance."""
 
         self.ttype = ttype
         self.value = value
         self.has_escape = has_escape
+        self.comment = comment
 
     def is_eof(self):
         return self.ttype == EOF
@@ -396,13 +397,13 @@ class Tokenizer:
                             if self.multiline:
                                 raise dns.exception.SyntaxError(
                                     'unbalanced parentheses')
-                            return Token(EOF)
+                            return Token(EOF, comment=token)
                         elif self.multiline:
                             self.skip_whitespace()
                             token = ''
                             continue
                         else:
-                            return Token(EOL, '\n')
+                            return Token(EOL, '\n', comment=token)
                     else:
                         # This code exists in case we ever want a
                         # delimiter to be returned.  It never produces
@@ -559,6 +560,25 @@ class Tokenizer:
             raise dns.exception.SyntaxError('expecting an identifier')
         return token.value
 
+    def get_remaining(self, max_tokens=None):
+        """Return the remaining tokens on the line, until an EOL or EOF is seen.
+
+        max_tokens: If not None, stop after this number of tokens.
+
+        Returns a list of tokens.
+        """
+
+        tokens = []
+        while True:
+            token = self.get()
+            if token.is_eol_or_eof():
+                self.unget(token)
+                break
+            tokens.append(token)
+            if len(tokens) == max_tokens:
+                break
+        return tokens
+
     def concatenate_remaining_identifiers(self):
         """Read the remaining tokens on the line, which should be identifiers.
 
@@ -572,6 +592,7 @@ class Tokenizer:
         while True:
             token = self.get().unescape()
             if token.is_eol_or_eof():
+                self.unget(token)
                 break
             if not token.is_identifier():
                 raise dns.exception.SyntaxError
@@ -601,7 +622,7 @@ class Tokenizer:
         token = self.get()
         return self.as_name(token, origin, relativize, relativize_to)
 
-    def get_eol(self):
+    def get_eol_as_token(self):
         """Read the next token and raise an exception if it isn't EOL or
         EOF.
 
@@ -613,7 +634,10 @@ class Tokenizer:
             raise dns.exception.SyntaxError(
                 'expected EOL or EOF, got %d "%s"' % (token.ttype,
                                                       token.value))
-        return token.value
+        return token
+
+    def get_eol(self):
+        return self.get_eol_as_token().value
 
     def get_ttl(self):
         """Read the next token and interpret it as a DNS TTL.
diff --git a/dns/zone.py b/dns/zone.py
index e8413c0..eee5fdb 100644
--- a/dns/zone.py
+++ b/dns/zone.py
@@ -532,7 +532,8 @@ class Zone:
                     for rdata in rds:
                         yield (name, rds.ttl, rdata)
 
-    def to_file(self, f, sorted=True, relativize=True, nl=None):
+    def to_file(self, f, sorted=True, relativize=True, nl=None,
+                want_comments=False):
         """Write a zone to a file.
 
         *f*, a file or `str`.  If *f* is a string, it is treated
@@ -550,6 +551,10 @@ class Zone:
         *nl*, a ``str`` or None.  The end of line string.  If not
         ``None``, the output will use the platform's native
         end-of-line marker (i.e. LF on POSIX, CRLF on Windows).
+
+        *want_comments*, a ``bool``.  If ``True``, emit end-of-line comments
+        as part of writing the file.  If ``False``, the default, do not
+        emit them.
         """
 
         with contextlib.ExitStack() as stack:
@@ -579,7 +584,8 @@ class Zone:
                 names = self.keys()
             for n in names:
                 l = self[n].to_text(n, origin=self.origin,
-                                    relativize=relativize)
+                                    relativize=relativize,
+                                    want_comments=want_comments)
                 if isinstance(l, str):
                     l_b = l.encode(file_enc)
                 else:
@@ -593,7 +599,8 @@ class Zone:
                     f.write(l)
                     f.write(nl)
 
-    def to_text(self, sorted=True, relativize=True, nl=None):
+    def to_text(self, sorted=True, relativize=True, nl=None,
+                want_comments=False):
         """Return a zone's text as though it were written to a file.
 
         *sorted*, a ``bool``.  If True, the default, then the file
@@ -609,10 +616,14 @@ class Zone:
         ``None``, the output will use the platform's native
         end-of-line marker (i.e. LF on POSIX, CRLF on Windows).
 
+        *want_comments*, a ``bool``.  If ``True``, emit end-of-line comments
+        as part of writing the file.  If ``False``, the default, do not
+        emit them.
+
         Returns a ``str``.
         """
         temp_buffer = io.StringIO()
-        self.to_file(temp_buffer, sorted, relativize, nl)
+        self.to_file(temp_buffer, sorted, relativize, nl, want_comments)
         return_value = temp_buffer.getvalue()
         temp_buffer.close()
         return return_value
diff --git a/tests/mx-2-0.pickle b/tests/mx-2-0.pickle
new file mode 100644
index 0000000..53d094c
--- /dev/null
+++ b/tests/mx-2-0.pickle
diff --git a/tests/test_message.py b/tests/test_message.py
index 4eb48d3..e64578b 100644
--- a/tests/test_message.py
+++ b/tests/test_message.py
@@ -29,8 +29,7 @@ import dns.rdatatype
 import dns.rrset
 import dns.update
 
-def here(filename):
-    return os.path.join(os.path.dirname(__file__), filename)
+from tests.util import here
 
 query_text = """id 1234
 opcode QUERY
diff --git a/tests/test_rdata.py b/tests/test_rdata.py
index 022642f..7960dd1 100644
--- a/tests/test_rdata.py
+++ b/tests/test_rdata.py
@@ -34,6 +34,7 @@ from dns.rdtypes.ANY.OPT import OPT
 
 import tests.stxt_module
 import tests.ttxt_module
+from tests.util import here
 
 class RdataTestCase(unittest.TestCase):
 
@@ -94,6 +95,15 @@ class RdataTestCase(unittest.TestCase):
             a1.replace(address="bogus")
         self.assertRaises(dns.exception.SyntaxError, bad)
 
+    def test_replace_comment(self):
+        a1 = dns.rdata.from_text(dns.rdataclass.IN, dns.rdatatype.A,
+                                 "1.2.3.4 ;foo")
+        self.assertEqual(a1.rdcomment, "foo")
+        a2 = a1.replace(rdcomment="bar")
+        self.assertEqual(a1, a2)
+        self.assertEqual(a1.rdcomment, "foo")
+        self.assertEqual(a2.rdcomment, "bar")
+
     def test_to_generic(self):
         a = dns.rdata.from_text(dns.rdataclass.IN, dns.rdatatype.A, "1.2.3.4")
         self.assertEqual(str(a.to_generic()), r'\# 4 01020304')
@@ -415,5 +425,13 @@ class RdataTestCase(unittest.TestCase):
         rdata = dns.rdata.from_wire('in', 'a', wire, 0, 4)
         self.assertEqual(rdata, dns.rdata.from_text('in', 'a', '1.2.3.4'))
 
+    def test_unpickle(self):
+        expected_mx = dns.rdata.from_text('in', 'mx', '10 mx.example.')
+        with open(here('mx-2-0.pickle'), 'rb') as f:
+            mx = pickle.load(f)
+        self.assertEqual(mx, expected_mx)
+        self.assertIsNone(mx.rdcomment)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/test_zone.py b/tests/test_zone.py
index 36428b1..9857f80 100644
--- a/tests/test_zone.py
+++ b/tests/test_zone.py
@@ -35,8 +35,7 @@ import dns.rrset
 import dns.zone
 import dns.node
 
-def here(filename):
-    return os.path.join(os.path.dirname(__file__), filename)
+from tests.util import here
 
 example_text = """$TTL 3600
 $ORIGIN example.
@@ -175,6 +174,23 @@ $ORIGIN example.
 @ 300 ns ns2
 """
 
+example_comments_text = """$TTL 3600
+$ORIGIN example.
+@ soa foo bar (1 ; not kept
+2 3 4 5) ; kept
+@ ns ns1
+@ ns ns2
+ns1 a 10.0.0.1 ; comment1
+ns2 a 10.0.0.2 ; comment2
+"""
+
+example_comments_text_output = """@ 3600 IN SOA foo bar 1 2 3 4 5 ; kept
+@ 3600 IN NS ns1
+@ 3600 IN NS ns2
+ns1 3600 IN A 10.0.0.1 ; comment1
+ns2 3600 IN A 10.0.0.2 ; comment2
+"""
+
 _keep_output = True
 
 def _rdata_sort(a):
@@ -746,5 +762,14 @@ class ZoneTestCase(unittest.TestCase):
         self.assertEqual(z._validate_name('foo.bar.example.'),
                          dns.name.from_text('foo.bar', None))
 
+    def testComments(self):
+        z = dns.zone.from_text(example_comments_text, 'example.',
+                               relativize=True)
+        f = StringIO()
+        z.to_file(f, want_comments=True)
+        out = f.getvalue()
+        f.close()
+        self.assertEqual(out, example_comments_text_output)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/util.py b/tests/util.py
new file mode 100644
index 0000000..df736df
--- /dev/null
+++ b/tests/util.py
@@ -0,0 +1,21 @@
+# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license
+
+# Copyright (C) 2003-2007, 2009-2011 Nominum, Inc.
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose with or without fee is hereby granted,
+# provided that the above copyright notice and this permission notice
+# appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import os.path
+
+def here(filename):
+    return os.path.join(os.path.dirname(__file__), filename)
diff --git a/util/generate-mx-pickle.py b/util/generate-mx-pickle.py
new file mode 100644
index 0000000..ad99942
--- /dev/null
+++ b/util/generate-mx-pickle.py
@@ -0,0 +1,19 @@
+import pickle
+import sys
+
+import dns.rdata
+import dns.version
+
+# Generate a pickled mx RR for the current dnspython version
+
+mx = dns.rdata.from_text('in', 'mx', '10 mx.example.')
+filename = f'pickled-{dns.version.MAJOR}-{dns.version.MINOR}.pickle'
+with open(filename, 'wb') as f:
+    pickle.dump(mx, f)
+with open(filename, 'rb') as f:
+    mx2 = pickle.load(f)
+if mx == mx2:
+    print('ok')
+else:
+    print('DIFFERENT!')
+    sys.exit(1)