fix lists within lists with comments

had to keep emit the comment and mark it for having been emitted, then reset on next run (as with maps)

10 files changed, 120 insertions, 63 deletions

diff --git a/_test/roundtrip.py b/_test/roundtrip.py
index 9774fc4..a00c4d4 100644
--- a/_test/roundtrip.py
+++ b/_test/roundtrip.py
@@ -44,9 +44,10 @@ def round_trip_load_all(inp, preserve_quotes=None, version=None):
     )
 
 
-def round_trip_dump(data, indent=None, block_seq_indent=None, top_level_colon_align=None,
+def round_trip_dump(data, stream=None,
+                    indent=None, block_seq_indent=None, top_level_colon_align=None,
                     prefix_colon=None, explicit_start=None, explicit_end=None, version=None):
-    return ruamel.yaml.round_trip_dump(data,
+    return ruamel.yaml.round_trip_dump(data, stream=stream,
                                        indent=indent, block_seq_indent=block_seq_indent,
                                        top_level_colon_align=top_level_colon_align,
                                        prefix_colon=prefix_colon,
diff --git a/_test/test_comments.py b/_test/test_comments.py
index 330ab58..b3e39d2 100644
--- a/_test/test_comments.py
+++ b/_test/test_comments.py
@@ -792,6 +792,21 @@ class TestEmptyValueBeforeComments:
           b: 1    #   comment 3
         """)
 
+    def test_flow_seq_within_seq(self):
+        round_trip("""\
+        # comment 1
+        - a
+        - b
+        # comment 2
+        - c
+        - d
+        # comment 3
+        - [e]
+        - f
+        # comment 4
+        - []
+        """)
+
 
 test_block_scalar_commented_line_template = """\
 y: p
diff --git a/_test/test_fail.py b/_test/test_fail.py
index 61b8ffb..fb85e5f 100644
--- a/_test/test_fail.py
+++ b/_test/test_fail.py
@@ -73,48 +73,6 @@ class TestCommentFailures:
 class TestIndentFailures:
 
     @pytest.mark.xfail(strict=True)
-    def test_roundtrip_four_space_indents(self):
-        s = (
-            'a:\n'
-            '-   foo\n'
-            '-   bar\n'
-        )
-        output = round_trip(s)
-        assert s == output
-
-    def test_roundtrip_four_space_indents_no_fail(self):
-        assert round_trip_dump(round_trip_load("""
-        a:
-        -   foo
-        -   bar
-        """)) == dedent("""
-        a:
-        - foo
-        - bar
-        """)
-
-    @pytest.mark.xfail(strict=True)
-    def test_roundtrip_four_space_indents_expl_indent(self):
-        s = (
-            'a:\n'
-            '- foo\n'
-            '- bar\n'
-        )
-        output = round_trip_dump(round_trip_load(s), indent=4)
-        assert s == output
-
-    def test_roundtrip_four_space_indents_expl_indent_no_fail(self):
-        assert round_trip_dump(round_trip_load("""
-        a:
-        -   foo
-        -   bar
-        """), indent=4) == dedent("""
-        a:
-        -   foo
-        -   bar
-        """)
-
-    @pytest.mark.xfail(strict=True)
     def test_indent_not_retained(self):
         round_trip("""
         verbosity: 1                  # 0 is minimal output, -1 none
diff --git a/_test/test_indentation.py b/_test/test_indentation.py
index 9e35091..4391013 100644
--- a/_test/test_indentation.py
+++ b/_test/test_indentation.py
@@ -5,14 +5,12 @@ from __future__ import print_function
 from __future__ import unicode_literals
 
 
-from textwrap import dedent
-
 import pytest  # NOQA
 
 import ruamel.yaml
 from ruamel.yaml.util import load_yaml_guess_indent
 
-from roundtrip import round_trip
+from roundtrip import round_trip, round_trip_load, round_trip_dump, dedent
 
 
 def rt(s):
@@ -169,6 +167,26 @@ class TestIndent:
             - 2
         """, indent=2, block_seq_indent=2)
 
+    # have to set indent!
+    def test_roundtrip_four_space_indents(self):
+        s = (
+            'a:\n'
+            '-   foo\n'
+            '-   bar\n'
+        )
+        round_trip(s, indent=4)
+
+    def test_roundtrip_four_space_indents_no_fail(self):
+        assert round_trip_dump(round_trip_load("""
+        a:
+        -   foo
+        -   bar
+        """)) == dedent("""
+        a:
+        - foo
+        - bar
+        """)
+
 
 class TestYpkgIndent:
     def test_00(self):
diff --git a/_test/test_z_check_debug_leftovers.py b/_test/test_z_check_debug_leftovers.py
new file mode 100644
index 0000000..f5be5df
--- /dev/null
+++ b/_test/test_z_check_debug_leftovers.py
@@ -0,0 +1,35 @@
+# coding: utf-8
+
+import sys
+import pytest  # NOQA
+
+from roundtrip import round_trip_load, round_trip_dump, dedent
+
+
+class TestLeftOverDebug:
+    # idea here is to capture round_trip_output via pytest stdout capture
+    # if there is are any leftover debug statements they should show up
+    def test_00(self, capsys):
+        s = dedent("""
+        a: 1
+        b: []
+        c: [a, 1]
+        d: {f: 3.14, g: 42}
+        """)
+        d = round_trip_load(s)
+        round_trip_dump(d, sys.stdout)
+        out, err = capsys.readouterr()
+        assert out == s
+
+    def test_01(self, capsys):
+        s = dedent("""
+        - 1
+        - []
+        - [a, 1]
+        - {f: 3.14, g: 42}
+        - - 123
+        """)
+        d = round_trip_load(s)
+        round_trip_dump(d, sys.stdout)
+        out, err = capsys.readouterr()
+        assert out == s
diff --git a/comments.py b/comments.py
index 8f69d0e..6cde817 100644
--- a/comments.py
+++ b/comments.py
@@ -951,10 +951,11 @@ def dump_comments(d, name='', sep='.', out=sys.stdout):
             print(name)
         print(d.ca, file=out)
         for k in d:
-            dump_comments(d[k], name=name + sep + k if name else k, sep=sep, out=out)
+            dump_comments(d[k], name=(name + sep + k) if name else k, sep=sep, out=out)
     elif isinstance(d, list):
         if name:
             print(name)
         print(d.ca, file=out)
         for idx, k in enumerate(d):
-            dump_comments(k, name=name + sep + str(idx) if name else k, sep=sep, out=out)
+            dump_comments(k, name=(name + sep + str(idx)) if name else str(idx),
+                          sep=sep, out=out)
diff --git a/emitter.py b/emitter.py
index 01ff3b8..52a5c82 100644
--- a/emitter.py
+++ b/emitter.py
@@ -1388,14 +1388,15 @@ class Emitter(object):
         if comments is None:
             return False
         try:
+            start_events = (MappingStartEvent, SequenceStartEvent)
             for comment in comments:
-                if isinstance(event, MappingStartEvent) and \
+                if isinstance(event, start_events) and \
                    getattr(comment, 'pre_done', None):
                     continue
                 if self.column != 0:
                     self.write_line_break()
                 self.write_comment(comment)
-                if isinstance(event, MappingStartEvent):
+                if isinstance(event, start_events):
                     comment.pre_done = True
         except TypeError:
             print('eventtt', type(event), event)
diff --git a/parser.py b/parser.py
index 4136687..1e79b4d 100644
--- a/parser.py
+++ b/parser.py
@@ -417,10 +417,11 @@ class Parser(object):
             )
             self.state = self.states.pop()
         elif self.scanner.check_token(FlowSequenceStartToken):
-            end_mark = self.scanner.peek_token().end_mark
+            pt = self.scanner.peek_token()
+            end_mark = pt.end_mark
             event = SequenceStartEvent(
                 anchor, tag, implicit,
-                start_mark, end_mark, flow_style=True)
+                start_mark, end_mark, flow_style=True, comment=pt.comment)
             self.state = self.parse_flow_sequence_first_entry
         elif self.scanner.check_token(FlowMappingStartToken):
             end_mark = self.scanner.peek_token().end_mark
diff --git a/representer.py b/representer.py
index 03dcbf4..7468060 100644
--- a/representer.py
+++ b/representer.py
@@ -844,6 +844,16 @@ class RoundTripRepresenter(SafeRepresenter):
         best_style = True
         try:
             comment = getattr(sequence, comment_attrib)
+            node.comment = comment.comment
+            # reset any comment already printed information
+            if node.comment and node.comment[1]:
+                for ct in node.comment[1]:
+                    ct.reset()
+            item_comments = comment.items
+            for v in item_comments.values():
+                if v and v[1]:
+                    for ct in v[1]:
+                        ct.reset()
             item_comments = comment.items
             node.comment = comment.comment
             try:
diff --git a/scanner.py b/scanner.py
index 5e6a5e5..9710b1e 100644
--- a/scanner.py
+++ b/scanner.py
@@ -43,6 +43,7 @@ __all__ = ['Scanner', 'RoundTripScanner', 'ScannerError']
 
 _THE_END = u'\0\r\n\x85\u2028\u2029'
 _THE_END_SPACE_TAB = u'\0 \t\r\n\x85\u2028\u2029'
+_SPACE_TAB = u' \t'
 
 
 class ScannerError(MarkedYAMLError):
@@ -80,6 +81,7 @@ class Scanner(object):
         if self.loader is not None and getattr(self.loader, '_scanner', None) is None:
             self.loader._scanner = self
         self.reset_scanner()
+        self.first_time = False
 
     def reset_scanner(self):
         # type: () -> None
@@ -612,7 +614,7 @@ class Scanner(object):
         else:
 
             # Block context needs additional checks.
-            # (Do we really need them? They will be catched by the parser
+            # (Do we really need them? They will be caught by the parser
             # anyway.)
             if not self.flow_level:
 
@@ -759,8 +761,12 @@ class Scanner(object):
     def check_value(self):
         # type: () -> Any
         # VALUE(flow context):  ':'
-        if bool(self.flow_level):
-            return True
+        if self.scanner_processing_version == (1, 1):
+            if bool(self.flow_level):
+                return True
+        else:
+            if bool(self.flow_level) and self.reader.peek(1) in '\'"':
+                return True
         # VALUE(block context): ':' (' '|'\n')
         return self.reader.peek(1) in _THE_END_SPACE_TAB
 
@@ -779,9 +785,22 @@ class Scanner(object):
         # '-' character) because we want the flow context to be space
         # independent.
         ch = self.reader.peek()
-        return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' or \
-            (self.reader.peek(1) not in _THE_END_SPACE_TAB and
-             (ch == u'-' or (not self.flow_level and ch in u'?:')))
+        if self.scanner_processing_version == (1, 1):
+            return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' or \
+                (self.reader.peek(1) not in _THE_END_SPACE_TAB and
+                 (ch == u'-' or (not self.flow_level and ch in u'?:')))
+        # YAML 1.2
+        if ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`':
+            # ###################                ^ ???
+            return True
+        ch1 = self.reader.peek(1)
+        if ch == '-' and ch1 not in _THE_END_SPACE_TAB:
+            return True
+        if ch == ':' and bool(self.flow_level) and ch1 not in _SPACE_TAB:
+            return True
+
+        return (self.reader.peek(1) not in _THE_END_SPACE_TAB and
+                (ch == u'-' or (not self.flow_level and ch in u'?:')))
 
     # Scanners.
 
@@ -1410,6 +1429,8 @@ class Scanner(object):
                 if (ch == u':' and
                    self.reader.peek(length + 1) not in _THE_END_SPACE_TAB):
                     pass
+                elif (ch == u'?' and self.scanner_processing_version != (1, 1)):
+                    pass
                 elif (ch in _THE_END_SPACE_TAB or
                       (not self.flow_level and ch == u':' and
                           self.reader.peek(length + 1) in _THE_END_SPACE_TAB) or
@@ -1624,7 +1645,6 @@ class RoundTripScanner(Scanner):
         if isinstance(self.tokens[0], CommentToken):
             comment = self.tokens.pop(0)
             self.tokens_taken += 1
-            # print('################ dropping', comment)
             comments.append(comment)
         while self.need_more_tokens():
             self.fetch_more_tokens()
@@ -1636,9 +1656,6 @@ class RoundTripScanner(Scanner):
                 # print 'dropping2', comment
                 comments.append(comment)
         if len(comments) >= 1:
-            # print('  len', len(comments), comments)
-            # print('  com', comments[0], comments[0].start_mark.line)
-            # print('  tok', self.tokens[0].end_mark.line)
             self.tokens[0].add_pre_comments(comments)
         # pull in post comment on e.g. ':'
         if not self.done and len(self.tokens) < 2: