keep start and end sequence when doing 'yaml rt'

4 files changed, 154 insertions, 49 deletions

diff --git a/py/__init__.py b/py/__init__.py
index d6e3661..f792d4c 100644
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -21,7 +21,7 @@ def _convert_version(tup):
     return ret_val
 
 
-version_info = (0, 9, 4)
+version_info = (0, 9, 5, 'alpha')
 __version__ = _convert_version(version_info)
 
 del _convert_version
diff --git a/py/yaml.py b/py/yaml.py
index fadc78d..48411e3 100644
--- a/py/yaml.py
+++ b/py/yaml.py
@@ -128,6 +128,17 @@ class YAML:
             x = ruamel.yaml.compose(input, ruamel.yaml.RoundTripLoader)
             x.dump()  # dump the node
 
+        def scan_file(file_name):
+            inp = open(file_name).read()
+            print('---------\n', file_name)
+            print('---', repr(self.first_non_empty_line(inp)))
+            print('<<<', repr(self.last_non_empty_line(inp)))
+
+        if True:
+            for x in self._args.file:
+                scan_file(x)
+            return
+
         input = dedent("""
         application: web2py
         version: 1
@@ -228,40 +239,110 @@ class YAML:
         with open(self._args.file) as fp:
             print(h2y(fp.read()))
 
+    def from_csv(self):
+        from .convert.from_csv import CSV2YAML
+        c2y = CSV2YAML(self._args)
+        c2y(self._args.file)
+
     def round_trip(self):
         errors = 0
         warnings = 0
         for file_name in self._args.file:
             inp = open(file_name).read()
-            outp = self.round_trip_single(inp)
-            if inp == outp:
+            e, w, stabilize, outp = self.round_trip_input(inp)
+            if w == 0:
                 if self._args.verbose > 0:
                     print(u"{0}: ok".format(file_name))
                 continue
-            warnings += 1
-            stabelize = []
-            if inp.split() != outp.split():
-                errors += 1
-                stabelize.append(u"drops info on round trip")
+            if self._args.save:
+                backup_file_name = file_name + '.orig'
+                if not os.path.exists(backup_file_name):
+                    os.rename(file_name, backup_file_name)
+                with open(file_name, 'w') as ofp:
+                    ofp.write(outp)
+            if not self._args.save or self._args.verbose > 0:
+                print("{0}:\n     {1}".format(file_name, u', '.join(stabilize)))
+                self.diff(inp, outp, file_name)
+            errors += e
+            warnings += w
+        if errors > 0:
+            return 2
+        if warnings > 0:
+            return 1
+        return 0
+
+    def round_trip_input(self, inp):
+        errors = 0
+        warnings = 0
+        stabilize = []
+        outp = self.round_trip_single(inp)
+        if inp == outp:
+            return errors, warnings, stabilize, outp
+        warnings += 1
+        if inp.split() != outp.split():
+            errors += 1
+            stabilize.append(u"drops info on round trip")
+        else:
+            if self.round_trip_single(outp) == outp:
+                stabilize.append(u"stabilizes on second round trip")
             else:
-                if self.round_trip_single(outp) == outp:
-                    stabelize.append(u"stabelizes on second round trip")
-                else:
-                    errors += 1
-            ncoutp = self.round_trip_single(inp, drop_comment=True)
-            if self.round_trip_single(ncoutp, drop_comment=True) == ncoutp:
-                stabelize.append(u"ok without comments")
-            print("{0}:\n     {1}".format(file_name, u', '.join(stabelize)))
-            self.diff(inp, outp, file_name)
-        return 2 if errors > 0 else 1 if warnings > 0 else 0
+                errors += 1
+        ncoutp = self.round_trip_single(inp, drop_comment=True)
+        if self.round_trip_single(ncoutp, drop_comment=True) == ncoutp:
+            stabilize.append(u"ok without comments")
+        return errors, warnings, stabilize, outp
 
     def round_trip_single(self, inp, drop_comment=False):
+        explicit_start=self.first_non_empty_line(inp) == '---'
+        explicit_end=self.last_non_empty_line(inp) == '...'
+        indent = self._args.indent
         loader = ruamel.yaml.SafeLoader if drop_comment else \
             ruamel.yaml.RoundTripLoader
         code = ruamel.yaml.load(inp, loader)
         dumper = ruamel.yaml.SafeDumper if drop_comment else \
             ruamel.yaml.RoundTripDumper
-        return ruamel.yaml.dump(code, Dumper=dumper)
+        return ruamel.yaml.dump(
+            code,
+            Dumper=dumper,
+            indent=indent,
+            explicit_start=explicit_start,
+            explicit_end=explicit_end,
+        )
+
+    def first_non_empty_line(self, txt):
+        """return the first non-empty line of a block of text (stripped)
+        do not split or strip the complete txt
+        """
+        pos = txt.find('\n')
+        prev_pos = 0
+        while pos >= 0:
+            segment = txt[prev_pos:pos].strip()
+            if segment:
+                break
+            # print (pos, repr(segment))
+            prev_pos = pos
+            pos = txt.find('\n', pos+1)
+        return segment
+
+    def last_non_empty_line(self, txt):
+        """return the last non-empty line of a block of text (stripped)
+        do not split or strip the complete txt
+        """
+        assert isinstance(txt, basestring)
+        pos = txt.rfind('\n')
+        prev_pos = len(txt)
+        maxloop = 10
+        while pos >= 0:
+            segment = txt[pos:prev_pos].strip()
+            if segment:
+                break
+            # print (pos, repr(segment))
+            prev_pos = pos
+            pos = txt.rfind('\n', 0, pos-1)
+            maxloop -= 1
+            if maxloop < 0:
+                break
+        return segment
 
     def diff(self, inp, outp, file_name):
         import difflib
@@ -293,6 +374,7 @@ class YAML_Cmd(ProgramBase):
     @option('--verbose', '-v',
             help='increase verbosity level', action=CountAction,
             const=1, nargs=0, default=0, global_option=True)
+    @option('--indent', type=int, global_option=True)
     @version('version: ' + __version__)
     def _pb_init(self):
         # special name for which attribs are included in help
@@ -321,6 +403,9 @@ class YAML_Cmd(ProgramBase):
         help='test round trip on YAML data',
         description='test round trip on YAML data',
     )
+    @option('--save', action='store_true', help="""save the rewritten data back
+    to the input file (if it doesn't exist a '.orig' backup will be made)
+    """)
     @option('file', nargs='+')
     def rt(self):
         return self._yaml.round_trip()
@@ -380,6 +465,20 @@ class YAML_Cmd(ProgramBase):
     def from_html(self):
         return self._yaml.from_html()
 
+    @sub_parser('from-csv',
+        aliases=['csv'],
+        help='convert CSV to YAML',
+        description="""convert CSV to YAML.
+        By default generates a list of rows, with the items in a 2nd level
+        list.
+        """,
+    )
+    @option('--delimiter')
+    #@option('--quotechar')
+    @option('file')
+    def from_csv(self):
+        return self._yaml.from_csv()
+
     if 'test' in sys.argv:
         @sub_parser(
             description='internal test function',
diff --git a/setup.py b/setup.py
index d576a66..8d0541c 100644
--- a/setup.py
+++ b/setup.py
@@ -98,6 +98,8 @@ class MyInstallLib(install_lib.install_lib):
 def check_extensions():
     """check if the C module can be build by trying to compile a small
     program against the libyaml development library"""
+    if sys.platform == "win32":
+        return None
     import tempfile
     import shutil
 
@@ -119,37 +121,41 @@ def check_extensions():
     }
     """)
     tmp_dir = tempfile.mkdtemp(prefix='tmp_ruamel_yaml_')
-    bin_file_name = os.path.join(tmp_dir, 'test_yaml')
-    file_name = bin_file_name + '.c'
-    with open(file_name, 'w') as fp:
-        fp.write(c_code)
-
-    # and try to compile it
-    compiler = distutils.ccompiler.new_compiler()
-    assert isinstance(compiler, distutils.ccompiler.CCompiler)
-    distutils.sysconfig.customize_compiler(compiler)
-
+    ret_val = None
     try:
-        compiler.link_executable(
-            compiler.compile([file_name]),
-            bin_file_name,
-            libraries=libraries,
-        )
-    except CompileError:
-        print('libyaml compile error')
-        ret_val = None
-    except LinkError:
-        print('libyaml link error')
-        ret_val = None
-    else:
-        ret_val = [
-            Extension(
-                '_yaml',
-                sources=['ext/_yaml.c'],
+        print('tmp_dir', tmp_dir)
+        bin_file_name = os.path.join(tmp_dir, 'test_yaml')
+        file_name = bin_file_name + '.c'
+        with open(file_name, 'w') as fp:
+            fp.write(c_code)
+
+        # and try to compile it
+        compiler = distutils.ccompiler.new_compiler()
+        assert isinstance(compiler, distutils.ccompiler.CCompiler)
+        distutils.sysconfig.customize_compiler(compiler)
+
+        try:
+            compiler.link_executable(
+                compiler.compile([file_name]),
+                bin_file_name,
                 libraries=libraries,
-                ),
-        ]
-    shutil.rmtree(tmp_dir)
+            )
+        except CompileError:
+            print('libyaml compile error')
+        except LinkError:
+            print('libyaml link error')
+        else:
+            ret_val = [
+                Extension(
+                    '_yaml',
+                    sources=['ext/_yaml.c'],
+                    libraries=libraries,
+                    ),
+            ]
+    except:
+        pass
+    finally:
+        shutil.rmtree(tmp_dir)
     return ret_val
 
 
diff --git a/test/test_util.py b/test/test_util.py
index 4701dc5..15bb7a9 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -71,7 +71,7 @@ class TestUtil:
         """, file_name, mp=monkeypatch, td=tmpdir)
         assert res.replace('\r\n', '\n') == dedent("""
         {file_name}:
-             stabelizes on second round trip, ok without comments
+             stabilizes on second round trip, ok without comments
         --- 01_second_rt_ok.yaml
         +++ round trip YAML
         @@ -1,3 +1,3 @@