Using pipe to handle tgz sources1.2.4

For tgz(or tar.bz2) sources, use pipe like `wget -O -
http://www.example.com/a.tar.gz | tar -xvf -` to save disk space
usage.

Change-Id: I59663aed098e8c96d8a41b2d84200f2a1e43a927
Fixes: bug #1192135

2 files changed, 94 insertions, 39 deletions

diff --git a/heat_cfntools/cfntools/cfn_helper.py b/heat_cfntools/cfntools/cfn_helper.py
index 5589133..5e0395c 100644
--- a/heat_cfntools/cfntools/cfn_helper.py
+++ b/heat_cfntools/cfntools/cfn_helper.py
@@ -573,49 +573,60 @@ class SourcesHandler(object):
 
         return '%s/%s' % (tempdir, sp[-1])
 
-    def _decompress(self, archive, dest_dir):
-        cmd_str = ''
-        LOG.debug("Decompressing")
-        (r, ext) = os.path.splitext(archive)
-        if ext == '.tgz':
-            cmd_str = 'tar -C %s -xzf %s' % (dest_dir, archive)
-        elif ext == '.tbz2':
-            cmd_str = 'tar -C %s -xjf %s' % (dest_dir, archive)
-        elif ext == '.zip':
-            cmd_str = 'unzip -d %s %s' % (dest_dir, archive)
-        elif ext == '.tar':
-            cmd_str = 'tar -C %s -xf %s' % (dest_dir, archive)
-        elif ext == '.gz':
-            (r, ext) = os.path.splitext(r)
-            if ext:
-                cmd_str = 'tar -C %s -xzf %s' % (dest_dir, archive)
-            else:
-                cmd_str = 'gunzip -c %s > %s' % (archive, dest_dir)
-        elif ext == 'bz2':
-            (r, ext) = os.path.splitext(r)
-            if ext:
-                cmd_str = 'tar -C %s -xjf %s' % (dest_dir, archive)
-            else:
-                cmd_str = 'bunzip2 -c %s > %s' % (archive, dest_dir)
+    def _splitext(self, path):
+        (r, ext) = os.path.splitext(path)
+        return (r, ext.lower())
+
+    def _source_type(self, name):
+        (r, ext) = self._splitext(name)
+        if ext == '.gz':
+            (r, ext2) = self._splitext(r)
+            if ext2 == '.tar':
+                ext = '.tgz'
+        elif ext == '.bz2':
+            (r, ext2) = self._splitext(r)
+            if ext2 == '.tar':
+                ext = '.tbz2'
+        return ext
+
+    def _apply_source_cmd(self, dest, url):
+        cmd = ""
+        basename = os.path.basename(url)
+        stype = self._source_type(basename)
+        if stype == '.tgz':
+            cmd = "wget -q -O - '%s' | gunzip | tar -xvf -" % url
+        elif stype == '.tbz2':
+            cmd = "wget -q -O - '%s' | bunzip2 | tar -xvf -" % url
+        elif stype == '.zip':
+            tmp = self._url_to_tmp_filename(url)
+            cmd = "wget -q -O '%s' '%s' && unzip -o '%s'" % (tmp, url, tmp)
+        elif stype == '.tar':
+            cmd = "wget -q -O - '%s' | tar -xvf -" % url
+        elif stype == '.gz':
+            (r, ext) = self._splitext(basename)
+            cmd = "wget -q -O - '%s' | gunzip > '%s'" % (url, r)
+        elif stype == '.bz2':
+            (r, ext) = self._splitext(basename)
+            cmd = "wget -q -O - '%s' | bunzip2 > '%s'" % (url, r)
         else:
             pass
-        return CommandRunner(cmd_str)
+
+        if cmd != '':
+            cmd = "mkdir -p '%s'; cd '%s'; %s" % (dest, dest, cmd)
+
+        return cmd
+
+    def _apply_source(self, dest, url):
+        cmd = self._apply_source_cmd(dest, url)
+        if cmd != '':
+            runner = CommandRunner(cmd)
+            runner.run()
 
     def apply_sources(self):
         if not self._sources:
             return
         for dest, url in self._sources.iteritems():
-            tmp_name = self._url_to_tmp_filename(url)
-            cmd_str = 'wget -O %s %s' % (tmp_name, url)
-            try:
-                os.makedirs(dest)
-            except OSError as e:
-                if e.errno == errno.EEXIST:
-                    LOG.debug(str(e))
-                else:
-                    LOG.exception(e)
-            decompress_command = self._decompress(tmp_name, dest)
-            CommandRunner(cmd_str, decompress_command).run()
+            self._apply_source(dest, url)
 
 
 class ServicesHandler(object):
diff --git a/heat_cfntools/tests/test_cfn_helper.py b/heat_cfntools/tests/test_cfn_helper.py
index 22b6b31..99f3a03 100644
--- a/heat_cfntools/tests/test_cfn_helper.py
+++ b/heat_cfntools/tests/test_cfn_helper.py
@@ -685,10 +685,10 @@ class TestSourcesHandler(MockPopenTestCase):
         td = os.path.dirname(end_file)
         self.m.StubOutWithMock(tempfile, 'mkdtemp')
         tempfile.mkdtemp().AndReturn(td)
-        cmd = ['su', 'root', '-c', 'wget -O %s %s' % (end_file, url)]
+        er = "mkdir -p '%s'; cd '%s'; wget -q -O - '%s' | gunzip | tar -xvf -"
+        cmd = ['su', 'root', '-c',
+               er % (dest, dest, url)]
         self.mock_cmd_run(cmd).AndReturn(FakePOpen('Wget good'))
-        cmd = ['su', 'root', '-c', 'tar -C %s -xzf %s' % (dest, end_file)]
-        self.mock_cmd_run(cmd).AndReturn(FakePOpen('Tarball good'))
         self.m.ReplayAll()
         sh = cfn_helper.SourcesHandler(sources)
         sh.apply_sources()
@@ -706,3 +706,47 @@ class TestSourcesHandler(MockPopenTestCase):
         self.addCleanup(os.rmdir, td)
         end_file = '%s/file.tar.gz' % td
         self._test_apply_sources(url, end_file)
+
+    def test_apply_source_cmd(self):
+        sh = cfn_helper.SourcesHandler({})
+        er = "mkdir -p '%s'; cd '%s'; wget -q -O - '%s' | %s | tar -xvf -"
+        dest = '/tmp'
+        # test tgz
+        url = 'http://www.example.com/a.tgz'
+        cmd = sh._apply_source_cmd(dest, url)
+        self.assertEqual(er % (dest, dest, url, "gunzip"), cmd)
+        # test tar.gz
+        url = 'http://www.example.com/a.tar.gz'
+        cmd = sh._apply_source_cmd(dest, url)
+        self.assertEqual(er % (dest, dest, url, "gunzip"), cmd)
+        # test tbz2
+        url = 'http://www.example.com/a.tbz2'
+        cmd = sh._apply_source_cmd(dest, url)
+        self.assertEqual(er % (dest, dest, url, "bunzip2"), cmd)
+        # test tar.bz2
+        url = 'http://www.example.com/a.tar.bz2'
+        cmd = sh._apply_source_cmd(dest, url)
+        self.assertEqual(er % (dest, dest, url, "bunzip2"), cmd)
+        # test zip
+        er = "mkdir -p '%s'; cd '%s'; wget -q -O '%s' '%s' && unzip -o '%s'"
+        url = 'http://www.example.com/a.zip'
+        d = "/tmp/tmp2I0yNK"
+        tmp = "%s/a.zip" % d
+        self.m.StubOutWithMock(tempfile, 'mkdtemp')
+        tempfile.mkdtemp().AndReturn(d)
+        self.m.ReplayAll()
+        cmd = sh._apply_source_cmd(dest, url)
+        self.assertEqual(er % (dest, dest, tmp, url, tmp), cmd)
+        # test gz
+        er = "mkdir -p '%s'; cd '%s'; wget -q -O - '%s' | %s > '%s'"
+        url = 'http://www.example.com/a.sh.gz'
+        cmd = sh._apply_source_cmd(dest, url)
+        self.assertEqual(er % (dest, dest, url, "gunzip", "a.sh"), cmd)
+        # test bz2
+        url = 'http://www.example.com/a.sh.bz2'
+        cmd = sh._apply_source_cmd(dest, url)
+        self.assertEqual(er % (dest, dest, url, "bunzip2", "a.sh"), cmd)
+        # test other
+        url = 'http://www.example.com/a.sh'
+        cmd = sh._apply_source_cmd(dest, url)
+        self.assertEqual("", cmd)