From 026146ae4e76f1884681278ec6db336f972e7219 Mon Sep 17 00:00:00 2001 From: JUN JIE NAN Date: Wed, 26 Jun 2013 15:23:15 +0800 Subject: Using pipe to handle tgz sources For tgz(or tar.bz2) sources, use pipe like `wget -O - http://www.example.com/a.tar.gz | tar -xvf -` to save disk space usage. Change-Id: I59663aed098e8c96d8a41b2d84200f2a1e43a927 Fixes: bug #1192135 --- heat_cfntools/cfntools/cfn_helper.py | 83 +++++++++++++++++++--------------- heat_cfntools/tests/test_cfn_helper.py | 50 ++++++++++++++++++-- 2 files changed, 94 insertions(+), 39 deletions(-) diff --git a/heat_cfntools/cfntools/cfn_helper.py b/heat_cfntools/cfntools/cfn_helper.py index 5589133..5e0395c 100644 --- a/heat_cfntools/cfntools/cfn_helper.py +++ b/heat_cfntools/cfntools/cfn_helper.py @@ -573,49 +573,60 @@ class SourcesHandler(object): return '%s/%s' % (tempdir, sp[-1]) - def _decompress(self, archive, dest_dir): - cmd_str = '' - LOG.debug("Decompressing") - (r, ext) = os.path.splitext(archive) - if ext == '.tgz': - cmd_str = 'tar -C %s -xzf %s' % (dest_dir, archive) - elif ext == '.tbz2': - cmd_str = 'tar -C %s -xjf %s' % (dest_dir, archive) - elif ext == '.zip': - cmd_str = 'unzip -d %s %s' % (dest_dir, archive) - elif ext == '.tar': - cmd_str = 'tar -C %s -xf %s' % (dest_dir, archive) - elif ext == '.gz': - (r, ext) = os.path.splitext(r) - if ext: - cmd_str = 'tar -C %s -xzf %s' % (dest_dir, archive) - else: - cmd_str = 'gunzip -c %s > %s' % (archive, dest_dir) - elif ext == 'bz2': - (r, ext) = os.path.splitext(r) - if ext: - cmd_str = 'tar -C %s -xjf %s' % (dest_dir, archive) - else: - cmd_str = 'bunzip2 -c %s > %s' % (archive, dest_dir) + def _splitext(self, path): + (r, ext) = os.path.splitext(path) + return (r, ext.lower()) + + def _source_type(self, name): + (r, ext) = self._splitext(name) + if ext == '.gz': + (r, ext2) = self._splitext(r) + if ext2 == '.tar': + ext = '.tgz' + elif ext == '.bz2': + (r, ext2) = self._splitext(r) + if ext2 == '.tar': + ext = '.tbz2' + return ext + + def _apply_source_cmd(self, dest, url): + cmd = "" + basename = os.path.basename(url) + stype = self._source_type(basename) + if stype == '.tgz': + cmd = "wget -q -O - '%s' | gunzip | tar -xvf -" % url + elif stype == '.tbz2': + cmd = "wget -q -O - '%s' | bunzip2 | tar -xvf -" % url + elif stype == '.zip': + tmp = self._url_to_tmp_filename(url) + cmd = "wget -q -O '%s' '%s' && unzip -o '%s'" % (tmp, url, tmp) + elif stype == '.tar': + cmd = "wget -q -O - '%s' | tar -xvf -" % url + elif stype == '.gz': + (r, ext) = self._splitext(basename) + cmd = "wget -q -O - '%s' | gunzip > '%s'" % (url, r) + elif stype == '.bz2': + (r, ext) = self._splitext(basename) + cmd = "wget -q -O - '%s' | bunzip2 > '%s'" % (url, r) else: pass - return CommandRunner(cmd_str) + + if cmd != '': + cmd = "mkdir -p '%s'; cd '%s'; %s" % (dest, dest, cmd) + + return cmd + + def _apply_source(self, dest, url): + cmd = self._apply_source_cmd(dest, url) + if cmd != '': + runner = CommandRunner(cmd) + runner.run() def apply_sources(self): if not self._sources: return for dest, url in self._sources.iteritems(): - tmp_name = self._url_to_tmp_filename(url) - cmd_str = 'wget -O %s %s' % (tmp_name, url) - try: - os.makedirs(dest) - except OSError as e: - if e.errno == errno.EEXIST: - LOG.debug(str(e)) - else: - LOG.exception(e) - decompress_command = self._decompress(tmp_name, dest) - CommandRunner(cmd_str, decompress_command).run() + self._apply_source(dest, url) class ServicesHandler(object): diff --git a/heat_cfntools/tests/test_cfn_helper.py b/heat_cfntools/tests/test_cfn_helper.py index 22b6b31..99f3a03 100644 --- a/heat_cfntools/tests/test_cfn_helper.py +++ b/heat_cfntools/tests/test_cfn_helper.py @@ -685,10 +685,10 @@ class TestSourcesHandler(MockPopenTestCase): td = os.path.dirname(end_file) self.m.StubOutWithMock(tempfile, 'mkdtemp') tempfile.mkdtemp().AndReturn(td) - cmd = ['su', 'root', '-c', 'wget -O %s %s' % (end_file, url)] + er = "mkdir -p '%s'; cd '%s'; wget -q -O - '%s' | gunzip | tar -xvf -" + cmd = ['su', 'root', '-c', + er % (dest, dest, url)] self.mock_cmd_run(cmd).AndReturn(FakePOpen('Wget good')) - cmd = ['su', 'root', '-c', 'tar -C %s -xzf %s' % (dest, end_file)] - self.mock_cmd_run(cmd).AndReturn(FakePOpen('Tarball good')) self.m.ReplayAll() sh = cfn_helper.SourcesHandler(sources) sh.apply_sources() @@ -706,3 +706,47 @@ class TestSourcesHandler(MockPopenTestCase): self.addCleanup(os.rmdir, td) end_file = '%s/file.tar.gz' % td self._test_apply_sources(url, end_file) + + def test_apply_source_cmd(self): + sh = cfn_helper.SourcesHandler({}) + er = "mkdir -p '%s'; cd '%s'; wget -q -O - '%s' | %s | tar -xvf -" + dest = '/tmp' + # test tgz + url = 'http://www.example.com/a.tgz' + cmd = sh._apply_source_cmd(dest, url) + self.assertEqual(er % (dest, dest, url, "gunzip"), cmd) + # test tar.gz + url = 'http://www.example.com/a.tar.gz' + cmd = sh._apply_source_cmd(dest, url) + self.assertEqual(er % (dest, dest, url, "gunzip"), cmd) + # test tbz2 + url = 'http://www.example.com/a.tbz2' + cmd = sh._apply_source_cmd(dest, url) + self.assertEqual(er % (dest, dest, url, "bunzip2"), cmd) + # test tar.bz2 + url = 'http://www.example.com/a.tar.bz2' + cmd = sh._apply_source_cmd(dest, url) + self.assertEqual(er % (dest, dest, url, "bunzip2"), cmd) + # test zip + er = "mkdir -p '%s'; cd '%s'; wget -q -O '%s' '%s' && unzip -o '%s'" + url = 'http://www.example.com/a.zip' + d = "/tmp/tmp2I0yNK" + tmp = "%s/a.zip" % d + self.m.StubOutWithMock(tempfile, 'mkdtemp') + tempfile.mkdtemp().AndReturn(d) + self.m.ReplayAll() + cmd = sh._apply_source_cmd(dest, url) + self.assertEqual(er % (dest, dest, tmp, url, tmp), cmd) + # test gz + er = "mkdir -p '%s'; cd '%s'; wget -q -O - '%s' | %s > '%s'" + url = 'http://www.example.com/a.sh.gz' + cmd = sh._apply_source_cmd(dest, url) + self.assertEqual(er % (dest, dest, url, "gunzip", "a.sh"), cmd) + # test bz2 + url = 'http://www.example.com/a.sh.bz2' + cmd = sh._apply_source_cmd(dest, url) + self.assertEqual(er % (dest, dest, url, "bunzip2", "a.sh"), cmd) + # test other + url = 'http://www.example.com/a.sh' + cmd = sh._apply_source_cmd(dest, url) + self.assertEqual("", cmd) -- cgit v1.2.1