diff options
author | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2011-12-06 19:11:49 +0000 |
---|---|---|
committer | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2011-12-06 19:11:49 +0000 |
commit | 8b104b27691c2d6b717c80d56914295f2bdba74d (patch) | |
tree | d685a3836eb087cd0ea767ba82a078f575b9df54 | |
parent | b5cb27c1a76b48b4fa318d8f3cc53e675c77b2dc (diff) | |
download | morph-8b104b27691c2d6b717c80d56914295f2bdba74d.tar.gz |
add memory profiling to create_chunk
-rw-r--r-- | morphlib/bins.py | 10 | ||||
-rw-r--r-- | morphlib/builder.py | 3 |
2 files changed, 10 insertions, 3 deletions
diff --git a/morphlib/bins.py b/morphlib/bins.py index 9dd52fef..aab684f5 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -29,7 +29,7 @@ import tarfile import morphlib -def create_chunk(rootdir, chunk_filename, regexps): +def create_chunk(rootdir, chunk_filename, regexps, dump_memory_profile=None): '''Create a chunk from the contents of a directory. Only files and directories that match at least one of the regular @@ -38,7 +38,9 @@ def create_chunk(rootdir, chunk_filename, regexps): filenames are relative to rootdir. ''' - + + dump_memory_profile = dump_memory_profile or (lambda msg: None ) + def mkrel(filename): assert filename.startswith(rootdir) if filename == rootdir: @@ -57,6 +59,7 @@ def create_chunk(rootdir, chunk_filename, regexps): logging.debug('Creating chunk file %s from %s with regexps %s' % (chunk_filename, rootdir, regexps)) + dump_memory_profile('at beginning of create_chunk') compiled = [re.compile(x) for x in regexps] include = set() @@ -74,6 +77,7 @@ def create_chunk(rootdir, chunk_filename, regexps): include.add(name) else: logging.debug('regexp MISMATCH: %s' % filename) + dump_memory_profile('after walking') include = sorted(include) @@ -81,6 +85,7 @@ def create_chunk(rootdir, chunk_filename, regexps): for filename in include: tar.add(filename, arcname=mkrel(filename), recursive=False) tar.close() + dump_memory_profile('after creating tarball') include.remove(rootdir) for filename in reversed(include): @@ -89,6 +94,7 @@ def create_chunk(rootdir, chunk_filename, regexps): os.rmdir(filename) else: os.remove(filename) + dump_memory_profile('after removing in create_chunks') def create_stratum(rootdir, stratum_filename): diff --git a/morphlib/builder.py b/morphlib/builder.py index cffac2ec..17bce5a8 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -230,7 +230,8 @@ class Chunk(BinaryBlob): patterns += [r'baserock/%s\.' % chunk_name] filename = self.filename(chunk_name) self.msg('Creating binary for %s' % chunk_name) - morphlib.bins.create_chunk(self.destdir, filename, patterns) + morphlib.bins.create_chunk(self.destdir, filename, patterns, + self.dump_memory_profile) ret[chunk_name] = filename files = os.listdir(self.destdir) if files: |