diff options
author | Richard Maw <richard.maw@codethink.co.uk> | 2013-02-27 14:25:19 +0000 |
---|---|---|
committer | Richard Maw <richard.maw@codethink.co.uk> | 2013-02-27 14:25:19 +0000 |
commit | 163d7e1dd8c84b93ddbb03b27f1a602cd660a402 (patch) | |
tree | 52cbcdba860884185dea6233b77acc05d2496615 /morphlib | |
parent | 872f039f76238378add89baa4a3e7fe0f2a51a61 (diff) | |
parent | b2a15b6c23451cbaf4339d69f1280ba2b7563ae2 (diff) | |
download | morph-163d7e1dd8c84b93ddbb03b27f1a602cd660a402.tar.gz |
Merge branch 'richardmaw/artifact-inspection-plugin-rebase-v2' of git://git.baserock.org/baserock/baserock/morph
Reviewed-by: Lars Wirzenius
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/__init__.py | 2 | ||||
-rw-r--r-- | morphlib/bins.py | 29 | ||||
-rw-r--r-- | morphlib/extractedtarball.py | 61 | ||||
-rw-r--r-- | morphlib/mountableimage.py | 81 | ||||
-rw-r--r-- | morphlib/plugins/artifact_inspection_plugin.py | 294 | ||||
-rw-r--r-- | morphlib/plugins/trebuchet_plugin.py | 60 |
6 files changed, 469 insertions, 58 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 271fa977..b20c3f01 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -53,10 +53,12 @@ import builder2 import cachedir import cachedrepo import cachekeycomputer +import extractedtarball import fsutils import git import localartifactcache import localrepocache +import mountableimage import morph2 import morphologyfactory import remoteartifactcache diff --git a/morphlib/bins.py b/morphlib/bins.py index 622aa165..ba5f713f 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 Codethink Limited +# Copyright (C) 2011-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,6 +21,7 @@ Binaries are chunks, strata, and system images. ''' +import cliapp import logging import os import re @@ -29,6 +30,11 @@ import stat import shutil import tarfile +import morphlib + +from morphlib.extractedtarball import ExtractedTarball +from morphlib.mountableimage import MountableImage + # Work around http://bugs.python.org/issue16477 def safe_makefile(self, tarinfo, targetpath): @@ -211,3 +217,24 @@ def unpack_binary_from_file(f, dirname): # pragma: no cover def unpack_binary(filename, dirname): with open(filename, "rb") as f: unpack_binary_from_file(f, dirname) + + +class ArtifactNotMountableError(cliapp.AppException): # pragma: no cover + + def __init__(self, filename): + cliapp.AppException.__init__( + self, 'Artifact %s cannot be extracted or mounted' % filename) + + +def call_in_artifact_directory(app, filename, callback): # pragma: no cover + '''Call a function in a directory the artifact is extracted/mounted in.''' + + try: + with ExtractedTarball(app, filename) as dirname: + callback(dirname) + except tarfile.TarError: + try: + with MountableImage(app, filename) as dirname: + callback(dirname) + except (IOError, OSError): + raise ArtifactNotMountableError(filename) diff --git a/morphlib/extractedtarball.py b/morphlib/extractedtarball.py new file mode 100644 index 00000000..e435b1ef --- /dev/null +++ b/morphlib/extractedtarball.py @@ -0,0 +1,61 @@ +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import gzip +import os +import tempfile +import shutil + +import morphlib + + +class ExtractedTarball(object): # pragma: no cover + + '''Tarball extracted in a temporary directory. + + This can be used e.g. to inspect the contents of a rootfs tarball. + + ''' + def __init__(self, app, tarball): + self.app = app + self.tarball = tarball + + def setup(self): + self.app.status(msg='Preparing tarball %(tarball)s', + tarball=os.path.basename(self.tarball), chatty=True) + self.app.status(msg=' Extracting...', chatty=True) + self.tempdir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) + try: + morphlib.bins.unpack_binary(self.tarball, self.tempdir) + except: + shutil.rmtree(self.tempdir) + raise + return self.tempdir + + def cleanup(self): + self.app.status(msg='Cleanup extracted tarball %(tarball)s', + tarball=os.path.basename(self.tarball), chatty=True) + try: + shutil.rmtree(self.tempdir) + except: + pass + + def __enter__(self): + return self.setup() + + def __exit__(self, exctype, excvalue, exctraceback): + self.cleanup() diff --git a/morphlib/mountableimage.py b/morphlib/mountableimage.py new file mode 100644 index 00000000..3d29a516 --- /dev/null +++ b/morphlib/mountableimage.py @@ -0,0 +1,81 @@ +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import os +import tempfile +import gzip + +import morphlib + + +class MountableImage(object): # pragma: no cover + + '''Mountable image (deals with decompression). + + Note, this is a read-only mount in the sense that the decompressed + image is not then recompressed after, instead any changes are discarded. + + ''' + def __init__(self, app, artifact_path): + self.app = app + self.artifact_path = artifact_path + + def setup(self, path): + self.app.status(msg='Preparing image %(path)s', path=path, chatty=True) + self.app.status(msg=' Decompressing...', chatty=True) + (tempfd, self.temp_path) = \ + tempfile.mkstemp(dir=self.app.settings['tempdir']) + + try: + with os.fdopen(tempfd, "wb") as outfh: + infh = gzip.open(path, "rb") + morphlib.util.copyfileobj(infh, outfh) + infh.close() + except: + os.unlink(self.temp_path) + raise + self.app.status(msg=' Mounting image at %(path)s', + path=self.temp_path, chatty=True) + part = morphlib.fsutils.setup_device_mapping(self.app.runcmd, + self.temp_path) + mount_point = tempfile.mkdtemp(dir=self.app.settings['tempdir']) + morphlib.fsutils.mount(self.app.runcmd, part, mount_point) + self.mount_point = mount_point + return mount_point + + def cleanup(self, path, mount_point): + self.app.status(msg='Clearing down image at %(path)s', path=path, + chatty=True) + try: + morphlib.fsutils.unmount(self.app.runcmd, mount_point) + except: + pass + try: + morphlib.fsutils.undo_device_mapping(self.app.runcmd, path) + except: + pass + try: + os.rmdir(mount_point) + os.unlink(path) + except: + pass + + def __enter__(self): + return self.setup(self.artifact_path) + + def __exit__(self, exctype, excvalue, exctraceback): + self.cleanup(self.temp_path, self.mount_point) diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py new file mode 100644 index 00000000..569fbb8a --- /dev/null +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -0,0 +1,294 @@ +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import glob +import json +import os +import re + +import morphlib + +from morphlib.bins import call_in_artifact_directory +from morphlib.extractedtarball import ExtractedTarball +from morphlib.mountableimage import MountableImage + + +class NotASystemArtifactError(cliapp.AppException): + + def __init__(self, artifact): + cliapp.AppException.__init__( + self, '%s is not a system artifact' % artifact) + + +class ProjectVersionGuesser(object): + + def __init__(self, app, lrc, rrc, interesting_files): + self.app = app + self.lrc = lrc + self.rrc = rrc + self.interesting_files = interesting_files + + def file_contents(self, repo, ref, tree): + filenames = [x for x in self.interesting_files if x in tree] + if filenames: + if self.lrc.has_repo(repo): + repository = self.lrc.get_repo(repo) + for filename in filenames: + yield filename, repository.cat(ref, filename) + elif self.rrc: + for filename in filenames: + yield filename, self.rrc.cat_file(repo, ref, filename) + + +class AutotoolsVersionGuesser(ProjectVersionGuesser): + + def __init__(self, app, lrc, rrc): + ProjectVersionGuesser.__init__(self, app, lrc, rrc, [ + 'configure.ac', + 'configure.in', + 'configure.ac.in', + 'configure.in.in', + ]) + + def guess_version(self, repo, ref, tree): + version = None + for filename, data in self.file_contents(repo, ref, tree): + # First, try to grep for AC_INIT() + version = self._check_ac_init(data) + if version: + self.app.status( + msg='%(repo)s: Version of %(ref)s detected ' + 'via %(filename)s:AC_INIT: %(version)s', + repo=repo, ref=ref, filename=filename, + version=version, chatty=True) + break + + # Then, try running autoconf against the configure script + version = self._check_autoconf_package_version(filename, data) + if version: + self.app.status( + msg='%(repo)s: Version of %(ref)s detected ' + 'by processing %(filename)s: %(version)s', + repo=repo, ref=ref, filename=filename, + version=version, chatty=True) + break + return version + + def _check_ac_init(self, data): + data = data.replace('\n', ' ') + for macro in ['AC_INIT', 'AM_INIT_AUTOMAKE']: + pattern = r'.*%s\((.*?)\).*' % macro + if not re.match(pattern, data): + continue + acinit = re.sub(pattern, r'\1', data) + if acinit: + version = acinit.split(',') + if macro == 'AM_INIT_AUTOMAKE' and len(version) == 1: + continue + version = version[0] if len(version) == 1 else version[1] + version = re.sub('[\[\]]', '', version).strip() + version = version.split()[0] + if version: + if version and version[0].isdigit(): + return version + return None + + def _check_autoconf_package_version(self, filename, data): + tempdir = morphlib.tempdir.Tempdir(self.app.settings['tempdir']) + with open(tempdir.join(filename), 'w') as f: + f.write(data) + exit_code, output, errors = self.app.runcmd_unchecked( + ['autoconf', filename], + ['grep', '^PACKAGE_VERSION='], + ['cut', '-d=', '-f2'], + ['sed', "s/'//g"], + cwd=tempdir.dirname) + tempdir.remove() + version = None + if output: + output = output.strip() + if output and output[0].isdigit(): + version = output + if exit_code != 0: + self.app.status( + msg='%(repo)s: Failed to detect version from ' + '%(ref)s:%(filename)s', + repo=repo, ref=ref, filename=filename, chatty=True) + return version + + +class VersionGuesser(object): + + def __init__(self, app): + self.app = app + self.lrc, self.rrc = morphlib.util.new_repo_caches(app) + self.guessers = [ + AutotoolsVersionGuesser(app, self.lrc, self.rrc) + ] + + def guess_version(self, repo, ref): + self.app.status(msg='%(repo)s: Guessing version of %(ref)s', + repo=repo, ref=ref, chatty=True) + version = None + try: + if self.lrc.has_repo(repo): + repository = self.lrc.get_repo(repo) + if not self.app.settings['no-git-update']: + repository.update() + tree = repository.ls_tree(ref) + elif self.rrc: + repository = None + tree = self.rrc.ls_tree(repo, ref) + else: + return None + for guesser in self.guessers: + version = guesser.guess_version(repo, ref, tree) + if version: + break + except cliapp.AppException, err: + self.app.status(msg='%(repo)s: Failed to list files in %(ref)s', + repo=repo, ref=ref, chatty=True) + return version + + +class ManifestGenerator(object): + + def __init__(self, app): + self.app = app + self.version_guesser = VersionGuesser(app) + + def generate(self, artifact, dirname): + # Try to find a directory with baserock metadata files. + metadirs = [ + os.path.join(dirname, 'factory', 'baserock'), + os.path.join(dirname, 'baserock') + ] + existing_metadirs = [x for x in metadirs if os.path.isdir(x)] + if not existing_metadirs: + raise NotASystemArtifactError(artifact) + metadir = existing_metadirs[0] + + # Collect all meta information about the system, its strata + # and its chunks that we are interested in. + artifacts = [] + for basename in glob.glob(os.path.join(metadir, '*.meta')): + metafile = os.path.join(metadir, basename) + metadata = json.load(open(metafile)) + + # Try to guess the version of this artifact + version = self.version_guesser.guess_version( + metadata['repo'], metadata['sha1']) or '-' + + artifacts.append({ + 'cache-key': metadata['cache-key'], + 'name': metadata['artifact-name'], + 'kind': metadata['kind'], + 'sha1': metadata['sha1'], + 'original_ref': metadata['original_ref'], + 'repo': metadata['repo'], + 'morphology': metadata['morphology'], + 'version': version, + }) + + # Generate a format string for dumping the information. + fmt = self._generate_output_format(artifacts) + + # Print information about system, strata and chunks. + self._print_artifacts(fmt, artifacts, 'system') + self._print_artifacts(fmt, artifacts, 'stratum') + self._print_artifacts(fmt, artifacts, 'chunk') + + def _generate_output_format(self, artifacts): + colwidths = {} + for artifact in artifacts: + for key, value in artifact.iteritems(): + colwidths[key] = max(colwidths.get(key, 0), len(value)) + + colwidths['first'] = sum([colwidths['cache-key'], + colwidths['kind'], + colwidths['name']]) + 1 + + return 'artifact=%%-%is\t' \ + 'version=%%-%is\t' \ + 'commit=%%-%is\t' \ + 'repository=%%-%is\t' \ + 'ref=%%-%is\t' \ + 'morphology=%%-%is\n' % ( + len('artifact=') + colwidths['first'], + len('version=') + colwidths['version'], + len('commit=') + colwidths['sha1'], + len('repository=') + colwidths['repo'], + len('ref=') + colwidths['original_ref'], + len('morphology=') + colwidths['morphology']) + + def _print_artifacts(self, fmt, artifacts, kind): + for artifact in sorted(artifacts, key=lambda x: x['name']): + if artifact['kind'] == kind: + self.app.output.write(fmt % ( + '%s.%s.%s' % (artifact['cache-key'], + artifact['kind'], + artifact['name']), + artifact['version'], + artifact['sha1'], + artifact['repo'], + artifact['original_ref'], + artifact['morphology'])) + + +class ArtifactInspectionPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand('run-in-artifact', + self.run_in_artifact, + arg_synopsis='ARTIFACT CMD') + self.app.add_subcommand('generate-manifest', + self.generate_manifest, + arg_synopsis='ROOTFS_ARTIFACT') + + def disable(self): + pass + + def run_in_artifact(self, args): + '''Run a command inside an extracted/mounted chunk or system.''' + + if len(args) < 2: + raise cliapp.AppException( + 'run-in-artifact requires arguments: a chunk or ' + 'system artifact and a a shell command') + + artifact, cmd = args[0], args[1:] + + def run_command_in_dir(dirname): + output = self.app.runcmd(cmd, cwd=dirname) + self.app.output.write(output) + + call_in_artifact_directory(self.app, artifact, run_command_in_dir) + + def generate_manifest(self, args): + '''Generate a content manifest for a system image.''' + + if len(args) != 1: + raise cliapp.AppException('morph generate-manifest expects ' + 'a system image as its input') + + artifact = args[0] + + def generate_manifest(dirname): + generator = ManifestGenerator(self.app) + generator.generate(artifact, dirname) + + call_in_artifact_directory(self.app, artifact, generate_manifest) diff --git a/morphlib/plugins/trebuchet_plugin.py b/morphlib/plugins/trebuchet_plugin.py index 2bdf4c3c..1ebffbf4 100644 --- a/morphlib/plugins/trebuchet_plugin.py +++ b/morphlib/plugins/trebuchet_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,62 +21,8 @@ import gzip import morphlib -class MountableImage(object): - '''Mountable image (deals with decompression). - - Note, this is a read-only mount in the sense that the decompressed - image is not then recompressed after, instead any changes are discarded. - - ''' - def __init__(self, app, artifact_path): - self.app = app - self.artifact_path = artifact_path - - def setup(self, path): - self.app.status(msg='Preparing image %(path)s', path=path) - self.app.status(msg=' Decompressing...', chatty=True) - (tempfd, self.temp_path) = \ - tempfile.mkstemp(dir=self.app.settings['tempdir']) - - try: - with os.fdopen(tempfd, "wb") as outfh: - infh = gzip.open(path, "rb") - morphlib.util.copyfileobj(infh, outfh) - infh.close() - except: - os.unlink(self.temp_path) - raise - self.app.status(msg=' Mounting image at %(path)s', - path=self.temp_path, chatty=True) - part = morphlib.fsutils.setup_device_mapping(self.app.runcmd, - self.temp_path) - mount_point = tempfile.mkdtemp(dir=self.app.settings['tempdir']) - morphlib.fsutils.mount(self.app.runcmd, part, mount_point) - self.mount_point = mount_point - return mount_point - - def cleanup(self, path, mount_point): - self.app.status(msg='Clearing down image at %(path)s', path=path, - chatty=True) - try: - morphlib.fsutils.unmount(self.app.runcmd, mount_point) - except: - pass - try: - morphlib.fsutils.undo_device_mapping(self.app.runcmd, path) - except: - pass - try: - os.rmdir(mount_point) - os.unlink(path) - except: - pass - - def __enter__(self): - return self.setup(self.artifact_path) - - def __exit__(self, exctype, excvalue, exctraceback): - self.cleanup(self.temp_path, self.mount_point) +from morphlib.mountableimage import MountableImage + class TrebuchetPlugin(cliapp.Plugin): |