diff options
Diffstat (limited to 'morphlib/ostreeartifactcache.py')
| -rw-r--r-- | morphlib/ostreeartifactcache.py | 147 |
1 files changed, 106 insertions, 41 deletions
diff --git a/morphlib/ostreeartifactcache.py b/morphlib/ostreeartifactcache.py index ea659c8f..230460f8 100644 --- a/morphlib/ostreeartifactcache.py +++ b/morphlib/ostreeartifactcache.py @@ -15,8 +15,10 @@ import collections +import contextlib import logging import os +import stat import shutil import tarfile import tempfile @@ -27,26 +29,48 @@ from gi.repository import GLib import morphlib from morphlib.artifactcachereference import ArtifactCacheReference + +class NotCachedError(morphlib.Error): + + def __init__(self, ref): + self.msg = 'Failed to checkout %s from the artifact cache.' % ref + + class OSTreeArtifactCache(object): """Class to provide the artifact cache API using an OSTree repo.""" - def __init__(self, cachedir, mode): + def __init__(self, cachedir, mode='bare', status_cb=None): repo_dir = os.path.join(cachedir, 'repo') self.repo = morphlib.ostree.OSTreeRepo(repo_dir, mode=mode) self.cachedir = cachedir + self.status_cb = status_cb + + def status(self, *args, **kwargs): + if self.status_cb is not None: + self.status_cb(*args, **kwargs) + @contextlib.contextmanager def _get_file_from_remote(self, artifact, remote, metadata_name=None): if metadata_name: handle = remote.get_artifact_metadata(artifact, metadata_name) + self.status( + msg='Downloading %(name)s %(metadata_name)s as a file.', + chatty=True, name=artifact.basename(), + metadata_name=metadata_name) else: handle = remote.get(artifact) - fd, path = tempfile.mkstemp() - with open(path, 'w+') as temp: - shutil.copyfileobj(handle, temp) - return path + self.status( + msg='Downloading %(name)s as a tarball.', chatty=True, + name=artifact.basename()) + + try: + temporary_download = tempfile.NamedTemporaryFile(dir=self.cachedir) + shutil.copyfileobj(handle, temporary_download) + yield temporary_download.name + finally: + temporary_download.close() def _get_artifact_cache_name(self, artifact): - logging.debug('LAC: %s' % artifact.basename()) cache_key, kind, name = artifact.basename().split('.', 2) suffix = name.split('-')[-1] return '%s-%s' % (cache_key, suffix) @@ -58,11 +82,13 @@ class OSTreeArtifactCache(object): contents of directory should be the contents of the artifact. """ + cache_key, kind, name = artifact.basename().split('.', 2) ref = self._get_artifact_cache_name(artifact) - subject = artifact.name + subject = name try: - logging.debug('Committing %s to artifact cache at %s.' % - (subject, ref)) + self.status( + msg='Committing %(subject)s to artifact cache at %(ref)s.', + chatty=True, subject=subject, ref=ref) self.repo.commit(subject, directory, ref) except GLib.GError as e: logging.debug('OSTree raised an exception: %s' % e) @@ -77,44 +103,71 @@ class OSTreeArtifactCache(object): else: filename = self.artifact_filename(artifact) shutil.copy(location, filename) - os.remove(location) + + def _remove_device_nodes(self, path): + for dirpath, dirnames, filenames in os.walk(path): + for f in filenames: + filepath = os.path.join(dirpath, f) + mode = os.lstat(filepath).st_mode + if stat.S_ISBLK(mode) or stat.S_ISCHR(mode): + logging.debug('Removing device node %s from artifact' % + filepath) + os.remove(filepath) + + def _copy_metadata_from_remote(self, artifact, remote): + """Copy a metadata file from a remote cache.""" + a, name = artifact.basename().split('.', 1) + with self._get_file_from_remote(ArtifactCacheReference(a), + remote, name) as location: + self.put_non_ostree_artifact(ArtifactCacheReference(a), + location, name) def copy_from_remote(self, artifact, remote): - """Get 'artifact' from remote artifact cache and store it locally.""" + """Get 'artifact' from remote artifact cache and store it locally. + + This takes an Artifact object and a RemoteArtifactCache. Note that + `remote` here is not the same as a `remote` for and OSTree repo. + + """ if remote.method == 'tarball': - logging.debug('Downloading artifact tarball for %s.' % - artifact.name) - location = self._get_file_from_remote(artifact, remote) - try: - tempdir = tempfile.mkdtemp() - with tarfile.open(name=location) as tf: - tf.extractall(path=tempdir) + with self._get_file_from_remote(artifact, remote) as location: try: + cache_key, kind, name = artifact.basename().split('.', 2) + except ValueError: + # We can't split the name properly, it must be metadata! + self._copy_metadata_from_remote(artifact, remote) + return + + if kind == 'stratum': + self.put_non_ostree_artifact(artifact, location) + return + try: + tempdir = tempfile.mkdtemp(dir=self.cachedir) + with tarfile.open(name=location) as tf: + tf.extractall(path=tempdir) + self._remove_device_nodes(tempdir) self.put(tempdir, artifact) + except tarfile.ReadError: + # Reading the tarball failed, and we expected a + # tarball artifact. Something must have gone + # wrong. + raise finally: - os.remove(location) shutil.rmtree(tempdir) - except tarfile.ReadError: - # Reading the artifact as a tarball failed, so it must be a - # single file (for example a stratum artifact). - self.put_non_ostree_artifact(artifact, location) elif remote.method == 'ostree': - logging.debug('Pulling artifact for %s from remote.' % - artifact.basename()) + self.status(msg='Pulling artifact for %(name)s from remote.', + chatty=True, name=artifact.basename()) try: ref = self._get_artifact_cache_name(artifact) - except Exception: + except ValueError: # if we can't split the name properly, we must want metadata - a, name = artifact.basename().split('.', 1) - location = self._get_file_from_remote( - ArtifactCacheReference(a), remote, name) - self.put_non_ostree_artifact(artifact, location, name) + self._copy_metadata_from_remote(artifact, remote) return if artifact.basename().split('.', 2)[1] == 'stratum': - location = self._get_file_from_remote(artifact, remote) - self.put_non_ostree_artifact(artifact, location) + with self._get_file_from_remote(artifact, remote) as location: + self.put_non_ostree_artifact(artifact, location) return try: @@ -126,7 +179,7 @@ class OSTreeArtifactCache(object): raise cliapp.AppException('Failed to pull %s from remote ' 'cache.' % ref) - def get(self, artifact, directory=None, status=lambda a: a): + def get(self, artifact, directory=None): """Checkout an artifact from the repo and return its location.""" cache_key, kind, name = artifact.basename().split('.', 2) if kind == 'stratum': @@ -136,11 +189,13 @@ class OSTreeArtifactCache(object): ref = self._get_artifact_cache_name(artifact) try: self.repo.checkout(ref, directory) + # We need to update the mtime and atime of the ref file in the + # repository so that we can decide which refs were least recently + # accessed when doing `morph gc`. self.repo.touch_ref(ref) except GLib.GError as e: logging.debug('OSTree raised an exception: %s' % e) - raise cliapp.AppException('Failed to checkout %s from artifact ' - 'cache.' % ref) + raise NotCachedError(ref) return directory def list_contents(self): @@ -173,16 +228,26 @@ class OSTreeArtifactCache(object): self.repo.prune() def has(self, artifact): - cachekey, kind, name = artifact.basename().split('.', 2) - logging.debug('OSTreeArtifactCache: got %s, %s, %s' % - (cachekey, kind, name)) + try: + cachekey, kind, name = artifact.basename().split('.', 2) + except ValueError: + # We couldn't split the basename properly, we must want metadata + cachekey, name = artifact.basename().split('.', 1) + if self.has_artifact_metadata(artifact, name): + return True + else: + return False + + if kind == 'stratum': + if self._has_file(self.artifact_filename(artifact)): + return True + else: + return False + sha = self.repo.resolve_rev(self._get_artifact_cache_name(artifact)) if sha: self.repo.touch_ref(self._get_artifact_cache_name(artifact)) return True - if kind == 'stratum' and \ - self._has_file(self.artifact_filename(artifact)): - return True return False def get_artifact_metadata(self, artifact, name): |