# Copyright (C) 2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import collections import contextlib import logging import os import shutil import tarfile import tempfile import cliapp from gi.repository import GLib import morphlib from morphlib.artifactcachereference import ArtifactCacheReference class OSTreeArtifactCache(object): """Class to provide the artifact cache API using an OSTree repo.""" def __init__(self, cachedir, mode='bare', status_cb=None): repo_dir = os.path.join(cachedir, 'repo') self.repo = morphlib.ostree.OSTreeRepo(repo_dir, mode=mode) self.cachedir = cachedir self.status_cb = status_cb def status(self, *args, **kwargs): if self.status_cb is not None: self.status_cb(*args, **kwargs) @contextlib.contextmanager def _get_file_from_remote(self, artifact, remote, metadata_name=None): if metadata_name: handle = remote.get_artifact_metadata(artifact, metadata_name) self.status( msg='Downloading %(name)s %(metadata_name)s as a file.', chatty=True, name=artifact.basename(), metadata_name=metadata_name) else: handle = remote.get(artifact) self.status( msg='Downloading %(name)s as a tarball.', chatty=True, name=artifact.basename()) try: temporary_download = tempfile.NamedTemporaryFile(dir=self.cachedir) shutil.copyfileobj(handle, temporary_download) yield temporary_download.name finally: temporary_download.close() def _get_artifact_cache_name(self, artifact): cache_key, kind, name = artifact.basename().split('.', 2) suffix = name.split('-')[-1] return '%s-%s' % (cache_key, suffix) def put(self, directory, artifact): """Commit the contents of 'directory' to the repo. This uses the artifact name and cache key to create the ref, so the contents of directory should be the contents of the artifact. """ cache_key, kind, name = artifact.basename().split('.', 2) ref = self._get_artifact_cache_name(artifact) subject = name try: self.status( msg='Committing %(subject)s to artifact cache at %(ref)s.', chatty=True, subject=subject, ref=ref) self.repo.commit(subject, directory, ref) except GLib.GError as e: logging.debug('OSTree raised an exception: %s' % e) raise cliapp.AppException('Failed to commit %s to artifact ' 'cache.' % ref) def put_non_ostree_artifact(self, artifact, location, metadata_name=None): """Store a single file in the artifact cachedir.""" if metadata_name: filename = self._artifact_metadata_filename(artifact, metadata_name) else: filename = self.artifact_filename(artifact) shutil.copy(location, filename) def copy_from_remote(self, artifact, remote): """Get 'artifact' from remote artifact cache and store it locally.""" if remote.method == 'tarball': with self._get_file_from_remote(artifact, remote) as location: try: tempdir = tempfile.mkdtemp(dir=self.cachedir) try: with tarfile.open(name=location) as tf: tf.extractall(path=tempdir) self.put(tempdir, artifact) finally: shutil.rmtree(tempdir) except tarfile.ReadError: # Reading the artifact as a tarball failed, so it must be a # single file (for example a stratum artifact). self.put_non_ostree_artifact(artifact, location) elif remote.method == 'ostree': self.status(msg='Pulling artifact for %(name)s from remote.', chatty=True, name=artifact.basename()) try: ref = self._get_artifact_cache_name(artifact) except Exception: # if we can't split the name properly, we must want metadata a, name = artifact.basename().split('.', 1) with self._get_file_from_remote(ArtifactCacheReference(a), remote, name) as location: self.put_non_ostree_artifact(artifact, location, name) return if artifact.basename().split('.', 2)[1] == 'stratum': with self._get_file_from_remote(artifact, remote) as location: self.put_non_ostree_artifact(artifact, location) return try: if not self.repo.has_remote(remote.name): self.repo.add_remote(remote.name, remote.ostree_url) self.repo.pull([ref], remote.name) except GLib.GError as e: logging.debug('OSTree raised an exception: %s' % e) raise cliapp.AppException('Failed to pull %s from remote ' 'cache.' % ref) def get(self, artifact, directory=None): """Checkout an artifact from the repo and return its location.""" cache_key, kind, name = artifact.basename().split('.', 2) if kind == 'stratum': return self.artifact_filename(artifact) if directory is None: directory = tempfile.mkdtemp() ref = self._get_artifact_cache_name(artifact) try: self.repo.checkout(ref, directory) self.repo.touch_ref(ref) except GLib.GError as e: logging.debug('OSTree raised an exception: %s' % e) raise cliapp.AppException('Failed to checkout %s from artifact ' 'cache.' % ref) return directory def list_contents(self): """Return the set of sources cached and related information. returns a [(cache_key, set(artifacts), last_used)] """ CacheInfo = collections.namedtuple('CacheInfo', ('artifacts', 'mtime')) contents = collections.defaultdict(lambda: CacheInfo(set(), 0)) for ref in self.repo.list_refs(): cachekey = ref[:63] artifact = ref[65:] artifacts, max_mtime = contents[cachekey] artifacts.add(artifact) ref_filename = os.path.join(self.repo.refsdir(), ref) mtime = os.path.getmtime(ref_filename) contents[cachekey] = CacheInfo(artifacts, max(max_mtime, mtime)) return ((cache_key, info.artifacts, info.mtime) for cache_key, info in contents.iteritems()) def remove(self, cachekey): """Remove all artifacts associated with the given cachekey.""" for ref in (r for r in self.repo.list_refs() if r.startswith(cachekey)): self.repo.delete_ref(ref) def prune(self): """Delete orphaned objects in the repo.""" self.repo.prune() def has(self, artifact): cachekey, kind, name = artifact.basename().split('.', 2) logging.debug('OSTreeArtifactCache: checking for %s, %s, %s' % (cachekey, kind, name)) if kind == 'stratum': if self._has_file(self.artifact_filename(artifact)): return True else: return False sha = self.repo.resolve_rev(self._get_artifact_cache_name(artifact)) if sha: self.repo.touch_ref(self._get_artifact_cache_name(artifact)) return True return False def get_artifact_metadata(self, artifact, name): filename = self._artifact_metadata_filename(artifact, name) os.utime(filename, None) return open(filename) def get_source_metadata_filename(self, source, cachekey, name): return self._source_metadata_filename(source, cachekey, name) def get_source_metadata(self, source, cachekey, name): filename = self._source_metadata_filename(source, cachekey, name) os.utime(filename, None) return open(filename) def artifact_filename(self, artifact): return os.path.join(self.cachedir, artifact.basename()) def _artifact_metadata_filename(self, artifact, name): return os.path.join(self.cachedir, artifact.metadata_basename(name)) def _source_metadata_filename(self, source, cachekey, name): return os.path.join(self.cachedir, '%s.%s' % (cachekey, name)) def put_artifact_metadata(self, artifact, name): filename = self._artifact_metadata_filename(artifact, name) return morphlib.savefile.SaveFile(filename, mode='w') def put_source_metadata(self, source, cachekey, name): filename = self._source_metadata_filename(source, cachekey, name) return morphlib.savefile.SaveFile(filename, mode='w') def _has_file(self, filename): if os.path.exists(filename): os.utime(filename, None) return True return False def has_artifact_metadata(self, artifact, name): filename = self._artifact_metadata_filename(artifact, name) return self._has_file(filename) def has_source_metadata(self, source, cachekey, name): filename = self._source_metadata_filename(source, cachekey, name) return self._has_file(filename)