From 5ae139d6427b2f114518d7fd1b8d8d29b46b7a81 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Wed, 1 Apr 2015 15:13:47 +0000 Subject: SUGGESTED: remove the chunk hardlink cache The OSTree checkout uses hardlinks anyway, so this is thankfully now redundant. --- morphlib/app.py | 3 +-- morphlib/buildcommand.py | 21 ++---------------- morphlib/stagingarea.py | 58 ++++-------------------------------------------- 3 files changed, 7 insertions(+), 75 deletions(-) diff --git a/morphlib/app.py b/morphlib/app.py index 800258e0..4073ee41 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -285,8 +285,7 @@ class Morph(cliapp.Application): sys.exit(0) tmpdir = self.settings['tempdir'] - for required_dir in (os.path.join(tmpdir, 'chunks'), - os.path.join(tmpdir, 'staging'), + for required_dir in (os.path.join(tmpdir, 'staging'), os.path.join(tmpdir, 'failed'), os.path.join(tmpdir, 'deployments'), self.settings['cachedir']): diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index c4d7a6f5..68f53906 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -491,32 +491,15 @@ class BuildCommand(object): if artifact.source.build_mode == 'bootstrap': if not self.in_same_stratum(artifact.source, target_source): continue + self.app.status( msg='Installing chunk %(chunk_name)s from cache %(cache)s', chunk_name=artifact.name, cache=artifact.source.cache_key[:7], chatty=True) - chunk_cache_dir = os.path.join(self.app.settings['tempdir'], - 'chunks') artifact_checkout = os.path.join( chunk_cache_dir, os.path.basename(artifact.basename()) + '.d') - if not os.path.exists(artifact_checkout): - self.app.status( - msg='Checking out %(chunk)s from cache.', - chunk=artifact.name - ) - temp_checkout = os.path.join(self.app.settings['tempdir'], - artifact.basename()) - try: - self.lac.get(artifact, temp_checkout) - except BaseException: - shutil.rmtree(temp_checkout) - raise - # TODO: This rename is not concurrency safe if two builds are - # extracting the same chunk, one build will fail because - # the other renamed its tempdir here first. - os.rename(temp_checkout, artifact_checkout) - staging_area.install_artifact(artifact, artifact_checkout) + staging_area.install_artifact(self.lac, artifact) if target_source.build_mode == 'staging': morphlib.builder.ldconfig(self.app.runcmd, staging_area.dirname) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 768ec643..df38a2e8 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -108,52 +108,6 @@ class StagingArea(object): assert filename.startswith(dirname) return filename[len(dirname) - 1:] # include leading slash - def hardlink_all_files(self, srcpath, destpath): # pragma: no cover - '''Hardlink every file in the path to the staging-area - - If an exception is raised, the staging-area is indeterminate. - - ''' - - file_stat = os.lstat(srcpath) - mode = file_stat.st_mode - - if stat.S_ISDIR(mode): - # Ensure directory exists in destination, then recurse. - if not os.path.lexists(destpath): - os.makedirs(destpath) - dest_stat = os.stat(os.path.realpath(destpath)) - if not stat.S_ISDIR(dest_stat.st_mode): - raise IOError('Destination not a directory. source has %s' - ' destination has %s' % (srcpath, destpath)) - - for entry in os.listdir(srcpath): - self.hardlink_all_files(os.path.join(srcpath, entry), - os.path.join(destpath, entry)) - elif stat.S_ISLNK(mode): - # Copy the symlink. - if os.path.lexists(destpath): - os.remove(destpath) - os.symlink(os.readlink(srcpath), destpath) - - elif stat.S_ISREG(mode): - # Hardlink the file. - if os.path.lexists(destpath): - os.remove(destpath) - os.link(srcpath, destpath) - - elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode): - # Block or character device. Put contents of st_dev in a mknod. - if os.path.lexists(destpath): - os.remove(destpath) - os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev) - os.chmod(destpath, file_stat.st_mode) - - else: - # Unsupported type. - raise IOError('Cannot extract %s into staging-area. Unsupported' - ' type.' % srcpath) - def create_devices(self, morphology): # pragma: no cover '''Creates device nodes if the morphology specifies them''' perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO @@ -178,17 +132,13 @@ class StagingArea(object): os.makedev(dev['major'], dev['minor'])) os.chown(destfile, dev['uid'], dev['gid']) - def install_artifact(self, artifact, artifact_checkout): - '''Install a build artifact into the staging area. - - We access the artifact via an open file handle. For now, we assume - the artifact is a tarball. - - ''' + def install_artifact(self, artifact_cache, artifact): + '''Install a build artifact into the staging area.''' if not os.path.exists(self.dirname): self._mkdir(self.dirname) - self.hardlink_all_files(artifact_checkout, self.dirname) + artifact_cache.get(artifact, directory=self.dirname) + self.create_devices(artifact.source.morphology) def remove(self): -- cgit v1.2.1