From 3ac7e7579bec1d3210af2ada6c03d4e7d5e454bf Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Wed, 1 Apr 2015 15:13:47 +0000 Subject: Remove the chunk hardlink cache The OSTree checkout uses hardlinks anyway, so this is thankfully now redundant. Most of this work was done by Sam Thursfield . Change-Id: I5be211ad3034331834f7b0542721f1bfc024b863 --- morphlib/app.py | 3 +-- morphlib/buildcommand.py | 23 ++--------------- morphlib/stagingarea.py | 58 +++---------------------------------------- morphlib/stagingarea_tests.py | 27 +++++++++++++++++--- 4 files changed, 30 insertions(+), 81 deletions(-) diff --git a/morphlib/app.py b/morphlib/app.py index 51a998b7..0d115bd7 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -285,8 +285,7 @@ class Morph(cliapp.Application): sys.exit(0) tmpdir = self.settings['tempdir'] - for required_dir in (os.path.join(tmpdir, 'chunks'), - os.path.join(tmpdir, 'staging'), + for required_dir in (os.path.join(tmpdir, 'staging'), os.path.join(tmpdir, 'failed'), os.path.join(tmpdir, 'deployments'), self.settings['cachedir']): diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 76f8bd99..cab38395 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -492,32 +492,13 @@ class BuildCommand(object): if artifact.source.build_mode == 'bootstrap': if not self.in_same_stratum(artifact.source, target_source): continue + self.app.status( msg='Installing chunk %(chunk_name)s from cache %(cache)s', chunk_name=artifact.name, cache=artifact.source.cache_key[:7], chatty=True) - chunk_cache_dir = os.path.join(self.app.settings['tempdir'], - 'chunks') - artifact_checkout = os.path.join( - chunk_cache_dir, os.path.basename(artifact.basename()) + '.d') - if not os.path.exists(artifact_checkout): - self.app.status( - msg='Checking out %(chunk)s from cache.', - chunk=artifact.name - ) - temp_checkout = os.path.join(self.app.settings['tempdir'], - artifact.basename()) - try: - self.lac.get(artifact, temp_checkout) - except BaseException: - shutil.rmtree(temp_checkout) - raise - # TODO: This rename is not concurrency safe if two builds are - # extracting the same chunk, one build will fail because - # the other renamed its tempdir here first. - os.rename(temp_checkout, artifact_checkout) - staging_area.install_artifact(artifact, artifact_checkout) + staging_area.install_artifact(self.lac, artifact) if target_source.build_mode == 'staging': morphlib.builder.ldconfig(self.app.runcmd, staging_area.dirname) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 768ec643..df38a2e8 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -108,52 +108,6 @@ class StagingArea(object): assert filename.startswith(dirname) return filename[len(dirname) - 1:] # include leading slash - def hardlink_all_files(self, srcpath, destpath): # pragma: no cover - '''Hardlink every file in the path to the staging-area - - If an exception is raised, the staging-area is indeterminate. - - ''' - - file_stat = os.lstat(srcpath) - mode = file_stat.st_mode - - if stat.S_ISDIR(mode): - # Ensure directory exists in destination, then recurse. - if not os.path.lexists(destpath): - os.makedirs(destpath) - dest_stat = os.stat(os.path.realpath(destpath)) - if not stat.S_ISDIR(dest_stat.st_mode): - raise IOError('Destination not a directory. source has %s' - ' destination has %s' % (srcpath, destpath)) - - for entry in os.listdir(srcpath): - self.hardlink_all_files(os.path.join(srcpath, entry), - os.path.join(destpath, entry)) - elif stat.S_ISLNK(mode): - # Copy the symlink. - if os.path.lexists(destpath): - os.remove(destpath) - os.symlink(os.readlink(srcpath), destpath) - - elif stat.S_ISREG(mode): - # Hardlink the file. - if os.path.lexists(destpath): - os.remove(destpath) - os.link(srcpath, destpath) - - elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode): - # Block or character device. Put contents of st_dev in a mknod. - if os.path.lexists(destpath): - os.remove(destpath) - os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev) - os.chmod(destpath, file_stat.st_mode) - - else: - # Unsupported type. - raise IOError('Cannot extract %s into staging-area. Unsupported' - ' type.' % srcpath) - def create_devices(self, morphology): # pragma: no cover '''Creates device nodes if the morphology specifies them''' perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO @@ -178,17 +132,13 @@ class StagingArea(object): os.makedev(dev['major'], dev['minor'])) os.chown(destfile, dev['uid'], dev['gid']) - def install_artifact(self, artifact, artifact_checkout): - '''Install a build artifact into the staging area. - - We access the artifact via an open file handle. For now, we assume - the artifact is a tarball. - - ''' + def install_artifact(self, artifact_cache, artifact): + '''Install a build artifact into the staging area.''' if not os.path.exists(self.dirname): self._mkdir(self.dirname) - self.hardlink_all_files(artifact_checkout, self.dirname) + artifact_cache.get(artifact, directory=self.dirname) + self.create_devices(artifact.source.morphology) def remove(self): diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index ffdf5eaa..3d378573 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -46,6 +46,25 @@ class FakeArtifact(object): self.source = FakeSource() +class FakeArtifactCache(object): + + def __init__(self, tempdir): + self.tempdir = tempdir + + def create_chunk(self, chunkdir): + if not chunkdir: + chunkdir = os.path.join(self.tempdir, 'chunk') + if not os.path.exists(chunkdir): + os.mkdir(chunkdir) + with open(os.path.join(chunkdir, 'file.txt'), 'w'): + pass + + return chunkdir + + def get(self, artifact, directory=None): + return self.create_chunk(directory) + + class FakeApplication(object): def __init__(self, cachedir, tempdir): @@ -141,14 +160,14 @@ class StagingAreaTests(unittest.TestCase): def test_installs_artifact(self): artifact = FakeArtifact() - chunkdir = self.create_chunk() - self.sa.install_artifact(artifact, chunkdir) + artifact_cache = FakeArtifactCache(self.tempdir) + self.sa.install_artifact(artifact_cache, artifact) self.assertEqual(self.list_tree(self.staging), ['/', '/file.txt']) def test_removes_everything(self): artifact = FakeArtifact() - chunkdir = self.create_chunk() - self.sa.install_artifact(artifact, chunkdir) + artifact_cache = FakeArtifactCache(self.tempdir) + self.sa.install_artifact(artifact_cache, artifact) self.sa.remove() self.assertFalse(os.path.exists(self.staging)) -- cgit v1.2.1