From 50247e57320de74e88049101e1ad47fb8e78b5a3 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Tue, 1 Sep 2015 10:15:36 +0000 Subject: Simplify StagingArea class Pass the Source to the staging area constructor so that we don't need to pass it as a parameter when we call some StagingArea methods. Also move the creation of the build and destdir directories to the constructor so that we can get rid of the chroot_open() and chroot_close() methods. Also, provide API to retrieve the relative locations for buildir and destdir. Change-Id: I6e8085392e19ff3d8df807f260acf90eec9e0901 --- morphlib/buildcommand.py | 11 ++-- morphlib/builder.py | 20 ++++--- morphlib/plugins/cross-bootstrap_plugin.py | 4 +- morphlib/stagingarea.py | 84 +++++++++--------------------- morphlib/stagingarea_tests.py | 33 ++++-------- 5 files changed, 54 insertions(+), 98 deletions(-) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 3ace34bd..bb354b2f 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -362,13 +362,14 @@ class BuildCommand(object): use_chroot = True setup_mounts = True - staging_area = self.create_staging_area(build_env, + staging_area = self.create_staging_area(source, + build_env, use_chroot, extra_env=extra_env, extra_path=extra_path) self.install_dependencies(staging_area, deps, source) else: - staging_area = self.create_staging_area(build_env, False) + staging_area = self.create_staging_area(source, build_env, False) self.build_and_cache(staging_area, source, setup_mounts) self.remove_staging_area(staging_area) @@ -440,15 +441,15 @@ class BuildCommand(object): name=artifact.name) fetch_files(to_fetch) - def create_staging_area(self, build_env, use_chroot=True, extra_env={}, - extra_path=[]): + def create_staging_area(self, source, build_env, use_chroot=True, + extra_env={}, extra_path=[]): '''Create the staging area for building a single artifact.''' self.app.status(msg='Creating staging area') staging_dir = tempfile.mkdtemp( dir=os.path.join(self.app.settings['tempdir'], 'staging')) staging_area = morphlib.stagingarea.StagingArea( - self.app, staging_dir, build_env, use_chroot, extra_env, + self.app, source, staging_dir, build_env, use_chroot, extra_env, extra_path) return staging_area diff --git a/morphlib/builder.py b/morphlib/builder.py index 58667159..bc674548 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -271,8 +271,8 @@ class ChunkBuilder(BuilderBase): def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): - builddir, destdir = self.staging_area.chroot_open( - self.source, self.setup_mounts) + builddir = self.staging_area.real_builddir() + destdir = self.staging_area.real_destdir() stdout = (self.app.output if self.app.settings['build-log-on-stdout'] else None) @@ -285,14 +285,13 @@ class ChunkBuilder(BuilderBase): try: self.get_sources(builddir) - self.run_commands(builddir, destdir, temppath, stdout) + self.run_commands(temppath, stdout) self.create_devices(destdir) os.rename(temppath, logpath) except BaseException as e: logging.error('Caught exception: %s' % str(e)) logging.info('Cleaning up staging area') - self.staging_area.chroot_close() if os.path.isfile(temppath): with open(temppath) as f: for line in f: @@ -304,21 +303,20 @@ class ChunkBuilder(BuilderBase): logging.error("Couldn't find build log at %s", temppath) raise - self.staging_area.chroot_close() built_artifacts = self.assemble_chunk_artifacts(destdir) self.save_build_times() return built_artifacts - def run_commands(self, builddir, destdir, - logfilepath, stdout=None): # pragma: no cover + def run_commands(self, logfilepath, stdout=None): # pragma: no cover m = self.source.morphology bs = morphlib.buildsystem.lookup_build_system(m['build-system']) - relative_builddir = self.staging_area.relative(builddir) - relative_destdir = self.staging_area.relative(destdir) - ccache_dir = self.staging_area.ccache_dir(self.source) + relative_builddir = self.staging_area.relative_builddir() + relative_destdir = self.staging_area.relative_destdir() + ccache_dir = self.staging_area.ccache_dir() + extra_env = { 'DESTDIR': relative_destdir } steps = [ @@ -552,7 +550,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover handle = self.local_artifact_cache.put(artifact) try: - fs_root = self.staging_area.destdir(self.source) + fs_root = self.staging_area.real_destdir() self.unpack_strata(fs_root) self.write_metadata(fs_root, a_name) self.run_system_integration_commands(fs_root) diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index b8da515e..265b273b 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -59,7 +59,7 @@ class BootstrapSystemBuilder(morphlib.builder.BuilderBase): with self.build_watch('overall-build'): for system_name, artifact in self.source.artifacts.iteritems(): handle = self.local_artifact_cache.put(artifact) - fs_root = self.staging_area.destdir(self.source) + fs_root = self.staging_area.real_destdir() try: self.unpack_binary_chunks(fs_root) self.unpack_sources(fs_root) @@ -301,7 +301,7 @@ class CrossBootstrapPlugin(cliapp.Plugin): system_artifact.source.cross_sources = cross_sources system_artifact.source.native_sources = native_sources staging_area = build_command.create_staging_area( - build_env, use_chroot=False) + system_artifact.source, build_env, use_chroot=False) builder = BootstrapSystemBuilder( self.app, staging_area, build_command.lac, build_command.rac, system_artifact.source, build_command.lrc, 1, False) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index e244a8d2..ba2bf39c 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -39,18 +39,20 @@ class StagingArea(object): _base_path = ['/sbin', '/usr/sbin', '/bin', '/usr/bin'] - def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}, - extra_path=[]): + def __init__(self, app, source, dirname, build_env, use_chroot=True, + extra_env={}, extra_path=[]): self._app = app + self.source = source self.dirname = dirname - self.builddirname = None - self.destdirname = None self._bind_readonly_mount = None self.use_chroot = use_chroot self.env = build_env.env self.env.update(extra_env) + os.makedirs(self.real_builddir()) + os.makedirs(self.real_destdir()) + if use_chroot: path = extra_path + build_env.extra_path + self._base_path else: @@ -78,47 +80,33 @@ class StagingArea(object): if not os.path.exists(dirname): os.makedirs(dirname) - # Wrapper to be overridden by unit tests. - def _mkdir(self, dirname): # pragma: no cover - os.makedirs(dirname) + def relative(self, path): + '''Return a path relative to the staging area.''' - def _dir_for_source(self, source, suffix): - dirname = os.path.join(self.dirname, - '%s.%s' % (str(source.name), suffix)) - self._mkdir(dirname) - return dirname + if self.use_chroot: + return os.path.join(os.sep, path) + else: + return os.path.join(self.dirname, path) - def builddir(self, source): - '''Create a build directory for a given source project. + def relative_builddir(self): + return self.relative('%s.build' % self.source.name) - Return path to directory. + def relative_destdir(self): + return self.relative('%s.inst' % self.source.name) - ''' + def real_builddir(self): + '''Build directory for a given source project ''' - return self._dir_for_source(source, 'build') + return os.path.join(self.dirname, '%s.build' % (self.source.name)) - def destdir(self, source): - '''Create an installation target directory for a given source project. + def real_destdir(self): + '''Installation target directory for a given source project. This is meant to be used as $DESTDIR when installing chunks. - Return path to directory. ''' - return self._dir_for_source(source, 'inst') - - def relative(self, filename): - '''Return a filename relative to the staging area.''' - - if not self.use_chroot: - return filename - - dirname = self.dirname - if not dirname.endswith('/'): - dirname += '/' - - assert filename.startswith(dirname) - return filename[len(dirname) - 1:] # include leading slash + return os.path.join(self.dirname, '%s.inst' % (self.source.name)) def hardlink_all_files(self, srcpath, destpath): # pragma: no cover '''Hardlink every file in the path to the staging-area @@ -209,12 +197,12 @@ class StagingArea(object): ) to_mount_in_bootstrap = () - def ccache_dir(self, source): #pragma: no cover + def ccache_dir(self): #pragma: no cover ccache_dir = self._app.settings['compiler-cache-dir'] if not os.path.isdir(ccache_dir): os.makedirs(ccache_dir) # Get a path for the repo's ccache - ccache_url = source.repo.url + ccache_url = self.source.repo.url ccache_path = urlparse(ccache_url).path ccache_repobase = os.path.basename(ccache_path) if ':' in ccache_repobase: # the basename is a repo-alias @@ -238,27 +226,6 @@ class StagingArea(object): os.makedirs(ccache_destdir) return ccache_repodir - def chroot_open(self, source, setup_mounts): # pragma: no cover - '''Setup staging area for use as a chroot.''' - - assert self.builddirname == None and self.destdirname == None - - builddir = self.builddir(source) - destdir = self.destdir(source) - self.builddirname = builddir - self.destdirname = destdir - - return builddir, destdir - - def chroot_close(self): # pragma: no cover - '''Undo changes by chroot_open. - - This should be called after the staging area is no longer needed. - - ''' - # No cleanup is currently required - pass - def runcmd(self, argv, **kwargs): # pragma: no cover '''Run a command in a chroot in the staging area.''' assert 'env' not in kwargs @@ -272,7 +239,8 @@ class StagingArea(object): chroot_dir = self.dirname if self.use_chroot else '/' temp_dir = kwargs["env"].get("TMPDIR", "/tmp") - staging_dirs = [self.builddirname, self.destdirname] + staging_dirs = [self.real_builddir(), self.real_destdir()] + if self.use_chroot: staging_dirs += ["dev", "proc", temp_dir.lstrip('/')] do_not_mount_dirs = [os.path.join(self.dirname, d) diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index f08a3989..2d7a7e33 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -64,6 +64,7 @@ class FakeApplication(object): class StagingAreaTests(unittest.TestCase): def setUp(self): + self.source = FakeSource() self.tempdir = tempfile.mkdtemp() self.cachedir = os.path.join(self.tempdir, 'cachedir') os.mkdir(self.cachedir) @@ -72,8 +73,8 @@ class StagingAreaTests(unittest.TestCase): self.created_dirs = [] self.build_env = FakeBuildEnvironment() self.sa = morphlib.stagingarea.StagingArea( - FakeApplication(self.cachedir, self.tempdir), self.staging, - self.build_env) + FakeApplication(self.cachedir, self.tempdir), self.source, + self.staging, self.build_env) def tearDown(self): shutil.rmtree(self.tempdir) @@ -98,35 +99,21 @@ class StagingAreaTests(unittest.TestCase): files.append(x[len(root):] or '/') return files - def fake_mkdir(self, dirname): - self.created_dirs.append(dirname) - def test_remembers_specified_directory(self): self.assertEqual(self.sa.dirname, self.staging) - def test_creates_build_directory(self): - source = FakeSource() - self.sa._mkdir = self.fake_mkdir - dirname = self.sa.builddir(source) - self.assertEqual(self.created_dirs, [dirname]) - self.assertTrue(dirname.startswith(self.staging)) - - def test_creates_install_directory(self): - source = FakeSource() - self.sa._mkdir = self.fake_mkdir - dirname = self.sa.destdir(source) - self.assertEqual(self.created_dirs, [dirname]) - self.assertTrue(dirname.startswith(self.staging)) - def test_makes_relative_name(self): - filename = os.path.join(self.staging, 'foobar') + filename = 'foobar' self.assertEqual(self.sa.relative(filename), '/foobar') def test_installs_artifact(self): chunk_tar = self.create_chunk() with open(chunk_tar, 'rb') as f: self.sa.install_artifact(f) - self.assertEqual(self.list_tree(self.staging), ['/', '/file.txt']) + self.assertEqual(self.list_tree(self.staging), + ['/', '/file.txt', + self.sa.relative_destdir(), + self.sa.relative_builddir()]) def test_removes_everything(self): chunk_tar = self.create_chunk() @@ -139,11 +126,13 @@ class StagingAreaTests(unittest.TestCase): class StagingAreaNonIsolatedTests(unittest.TestCase): def setUp(self): + self.source = FakeSource() self.tempdir = tempfile.mkdtemp() self.staging = os.path.join(self.tempdir, 'staging') self.build_env = FakeBuildEnvironment() self.sa = morphlib.stagingarea.StagingArea( - object(), self.staging, self.build_env, use_chroot=False) + object(), self.source, self.staging, self.build_env, + use_chroot=False) def tearDown(self): shutil.rmtree(self.tempdir) -- cgit v1.2.1