From 6a0bc4c29b2bd3bbbdf8e765da8dce8490e34ddd Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 11 Nov 2014 17:06:58 +0000 Subject: Make temporary build branches optional Not everyone is a fan of the `morph build` magic that collects up your changes and puts them in a temporary branch. Now you can disable it by setting 'local-changes=ignore' in your morph.conf file. This speeds up `morph build` and `morph deploy` by 5-10 seconds on my machine. I looked an option to make `morph build` warn if there are uncommitted changes. I found that with a cold cache, it takes about 5 seconds on my machine to verify that there are no uncommitted changes to a checkout of definitions.git. That defeats the main purpose of this patch for me, so I didn't include the option. --- morphlib/app.py | 8 +++++ morphlib/plugins/build_plugin.py | 62 ++++++++++++++++++++++++++++++++++----- morphlib/plugins/deploy_plugin.py | 51 ++++++++++++++++++++------------ 3 files changed, 95 insertions(+), 26 deletions(-) diff --git a/morphlib/app.py b/morphlib/app.py index 177bce45..0c87f814 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -139,6 +139,14 @@ class Morph(cliapp.Application): 'always push temporary build branches to the ' 'remote repository', group=group_build) + self.settings.choice (['local-changes'], + ['include', 'ignore'], + 'the `build` and `deploy` commands detect ' + 'uncommitted/unpushed local changes and operate ' + 'operate from a temporary branch containing ' + 'those changes. Disable this behaviour with the ' + '`ignore` setting.', + group=group_build) group_storage = 'Storage Options' self.settings.string(['tempdir'], diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 218bd819..c5adffb7 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2013,2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -135,11 +135,13 @@ class BuildPlugin(cliapp.Plugin): The location of the resulting system image artifact is printed at the end of the build output. - You do not need to commit your changes before building, Morph - does that for you, in a temporary branch for each build. However, - note that Morph does not untracked files to the temporary branch, - only uncommitted changes to files git already knows about. You - need to `git add` and commit each new file yourself. + If the 'local-changes' setting is set to 'include', you do not need + to commit your changes before building. Morph does that for you, in a + temporary branch for each build. Note that any system produced this way + will not be reproducible later on as the branch it is built from will + have been deleted. Also note that Morph does not add untracked files to + the temporary branch, only uncommitted changes to files git already + knows about. You need to `git add` and commit each new file yourself. Example: @@ -163,8 +165,6 @@ class BuildPlugin(cliapp.Plugin): ws = morphlib.workspace.open('.') sb = morphlib.sysbranchdir.open_from_within('.') - build_uuid = uuid.uuid4().hex - if self.use_distbuild: addr = self.app.settings['controller-initiator-address'] port = self.app.settings['controller-initiator-port'] @@ -174,6 +174,22 @@ class BuildPlugin(cliapp.Plugin): else: build_command = morphlib.buildcommand.BuildCommand(self.app) + if self.app.settings['local-changes'] == 'include': + self._build_with_local_changes(build_command, sb, system_filename) + else: + self._build_local_commit(build_command, sb, system_filename) + + def _build_with_local_changes(self, build_command, sb, system_filename): + '''Construct a branch including user's local changes, and build that. + + It is often a slow process to check all repos in the system branch for + local changes. However, when using a distributed build cluster, all + code being built must be pushed to the associated Trove, and it can be + helpful to have this automated as part of the `morph build` command. + + ''' + build_uuid = uuid.uuid4().hex + loader = morphlib.morphloader.MorphologyLoader() push = self.app.settings['push-build-branches'] name = morphlib.git.get_user_name(self.app.runcmd) @@ -194,3 +210,33 @@ class BuildPlugin(cliapp.Plugin): with pbb as (repo, commit, original_ref): build_command.build(repo, commit, system_filename, original_ref=original_ref) + + def _build_local_commit(self, build_command, sb, system_filename): + '''Build whatever commit the user has checked-out locally. + + This ignores any uncommitted changes. Also, if the user has a commit + checked out locally that hasn't been pushed to the Trove that Morph is + configured to work with, the build will fail in this sort of way: + + ERROR: Ref c55b853d92a52a5b5fe62edbfbf351169eb79c0a is an invalid + reference for repo + git://git.baserock.org/baserock/baserock/definitions + + The build process doesn't use the checked-out definitions repo at all, + except to resolve the checked-out commit (HEAD). Instead, it uses the + cached version of the definitions repo, updating the cache if + necessary. + + We don't detect and warn the user about any uncommitted changes because + doing so is slow when there are no changes (around 5 seconds on my + machine for Baserock's definitions.git). + + ''' + root_repo_url = sb.get_config('branch.root') + ref = sb.get_config('branch.name') + + definitions_repo_path = sb.get_git_directory_name(root_repo_url) + definitions_repo = morphlib.gitdir.GitDirectory(definitions_repo_path) + commit = definitions_repo.resolve_ref_to_commit(ref) + + build_command.build(root_repo_url, commit, system_filename) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 0121c110..87fd259f 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013, 2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -376,23 +376,24 @@ class DeployPlugin(cliapp.Plugin): self.validate_deployment_options( env_vars, all_deployments, all_subsystems) - bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix) - pbb = morphlib.buildbranch.pushed_build_branch( - bb, loader=loader, changes_need_pushing=False, - name=name, email=email, build_uuid=build_uuid, - status=self.app.status) - with pbb as (repo, commit, original_ref): - # Create a tempdir for this deployment to work in - deploy_tempdir = tempfile.mkdtemp( - dir=os.path.join(self.app.settings['tempdir'], 'deployments')) - try: - for system in cluster_morphology['systems']: - self.deploy_system(build_command, deploy_tempdir, - root_repo_dir, repo, commit, system, - env_vars, deployments, - parent_location='') - finally: - shutil.rmtree(deploy_tempdir) + if self.app.settings['local-changes'] == 'include': + bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix) + pbb = morphlib.buildbranch.pushed_build_branch( + bb, loader=loader, changes_need_pushing=False, + name=name, email=email, build_uuid=build_uuid, + status=self.app.status) + with pbb as (repo, commit, original_ref): + self.deploy_cluster(build_command, cluster_morphology, + root_repo_dir, repo, commit, env_vars, + deployments) + else: + repo = sb.get_config('branch.root') + ref = sb.get_config('branch.name') + commit = root_repo_dir.resolve_ref_to_commit(ref) + + self.deploy_cluster(build_command, cluster_morphology, + root_repo_dir, repo, commit, env_vars, + deployments) self.app.status(msg='Finished deployment') @@ -412,6 +413,20 @@ class DeployPlugin(cliapp.Plugin): 'Variable referenced a non-existent deployment ' 'name: %s' % var) + def deploy_cluster(self, build_command, cluster_morphology, root_repo_dir, + repo, commit, env_vars, deployments): + # Create a tempdir for this deployment to work in + deploy_tempdir = tempfile.mkdtemp( + dir=os.path.join(self.app.settings['tempdir'], 'deployments')) + try: + for system in cluster_morphology['systems']: + self.deploy_system(build_command, deploy_tempdir, + root_repo_dir, repo, commit, system, + env_vars, deployments, + parent_location='') + finally: + shutil.rmtree(deploy_tempdir) + def deploy_system(self, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, system, env_vars, deployment_filter, parent_location): -- cgit v1.2.1