From 8df5d7df210f9c712c2377cecc6f3f3a7efce01a Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 24 Oct 2014 16:09:45 +0000 Subject: build: Restructure how BuildCommand.build() is called Rather than take a list of triplets to build, the BuildCommand.build() function now takes a single repo/ref/morph triplet. Iterating through multiple sets of triplets is now done in the build plugin. There are a couple of cosmetic changes to the status output at the start and end of a build as a result. --- morphlib/buildcommand.py | 26 ++++++++++++++------------ morphlib/plugins/build_plugin.py | 8 +++++--- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 7eb6c0ab..650ee13a 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -50,21 +50,23 @@ class BuildCommand(object): self.lac, self.rac = self.new_artifact_caches() self.lrc, self.rrc = self.new_repo_caches() - def build(self, args): - '''Build triplets specified on command line.''' + def build(self, repo_name, ref, filename): + '''Build a given system morphology.''' - self.app.status(msg='Build starts', chatty=True) + self.app.status( + msg='Building %(repo_name)s %(ref)s %(filename)s', + repo_name=repo_name, ref=ref, filename=filename) - for repo_name, ref, filename in self.app.itertriplets(args): - self.app.status(msg='Building %(repo_name)s %(ref)s %(filename)s', - repo_name=repo_name, ref=ref, filename=filename) - self.app.status(msg='Deciding on task order') - srcpool = self.create_source_pool(repo_name, ref, filename) - self.validate_sources(srcpool) - root_artifact = self.resolve_artifacts(srcpool) - self.build_in_order(root_artifact) + self.app.status(msg='Deciding on task order') + srcpool = self.create_source_pool(repo_name, ref, filename) + self.validate_sources(srcpool) + root_artifact = self.resolve_artifacts(srcpool) + self.build_in_order(root_artifact) - self.app.status(msg='Build ends successfully') + self.app.status( + msg='Build of %(repo_name)s %(ref)s %(filename)s ended ' + 'successfully', + repo_name=repo_name, ref=ref, filename=filename) def new_artifact_caches(self): '''Create interfaces for the build artifact caches. diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 64630c2b..d4f53e2b 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -56,7 +56,8 @@ class BuildPlugin(cliapp.Plugin): build_command = morphlib.buildcommand.InitiatorBuildCommand( self.app, addr, port) - build_command.build(args) + for repo_name, ref, filename in self.app.itertriplets(args): + build_command.build(repo_name, ref, filename) def distbuild(self, args): '''Distbuild a system image in the current system branch @@ -116,7 +117,8 @@ class BuildPlugin(cliapp.Plugin): self.app.settings['cachedir-min-space']) build_command = morphlib.buildcommand.BuildCommand(self.app) - build_command.build(args) + for repo_name, ref, filename in self.app.itertriplets(args): + build_command.build(repo_name, ref, filename) def build(self, args): '''Build a system image in the current system branch @@ -190,4 +192,4 @@ class BuildPlugin(cliapp.Plugin): name=name, email=email, build_uuid=build_uuid, status=self.app.status) with pbb as (repo, ref): - build_command.build([repo, ref, system_filename]) + build_command.build(repo, ref, system_filename) -- cgit v1.2.1 From 35178357f0ba7eafdeebfc621d70ba69f2cd270a Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 24 Oct 2014 16:28:30 +0000 Subject: build: Allow passing in the name of the original ref to create_source_pool() This means that we can force the building of a specific commit without losing the original branch name in the metadata of the resulting system. --- morphlib/app.py | 12 +++++++++--- morphlib/buildcommand.py | 10 ++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/morphlib/app.py b/morphlib/app.py index 9ab102b3..c3c9c970 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -290,7 +290,8 @@ class Morph(cliapp.Application): morphlib.util.sanitise_morphology_path(args[2])) args = args[3:] - def create_source_pool(self, lrc, rrc, repo, ref, filename): + def create_source_pool(self, lrc, rrc, repo, ref, filename, + original_ref=None): pool = morphlib.sourcepool.SourcePool() def add_to_pool(reponame, ref, filename, absref, tree, morphology): @@ -302,7 +303,8 @@ class Morph(cliapp.Application): self.traverse_morphs(repo, ref, [filename], lrc, rrc, update=not self.settings['no-git-update'], - visit=add_to_pool) + visit=add_to_pool, + definitions_original_ref=original_ref) return pool def resolve_ref(self, lrc, rrc, reponame, ref, update=True): @@ -346,7 +348,8 @@ class Morph(cliapp.Application): def traverse_morphs(self, definitions_repo, definitions_ref, system_filenames, lrc, rrc, update=True, - visit=lambda rn, rf, fn, arf, m: None): + visit=lambda rn, rf, fn, arf, m: None, + definitions_original_ref=None): morph_factory = morphlib.morphologyfactory.MorphologyFactory(lrc, rrc, self) definitions_queue = collections.deque(system_filenames) @@ -359,6 +362,9 @@ class Morph(cliapp.Application): definitions_absref, definitions_tree = self.resolve_ref( lrc, rrc, definitions_repo, definitions_ref, update) + if definitions_original_ref: + definitions_ref = definitions_original_ref + while definitions_queue: filename = definitions_queue.popleft() diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 650ee13a..544d88d8 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -50,7 +50,7 @@ class BuildCommand(object): self.lac, self.rac = self.new_artifact_caches() self.lrc, self.rrc = self.new_repo_caches() - def build(self, repo_name, ref, filename): + def build(self, repo_name, ref, filename, original_ref=None): '''Build a given system morphology.''' self.app.status( @@ -58,7 +58,8 @@ class BuildCommand(object): repo_name=repo_name, ref=ref, filename=filename) self.app.status(msg='Deciding on task order') - srcpool = self.create_source_pool(repo_name, ref, filename) + srcpool = self.create_source_pool( + repo_name, ref, filename, original_ref) self.validate_sources(srcpool) root_artifact = self.resolve_artifacts(srcpool) self.build_in_order(root_artifact) @@ -84,7 +85,7 @@ class BuildCommand(object): return morphlib.buildenvironment.BuildEnvironment(self.app.settings, arch) - def create_source_pool(self, repo_name, ref, filename): + def create_source_pool(self, repo_name, ref, filename, original_ref=None): '''Find the source objects required for building a the given artifact The SourcePool will contain every stratum and chunk dependency of the @@ -94,7 +95,8 @@ class BuildCommand(object): ''' self.app.status(msg='Creating source pool', chatty=True) srcpool = self.app.create_source_pool( - self.lrc, self.rrc, repo_name, ref, filename) + self.lrc, self.rrc, repo_name, ref, filename, + original_ref=original_ref) return srcpool -- cgit v1.2.1 From c255fc68775fe4c5975155c2932b3ed6ee3625c1 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 23 Oct 2014 12:08:34 +0100 Subject: build: Resolve the ref being built in the user's local definitions.git Most usefully, this patch means that Morph no longer updates its cached copy of definitions.git every time you run `morph build`. Also, it prevents confusion in the following situation. Imagine I have run: morph checkout baserock:baserock/definitions master I then wait a while, during which time someone pushes to 'master' in the definitions.git repo that I cloned from. Now I run: cd master morph build systems/whatever.morph Which commit does it build, the local head of 'master' or the remote head of 'master'? The answer, both before and after this patch, is that it builds the local version of master. But previously, this only happened because of the magic that we have to detect local changes. With this patch, the local change detection could be disabled and `morph build` would still build what the user had checked out as 'master' locally, not whatever 'master' pointed to in the remote repo. --- morphlib/buildbranch.py | 16 +++++++++++++--- morphlib/plugins/build_plugin.py | 5 +++-- morphlib/plugins/deploy_plugin.py | 4 ++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index 638350e3..cfc4a67f 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -253,6 +253,10 @@ class BuildBranch(object): def root_ref(self): return self._sb.get_config('branch.name') + @property + def root_commit(self): + return self._root.resolve_ref_to_commit(self.root_ref) + @property def root_local_repo_url(self): return urlparse.urljoin('file://', self._root.dirname) @@ -291,7 +295,13 @@ def pushed_build_branch(bb, loader, changes_need_pushing, name, email, unpushed = any(bb.get_unpushed_branches()) if not changes_made and not unpushed: - yield bb.root_repo_url, bb.root_ref + # We resolve the system branch ref to the commit SHA1 here, so that + # the build uses whatever commit the user's copy of the root repo + # considers the head of that branch to be. If we returned a named + # ref, we risk building what the remote considers the head of that + # branch to be instead, and we also trigger a needless update in + # the cached copy of the root repo. + yield bb.root_repo_url, bb.root_commit, bb.root_ref return def report_inject(gd): @@ -318,6 +328,6 @@ def pushed_build_branch(bb, loader, changes_need_pushing, name, email, remote=remote.get_push_url(), chatty=True) bb.push_build_branches(push_cb=report_push) - yield bb.root_repo_url, bb.root_build_ref + yield bb.root_repo_url, bb.root_build_ref, bb.root_build_ref else: - yield bb.root_local_repo_url, bb.root_build_ref + yield bb.root_local_repo_url, bb.root_build_ref, bb.root_build_ref diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index d4f53e2b..218bd819 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -191,5 +191,6 @@ class BuildPlugin(cliapp.Plugin): bb, loader=loader, changes_need_pushing=push, name=name, email=email, build_uuid=build_uuid, status=self.app.status) - with pbb as (repo, ref): - build_command.build(repo, ref, system_filename) + with pbb as (repo, commit, original_ref): + build_command.build(repo, commit, system_filename, + original_ref=original_ref) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 2bc53a0d..e795e637 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -328,14 +328,14 @@ class DeployPlugin(cliapp.Plugin): bb, loader=loader, changes_need_pushing=False, name=name, email=email, build_uuid=build_uuid, status=self.app.status) - with pbb as (repo, ref): + with pbb as (repo, commit, original_ref): # Create a tempdir for this deployment to work in deploy_tempdir = tempfile.mkdtemp( dir=os.path.join(self.app.settings['tempdir'], 'deployments')) try: for system in cluster_morphology['systems']: self.deploy_system(build_command, deploy_tempdir, - root_repo_dir, repo, ref, system, + root_repo_dir, repo, commit, system, env_vars, deployments, parent_location='') finally: -- cgit v1.2.1