summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRichard Maw <richard.maw@codethink.co.uk>2015-05-01 14:00:51 (GMT)
committerRichard Maw <richard.maw@codethink.co.uk>2015-05-01 14:00:51 (GMT)
commit8956e5e47bb569368e6d1963ecf35deee3ab7e19 (patch)
tree6f6250c6031b56f77fb2ac3ecd6f978b075ca88a
parentb477acbbda4c8c0ea7c5dd1c282dad4e45cfdddf (diff)
downloadmorph-8956e5e47bb569368e6d1963ecf35deee3ab7e19.tar.gz
Add `morph anchor` command
Change-Id: If9d92d7c75b9c4276b69c482c076c6fc1d4ccbbf
-rw-r--r--morphlib/plugins/anchor_plugin.py226
-rw-r--r--without-test-modules1
2 files changed, 227 insertions, 0 deletions
diff --git a/morphlib/plugins/anchor_plugin.py b/morphlib/plugins/anchor_plugin.py
new file mode 100644
index 0000000..bd2449b
--- /dev/null
+++ b/morphlib/plugins/anchor_plugin.py
@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+# Copyright © 2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+from collections import defaultdict
+
+import cliapp
+
+from morphlib.branchmanager import RemoteRefManager
+from morphlib.buildcommand import BuildCommand
+from morphlib.gitdir import (PushFailureError, RefSpec, Remote)
+from morphlib.repoaliasresolver import RepoAliasResolver
+
+
+TAG_PUSH_ERR = 'error: Trying to write non-commit object'
+
+
+INVALID_TAGS_MESSAGE = '''\
+
+Some refs pointed to tag objects and the anchor-ref-format option was
+configured to push anchor refs to refs/heads.
+Unfortunately git does not allow tags to be pushed there, so the commit objects
+they point to have been pushed instead.
+As a result the anchored systems cannot be built with the current definitions.
+
+To remedy this either:
+
+1. Re-configure the git server to allow pushes to outside of refs/heads and
+ change anchor-ref-format to the new namespace
+
+ i.e. Allow pushes to refs/tags/{trove_id} and run with
+ --anchor-ref-format=refs/tags/{trove_id}/anchors/{name}/{commit_sha1}
+
+2. Amend definitions so that the following refs are replaced with their listed
+ commit. The anchors that were pushed will remain appropriate to preserve
+ reproducibility with the amended definitions.
+
+---
+'''
+
+
+class AnchorPlugin(cliapp.Plugin):
+
+ def enable(self):
+ self.app.add_subcommand(
+ 'anchor', self.anchor, arg_synopsis='NAME REPO REF SYSTEM...')
+ self.app.settings.string(
+ ['anchor-ref-format'],
+ 'python format string with {trove_id}, {name} and '
+ '{commit_sha1} interpolated in to produce the name of the '
+ 'anchor refs to push',
+ metavar='FORMAT',
+ default='refs/heads/{trove_id}/anchors/{name}/{commit_sha1}',
+ group='anchor options')
+
+ def disable(self):
+ pass
+
+ @staticmethod
+ def _push(status, rrm, remote, refspecs):
+ status(msg='Pushing %(targets)s to %(repo)s',
+ targets=', '.join(refspec.target for refspec in refspecs),
+ repo=remote.get_push_url())
+ rrm.push(remote, *refspecs)
+
+ def anchor(self, args):
+ '''Push anchoring commits for listed systems.
+
+ Command line arguments:
+
+ * `NAME` - semantic name to anchor systems by.
+
+ If the purpose of anchoring is to make sure a release can be
+ reproducibly built, then you would use `NAME` to know which
+ anchor refs are used to keep a release buildable, so when
+ support for the release is dropped, the anchors may be
+ dropped.
+
+ * `REPO` - repository url to definitions repository.
+
+ For a locally checked out repository, this would be
+ `file://$(pwd)`
+
+ * `REF` - name of branch in definitions repository.
+
+ For a locally checked out repository, this would be `HEAD`
+
+ * `SYSTEM...` - list of systems to create anchors for
+
+ '''
+ if len(args) < 4:
+ raise cliapp.AppException(
+ 'Insufficient args to anchor command, '
+ 'see morph help anchor')
+ anchor_name, branch_repo, branch_ref = args[0:3]
+ systems = args[3:]
+ anchor_ref_format = self.app.settings['anchor-ref-format']
+
+ trove_ids = self.app.settings['trove-id']
+ if not trove_ids and '{trove_id}' in anchor_ref_format:
+ raise cliapp.AppException(
+ 'No trove-id configured and anchor-ref-format has not '
+ 'been adjusted to remove it.')
+ trove_id = trove_ids[0]
+
+ resolver = RepoAliasResolver(self.app.settings['repo-alias'])
+
+ with RemoteRefManager(cleanup_on_success=False) as rrm:
+ unpushable_tags = set()
+ # We must use the build logic to resolve the build graph as a
+ # naive traversal of the morphologies would not determine that
+ # some sources weren't required because even though it is in a
+ # stratum morphology we used, none of its artifacts were used
+ # to construct our system.
+ sources_by_reponame = defaultdict(set)
+ for system in systems:
+ bc = BuildCommand(self.app)
+ srcpool = bc.create_source_pool(branch_repo, branch_ref,
+ system)
+ artifact = bc.resolve_artifacts(srcpool)
+ sources = set(a.source for a in artifact.walk())
+ for source in sources:
+ sources_by_reponame[source.repo_name].add(source)
+
+ for reponame, sources in sources_by_reponame.iteritems():
+ # UGLY HACK we need to push *FROM* our local repo cache to
+ # avoid cloning everything multiple times.
+ # This uses cache_repo rather than get_repo because the
+ # BuildCommand.create_source_pool won't cache the
+ # repositories locally if it can use a remote cache
+ # instead.
+ repo = bc.lrc.cache_repo(reponame)
+ remote = Remote(repo.gitdir)
+
+ push_url = resolver.push_url(reponame)
+ remote.set_push_url(push_url)
+
+ lsinfo = dict((ref, sha1) for (sha1, ref) in remote.ls())
+ refspecparams = defaultdict(set)
+ for source in sources:
+ sha1 = source.sha1
+ anchor_ref_name = anchor_ref_format.format(
+ name=anchor_name, trove_id=trove_id,
+ commit_sha1=sha1)
+ existing_anchor = lsinfo.get(anchor_ref_name)
+ params = (sha1, anchor_ref_name, existing_anchor)
+ refspecparams[params].add(source.original_ref)
+
+ refspecs = dict(
+ (RefSpec(source=sha1, target=anchor_ref_name,
+ require=existing_anchor),
+ original_refs)
+ for ((sha1, anchor_ref_name, existing_anchor),
+ original_refs)
+ in refspecparams.iteritems())
+
+ try:
+ self._push(status=self.app.status, rrm=rrm,
+ remote=remote, refspecs=refspecs.keys())
+ except PushFailureError as e:
+ if TAG_PUSH_ERR not in e.stderr:
+ raise
+ results = set(e.results)
+ newrefspecs = set()
+
+ # We need to check the state of the remote so that we can
+ # re-send updates if one of the updates failed.
+ lsinfo = dict((ref, sha1) for (sha1, ref) in remote.ls())
+
+ for flag, sha1, target, summary, reason in results:
+ commit = repo.gitdir.resolve_ref_to_commit(sha1)
+
+ # Fail if we failed to push something other than a tag
+ # pushed to a branch
+ if (flag == '!' and
+ not (commit != sha1
+ and target.startswith('refs/heads/'))):
+ raise
+
+ for rs, original_refs in refspecs.iteritems():
+ if rs.source == sha1 and rs.target == target:
+ break
+
+ if flag != '!':
+ # We assert that if any of the pushes failed, then
+ # they were all rolled back, as otherwise we can't
+ # clean up properly.
+ assert (rs.source == rs.require
+ or lsinfo.get(target) == rs.require)
+ # Because even successfull pushes were rolled back
+ # in the case of failure, we need to re-push the
+ # changes that had succeeded.
+ newrefspecs.add(
+ RefSpec(source=sha1, target=target,
+ require=lsinfo.get(target)))
+ continue
+
+ unpushable_tags.add(
+ (remote, anchor_ref_name,
+ tuple(original_refs), sha1, commit))
+ newrefspecs.add(
+ RefSpec(source=commit, target=target,
+ require=lsinfo.get(target)))
+
+ # Re-attempt the push with the new refspecs
+ self._push(status=self.app.status, rrm=rrm,
+ remote=remote, refspecs=newrefspecs)
+
+ if unpushable_tags:
+ self.app.status(msg=INVALID_TAGS_MESSAGE, error='very yes')
+ for remote, pushed_ref, refs, tag, commit in unpushable_tags:
+ self.app.output.write(
+ 'Replace {} with {}\n'.format(', '.join(refs), commit))
+ self.app.output.flush()
diff --git a/without-test-modules b/without-test-modules
index 189f130..fb8bca1 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -36,6 +36,7 @@ morphlib/plugins/certify_plugin.py
morphlib/plugins/get_repo_plugin.py
morphlib/plugins/system_manifests_plugin.py
morphlib/plugins/show_build_log_plugin.py
+morphlib/plugins/anchor_plugin.py
distbuild/__init__.py
distbuild/build_controller.py
distbuild/connection_machine.py