From c2bf2f9961bea6c382fe17bbf1057a58ce6a3d2b Mon Sep 17 00:00:00 2001 From: Firehose merge bot Date: Thu, 7 Aug 2014 09:25:45 +0100 Subject: More firehose bits --- .gitignore | 1 + examples/cross-bootstrap-morph.yaml | 4 +- examples/linux-master.yaml | 4 +- examples/linux.yaml | 23 ---- examples/linux.yaml.disabled | 23 ++++ examples/morph.yaml | 4 +- examples/screen.yaml | 4 +- examples/vim.yaml | 6 +- firehose/__init__.py | 0 firehose/config.py | 48 ++++++++ morph | 17 +++ plugin/firehose_plugin.py | 231 ++++++++++++++++++++++++++++++++++++ 12 files changed, 331 insertions(+), 34 deletions(-) delete mode 100644 examples/linux.yaml create mode 100644 examples/linux.yaml.disabled create mode 100644 firehose/__init__.py create mode 100644 firehose/config.py create mode 100755 morph create mode 100644 plugin/firehose_plugin.py diff --git a/.gitignore b/.gitignore index b25c15b..2f836aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *~ +*.pyc diff --git a/examples/cross-bootstrap-morph.yaml b/examples/cross-bootstrap-morph.yaml index a2cc66d..7d112bd 100644 --- a/examples/cross-bootstrap-morph.yaml +++ b/examples/cross-bootstrap-morph.yaml @@ -9,8 +9,8 @@ description: | landing: repo: baserock:baserock/definitions - base-ref: master - my-ref: baserock/firehose + baseref: master + myref: baserock/firehose stratum: cross-bootstrap chunk: morph method: absolute-sha1 diff --git a/examples/linux-master.yaml b/examples/linux-master.yaml index c0c1732..81cf151 100644 --- a/examples/linux-master.yaml +++ b/examples/linux-master.yaml @@ -6,8 +6,8 @@ description: | landing: repo: baserock:baserock/definitions - base-ref: master - my-ref: baserock/firehose + baseref: master + myref: baserock/firehose stratum: bsp-x86_64-generic chunk: linux-x86-64-generic method: absolute-sha1 diff --git a/examples/linux.yaml b/examples/linux.yaml deleted file mode 100644 index 7d7e9d0..0000000 --- a/examples/linux.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: linux -kind: firehose -description: | - This is an example for tracking Linux kernel tags. We attempt to follow any - tag which happens to be made and then we update the chunk listed in the - x86_64 BSP. - -landing: - repo: baserock:baserock/definitions - base-ref: master - my-ref: baserock/firehose - stratum: bsp-x86_64-generic - chunk: linux-x86-64-generic - method: absolute-sha1 - -tracking: - mode: refs - filters: - - ^refs/tags/ - # Turn vX.Y-rcZ into vX.Y~rcZ so that versions can be ordered - transforms: - - match: (.*)-rc(.*) - replacement: $1~rc$2 diff --git a/examples/linux.yaml.disabled b/examples/linux.yaml.disabled new file mode 100644 index 0000000..fa9c5f3 --- /dev/null +++ b/examples/linux.yaml.disabled @@ -0,0 +1,23 @@ +name: linux +kind: firehose +description: | + This is an example for tracking Linux kernel tags. We attempt to follow any + tag which happens to be made and then we update the chunk listed in the + x86_64 BSP. + +landing: + repo: baserock:baserock/definitions + baseref: master + myref: baserock/firehose + stratum: bsp-x86_64-generic + chunk: linux-x86-64-generic + method: absolute-sha1 + +tracking: + mode: refs + filters: + - ^refs/tags/ + # Turn vX.Y-rcZ into vX.Y~rcZ so that versions can be ordered + transforms: + - match: (.*)-rc(.*) + replacement: $1~rc$2 diff --git a/examples/morph.yaml b/examples/morph.yaml index 86e2592..d43ba89 100644 --- a/examples/morph.yaml +++ b/examples/morph.yaml @@ -6,8 +6,8 @@ description: | landing: repo: baserock:baserock/definitions - base-ref: master - my-ref: baserock/firehose + baseref: master + myref: baserock/firehose stratum: tools chunk: morph method: absolute-sha1 diff --git a/examples/screen.yaml b/examples/screen.yaml index a847a52..f49baeb 100644 --- a/examples/screen.yaml +++ b/examples/screen.yaml @@ -6,8 +6,8 @@ description: | landing: repo: baserock:baserock/definitions - base-ref: master - my-ref: baserock/firehose + baseref: master + myref: baserock/firehose stratum: tools chunk: screen method: absolute-sha1 diff --git a/examples/vim.yaml b/examples/vim.yaml index b7543e2..5f71731 100644 --- a/examples/vim.yaml +++ b/examples/vim.yaml @@ -7,8 +7,8 @@ description: | landing: repo: baserock:baserock/definitions - base-ref: master - my-ref: baserock/firehose + baseref: master + myref: baserock/firehose stratum: tools chunk: vim method: absolute-sha1 @@ -20,5 +20,5 @@ tracking: # Turns vX-YaZ into vX-Y~aZ so that tags can be usefully ordered transforms: - match: v([0-9]-[0-9])([^-].*) - replacement: $1~$2 + replacement: \1~\2 diff --git a/firehose/__init__.py b/firehose/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/firehose/config.py b/firehose/config.py new file mode 100644 index 0000000..be034fe --- /dev/null +++ b/firehose/config.py @@ -0,0 +1,48 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import yaml + +class FirehoseConfigError(Exception): + def __init__(self, config, what, path): + self.config = config + self.what = what + self.path = path + + def __repr__(self): + return "" % ( + self.config.sourcename, self.what, (".").join(self.path)) + + def __str__(self): + return repr(self) + +class FirehoseConfig: + def __init__(self, sourcename, readfrom): + self.sourcename = sourcename + self.content = yaml.safe_load(readfrom) + assert(self.content.get("kind") == "firehose") + + def __getattr__(self, attrname): + attrpath = attrname.split("_") + node = self.content + pathused = [] + while attrpath: + elem = attrpath.pop(0) + pathused.append(elem) + if node.get(elem) is None: + raise FirehoseConfigError(self, "Unknown element", pathused) + else: + node = node.get(elem) + return node diff --git a/morph b/morph new file mode 100755 index 0000000..d92fe4c --- /dev/null +++ b/morph @@ -0,0 +1,17 @@ +#!/bin/sh + +MORPH="${MORPH:-$(which morph)}" + +BASE="$(dirname $0)" +BASE="${BASE:-.}" +BASE="$(realpath ${BASE})" + +MORPH_PLUGIN_PATH="${BASE}/plugin" + +export MORPH_PLUGIN_PATH + +PYTHONPATH="${BASE}${PYTHONPATH:+:${PYTHONPATH}}:" + +export PYTHONPATH + +exec ${MORPH} "$@" diff --git a/plugin/firehose_plugin.py b/plugin/firehose_plugin.py new file mode 100644 index 0000000..6f9cc12 --- /dev/null +++ b/plugin/firehose_plugin.py @@ -0,0 +1,231 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +from contextlib import closing, contextmanager +from fs.tempfs import TempFS +import os +import re + +from firehose.config import FirehoseConfig +import morphlib + +from debian.debian_support import Version + +@contextmanager +def firehose_git(app): + try: + username = (app.runcmd_unchecked(["git", "config", "--global", "user.name"]))[1].strip() + email = (app.runcmd_unchecked(["git", "config", "--global", "user.email"]))[1].strip() + app.runcmd(["git", "config", "--global", "user.name", "Firehose merge bot"]) + app.runcmd(["git", "config", "--global", "user.email", "firehose@merge.bot"]) + yield () + finally: + app.runcmd(["git", "config", "--global", "user.name", username]) + app.runcmd(["git", "config", "--global", "user.email", email]) + + + +class FirehosePlugin(cliapp.Plugin): + def enable(self): + self.app.add_subcommand('firehose', self.firehose_cmd, + arg_synopsis='some-firehose.yaml...') + + def disable(self): + pass + + def firehose_cmd(self, args): + confs = [] + if len(args) == 0: + raise cliapp.AppException("Expected list of firehoses on command line") + for fname in args: + with open(fname, "r") as fh: + confs.append(FirehoseConfig(fname, fh)) + + # Ensure all incoming configurations are based on, and landing in, the + # same repository. This is because we're only supporting an aggregated + # integration mode for now. + + if len(set(c.landing_repo for c in confs)) > 1: + raise cliapp.AppException("Not all firehoses have the same landing repo") + if len(set(c.landing_baseref for c in confs)) > 1: + raise cliapp.AppException("Not all firehoses have the same landing baseref") + if len(set(c.landing_myref for c in confs)) > 1: + raise cliapp.AppException("Not all firehoses have the same landing myref") + + + # Ensure that all incoming configurations have unique things they are + # integrating into. Note: this allows for the same upstream to be + # tracked in multiple configs, providing they are all targetting a + # different part of the system. (e.g. linux kernels for multiple BSPs) + + if len(set("%s:%s" % (c.landing_stratum, + c.landing_chunk) for c in confs)) != len(confs): + raise cliapp.AppException("Not all firehoses have unique landing locations") + + with closing(TempFS(temp_dir=self.app.settings['tempdir'])) as tfs, \ + firehose_git(self.app): + self.base_path = tfs.getsyspath("/") + self.make_workspace() + self.make_branch(confs[0].landing) + self.reset_to_tracking(confs[0].landing) + self.load_morphologies() + for c in confs: + self.update_for_conf(c) + if self.updated_morphologies(): + print self.app.runcmd_unchecked(["git", "diff"], cwd=self.gitpath)[1] + + def make_path(self, *subpath): + return os.path.join(self.base_path, *subpath) + + def make_workspace(self): + self.app.subcommands['init']([self.make_path("ws")]) + + def make_branch(self, root): + os.chdir(self.make_path("ws")) + try: + self.app.subcommands['branch']([root['repo'], root['myref'], root['baseref']]) + except cliapp.AppException, ae: + if "already exists in" in str(ae): + self.app.subcommands['checkout']([root['repo'], root['myref']]) + else: + raise + repopath = root['repo'].replace(':', '/') + self.gitpath = self.make_path("ws", root['myref'], repopath) + + def reset_to_tracking(self, root): + branch_head_sha = self.app.runcmd(['git', 'rev-parse', 'HEAD'], + cwd=self.gitpath).strip() + self.app.runcmd(['git', 'reset', '--hard', 'origin/%s' % root['baseref']], + cwd=self.gitpath) + self.app.runcmd(['git', 'reset', '--soft', branch_head_sha], + cwd=self.gitpath) + + def load_morphologies(self): + ws = morphlib.workspace.open(self.gitpath) + sb = morphlib.sysbranchdir.open_from_within(self.gitpath) + loader = morphlib.morphloader.MorphologyLoader() + morphs = morphlib.morphset.MorphologySet() + for morph in sb.load_all_morphologies(loader): + morphs.add_morphology(morph) + + self.ws = ws + self.sb = sb + self.loader = loader + self.morphs = morphs + self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + + def find_cached_repo(self, stratum, chunk): + urls = [] + def wanted_spec(m, kind, spec): + if not m['kind'] == 'stratum' and kind == 'chunks': + return False + if m.get('name') == stratum and spec.get('name') == chunk: + return True + def process_spec(m, kind, spec): + urls.append(spec['repo']) + return False + self.morphs.traverse_specs(process_spec, wanted_spec) + if len(urls) != 1: + raise cliapp.AppException( + "Woah! expected 1 chunk matching %s:%s (got %d)" % ( + stratum, chunk, len(urls))) + return self.lrc.get_updated_repo(urls[0]) + + def git_in_repo_unchecked(self, repo, *args): + args = list(args) + args.insert(0, "git") + return self.app.runcmd_unchecked(args, cwd=repo.path) + def git_in_repo(self, repo, *args): + args = list(args) + args.insert(0, "git") + return self.app.runcmd(args, cwd=repo.path) + + def all_shas_for_refs(self, repo): + all_lines = self.git_in_repo(repo, "for-each-ref").strip().split("\n") + for refline in all_lines: + (sha, objtype, name) = refline.split(None, 2) + if objtype == 'tag': + sha = self.git_in_repo(repo, "rev-list", "-1", sha).strip() + yield name, sha + + def interested_in_ref(self, conf, refname): + if conf.tracking_mode == 'follow-tip': + return refname == conf.tracking_ref + elif conf.tracking_mode == 'refs': + return any(re.match(filterstr, refname) + for filterstr in conf.tracking_filters) + else: + raise FirehoseConfigError(conf, "Unknown value: %s" % + conf.tracking_mode, ["tracking", "mode"]) + + def rewrite_ref(self, conf, ref): + if conf.tracking_mode == 'refs': + for transform in conf.tracking_transforms: + ref = re.sub(transform['match'], transform['replacement'], ref) + return ref + + def compare_refs(self, ref1, ref2): + if ref1[0] == ref2[0]: + return 0 + v1 = Version(ref1[0].replace("/", "-")) + v2 = Version(ref2[0].replace("/", "-")) + if v1 < v2: + return -1 + return 1 + + def sanitise_refname(self, refname): + if refname.startswith("refs/"): + return "/".join((refname.split("/"))[2:]) + else: + return refname + + def update_refs(self, stratum, chunk, sha, refname): + def wanted_spec(m, kind, spec): + if not m['kind'] == 'stratum' and kind == 'chunks': + return False + if m.get('name') == stratum and spec.get('name') == chunk: + return True + def process_spec(m, kind, spec): + spec['ref'] = sha + spec['unpetrify-ref'] = refname + return True + self.morphs.traverse_specs(process_spec, wanted_spec) + + def update_for_conf(self, conf): + stratum = conf.landing_stratum + chunk = conf.landing_chunk + crc = self.find_cached_repo(stratum, chunk) + interesting_refs = [ + (self.rewrite_ref(conf, name), name, sha) + for (name, sha) in self.all_shas_for_refs(crc) + if self.interested_in_ref(conf, name)] + interesting_refs.sort(self.compare_refs) + (_, refname, sha) = interesting_refs.pop() + refname = self.sanitise_refname(refname) + self.update_refs(stratum, chunk, sha, refname) + + def updated_morphologies(self): + if not any(m.dirty for m in self.morphs.morphologies): + return False + for morph in self.morphs.morphologies: + if morph.dirty: + self.loader.unset_defaults(morph) + self.loader.save_to_file( + self.sb.get_filename(morph.repo_url, morph.filename), morph) + morph.dirty = False + + return True -- cgit v1.2.1