From 779fd24b8e9926e5b8cc2949589544a58abbda80 Mon Sep 17 00:00:00 2001 From: Lauren Perry Date: Thu, 11 Dec 2014 16:36:04 +0000 Subject: Add tools for installing Firehose and its dependencies --- firehose.sh | 21 ++++ firehose/plugin/firehose_plugin.py | 240 +++++++++++++++++++++++++++++++++++++ plugin/firehose_plugin.py | 240 ------------------------------------- setup.py | 16 +++ 4 files changed, 277 insertions(+), 240 deletions(-) create mode 100755 firehose.sh create mode 100644 firehose/plugin/firehose_plugin.py delete mode 100644 plugin/firehose_plugin.py create mode 100644 setup.py diff --git a/firehose.sh b/firehose.sh new file mode 100755 index 0000000..4d272d3 --- /dev/null +++ b/firehose.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +MORPH="${MORPH:-$(which morph)}" + +BASE="$(dirname $0)" +BASE="${BASE:-.}" +BASE="$(realpath ${BASE})" + +# Currently the only way to get this to work is to hardcode the path of the plugin. +# Obviously this is not ideal, but until a working method is obtained this remains +# our best option at present. + +MORPH_PLUGIN_PATH="/usr/lib/python2.7/site-packages/firehose-0.1-py2.7.egg/firehose/plugin" + +export MORPH_PLUGIN_PATH + +PYTHONPATH="${BASE}${PYTHONPATH:+:${PYTHONPATH}}:" + +export PYTHONPATH + +exec ${MORPH} firehose "$@" diff --git a/firehose/plugin/firehose_plugin.py b/firehose/plugin/firehose_plugin.py new file mode 100644 index 0000000..25e2eb4 --- /dev/null +++ b/firehose/plugin/firehose_plugin.py @@ -0,0 +1,240 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +from contextlib import closing, contextmanager +from fs.tempfs import TempFS +import os +import re + +from firehose.config import FirehoseConfig +import morphlib + +from debian.debian_support import Version + +@contextmanager +def firehose_git(app): + try: + username = (app.runcmd_unchecked(["git", "config", "--global", "user.name"]))[1].strip() + email = (app.runcmd_unchecked(["git", "config", "--global", "user.email"]))[1].strip() + app.runcmd(["git", "config", "--global", "user.name", "Firehose merge bot"]) + app.runcmd(["git", "config", "--global", "user.email", "firehose@merge.bot"]) + yield () + finally: + app.runcmd(["git", "config", "--global", "user.name", username]) + app.runcmd(["git", "config", "--global", "user.email", email]) + + + +class FirehosePlugin(cliapp.Plugin): + def enable(self): + self.app.add_subcommand('firehose', self.firehose_cmd, + arg_synopsis='some-firehose.yaml...') + + def disable(self): + pass + + def firehose_cmd(self, args): + confs = [] + if len(args) == 0: + raise cliapp.AppException("Expected list of firehoses on command line") + for fname in args: + with open(fname, "r") as fh: + confs.append(FirehoseConfig(fname, fh)) + + # Ensure all incoming configurations are based on, and landing in, the + # same repository. This is because we're only supporting an aggregated + # integration mode for now. + + if len(set(c.landing_repo for c in confs)) > 1: + raise cliapp.AppException("Not all firehoses have the same landing repo") + if len(set(c.landing_baseref for c in confs)) > 1: + raise cliapp.AppException("Not all firehoses have the same landing baseref") + if len(set(c.landing_myref for c in confs)) > 1: + raise cliapp.AppException("Not all firehoses have the same landing myref") + + + # Ensure that all incoming configurations have unique things they are + # integrating into. Note: this allows for the same upstream to be + # tracked in multiple configs, providing they are all targetting a + # different part of the system. (e.g. linux kernels for multiple BSPs) + + if len(set("%s:%s" % (c.landing_stratum, + c.landing_chunk) for c in confs)) != len(confs): + raise cliapp.AppException("Not all firehoses have unique landing locations") + + with closing(TempFS(temp_dir=self.app.settings['tempdir'])) as tfs, \ + firehose_git(self.app): + self.base_path = tfs.getsyspath("/") + self.make_workspace() + self.make_branch(confs[0].landing) + self.reset_to_tracking(confs[0].landing) + self.load_morphologies() + for c in confs: + self.update_for_conf(c) + if self.updated_morphologies(): + self.commit_and_push() + + def make_path(self, *subpath): + return os.path.join(self.base_path, *subpath) + + def make_workspace(self): + self.app.subcommands['init']([self.make_path("ws")]) + + def make_branch(self, root): + os.chdir(self.make_path("ws")) + try: + self.app.subcommands['branch']([root['repo'], root['myref'], root['baseref']]) + except cliapp.AppException, ae: + if "already exists in" in str(ae): + self.app.subcommands['checkout']([root['repo'], root['myref']]) + else: + raise + repopath = root['repo'].replace(':', '/') + self.gitpath = self.make_path("ws", root['myref'], repopath) + + def reset_to_tracking(self, root): + branch_head_sha = self.app.runcmd(['git', 'rev-parse', 'HEAD'], + cwd=self.gitpath).strip() + self.app.runcmd(['git', 'reset', '--hard', 'origin/%s' % root['baseref']], + cwd=self.gitpath) + self.app.runcmd(['git', 'reset', '--soft', branch_head_sha], + cwd=self.gitpath) + + def load_morphologies(self): + ws = morphlib.workspace.open(self.gitpath) + sb = morphlib.sysbranchdir.open_from_within(self.gitpath) + loader = morphlib.morphloader.MorphologyLoader() + morphs = morphlib.morphset.MorphologySet() + for morph in sb.load_all_morphologies(loader): + morphs.add_morphology(morph) + + self.ws = ws + self.sb = sb + self.loader = loader + self.morphs = morphs + self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + + def find_cached_repo(self, stratum, chunk): + urls = [] + def wanted_spec(m, kind, spec): + if not m['kind'] == 'stratum' and kind == 'chunks': + return False + if m.get('name') == stratum and spec.get('name') == chunk: + return True + def process_spec(m, kind, spec): + urls.append(spec['repo']) + return False + self.morphs.traverse_specs(process_spec, wanted_spec) + if len(urls) != 1: + raise cliapp.AppException( + "Woah! expected 1 chunk matching %s:%s (got %d)" % ( + stratum, chunk, len(urls))) + return self.lrc.get_updated_repo(urls[0]) + + def git_in_repo_unchecked(self, repo, *args): + args = list(args) + args.insert(0, "git") + return self.app.runcmd_unchecked(args, cwd=repo.path) + def git_in_repo(self, repo, *args): + args = list(args) + args.insert(0, "git") + return self.app.runcmd(args, cwd=repo.path) + + def all_shas_for_refs(self, repo): + all_lines = self.git_in_repo(repo, "for-each-ref").strip().split("\n") + for refline in all_lines: + (sha, objtype, name) = refline.split(None, 2) + if objtype == 'tag': + sha = self.git_in_repo(repo, "rev-list", "-1", sha).strip() + yield name, sha + + def interested_in_ref(self, conf, refname): + if conf.tracking_mode == 'follow-tip': + return refname == conf.tracking_ref + elif conf.tracking_mode == 'refs': + return any(re.match(filterstr, refname) + for filterstr in conf.tracking_filters) + else: + raise FirehoseConfigError(conf, "Unknown value: %s" % + conf.tracking_mode, ["tracking", "mode"]) + + def rewrite_ref(self, conf, ref): + if conf.tracking_mode == 'refs': + for transform in conf.tracking_transforms: + ref = re.sub(transform['match'], transform['replacement'], ref) + return ref + + def compare_refs(self, ref1, ref2): + if ref1[0] == ref2[0]: + return 0 + v1 = Version(ref1[0].replace("/", "-")) + v2 = Version(ref2[0].replace("/", "-")) + if v1 < v2: + return -1 + return 1 + + def sanitise_refname(self, refname): + if refname.startswith("refs/"): + return "/".join((refname.split("/"))[2:]) + else: + return refname + + def update_refs(self, stratum, chunk, sha, refname): + def wanted_spec(m, kind, spec): + if not m['kind'] == 'stratum' and kind == 'chunks': + return False + if m.get('name') == stratum and spec.get('name') == chunk: + return True + def process_spec(m, kind, spec): + spec['ref'] = sha + spec['unpetrify-ref'] = refname + return True + self.morphs.traverse_specs(process_spec, wanted_spec) + + def update_for_conf(self, conf): + stratum = conf.landing_stratum + chunk = conf.landing_chunk + crc = self.find_cached_repo(stratum, chunk) + interesting_refs = [ + (self.rewrite_ref(conf, name), name, sha) + for (name, sha) in self.all_shas_for_refs(crc) + if self.interested_in_ref(conf, name)] + interesting_refs.sort(self.compare_refs) + (_, refname, sha) = interesting_refs.pop() + refname = self.sanitise_refname(refname) + self.update_refs(stratum, chunk, sha, refname) + + def updated_morphologies(self): + if not any(m.dirty for m in self.morphs.morphologies): + return False + for morph in self.morphs.morphologies: + if morph.dirty: + self.loader.unset_defaults(morph) + self.loader.save_to_file( + self.sb.get_filename(morph.repo_url, morph.filename), morph) + morph.dirty = False + + return True + + def commit_and_push(self): + (code, out, err) = self.app.runcmd_unchecked( + ['git', 'commit', '-a', '-m', 'Firehose test commit'], + cwd=self.gitpath) + if code == 0: + self.app.runcmd(['git', 'push', 'origin', 'HEAD'], + cwd=self.gitpath) + diff --git a/plugin/firehose_plugin.py b/plugin/firehose_plugin.py deleted file mode 100644 index 25e2eb4..0000000 --- a/plugin/firehose_plugin.py +++ /dev/null @@ -1,240 +0,0 @@ -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -from contextlib import closing, contextmanager -from fs.tempfs import TempFS -import os -import re - -from firehose.config import FirehoseConfig -import morphlib - -from debian.debian_support import Version - -@contextmanager -def firehose_git(app): - try: - username = (app.runcmd_unchecked(["git", "config", "--global", "user.name"]))[1].strip() - email = (app.runcmd_unchecked(["git", "config", "--global", "user.email"]))[1].strip() - app.runcmd(["git", "config", "--global", "user.name", "Firehose merge bot"]) - app.runcmd(["git", "config", "--global", "user.email", "firehose@merge.bot"]) - yield () - finally: - app.runcmd(["git", "config", "--global", "user.name", username]) - app.runcmd(["git", "config", "--global", "user.email", email]) - - - -class FirehosePlugin(cliapp.Plugin): - def enable(self): - self.app.add_subcommand('firehose', self.firehose_cmd, - arg_synopsis='some-firehose.yaml...') - - def disable(self): - pass - - def firehose_cmd(self, args): - confs = [] - if len(args) == 0: - raise cliapp.AppException("Expected list of firehoses on command line") - for fname in args: - with open(fname, "r") as fh: - confs.append(FirehoseConfig(fname, fh)) - - # Ensure all incoming configurations are based on, and landing in, the - # same repository. This is because we're only supporting an aggregated - # integration mode for now. - - if len(set(c.landing_repo for c in confs)) > 1: - raise cliapp.AppException("Not all firehoses have the same landing repo") - if len(set(c.landing_baseref for c in confs)) > 1: - raise cliapp.AppException("Not all firehoses have the same landing baseref") - if len(set(c.landing_myref for c in confs)) > 1: - raise cliapp.AppException("Not all firehoses have the same landing myref") - - - # Ensure that all incoming configurations have unique things they are - # integrating into. Note: this allows for the same upstream to be - # tracked in multiple configs, providing they are all targetting a - # different part of the system. (e.g. linux kernels for multiple BSPs) - - if len(set("%s:%s" % (c.landing_stratum, - c.landing_chunk) for c in confs)) != len(confs): - raise cliapp.AppException("Not all firehoses have unique landing locations") - - with closing(TempFS(temp_dir=self.app.settings['tempdir'])) as tfs, \ - firehose_git(self.app): - self.base_path = tfs.getsyspath("/") - self.make_workspace() - self.make_branch(confs[0].landing) - self.reset_to_tracking(confs[0].landing) - self.load_morphologies() - for c in confs: - self.update_for_conf(c) - if self.updated_morphologies(): - self.commit_and_push() - - def make_path(self, *subpath): - return os.path.join(self.base_path, *subpath) - - def make_workspace(self): - self.app.subcommands['init']([self.make_path("ws")]) - - def make_branch(self, root): - os.chdir(self.make_path("ws")) - try: - self.app.subcommands['branch']([root['repo'], root['myref'], root['baseref']]) - except cliapp.AppException, ae: - if "already exists in" in str(ae): - self.app.subcommands['checkout']([root['repo'], root['myref']]) - else: - raise - repopath = root['repo'].replace(':', '/') - self.gitpath = self.make_path("ws", root['myref'], repopath) - - def reset_to_tracking(self, root): - branch_head_sha = self.app.runcmd(['git', 'rev-parse', 'HEAD'], - cwd=self.gitpath).strip() - self.app.runcmd(['git', 'reset', '--hard', 'origin/%s' % root['baseref']], - cwd=self.gitpath) - self.app.runcmd(['git', 'reset', '--soft', branch_head_sha], - cwd=self.gitpath) - - def load_morphologies(self): - ws = morphlib.workspace.open(self.gitpath) - sb = morphlib.sysbranchdir.open_from_within(self.gitpath) - loader = morphlib.morphloader.MorphologyLoader() - morphs = morphlib.morphset.MorphologySet() - for morph in sb.load_all_morphologies(loader): - morphs.add_morphology(morph) - - self.ws = ws - self.sb = sb - self.loader = loader - self.morphs = morphs - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) - - def find_cached_repo(self, stratum, chunk): - urls = [] - def wanted_spec(m, kind, spec): - if not m['kind'] == 'stratum' and kind == 'chunks': - return False - if m.get('name') == stratum and spec.get('name') == chunk: - return True - def process_spec(m, kind, spec): - urls.append(spec['repo']) - return False - self.morphs.traverse_specs(process_spec, wanted_spec) - if len(urls) != 1: - raise cliapp.AppException( - "Woah! expected 1 chunk matching %s:%s (got %d)" % ( - stratum, chunk, len(urls))) - return self.lrc.get_updated_repo(urls[0]) - - def git_in_repo_unchecked(self, repo, *args): - args = list(args) - args.insert(0, "git") - return self.app.runcmd_unchecked(args, cwd=repo.path) - def git_in_repo(self, repo, *args): - args = list(args) - args.insert(0, "git") - return self.app.runcmd(args, cwd=repo.path) - - def all_shas_for_refs(self, repo): - all_lines = self.git_in_repo(repo, "for-each-ref").strip().split("\n") - for refline in all_lines: - (sha, objtype, name) = refline.split(None, 2) - if objtype == 'tag': - sha = self.git_in_repo(repo, "rev-list", "-1", sha).strip() - yield name, sha - - def interested_in_ref(self, conf, refname): - if conf.tracking_mode == 'follow-tip': - return refname == conf.tracking_ref - elif conf.tracking_mode == 'refs': - return any(re.match(filterstr, refname) - for filterstr in conf.tracking_filters) - else: - raise FirehoseConfigError(conf, "Unknown value: %s" % - conf.tracking_mode, ["tracking", "mode"]) - - def rewrite_ref(self, conf, ref): - if conf.tracking_mode == 'refs': - for transform in conf.tracking_transforms: - ref = re.sub(transform['match'], transform['replacement'], ref) - return ref - - def compare_refs(self, ref1, ref2): - if ref1[0] == ref2[0]: - return 0 - v1 = Version(ref1[0].replace("/", "-")) - v2 = Version(ref2[0].replace("/", "-")) - if v1 < v2: - return -1 - return 1 - - def sanitise_refname(self, refname): - if refname.startswith("refs/"): - return "/".join((refname.split("/"))[2:]) - else: - return refname - - def update_refs(self, stratum, chunk, sha, refname): - def wanted_spec(m, kind, spec): - if not m['kind'] == 'stratum' and kind == 'chunks': - return False - if m.get('name') == stratum and spec.get('name') == chunk: - return True - def process_spec(m, kind, spec): - spec['ref'] = sha - spec['unpetrify-ref'] = refname - return True - self.morphs.traverse_specs(process_spec, wanted_spec) - - def update_for_conf(self, conf): - stratum = conf.landing_stratum - chunk = conf.landing_chunk - crc = self.find_cached_repo(stratum, chunk) - interesting_refs = [ - (self.rewrite_ref(conf, name), name, sha) - for (name, sha) in self.all_shas_for_refs(crc) - if self.interested_in_ref(conf, name)] - interesting_refs.sort(self.compare_refs) - (_, refname, sha) = interesting_refs.pop() - refname = self.sanitise_refname(refname) - self.update_refs(stratum, chunk, sha, refname) - - def updated_morphologies(self): - if not any(m.dirty for m in self.morphs.morphologies): - return False - for morph in self.morphs.morphologies: - if morph.dirty: - self.loader.unset_defaults(morph) - self.loader.save_to_file( - self.sb.get_filename(morph.repo_url, morph.filename), morph) - morph.dirty = False - - return True - - def commit_and_push(self): - (code, out, err) = self.app.runcmd_unchecked( - ['git', 'commit', '-a', '-m', 'Firehose test commit'], - cwd=self.gitpath) - if code == 0: - self.app.runcmd(['git', 'push', 'origin', 'HEAD'], - cwd=self.gitpath) - diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..418f4c6 --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup +setup( + name = "firehose", + version = "0.1", + packages = ["firehose", "debian"], + package_data={ + "firehose": + ["plugin/*.py"] + }, + scripts = [ + "firehose.sh" + ], + # This is because morph won't find it as a plugin if it is + # installed as a .egg file + zip_safe = False +) -- cgit v1.2.1