From 052fa53b99378d864f21761b7e6f02d23f9156e4 Mon Sep 17 00:00:00 2001 From: Paul Sherwood Date: Sat, 20 Dec 2014 20:29:19 +0000 Subject: WIP hack of ybd into new morph assemble command --- .coverage | 1 + .gitignore | 1 + morphlib/__init__.py | 5 + morphlib/buildcommand.py | 25 +++++ morphlib/cache.py | 81 ++++++++++++++++ morphlib/definitions.py | 133 ++++++++++++++++++++++++++ morphlib/plugins/build_plugin.py | 9 ++ morphlib/repos.py | 202 +++++++++++++++++++++++++++++++++++++++ morphlib/yapp.py | 120 +++++++++++++++++++++++ 9 files changed, 577 insertions(+) create mode 100644 .coverage create mode 100644 morphlib/cache.py create mode 100644 morphlib/definitions.py create mode 100644 morphlib/repos.py create mode 100644 morphlib/yapp.py diff --git a/.coverage b/.coverage new file mode 100644 index 00000000..f2b0d15d --- /dev/null +++ b/.coverage @@ -0,0 +1 @@ +€}q(U collectorqUcoverage v3.5.2b1qUlinesq}u. \ No newline at end of file diff --git a/.gitignore b/.gitignore index f3d74a9a..55787803 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +._* *.pyc *~ diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 8319430c..ef9f524d 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -91,4 +91,9 @@ import yamlparse import writeexts +import cache +import definitions +import repos +import yapp + import app # this needs to be last diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 527163f6..bcc7f5e8 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -50,6 +50,31 @@ class BuildCommand(object): self.lac, self.rac = self.new_artifact_caches() self.lrc, self.rrc = self.new_repo_caches() + def assemble(self, target): + defs = morphlib.definitions.Definitions() + this = defs.get(target) + if defs.lookup(this, 'repo') != [] and defs.lookup(this, 'tree') == []: + this['tree'] = repos.get_tree(this) + + if morphlib.cache.get_cache(this): + yapp.log(this, 'Cache found', morphlib.cache.get_cache(this)) + return + + with yapp.timer(this, 'Starting assembly'): + build_env = BuildEnvironment(yapp.settings) + stage = StagingArea(this, build_env) + for dependency in defs.lookup(this, 'build-depends'): + assemble(defs.get(dependency)) + stage.install_artifact(dependency) + + # if we're distbuilding, wait here for all dependencies to complete + # how do we know when that happens? + + for component in defs.lookup(this, 'contents'): + assemble(defs.get(component)) + + build(this) + def build(self, repo_name, ref, filename, original_ref=None): '''Build a given system morphology.''' diff --git a/morphlib/cache.py b/morphlib/cache.py new file mode 100644 index 00000000..ed3630e0 --- /dev/null +++ b/morphlib/cache.py @@ -0,0 +1,81 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + +import os +import shutil +import yapp +import re +import hashlib +import json +import definitions +import repos + + +def cache_key(this): + defs = definitions.Definitions() + definition = defs.get(this) + + if defs.lookup(definition, 'tree') == []: + definition['tree'] = repos.get_tree(definition) + + if defs.lookup(definition, 'cache') != []: + return definition['cache'] + + hash_factors = { 'arch': app.settings['arch'] } + + for factor in ['build-depends', 'contents']: + for it in defs.lookup(definition, factor): + component = defs.get(it) + + if definition['name'] == component['name']: + yapp.log(this, 'ERROR: recursion loop') + raise SystemExit + + hash_factors[component['name']] = cache_key(component) + + for factor in ['tree', 'configure-commands', 'build-commands', + 'install-commands']: + + if defs.lookup(definition, factor) != []: + hash_factors[factor] = definition[factor] + + result = json.dumps(hash_factors, sort_keys=True).encode('utf-8') + + safename = definition['name'].replace('/', '-') + definition['cache'] = safename + "@" + hashlib.sha256(result).hexdigest() + yapp.log(definition, 'Cache_key is', definition['cache']) + return definition['cache'] + + +def cache(this): + cachefile = os.path.join(yapp.settings['artifacts'], + cache_key(this)) + + shutil.make_archive(cachefile, 'gztar', this['install']) + yapp.log(this, 'Now cached as', cache_key(this)) + + +def get_cache(this): + ''' Check if a cached artifact exists for the hashed version of this. ''' + + cachefile = os.path.join(yapp.settings['artifacts'], + cache_key(this) + '.tar.gz') + + if os.path.exists(cachefile): + return cachefile + + return False diff --git a/morphlib/definitions.py b/morphlib/definitions.py new file mode 100644 index 00000000..09838236 --- /dev/null +++ b/morphlib/definitions.py @@ -0,0 +1,133 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + +import yaml +import os +import yapp +import cache +from subprocess import check_output + + +class Definitions(): + __definitions = [] + __trees = {} + + def __init__(self): + ''' Load all definitions from `cwd` tree. ''' + if self.__definitions != []: + return + + for dirname, dirnames, filenames in os.walk(os.getcwd()): + for filename in filenames: + if not (filename.endswith('.def') or + filename.endswith('.morph')): + continue + + this = self._load(dirname, filename) + name = self.lookup(this, 'name') + if this.get('name'): + self._insert(this) + + for dependency in self.lookup(this, 'build-depends'): + if self.lookup(dependency, 'repo') != []: + self._insert(dependency) + + for component in (self.lookup(this, 'contents') + + self.lookup(this, 'chunks') + + self.lookup(this, 'strata')): + component['build-depends'] = ( + self.lookup(component, 'build-depends')) + for dependency in self.lookup(this, 'build-depends'): + component['build-depends'].extend( + [self.lookup(dependency, 'name')]) + self._insert(component) + + if '.git' in dirnames: + dirnames.remove('.git') + + try: + self.__trees = self._load(os.getcwd(), ".trees") + for definition in self.__definitions: + definition['tree'] = self.__trees.get(definition['name']) + + except Exception, e: + return + + def _load(self, path, name): + ''' Load a single definition file, and create a hash for it. ''' + try: + filename = os.path.join(path, name) + with open(filename) as f: + text = f.read() + + definition = yaml.safe_load(text) + except ValueError: + flog(this, 'ERROR: problem loading', filename) + + return definition + + def _insert(self, this): + for i, definition in enumerate(self.__definitions): + if definition['name'] == this['name']: + if (self.lookup(definition, 'ref') == [] + or self.lookup(this, 'ref') == []): + for key in this: + definition[key] = this[key] + + return + + for key in this: + if key == 'build-depends' or key == 'morph': + continue + if definition.get(key) != this[key]: + yapp.log(this, 'WARNING: multiple definitions of', key) + yapp.log(this, '%s | %s' % (definition.get(key), + this[key])) + + self.__definitions.append(this) + + def get(self, this): + for definition in self.__definitions: + if (definition['name'] == this or + definition['name'] == self.lookup(this, 'name')): + return definition + + yapp.log(this, 'ERROR: no definition found for', this) + raise SystemExit + + def lookup(self, thing, value): + ''' Look up value from thing, return [] if none. ''' + val = [] + try: + val = thing[value] + except Exception, e: + pass + return val + + def version(self, this): + try: + return this['name'].split('@')[1] + except Exception, e: + return False + + def save_trees(self): + self.__trees = {} + for definition in self.__definitions: + if definition.get('tree') is not None: + self.__trees[definition['name']] = definition.get('tree') + with open(os.path.join(os.getcwd(), '.trees'), 'w') as f: + f.write(yaml.dump(self.__trees, default_flow_style=False)) diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 218bd819..ce9e51ad 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -17,6 +17,7 @@ import cliapp import contextlib import uuid +import os import morphlib @@ -26,6 +27,8 @@ class BuildPlugin(cliapp.Plugin): def enable(self): self.app.add_subcommand('build-morphology', self.build_morphology, arg_synopsis='(REPO REF FILENAME)...') + self.app.add_subcommand('assemble', self.assemble, + arg_synopsis='DEFINITION') self.app.add_subcommand('build', self.build, arg_synopsis='SYSTEM') self.app.add_subcommand('distbuild-morphology', @@ -38,6 +41,12 @@ class BuildPlugin(cliapp.Plugin): def disable(self): self.use_distbuild = False + def assemble(self, args): + build_command = morphlib.buildcommand.BuildCommand(self.app) + path, target = os.path.split(args[0]) + target = target.replace('.morph', '') + build_command.assemble(target) + def distbuild_morphology(self, args): '''Distbuild a system, outside of a system branch. diff --git a/morphlib/repos.py b/morphlib/repos.py new file mode 100644 index 00000000..a2c5967b --- /dev/null +++ b/morphlib/repos.py @@ -0,0 +1,202 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + +import os +import app +import re +from subprocess import call +from subprocess import check_output +import string +import definitions +import urllib2 +import json + + +def get_repo_url(this): + url = this['repo'] + url = url.replace('upstream:', 'git://git.baserock.org/delta/') + url = url.replace('baserock:baserock/', + 'git://git.baserock.org/baserock/baserock/') + url = url.replace('freedesktop:', 'git://anongit.freedesktop.org/') + url = url.replace('github:', 'git://github.com/') + url = url.replace('gnome:', 'git://git.gnome.org') + if url.endswith('.git'): + url = url[:-4] + return url + + +def quote_url(url): + ''' Convert URIs to strings that only contain digits, letters, % and _. + + NOTE: When changing the code of this function, make sure to also apply + the same to the quote_url() function of lorry. Otherwise the git tarballs + generated by lorry may no longer be found by morph. + + ''' + valid_chars = string.digits + string.ascii_letters + '%_' + transl = lambda x: x if x in valid_chars else '_' + return ''.join([transl(x) for x in url]) + + +def get_repo_name(this): + return quote_url(get_repo_url(this)) +# return re.split('[:/]', this['repo'])[-1] + + +def get_tree(this): + defs = definitions.Definitions() + + if defs.lookup(this, 'repo') == []: + return None + + if defs.lookup(this, 'git') == []: + this['git'] = (os.path.join(app.settings['gits'], + get_repo_name(this))) + + ref = defs.lookup(this, 'ref') + if not os.path.exists(this['git']): + try: + url = (app.settings['cache-server-url'] + 'repo=' + + get_repo_url(this) + '&ref=' + ref) + with urllib2.urlopen(url) as response: + tree = json.loads(response.read().decode())['tree'] + return tree + except Exception, e: + yapp.log(this, 'WARNING: no tree from cache-server', tree) + mirror(this) + + with app.chdir(this['git']), open(os.devnull, "w") as fnull: + if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull, + stderr=fnull): + # can't resolve this ref. is it upstream? + call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull) + + try: + tree = check_output(['git', 'rev-parse', ref + '^{tree}'], + universal_newlines=True)[0:-1] + return tree + + except Exception, e: + # either we don't have a git dir, or ref is not unique + # or ref does not exist + + yapp.log(this, 'ERROR: could not find tree for ref', ref) + raise SystemExit + + +def copy_repo(repo, destdir): + '''Copies a cached repository into a directory using cp. + + This also fixes up the repository afterwards, so that it can contain + code etc. It does not leave any given branch ready for use. + + ''' + + # core.bare should be false so that git believes work trees are possible + # we do not want the origin remote to behave as a mirror for pulls + # we want a traditional refs/heads -> refs/remotes/origin ref mapping + # set the origin url to the cached repo so that we can quickly clean up + # by packing the refs, we can then edit then en-masse easily + call(['cp', '-a', repo, os.path.join(destdir, '.git')]) + call(['git', 'config', 'core.bare', 'false']) + call(['git', 'config', '--unset', 'remote.origin.mirror']) + with open(os.devnull, "w") as fnull: + call(['git', 'config', 'remote.origin.fetch', + '+refs/heads/*:refs/remotes/origin/*'], + stdout=fnull, + stderr=fnull) + call(['git', 'config', 'remote.origin.url', repo]) + call(['git', 'pack-refs', '--all', '--prune']) + + # turn refs/heads/* into refs/remotes/origin/* in the packed refs + # so that the new copy behaves more like a traditional clone. + with open(os.path.join(destdir, ".git", "packed-refs"), "r") as ref_fh: + pack_lines = ref_fh.read().split("\n") + with open(os.path.join(destdir, ".git", "packed-refs"), "w") as ref_fh: + ref_fh.write(pack_lines.pop(0) + "\n") + for refline in pack_lines: + if ' refs/remotes/' in refline: + continue + if ' refs/heads/' in refline: + sha, ref = refline[:40], refline[41:] + if ref.startswith("refs/heads/"): + ref = "refs/remotes/origin/" + ref[11:] + refline = "%s %s" % (sha, ref) + ref_fh.write("%s\n" % (refline)) + # Finally run a remote update to clear up the refs ready for use. + with open(os.devnull, "w") as fnull: + call(['git', 'remote', 'update', 'origin', '--prune'], + stdout=fnull, + stderr=fnull) + + +def mirror(this): + # try tarball first + try: + os.makedirs(this['git']) + with app.chdir(this['git']): + yapp.log(this, 'Try fetching tarball') + repo_url = get_repo_url(this) + tar_file = quote_url(repo_url) + '.tar' + tar_url = os.path.join("http://git.baserock.org/tarballs", + tar_file) + with open(os.devnull, "w") as fnull: + call(['wget', tar_url], stdout=fnull, stderr=fnull) + call(['tar', 'xf', tar_file], stdout=fnull, stderr=fnull) + os.remove(tar_file) + call(['git', 'config', 'remote.origin.url', repo_url], + stdout=fnull, stderr=fnull) + call(['git', 'config', 'remote.origin.mirror', 'true'], + stdout=fnull, stderr=fnull) + if call(['git', 'config', 'remote.origin.fetch', + '+refs/*:refs/*'], + stdout=fnull, stderr=fnull) != 0: + raise BaseException('Did not get a valid git repo') + call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull) + except Exception, e: + yapp.log(this, 'Using git clone', get_repo_url(this)) + try: + with open(os.devnull, "w") as fnull: + call(['git', 'clone', '--mirror', '-n', get_repo_url(this), + this['git']], stdout=fnull, stderr=fnull) + except Exception, e: + yapp.log(this, 'ERROR: failed to clone', get_repo_url(this)) + raise SystemExit + + yapp.log(this, 'Git repo is mirrored at', this['git']) + + +def fetch(repo): + with app.chdir(repo), open(os.devnull, "w") as fnull: + call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull) + + +def checkout(this): + # checkout the required version of this from git + with app.chdir(this['build']): + yapp.log(this, 'Git checkout') + if not this.get('git'): + this['git'] = (os.path.join(app.settings['gits'], + get_repo_name(this))) + if not os.path.exists(this['git']): + mirror(this) + copy_repo(this['git'], this['build']) + with open(os.devnull, "w") as fnull: + if call(['git', 'checkout', this['ref']], + stdout=fnull, stderr=fnull) != 0: + yapp.log(this, 'ERROR: git checkout failed for', this['tree']) + raise SystemExit diff --git a/morphlib/yapp.py b/morphlib/yapp.py new file mode 100644 index 00000000..c0e41fdb --- /dev/null +++ b/morphlib/yapp.py @@ -0,0 +1,120 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + +import contextlib +import os +import datetime +import shutil +from subprocess import check_output +from subprocess import call +from multiprocessing import cpu_count +settings = {} + + +def log(component, message='', data=''): + ''' Print a timestamped log. ''' + name = component + try: + name = component['name'] + except Exception, e: + pass + + timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + log_entry = '%s [%s] %s %s\n' % (timestamp, name, message, data) + print(log_entry), + + +def run_cmd(this, command): + log(this, 'Running command\n\n', command) + with open(settings['logfile'], "a") as logfile: + if call(['sh', '-c', command], stdout=logfile, stderr=logfile): + log(this, 'ERROR: in directory %s command failed:' % os.getcwd(), + command) + raise SystemExit + + +@contextlib.contextmanager +def setup(target, arch): + try: + settings['arch'] = arch + settings['no-ccache'] = True + settings['cache-server-url'] = \ + 'http://git.baserock.org:8080/1.0/sha1s?' + settings['base'] = os.path.expanduser('~/.ybd/') + if os.path.exists('/src'): + settings['base'] = '/src' + settings['caches'] = os.path.join(settings['base'], 'cache') + settings['artifacts'] = os.path.join(settings['caches'], + 'ybd-artifacts') + settings['gits'] = os.path.join(settings['caches'], 'gits') + settings['staging'] = os.path.join(settings['base'], 'staging') + timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') + settings['assembly'] = os.path.join(settings['staging'], + target + '-' + timestamp) + settings['max_jobs'] = max(int(cpu_count() * 1.5 + 0.5), 1) + + for directory in ['base', 'caches', 'artifacts', 'gits', + 'staging', 'assembly']: + if not os.path.exists(settings[directory]): + os.mkdir(settings[directory]) + + # git replace means we can't trust that just the sha1 of a branch + # is enough to say what it contains, so we turn it off by setting + # the right flag in an environment variable. + os.environ['GIT_NO_REPLACE_OBJECTS'] = '1' + + + yield + + finally: + # assuming success, we can remove the 'assembly' directory + # shutil.rmtree(settings['assembly']) + log(target, 'Assembly directory is still at', settings['assembly']) + + +@contextlib.contextmanager +def chdir(dirname=None, env={}): + currentdir = os.getcwd() + currentenv = {} + try: + if dirname is not None: + os.chdir(dirname) + for key, value in env.items(): + currentenv[key] = os.environ.get(key) + os.environ[key] = value + yield + finally: + for key, value in currentenv.items(): + if value: + os.environ[key] = value + else: + del os.environ[key] + os.chdir(currentdir) + + +@contextlib.contextmanager +def timer(this, start_message=''): + starttime = datetime.datetime.now() + log(this, start_message) + try: + yield + finally: + td = datetime.datetime.now() - starttime + hours, remainder = divmod(int(td.total_seconds()), 60*60) + minutes, seconds = divmod(remainder, 60) + td_string = "%02d:%02d:%02d" % (hours, minutes, seconds) + log(this, 'Elapsed time', td_string) -- cgit v1.2.1