summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPaul Sherwood <paul.sherwood@codethink.co.uk>2014-12-20 20:29:19 (GMT)
committerPaul Sherwood <paul.sherwood@codethink.co.uk>2014-12-20 21:10:15 (GMT)
commit052fa53b99378d864f21761b7e6f02d23f9156e4 (patch)
treead05147ef69bff58b19df8e4ecc65e4fbbe80965
parentf33748d6e6795751e7ea628d5f4e8478353a88ee (diff)
downloadmorph-baserock/ps/wip-ybd-hack.tar.gz
WIP hack of ybd into new morph assemble commandbaserock/ps/wip-ybd-hack
-rw-r--r--.coverage1
-rw-r--r--.gitignore1
-rw-r--r--morphlib/__init__.py5
-rw-r--r--morphlib/buildcommand.py25
-rw-r--r--morphlib/cache.py81
-rw-r--r--morphlib/definitions.py133
-rw-r--r--morphlib/plugins/build_plugin.py9
-rw-r--r--morphlib/repos.py202
-rw-r--r--morphlib/yapp.py120
9 files changed, 577 insertions, 0 deletions
diff --git a/.coverage b/.coverage
new file mode 100644
index 0000000..f2b0d15
--- /dev/null
+++ b/.coverage
@@ -0,0 +1 @@
+}q(U collectorqUcoverage v3.5.2b1qUlinesq}u. \ No newline at end of file
diff --git a/.gitignore b/.gitignore
index f3d74a9..5578780 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
+._*
*.pyc
*~
diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index 8319430..ef9f524 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -91,4 +91,9 @@ import yamlparse
import writeexts
+import cache
+import definitions
+import repos
+import yapp
+
import app # this needs to be last
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 527163f..bcc7f5e 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -50,6 +50,31 @@ class BuildCommand(object):
self.lac, self.rac = self.new_artifact_caches()
self.lrc, self.rrc = self.new_repo_caches()
+ def assemble(self, target):
+ defs = morphlib.definitions.Definitions()
+ this = defs.get(target)
+ if defs.lookup(this, 'repo') != [] and defs.lookup(this, 'tree') == []:
+ this['tree'] = repos.get_tree(this)
+
+ if morphlib.cache.get_cache(this):
+ yapp.log(this, 'Cache found', morphlib.cache.get_cache(this))
+ return
+
+ with yapp.timer(this, 'Starting assembly'):
+ build_env = BuildEnvironment(yapp.settings)
+ stage = StagingArea(this, build_env)
+ for dependency in defs.lookup(this, 'build-depends'):
+ assemble(defs.get(dependency))
+ stage.install_artifact(dependency)
+
+ # if we're distbuilding, wait here for all dependencies to complete
+ # how do we know when that happens?
+
+ for component in defs.lookup(this, 'contents'):
+ assemble(defs.get(component))
+
+ build(this)
+
def build(self, repo_name, ref, filename, original_ref=None):
'''Build a given system morphology.'''
diff --git a/morphlib/cache.py b/morphlib/cache.py
new file mode 100644
index 0000000..ed3630e
--- /dev/null
+++ b/morphlib/cache.py
@@ -0,0 +1,81 @@
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+import os
+import shutil
+import yapp
+import re
+import hashlib
+import json
+import definitions
+import repos
+
+
+def cache_key(this):
+ defs = definitions.Definitions()
+ definition = defs.get(this)
+
+ if defs.lookup(definition, 'tree') == []:
+ definition['tree'] = repos.get_tree(definition)
+
+ if defs.lookup(definition, 'cache') != []:
+ return definition['cache']
+
+ hash_factors = { 'arch': app.settings['arch'] }
+
+ for factor in ['build-depends', 'contents']:
+ for it in defs.lookup(definition, factor):
+ component = defs.get(it)
+
+ if definition['name'] == component['name']:
+ yapp.log(this, 'ERROR: recursion loop')
+ raise SystemExit
+
+ hash_factors[component['name']] = cache_key(component)
+
+ for factor in ['tree', 'configure-commands', 'build-commands',
+ 'install-commands']:
+
+ if defs.lookup(definition, factor) != []:
+ hash_factors[factor] = definition[factor]
+
+ result = json.dumps(hash_factors, sort_keys=True).encode('utf-8')
+
+ safename = definition['name'].replace('/', '-')
+ definition['cache'] = safename + "@" + hashlib.sha256(result).hexdigest()
+ yapp.log(definition, 'Cache_key is', definition['cache'])
+ return definition['cache']
+
+
+def cache(this):
+ cachefile = os.path.join(yapp.settings['artifacts'],
+ cache_key(this))
+
+ shutil.make_archive(cachefile, 'gztar', this['install'])
+ yapp.log(this, 'Now cached as', cache_key(this))
+
+
+def get_cache(this):
+ ''' Check if a cached artifact exists for the hashed version of this. '''
+
+ cachefile = os.path.join(yapp.settings['artifacts'],
+ cache_key(this) + '.tar.gz')
+
+ if os.path.exists(cachefile):
+ return cachefile
+
+ return False
diff --git a/morphlib/definitions.py b/morphlib/definitions.py
new file mode 100644
index 0000000..0983823
--- /dev/null
+++ b/morphlib/definitions.py
@@ -0,0 +1,133 @@
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+import yaml
+import os
+import yapp
+import cache
+from subprocess import check_output
+
+
+class Definitions():
+ __definitions = []
+ __trees = {}
+
+ def __init__(self):
+ ''' Load all definitions from `cwd` tree. '''
+ if self.__definitions != []:
+ return
+
+ for dirname, dirnames, filenames in os.walk(os.getcwd()):
+ for filename in filenames:
+ if not (filename.endswith('.def') or
+ filename.endswith('.morph')):
+ continue
+
+ this = self._load(dirname, filename)
+ name = self.lookup(this, 'name')
+ if this.get('name'):
+ self._insert(this)
+
+ for dependency in self.lookup(this, 'build-depends'):
+ if self.lookup(dependency, 'repo') != []:
+ self._insert(dependency)
+
+ for component in (self.lookup(this, 'contents') +
+ self.lookup(this, 'chunks') +
+ self.lookup(this, 'strata')):
+ component['build-depends'] = (
+ self.lookup(component, 'build-depends'))
+ for dependency in self.lookup(this, 'build-depends'):
+ component['build-depends'].extend(
+ [self.lookup(dependency, 'name')])
+ self._insert(component)
+
+ if '.git' in dirnames:
+ dirnames.remove('.git')
+
+ try:
+ self.__trees = self._load(os.getcwd(), ".trees")
+ for definition in self.__definitions:
+ definition['tree'] = self.__trees.get(definition['name'])
+
+ except Exception, e:
+ return
+
+ def _load(self, path, name):
+ ''' Load a single definition file, and create a hash for it. '''
+ try:
+ filename = os.path.join(path, name)
+ with open(filename) as f:
+ text = f.read()
+
+ definition = yaml.safe_load(text)
+ except ValueError:
+ flog(this, 'ERROR: problem loading', filename)
+
+ return definition
+
+ def _insert(self, this):
+ for i, definition in enumerate(self.__definitions):
+ if definition['name'] == this['name']:
+ if (self.lookup(definition, 'ref') == []
+ or self.lookup(this, 'ref') == []):
+ for key in this:
+ definition[key] = this[key]
+
+ return
+
+ for key in this:
+ if key == 'build-depends' or key == 'morph':
+ continue
+ if definition.get(key) != this[key]:
+ yapp.log(this, 'WARNING: multiple definitions of', key)
+ yapp.log(this, '%s | %s' % (definition.get(key),
+ this[key]))
+
+ self.__definitions.append(this)
+
+ def get(self, this):
+ for definition in self.__definitions:
+ if (definition['name'] == this or
+ definition['name'] == self.lookup(this, 'name')):
+ return definition
+
+ yapp.log(this, 'ERROR: no definition found for', this)
+ raise SystemExit
+
+ def lookup(self, thing, value):
+ ''' Look up value from thing, return [] if none. '''
+ val = []
+ try:
+ val = thing[value]
+ except Exception, e:
+ pass
+ return val
+
+ def version(self, this):
+ try:
+ return this['name'].split('@')[1]
+ except Exception, e:
+ return False
+
+ def save_trees(self):
+ self.__trees = {}
+ for definition in self.__definitions:
+ if definition.get('tree') is not None:
+ self.__trees[definition['name']] = definition.get('tree')
+ with open(os.path.join(os.getcwd(), '.trees'), 'w') as f:
+ f.write(yaml.dump(self.__trees, default_flow_style=False))
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index 218bd81..ce9e51a 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -17,6 +17,7 @@
import cliapp
import contextlib
import uuid
+import os
import morphlib
@@ -26,6 +27,8 @@ class BuildPlugin(cliapp.Plugin):
def enable(self):
self.app.add_subcommand('build-morphology', self.build_morphology,
arg_synopsis='(REPO REF FILENAME)...')
+ self.app.add_subcommand('assemble', self.assemble,
+ arg_synopsis='DEFINITION')
self.app.add_subcommand('build', self.build,
arg_synopsis='SYSTEM')
self.app.add_subcommand('distbuild-morphology',
@@ -38,6 +41,12 @@ class BuildPlugin(cliapp.Plugin):
def disable(self):
self.use_distbuild = False
+ def assemble(self, args):
+ build_command = morphlib.buildcommand.BuildCommand(self.app)
+ path, target = os.path.split(args[0])
+ target = target.replace('.morph', '')
+ build_command.assemble(target)
+
def distbuild_morphology(self, args):
'''Distbuild a system, outside of a system branch.
diff --git a/morphlib/repos.py b/morphlib/repos.py
new file mode 100644
index 0000000..a2c5967
--- /dev/null
+++ b/morphlib/repos.py
@@ -0,0 +1,202 @@
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+import os
+import app
+import re
+from subprocess import call
+from subprocess import check_output
+import string
+import definitions
+import urllib2
+import json
+
+
+def get_repo_url(this):
+ url = this['repo']
+ url = url.replace('upstream:', 'git://git.baserock.org/delta/')
+ url = url.replace('baserock:baserock/',
+ 'git://git.baserock.org/baserock/baserock/')
+ url = url.replace('freedesktop:', 'git://anongit.freedesktop.org/')
+ url = url.replace('github:', 'git://github.com/')
+ url = url.replace('gnome:', 'git://git.gnome.org')
+ if url.endswith('.git'):
+ url = url[:-4]
+ return url
+
+
+def quote_url(url):
+ ''' Convert URIs to strings that only contain digits, letters, % and _.
+
+ NOTE: When changing the code of this function, make sure to also apply
+ the same to the quote_url() function of lorry. Otherwise the git tarballs
+ generated by lorry may no longer be found by morph.
+
+ '''
+ valid_chars = string.digits + string.ascii_letters + '%_'
+ transl = lambda x: x if x in valid_chars else '_'
+ return ''.join([transl(x) for x in url])
+
+
+def get_repo_name(this):
+ return quote_url(get_repo_url(this))
+# return re.split('[:/]', this['repo'])[-1]
+
+
+def get_tree(this):
+ defs = definitions.Definitions()
+
+ if defs.lookup(this, 'repo') == []:
+ return None
+
+ if defs.lookup(this, 'git') == []:
+ this['git'] = (os.path.join(app.settings['gits'],
+ get_repo_name(this)))
+
+ ref = defs.lookup(this, 'ref')
+ if not os.path.exists(this['git']):
+ try:
+ url = (app.settings['cache-server-url'] + 'repo='
+ + get_repo_url(this) + '&ref=' + ref)
+ with urllib2.urlopen(url) as response:
+ tree = json.loads(response.read().decode())['tree']
+ return tree
+ except Exception, e:
+ yapp.log(this, 'WARNING: no tree from cache-server', tree)
+ mirror(this)
+
+ with app.chdir(this['git']), open(os.devnull, "w") as fnull:
+ if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
+ stderr=fnull):
+ # can't resolve this ref. is it upstream?
+ call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
+
+ try:
+ tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
+ universal_newlines=True)[0:-1]
+ return tree
+
+ except Exception, e:
+ # either we don't have a git dir, or ref is not unique
+ # or ref does not exist
+
+ yapp.log(this, 'ERROR: could not find tree for ref', ref)
+ raise SystemExit
+
+
+def copy_repo(repo, destdir):
+ '''Copies a cached repository into a directory using cp.
+
+ This also fixes up the repository afterwards, so that it can contain
+ code etc. It does not leave any given branch ready for use.
+
+ '''
+
+ # core.bare should be false so that git believes work trees are possible
+ # we do not want the origin remote to behave as a mirror for pulls
+ # we want a traditional refs/heads -> refs/remotes/origin ref mapping
+ # set the origin url to the cached repo so that we can quickly clean up
+ # by packing the refs, we can then edit then en-masse easily
+ call(['cp', '-a', repo, os.path.join(destdir, '.git')])
+ call(['git', 'config', 'core.bare', 'false'])
+ call(['git', 'config', '--unset', 'remote.origin.mirror'])
+ with open(os.devnull, "w") as fnull:
+ call(['git', 'config', 'remote.origin.fetch',
+ '+refs/heads/*:refs/remotes/origin/*'],
+ stdout=fnull,
+ stderr=fnull)
+ call(['git', 'config', 'remote.origin.url', repo])
+ call(['git', 'pack-refs', '--all', '--prune'])
+
+ # turn refs/heads/* into refs/remotes/origin/* in the packed refs
+ # so that the new copy behaves more like a traditional clone.
+ with open(os.path.join(destdir, ".git", "packed-refs"), "r") as ref_fh:
+ pack_lines = ref_fh.read().split("\n")
+ with open(os.path.join(destdir, ".git", "packed-refs"), "w") as ref_fh:
+ ref_fh.write(pack_lines.pop(0) + "\n")
+ for refline in pack_lines:
+ if ' refs/remotes/' in refline:
+ continue
+ if ' refs/heads/' in refline:
+ sha, ref = refline[:40], refline[41:]
+ if ref.startswith("refs/heads/"):
+ ref = "refs/remotes/origin/" + ref[11:]
+ refline = "%s %s" % (sha, ref)
+ ref_fh.write("%s\n" % (refline))
+ # Finally run a remote update to clear up the refs ready for use.
+ with open(os.devnull, "w") as fnull:
+ call(['git', 'remote', 'update', 'origin', '--prune'],
+ stdout=fnull,
+ stderr=fnull)
+
+
+def mirror(this):
+ # try tarball first
+ try:
+ os.makedirs(this['git'])
+ with app.chdir(this['git']):
+ yapp.log(this, 'Try fetching tarball')
+ repo_url = get_repo_url(this)
+ tar_file = quote_url(repo_url) + '.tar'
+ tar_url = os.path.join("http://git.baserock.org/tarballs",
+ tar_file)
+ with open(os.devnull, "w") as fnull:
+ call(['wget', tar_url], stdout=fnull, stderr=fnull)
+ call(['tar', 'xf', tar_file], stdout=fnull, stderr=fnull)
+ os.remove(tar_file)
+ call(['git', 'config', 'remote.origin.url', repo_url],
+ stdout=fnull, stderr=fnull)
+ call(['git', 'config', 'remote.origin.mirror', 'true'],
+ stdout=fnull, stderr=fnull)
+ if call(['git', 'config', 'remote.origin.fetch',
+ '+refs/*:refs/*'],
+ stdout=fnull, stderr=fnull) != 0:
+ raise BaseException('Did not get a valid git repo')
+ call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
+ except Exception, e:
+ yapp.log(this, 'Using git clone', get_repo_url(this))
+ try:
+ with open(os.devnull, "w") as fnull:
+ call(['git', 'clone', '--mirror', '-n', get_repo_url(this),
+ this['git']], stdout=fnull, stderr=fnull)
+ except Exception, e:
+ yapp.log(this, 'ERROR: failed to clone', get_repo_url(this))
+ raise SystemExit
+
+ yapp.log(this, 'Git repo is mirrored at', this['git'])
+
+
+def fetch(repo):
+ with app.chdir(repo), open(os.devnull, "w") as fnull:
+ call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
+
+
+def checkout(this):
+ # checkout the required version of this from git
+ with app.chdir(this['build']):
+ yapp.log(this, 'Git checkout')
+ if not this.get('git'):
+ this['git'] = (os.path.join(app.settings['gits'],
+ get_repo_name(this)))
+ if not os.path.exists(this['git']):
+ mirror(this)
+ copy_repo(this['git'], this['build'])
+ with open(os.devnull, "w") as fnull:
+ if call(['git', 'checkout', this['ref']],
+ stdout=fnull, stderr=fnull) != 0:
+ yapp.log(this, 'ERROR: git checkout failed for', this['tree'])
+ raise SystemExit
diff --git a/morphlib/yapp.py b/morphlib/yapp.py
new file mode 100644
index 0000000..c0e41fd
--- /dev/null
+++ b/morphlib/yapp.py
@@ -0,0 +1,120 @@
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+import contextlib
+import os
+import datetime
+import shutil
+from subprocess import check_output
+from subprocess import call
+from multiprocessing import cpu_count
+settings = {}
+
+
+def log(component, message='', data=''):
+ ''' Print a timestamped log. '''
+ name = component
+ try:
+ name = component['name']
+ except Exception, e:
+ pass
+
+ timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+ log_entry = '%s [%s] %s %s\n' % (timestamp, name, message, data)
+ print(log_entry),
+
+
+def run_cmd(this, command):
+ log(this, 'Running command\n\n', command)
+ with open(settings['logfile'], "a") as logfile:
+ if call(['sh', '-c', command], stdout=logfile, stderr=logfile):
+ log(this, 'ERROR: in directory %s command failed:' % os.getcwd(),
+ command)
+ raise SystemExit
+
+
+@contextlib.contextmanager
+def setup(target, arch):
+ try:
+ settings['arch'] = arch
+ settings['no-ccache'] = True
+ settings['cache-server-url'] = \
+ 'http://git.baserock.org:8080/1.0/sha1s?'
+ settings['base'] = os.path.expanduser('~/.ybd/')
+ if os.path.exists('/src'):
+ settings['base'] = '/src'
+ settings['caches'] = os.path.join(settings['base'], 'cache')
+ settings['artifacts'] = os.path.join(settings['caches'],
+ 'ybd-artifacts')
+ settings['gits'] = os.path.join(settings['caches'], 'gits')
+ settings['staging'] = os.path.join(settings['base'], 'staging')
+ timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
+ settings['assembly'] = os.path.join(settings['staging'],
+ target + '-' + timestamp)
+ settings['max_jobs'] = max(int(cpu_count() * 1.5 + 0.5), 1)
+
+ for directory in ['base', 'caches', 'artifacts', 'gits',
+ 'staging', 'assembly']:
+ if not os.path.exists(settings[directory]):
+ os.mkdir(settings[directory])
+
+ # git replace means we can't trust that just the sha1 of a branch
+ # is enough to say what it contains, so we turn it off by setting
+ # the right flag in an environment variable.
+ os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'
+
+
+ yield
+
+ finally:
+ # assuming success, we can remove the 'assembly' directory
+ # shutil.rmtree(settings['assembly'])
+ log(target, 'Assembly directory is still at', settings['assembly'])
+
+
+@contextlib.contextmanager
+def chdir(dirname=None, env={}):
+ currentdir = os.getcwd()
+ currentenv = {}
+ try:
+ if dirname is not None:
+ os.chdir(dirname)
+ for key, value in env.items():
+ currentenv[key] = os.environ.get(key)
+ os.environ[key] = value
+ yield
+ finally:
+ for key, value in currentenv.items():
+ if value:
+ os.environ[key] = value
+ else:
+ del os.environ[key]
+ os.chdir(currentdir)
+
+
+@contextlib.contextmanager
+def timer(this, start_message=''):
+ starttime = datetime.datetime.now()
+ log(this, start_message)
+ try:
+ yield
+ finally:
+ td = datetime.datetime.now() - starttime
+ hours, remainder = divmod(int(td.total_seconds()), 60*60)
+ minutes, seconds = divmod(remainder, 60)
+ td_string = "%02d:%02d:%02d" % (hours, minutes, seconds)
+ log(this, 'Elapsed time', td_string)