WIP hack of ybd into new morph assemble commandbaserock/ps/wip-ybd-hack

9 files changed, 577 insertions, 0 deletions

diff --git a/.coverage b/.coverage
new file mode 100644
index 00000000..f2b0d15d
--- /dev/null
+++ b/.coverage
@@ -0,0 +1 @@
+€}q
(U	collectorqUcoverage v3.5.2b1qUlinesq}u.
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index f3d74a9a..55787803 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
+._*
 *.pyc
 *~
diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index 8319430c..ef9f524d 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -91,4 +91,9 @@ import yamlparse
 
 import writeexts
 
+import cache
+import definitions
+import repos
+import yapp
+
 import app  # this needs to be last
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 527163f6..bcc7f5e8 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -50,6 +50,31 @@ class BuildCommand(object):
         self.lac, self.rac = self.new_artifact_caches()
         self.lrc, self.rrc = self.new_repo_caches()
 
+    def assemble(self, target):
+        defs = morphlib.definitions.Definitions()
+        this = defs.get(target)
+        if defs.lookup(this, 'repo') != [] and defs.lookup(this, 'tree') == []:
+            this['tree'] = repos.get_tree(this)
+
+        if morphlib.cache.get_cache(this):
+            yapp.log(this, 'Cache found', morphlib.cache.get_cache(this))
+            return
+
+        with yapp.timer(this, 'Starting assembly'):
+            build_env = BuildEnvironment(yapp.settings)
+            stage = StagingArea(this, build_env)
+            for dependency in defs.lookup(this, 'build-depends'):
+                assemble(defs.get(dependency))
+                stage.install_artifact(dependency)
+
+            # if we're distbuilding, wait here for all dependencies to complete
+            # how do we know when that happens?
+
+            for component in defs.lookup(this, 'contents'):
+                assemble(defs.get(component))
+
+            build(this)
+
     def build(self, repo_name, ref, filename, original_ref=None):
         '''Build a given system morphology.'''
 
diff --git a/morphlib/cache.py b/morphlib/cache.py
new file mode 100644
index 00000000..ed3630e0
--- /dev/null
+++ b/morphlib/cache.py
@@ -0,0 +1,81 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+import os
+import shutil
+import yapp
+import re
+import hashlib
+import json
+import definitions
+import repos
+
+
+def cache_key(this):
+    defs = definitions.Definitions()
+    definition = defs.get(this)
+
+    if defs.lookup(definition, 'tree') == []:
+        definition['tree'] = repos.get_tree(definition)
+
+    if defs.lookup(definition, 'cache') != []:
+        return definition['cache']
+
+    hash_factors = { 'arch': app.settings['arch'] }
+
+    for factor in ['build-depends', 'contents']:
+        for it in defs.lookup(definition, factor):
+            component = defs.get(it)
+
+            if definition['name'] == component['name']:
+                yapp.log(this, 'ERROR: recursion loop')
+                raise SystemExit
+
+            hash_factors[component['name']] = cache_key(component)
+
+    for factor in ['tree', 'configure-commands', 'build-commands',
+                   'install-commands']:
+
+        if defs.lookup(definition, factor) != []:
+            hash_factors[factor] = definition[factor]
+
+    result = json.dumps(hash_factors, sort_keys=True).encode('utf-8')
+
+    safename = definition['name'].replace('/', '-')
+    definition['cache'] = safename + "@" + hashlib.sha256(result).hexdigest()
+    yapp.log(definition, 'Cache_key is', definition['cache'])
+    return definition['cache']
+
+
+def cache(this):
+    cachefile = os.path.join(yapp.settings['artifacts'],
+                             cache_key(this))
+
+    shutil.make_archive(cachefile, 'gztar', this['install'])
+    yapp.log(this, 'Now cached as', cache_key(this))
+
+
+def get_cache(this):
+    ''' Check if a cached artifact exists for the hashed version of this. '''
+
+    cachefile = os.path.join(yapp.settings['artifacts'],
+                             cache_key(this) + '.tar.gz')
+
+    if os.path.exists(cachefile):
+        return cachefile
+
+    return False
diff --git a/morphlib/definitions.py b/morphlib/definitions.py
new file mode 100644
index 00000000..09838236
--- /dev/null
+++ b/morphlib/definitions.py
@@ -0,0 +1,133 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+import yaml
+import os
+import yapp
+import cache
+from subprocess import check_output
+
+
+class Definitions():
+    __definitions = []
+    __trees = {}
+
+    def __init__(self):
+        ''' Load all definitions from `cwd` tree. '''
+        if self.__definitions != []:
+            return
+
+        for dirname, dirnames, filenames in os.walk(os.getcwd()):
+            for filename in filenames:
+                if not (filename.endswith('.def') or
+                        filename.endswith('.morph')):
+                    continue
+
+                this = self._load(dirname, filename)
+                name = self.lookup(this, 'name')
+                if this.get('name'):
+                    self._insert(this)
+
+                    for dependency in self.lookup(this, 'build-depends'):
+                        if self.lookup(dependency, 'repo') != []:
+                            self._insert(dependency)
+
+                    for component in (self.lookup(this, 'contents') +
+                                      self.lookup(this, 'chunks') +
+                                      self.lookup(this, 'strata')):
+                        component['build-depends'] = (
+                            self.lookup(component, 'build-depends'))
+                        for dependency in self.lookup(this, 'build-depends'):
+                            component['build-depends'].extend(
+                                [self.lookup(dependency, 'name')])
+                        self._insert(component)
+
+            if '.git' in dirnames:
+                dirnames.remove('.git')
+
+        try:
+            self.__trees = self._load(os.getcwd(), ".trees")
+            for definition in self.__definitions:
+                definition['tree'] = self.__trees.get(definition['name'])
+
+        except Exception, e:
+            return
+
+    def _load(self, path, name):
+        ''' Load a single definition file, and create a hash for it. '''
+        try:
+            filename = os.path.join(path, name)
+            with open(filename) as f:
+                text = f.read()
+
+            definition = yaml.safe_load(text)
+        except ValueError:
+            flog(this, 'ERROR: problem loading', filename)
+
+        return definition
+
+    def _insert(self, this):
+        for i, definition in enumerate(self.__definitions):
+            if definition['name'] == this['name']:
+                if (self.lookup(definition, 'ref') == []
+                        or self.lookup(this, 'ref') == []):
+                    for key in this:
+                        definition[key] = this[key]
+
+                    return
+
+                for key in this:
+                    if key == 'build-depends' or key == 'morph':
+                        continue
+                    if definition.get(key) != this[key]:
+                        yapp.log(this, 'WARNING: multiple definitions of', key)
+                        yapp.log(this, '%s | %s' % (definition.get(key),
+                                                    this[key]))
+
+        self.__definitions.append(this)
+
+    def get(self, this):
+        for definition in self.__definitions:
+            if (definition['name'] == this or
+                    definition['name'] == self.lookup(this, 'name')):
+                return definition
+
+        yapp.log(this, 'ERROR: no definition found for', this)
+        raise SystemExit
+
+    def lookup(self, thing, value):
+        ''' Look up value from thing, return [] if none. '''
+        val = []
+        try:
+            val = thing[value]
+        except Exception, e:
+            pass
+        return val
+
+    def version(self, this):
+        try:
+            return this['name'].split('@')[1]
+        except Exception, e:
+            return False
+
+    def save_trees(self):
+        self.__trees = {}
+        for definition in self.__definitions:
+            if definition.get('tree') is not None:
+                self.__trees[definition['name']] = definition.get('tree')
+        with open(os.path.join(os.getcwd(), '.trees'), 'w') as f:
+            f.write(yaml.dump(self.__trees, default_flow_style=False))
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index 218bd819..ce9e51ad 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -17,6 +17,7 @@
 import cliapp
 import contextlib
 import uuid
+import os
 
 import morphlib
 
@@ -26,6 +27,8 @@ class BuildPlugin(cliapp.Plugin):
     def enable(self):
         self.app.add_subcommand('build-morphology', self.build_morphology,
                                 arg_synopsis='(REPO REF FILENAME)...')
+        self.app.add_subcommand('assemble', self.assemble,
+                                arg_synopsis='DEFINITION')
         self.app.add_subcommand('build', self.build,
                                 arg_synopsis='SYSTEM')
         self.app.add_subcommand('distbuild-morphology',
@@ -38,6 +41,12 @@ class BuildPlugin(cliapp.Plugin):
     def disable(self):
         self.use_distbuild = False
 
+    def assemble(self, args):
+        build_command = morphlib.buildcommand.BuildCommand(self.app)
+        path, target = os.path.split(args[0])
+        target = target.replace('.morph', '')
+        build_command.assemble(target)
+
     def distbuild_morphology(self, args):
         '''Distbuild a system, outside of a system branch.
 
diff --git a/morphlib/repos.py b/morphlib/repos.py
new file mode 100644
index 00000000..a2c5967b
--- /dev/null
+++ b/morphlib/repos.py
@@ -0,0 +1,202 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+import os
+import app
+import re
+from subprocess import call
+from subprocess import check_output
+import string
+import definitions
+import urllib2
+import json
+
+
+def get_repo_url(this):
+    url = this['repo']
+    url = url.replace('upstream:', 'git://git.baserock.org/delta/')
+    url = url.replace('baserock:baserock/',
+                      'git://git.baserock.org/baserock/baserock/')
+    url = url.replace('freedesktop:', 'git://anongit.freedesktop.org/')
+    url = url.replace('github:', 'git://github.com/')
+    url = url.replace('gnome:', 'git://git.gnome.org')
+    if url.endswith('.git'):
+        url = url[:-4]
+    return url
+
+
+def quote_url(url):
+    ''' Convert URIs to strings that only contain digits, letters, % and _.
+
+    NOTE: When changing the code of this function, make sure to also apply
+    the same to the quote_url() function of lorry. Otherwise the git tarballs
+    generated by lorry may no longer be found by morph.
+
+    '''
+    valid_chars = string.digits + string.ascii_letters + '%_'
+    transl = lambda x: x if x in valid_chars else '_'
+    return ''.join([transl(x) for x in url])
+
+
+def get_repo_name(this):
+    return quote_url(get_repo_url(this))
+#    return re.split('[:/]', this['repo'])[-1]
+
+
+def get_tree(this):
+    defs = definitions.Definitions()
+
+    if defs.lookup(this, 'repo') == []:
+        return None
+
+    if defs.lookup(this, 'git') == []:
+        this['git'] = (os.path.join(app.settings['gits'],
+                       get_repo_name(this)))
+
+    ref = defs.lookup(this, 'ref')
+    if not os.path.exists(this['git']):
+        try:
+            url = (app.settings['cache-server-url'] + 'repo='
+                   + get_repo_url(this) + '&ref=' + ref)
+            with urllib2.urlopen(url) as response:
+                tree = json.loads(response.read().decode())['tree']
+                return tree
+        except Exception, e:
+            yapp.log(this, 'WARNING: no tree from cache-server', tree)
+            mirror(this)
+
+    with app.chdir(this['git']), open(os.devnull, "w") as fnull:
+        if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
+                stderr=fnull):
+            # can't resolve this ref. is it upstream?
+            call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
+
+        try:
+            tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
+                                universal_newlines=True)[0:-1]
+            return tree
+
+        except Exception, e:
+            # either we don't have a git dir, or ref is not unique
+            # or ref does not exist
+
+            yapp.log(this, 'ERROR: could not find tree for ref', ref)
+            raise SystemExit
+
+
+def copy_repo(repo, destdir):
+    '''Copies a cached repository into a directory using cp.
+
+    This also fixes up the repository afterwards, so that it can contain
+    code etc.  It does not leave any given branch ready for use.
+
+    '''
+
+    # core.bare should be false so that git believes work trees are possible
+    # we do not want the origin remote to behave as a mirror for pulls
+    # we want a traditional refs/heads -> refs/remotes/origin ref mapping
+    # set the origin url to the cached repo so that we can quickly clean up
+    # by packing the refs, we can then edit then en-masse easily
+    call(['cp', '-a', repo, os.path.join(destdir, '.git')])
+    call(['git', 'config', 'core.bare', 'false'])
+    call(['git', 'config', '--unset', 'remote.origin.mirror'])
+    with open(os.devnull, "w") as fnull:
+        call(['git', 'config', 'remote.origin.fetch',
+              '+refs/heads/*:refs/remotes/origin/*'],
+             stdout=fnull,
+             stderr=fnull)
+    call(['git',  'config', 'remote.origin.url', repo])
+    call(['git',  'pack-refs', '--all', '--prune'])
+
+    # turn refs/heads/* into refs/remotes/origin/* in the packed refs
+    # so that the new copy behaves more like a traditional clone.
+    with open(os.path.join(destdir, ".git", "packed-refs"), "r") as ref_fh:
+        pack_lines = ref_fh.read().split("\n")
+    with open(os.path.join(destdir, ".git", "packed-refs"), "w") as ref_fh:
+        ref_fh.write(pack_lines.pop(0) + "\n")
+        for refline in pack_lines:
+            if ' refs/remotes/' in refline:
+                continue
+            if ' refs/heads/' in refline:
+                sha, ref = refline[:40], refline[41:]
+                if ref.startswith("refs/heads/"):
+                    ref = "refs/remotes/origin/" + ref[11:]
+                refline = "%s %s" % (sha, ref)
+            ref_fh.write("%s\n" % (refline))
+    # Finally run a remote update to clear up the refs ready for use.
+    with open(os.devnull, "w") as fnull:
+        call(['git', 'remote', 'update', 'origin', '--prune'],
+             stdout=fnull,
+             stderr=fnull)
+
+
+def mirror(this):
+    # try tarball first
+    try:
+        os.makedirs(this['git'])
+        with app.chdir(this['git']):
+            yapp.log(this, 'Try fetching tarball')
+            repo_url = get_repo_url(this)
+            tar_file = quote_url(repo_url) + '.tar'
+            tar_url = os.path.join("http://git.baserock.org/tarballs",
+                                   tar_file)
+            with open(os.devnull, "w") as fnull:
+                call(['wget', tar_url], stdout=fnull, stderr=fnull)
+                call(['tar', 'xf', tar_file], stdout=fnull, stderr=fnull)
+                os.remove(tar_file)
+                call(['git', 'config', 'remote.origin.url', repo_url],
+                     stdout=fnull, stderr=fnull)
+                call(['git', 'config', 'remote.origin.mirror', 'true'],
+                     stdout=fnull, stderr=fnull)
+                if call(['git', 'config', 'remote.origin.fetch',
+                         '+refs/*:refs/*'],
+                        stdout=fnull, stderr=fnull) != 0:
+                    raise BaseException('Did not get a valid git repo')
+                call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
+    except Exception, e:
+        yapp.log(this, 'Using git clone', get_repo_url(this))
+        try:
+            with open(os.devnull, "w") as fnull:
+                call(['git', 'clone', '--mirror', '-n', get_repo_url(this),
+                      this['git']], stdout=fnull, stderr=fnull)
+        except Exception, e:
+            yapp.log(this, 'ERROR: failed to clone', get_repo_url(this))
+            raise SystemExit
+
+    yapp.log(this, 'Git repo is mirrored at', this['git'])
+
+
+def fetch(repo):
+    with app.chdir(repo), open(os.devnull, "w") as fnull:
+        call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
+
+
+def checkout(this):
+    # checkout the required version of this from git
+    with app.chdir(this['build']):
+        yapp.log(this, 'Git checkout')
+        if not this.get('git'):
+            this['git'] = (os.path.join(app.settings['gits'],
+                           get_repo_name(this)))
+        if not os.path.exists(this['git']):
+            mirror(this)
+        copy_repo(this['git'], this['build'])
+        with open(os.devnull, "w") as fnull:
+            if call(['git', 'checkout', this['ref']],
+                    stdout=fnull, stderr=fnull) != 0:
+                yapp.log(this, 'ERROR: git checkout failed for', this['tree'])
+                raise SystemExit
diff --git a/morphlib/yapp.py b/morphlib/yapp.py
new file mode 100644
index 00000000..c0e41fdb
--- /dev/null
+++ b/morphlib/yapp.py
@@ -0,0 +1,120 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+import contextlib
+import os
+import datetime
+import shutil
+from subprocess import check_output
+from subprocess import call
+from multiprocessing import cpu_count
+settings = {}
+
+
+def log(component, message='', data=''):
+    ''' Print a timestamped log. '''
+    name = component
+    try:
+        name = component['name']
+    except Exception, e:
+        pass
+
+    timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    log_entry = '%s [%s] %s %s\n' % (timestamp, name, message, data)
+    print(log_entry),
+
+
+def run_cmd(this, command):
+    log(this, 'Running command\n\n', command)
+    with open(settings['logfile'], "a") as logfile:
+        if call(['sh', '-c', command], stdout=logfile, stderr=logfile):
+            log(this, 'ERROR: in directory %s command failed:' % os.getcwd(),
+                command)
+            raise SystemExit
+
+
+@contextlib.contextmanager
+def setup(target, arch):
+    try:
+        settings['arch'] = arch
+        settings['no-ccache'] = True
+        settings['cache-server-url'] = \
+            'http://git.baserock.org:8080/1.0/sha1s?'
+        settings['base'] = os.path.expanduser('~/.ybd/')
+        if os.path.exists('/src'):
+            settings['base'] = '/src'
+        settings['caches'] = os.path.join(settings['base'], 'cache')
+        settings['artifacts'] = os.path.join(settings['caches'],
+                                             'ybd-artifacts')
+        settings['gits'] = os.path.join(settings['caches'], 'gits')
+        settings['staging'] = os.path.join(settings['base'], 'staging')
+        timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
+        settings['assembly'] = os.path.join(settings['staging'],
+                                            target + '-' + timestamp)
+        settings['max_jobs'] = max(int(cpu_count() * 1.5 + 0.5), 1)
+
+        for directory in ['base', 'caches', 'artifacts', 'gits',
+                          'staging', 'assembly']:
+            if not os.path.exists(settings[directory]):
+                os.mkdir(settings[directory])
+
+        # git replace means we can't trust that just the sha1 of a branch
+        # is enough to say what it contains, so we turn it off by setting
+        # the right flag in an environment variable.
+        os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'
+
+
+        yield
+
+    finally:
+        # assuming success, we can remove the 'assembly' directory
+        # shutil.rmtree(settings['assembly'])
+        log(target, 'Assembly directory is still at', settings['assembly'])
+
+
+@contextlib.contextmanager
+def chdir(dirname=None, env={}):
+    currentdir = os.getcwd()
+    currentenv = {}
+    try:
+        if dirname is not None:
+            os.chdir(dirname)
+        for key, value in env.items():
+            currentenv[key] = os.environ.get(key)
+            os.environ[key] = value
+        yield
+    finally:
+        for key, value in currentenv.items():
+            if value:
+                os.environ[key] = value
+            else:
+                del os.environ[key]
+        os.chdir(currentdir)
+
+
+@contextlib.contextmanager
+def timer(this, start_message=''):
+    starttime = datetime.datetime.now()
+    log(this, start_message)
+    try:
+        yield
+    finally:
+        td = datetime.datetime.now() - starttime
+        hours, remainder = divmod(int(td.total_seconds()), 60*60)
+        minutes, seconds = divmod(remainder, 60)
+        td_string = "%02d:%02d:%02d" % (hours, minutes, seconds)
+        log(this, 'Elapsed time', td_string)