From 9ba282863b41bea8d9fe990a9aba92677e2f4501 Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Tue, 17 Apr 2012 15:29:51 +0100 Subject: Initial import implementing /files and /sha1s. --- README | 21 ++++++++++ morph-cache-server | 83 ++++++++++++++++++++++++++++++++++++++ morphcacheserver/__init__.py | 17 ++++++++ morphcacheserver/repocache.py | 92 +++++++++++++++++++++++++++++++++++++++++++ setup.py | 46 ++++++++++++++++++++++ 5 files changed, 259 insertions(+) create mode 100644 README create mode 100755 morph-cache-server create mode 100644 morphcacheserver/__init__.py create mode 100644 morphcacheserver/repocache.py create mode 100644 setup.py diff --git a/README b/README new file mode 100644 index 00000000..a57e9a06 --- /dev/null +++ b/README @@ -0,0 +1,21 @@ +README for morph-cache-server +============================= + +Legalese +-------- + +Copyright (C) 2012 Codethink Limited + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + diff --git a/morph-cache-server b/morph-cache-server new file mode 100755 index 00000000..777e6276 --- /dev/null +++ b/morph-cache-server @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import logging +import urllib + +from bottle import Bottle, request, response, run +from morphcacheserver.repocache import RepoCache + + +defaults = { + 'repo-dir': '/var/cache/morph-cache-server/gits', +} + + +class MorphCacheServer(cliapp.Application): + + def add_settings(self): + self.settings.string(['repo-dir'], + 'path to the repository cache directory', + metavar='PATH', + default=defaults['repo-dir']) + + def process_args(self, args): + app = Bottle() + + repo_cache = RepoCache(self, self.settings['repo-dir']) + + @app.get('/sha1s') + def sha1(): + repo = self._unescape_parameter(request.query.repo) + ref = self._unescape_parameter(request.query.ref) + try: + sha1 = repo_cache.resolve_ref(repo, ref) + return { + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'sha1': '%s' % sha1 + } + except Exception, e: + response.status = 404 + logging.debug('%s' % e) + + @app.get('/files') + def file(): + repo = self._unescape_parameter(request.query.repo) + ref = self._unescape_parameter(request.query.ref) + filename = self._unescape_parameter(request.query.filename) + try: + content = repo_cache.cat_file(repo, ref, filename) + response.set_header('Content-Type', 'application/octet-stream') + return content + except Exception, e: + response.status = 404 + logging.debug('%s' % e) + + root = Bottle() + root.mount(app, '/1.0') + + run(root, host='0.0.0.0', port=8080, reloader=True) + + def _unescape_parameter(self, param): + return urllib.unquote(param) + + +if __name__ == '__main__': + MorphCacheServer().run() diff --git a/morphcacheserver/__init__.py b/morphcacheserver/__init__.py new file mode 100644 index 00000000..9ad5a305 --- /dev/null +++ b/morphcacheserver/__init__.py @@ -0,0 +1,17 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import repocache diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py new file mode 100644 index 00000000..49b82001 --- /dev/null +++ b/morphcacheserver/repocache.py @@ -0,0 +1,92 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import os +import string + + +class InvalidReferenceError(cliapp.AppException): + + def __init__(self, repo, ref): + cliapp.AppException.__init__( + self, 'Ref %s is an invalid reference for repo %s' % + (ref, repo)) + + +class UnresolvedNamedReferenceError(cliapp.AppException): + + def __init__(self, repo, ref): + cliapp.AppException.__init__( + self, 'Ref %s is not a SHA1 ref for repo %s' % + (ref, repo)) + + +class RepoCache(object): + + def __init__(self, app, dirname): + self.app = app + self.dirname = dirname + + def resolve_ref(self, repo_url, ref): + quoted_url = self._quote_url(repo_url) + repo_dir = os.path.join(self.dirname, quoted_url) + try: + refs = self._show_ref(repo_dir, ref).split('\n') + refs = [x.split() for x in refs if 'origin' in x] + return refs[0][0] + except cliapp.AppException: + pass + if not self._is_valid_sha1(ref): + raise InvalidReferenceError(repo_url, ref) + try: + return self._rev_list(ref).strip() + except: + raise InvalidReferenceError(repo_url, ref) + + def cat_file(self, repo_url, ref, filename): + quoted_url = self._quote_url(repo_url) + repo_dir = os.path.join(self.dirname, quoted_url) + + if not self._is_valid_sha1(ref): + raise UnresolvedNamedReferenceError(repo_url, ref) + try: + sha1 = self._rev_list(repo_dir, ref).strip() + except: + raise InvalidReferenceError(repo_url, ref) + + return self._cat_file(repo_dir, sha1, filename) + + def _quote_url(self, url): + valid_chars = string.digits + string.letters + '%_' + transl = lambda x: x if x in valid_chars else '_' + return ''.join([transl(x) for x in url]) + + def _show_ref(self, repo_dir, ref): + return self.app.runcmd(['git', 'show-ref', ref], cwd=repo_dir) + + def _rev_list(self, repo_dir, ref): + return self.app.runcmd( + ['git', 'rev-list', '--no-walk', ref], cwd=repo_dir) + + def _cat_file(self, repo_dir, sha1, filename): + return self.app.runcmd( + ['git', 'cat-file', 'blob', '%s:%s' % (sha1, filename)], + cwd=repo_dir) + + def _is_valid_sha1(self, ref): + valid_chars = 'abcdefABCDEF0123456789' + return len(ref) == 40 and all([x in valid_chars for x in ref]) diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..e861f392 --- /dev/null +++ b/setup.py @@ -0,0 +1,46 @@ +#!/usr/bin/python +# +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +from distutils.core import setup + + +setup(name='morph-cache-server', + description='FIXME', + long_description='''\ +FIXME +''', + classifiers=[ + 'Development Status :: 2 - Pre-Alpha', + 'Environment :: Console', + 'Environment :: Web Environment', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Topic :: Software Development :: Build Tools', + 'Topic :: Software Development :: Embedded Systems', + 'Topic :: System :: Archiving :: Packaging', + 'Topic :: System :: Software Distribution', + ], + author='Jannis Pohlmann', + author_email='jannis.pohlmann@codethink.co.uk', + url='http://www.baserock.org/', + scripts=['morph-cache-server'], + packages=['morphcacheserver'], + ) -- cgit v1.2.1 From 63927c35611bf56a1fce03750e953ec5250fb282 Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Wed, 18 Apr 2012 13:55:32 +0100 Subject: Raise a RepositoryNotFoundError if a repo does not exist in the cache. --- morph-cache-server | 2 +- morphcacheserver/repocache.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index 777e6276..b13241ba 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -41,7 +41,7 @@ class MorphCacheServer(cliapp.Application): app = Bottle() repo_cache = RepoCache(self, self.settings['repo-dir']) - + @app.get('/sha1s') def sha1(): repo = self._unescape_parameter(request.query.repo) diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index 49b82001..1b6862cb 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -19,6 +19,13 @@ import os import string +class RepositoryNotFoundError(cliapp.AppException): + + def __init__(self, repo): + cliapp.AppException.__init__( + self, 'Repository %s does not exist in the cache' % repo) + + class InvalidReferenceError(cliapp.AppException): def __init__(self, repo, ref): @@ -44,6 +51,8 @@ class RepoCache(object): def resolve_ref(self, repo_url, ref): quoted_url = self._quote_url(repo_url) repo_dir = os.path.join(self.dirname, quoted_url) + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) try: refs = self._show_ref(repo_dir, ref).split('\n') refs = [x.split() for x in refs if 'origin' in x] @@ -60,9 +69,10 @@ class RepoCache(object): def cat_file(self, repo_url, ref, filename): quoted_url = self._quote_url(repo_url) repo_dir = os.path.join(self.dirname, quoted_url) - if not self._is_valid_sha1(ref): raise UnresolvedNamedReferenceError(repo_url, ref) + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) try: sha1 = self._rev_list(repo_dir, ref).strip() except: -- cgit v1.2.1 From 8f3b7be2e1130a94cf4de5e9e1b5a18d382efff8 Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Wed, 18 Apr 2012 13:57:36 +0100 Subject: Use "Cache-Control: no-cache" to avoid caching of /sha1s results. Resolving a ref may result in a different SHA1 between any two requests, so we simply should never allow the results to be cached by an HTTP cache. --- morph-cache-server | 1 + 1 file changed, 1 insertion(+) diff --git a/morph-cache-server b/morph-cache-server index b13241ba..60c31053 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -47,6 +47,7 @@ class MorphCacheServer(cliapp.Application): repo = self._unescape_parameter(request.query.repo) ref = self._unescape_parameter(request.query.ref) try: + response.set_header('Cache-Control', 'no-cache') sha1 = repo_cache.resolve_ref(repo, ref) return { 'repo': '%s' % repo, -- cgit v1.2.1 From 6b762c363224860833ea20c9ba8109c0c210d419 Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Wed, 18 Apr 2012 17:14:11 +0100 Subject: Add untested support for bundles. --- morph-cache-server | 18 ++++++++++++++++-- morphcacheserver/repocache.py | 13 +++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index 60c31053..5554481c 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -20,12 +20,14 @@ import cliapp import logging import urllib -from bottle import Bottle, request, response, run +from bottle import Bottle, request, response, run, static_file + from morphcacheserver.repocache import RepoCache defaults = { 'repo-dir': '/var/cache/morph-cache-server/gits', + 'bundle-dir': '/var/cache/morph-cache-server/bundles', } @@ -36,11 +38,17 @@ class MorphCacheServer(cliapp.Application): 'path to the repository cache directory', metavar='PATH', default=defaults['repo-dir']) + self.settings.string(['bundle-dir'], + 'path to the bundle cache directory', + metavar='PATH', + default=defaults['bundle-dir']) def process_args(self, args): app = Bottle() - repo_cache = RepoCache(self, self.settings['repo-dir']) + repo_cache = RepoCache(self, + self.settings['repo-dir'], + self.settings['bundles']) @app.get('/sha1s') def sha1(): @@ -70,6 +78,12 @@ class MorphCacheServer(cliapp.Application): except Exception, e: response.status = 404 logging.debug('%s' % e) + + @app.get('/bundles') + def bundle(): + repo = self._unescape_parameter(request.query.repo) + filename = repo_cache.get_bundle_filename(repo) + return static_file(filename, download=True) root = Bottle() root.mount(app, '/1.0') diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index 1b6862cb..e6ab6401 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -44,13 +44,14 @@ class UnresolvedNamedReferenceError(cliapp.AppException): class RepoCache(object): - def __init__(self, app, dirname): + def __init__(self, app, repo_cache_dir, bundle_cache_dir): self.app = app - self.dirname = dirname + self.repo_cache_dir = repo_cache_dir + self.bundle_cache_dir = bundle_cache_dir def resolve_ref(self, repo_url, ref): quoted_url = self._quote_url(repo_url) - repo_dir = os.path.join(self.dirname, quoted_url) + repo_dir = os.path.join(self.repo_cache_dir, quoted_url) if not os.path.exists(repo_dir): raise RepositoryNotFoundError(repo_url) try: @@ -68,7 +69,7 @@ class RepoCache(object): def cat_file(self, repo_url, ref, filename): quoted_url = self._quote_url(repo_url) - repo_dir = os.path.join(self.dirname, quoted_url) + repo_dir = os.path.join(self.repo_cache_dir, quoted_url) if not self._is_valid_sha1(ref): raise UnresolvedNamedReferenceError(repo_url, ref) if not os.path.exists(repo_dir): @@ -79,6 +80,10 @@ class RepoCache(object): raise InvalidReferenceError(repo_url, ref) return self._cat_file(repo_dir, sha1, filename) + + def get_bundle_filename(self, repo_url): + quoted_url = self._quote_url(repo_url) + return os.path.join(self.bundle_dir, '%s.bndl' % quoted_url) def _quote_url(self, url): valid_chars = string.digits + string.letters + '%_' -- cgit v1.2.1 From a58f373cec3b3bd4420478c99dde6e7fa2cba31b Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Wed, 18 Apr 2012 17:36:00 +0100 Subject: Fix various small issues preventing bundles from working. --- morph-cache-server | 7 +++++-- morphcacheserver/repocache.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index 5554481c..57713e25 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -18,6 +18,7 @@ import cliapp import logging +import os import urllib from bottle import Bottle, request, response, run, static_file @@ -48,7 +49,7 @@ class MorphCacheServer(cliapp.Application): repo_cache = RepoCache(self, self.settings['repo-dir'], - self.settings['bundles']) + self.settings['bundle-dir']) @app.get('/sha1s') def sha1(): @@ -83,7 +84,9 @@ class MorphCacheServer(cliapp.Application): def bundle(): repo = self._unescape_parameter(request.query.repo) filename = repo_cache.get_bundle_filename(repo) - return static_file(filename, download=True) + dirname = os.path.dirname(filename) + basename = os.path.basename(filename) + return static_file(basename, root=dirname, download=True) root = Bottle() root.mount(app, '/1.0') diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index e6ab6401..3bb348ff 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -83,7 +83,7 @@ class RepoCache(object): def get_bundle_filename(self, repo_url): quoted_url = self._quote_url(repo_url) - return os.path.join(self.bundle_dir, '%s.bndl' % quoted_url) + return os.path.join(self.bundle_cache_dir, '%s.bndl' % quoted_url) def _quote_url(self, url): valid_chars = string.digits + string.letters + '%_' -- cgit v1.2.1 From d751e8da557741eb21d0856293e80d57474721b7 Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Wed, 18 Apr 2012 18:00:30 +0100 Subject: Add support for /artifacts. --- morph-cache-server | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/morph-cache-server b/morph-cache-server index 57713e25..4639a33a 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -29,6 +29,7 @@ from morphcacheserver.repocache import RepoCache defaults = { 'repo-dir': '/var/cache/morph-cache-server/gits', 'bundle-dir': '/var/cache/morph-cache-server/bundles', + 'artifact-dir': '/var/cache/morph-cache-server/artifacts', } @@ -43,6 +44,10 @@ class MorphCacheServer(cliapp.Application): 'path to the bundle cache directory', metavar='PATH', default=defaults['bundle-dir']) + self.settings.string(['artifact-dir'], + 'path to the artifact cache directory', + metavar='PATH', + default=defaults['artifact-dir']) def process_args(self, args): app = Bottle() @@ -87,6 +92,18 @@ class MorphCacheServer(cliapp.Application): dirname = os.path.dirname(filename) basename = os.path.basename(filename) return static_file(basename, root=dirname, download=True) + + @app.get('/artifacts') + def artifact(): + basename = self._unescape_parameter(request.query.filename) + filename = os.path.join(self.settings['artifact-dir'], basename) + if os.path.exists(filename): + return static_file(basename, + root=self.settings['artifact-dir'], + download=True) + else: + response.status = 404 + logging.debug('artifact %s does not exist' % basename) root = Bottle() root.mount(app, '/1.0') -- cgit v1.2.1 From f1fba299bd07510346082ef985ef08a494dca9d9 Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Wed, 18 Apr 2012 18:02:44 +0100 Subject: Use the desired artifact filename as the download filename. --- morph-cache-server | 1 + 1 file changed, 1 insertion(+) diff --git a/morph-cache-server b/morph-cache-server index 4639a33a..7618f5b9 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -100,6 +100,7 @@ class MorphCacheServer(cliapp.Application): if os.path.exists(filename): return static_file(basename, root=self.settings['artifact-dir'], + filename=basename, download=True) else: response.status = 404 -- cgit v1.2.1 From eb4c1530c57b6ae200643b259e6ed95904951681 Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Fri, 20 Apr 2012 15:46:53 +0100 Subject: Add /trees which serves the contents of a git tree using ls-tree. /trees queries take repo URI, a SHA1 ref and an optional path parameter. The result is a JSON dictionary of the form { "repo": "", "ref": "", "tree": { "filename1": { "mode": "100644", "kind": "blob", "sha1": "FOOBARBAZ" }, ... } } --- morph-cache-server | 16 ++++++++++++++++ morphcacheserver/repocache.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/morph-cache-server b/morph-cache-server index 7618f5b9..86a1fe26 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -85,6 +85,22 @@ class MorphCacheServer(cliapp.Application): response.status = 404 logging.debug('%s' % e) + @app.get('/trees') + def tree(): + repo = self._unescape_parameter(request.query.repo) + ref = self._unescape_parameter(request.query.ref) + path = self._unescape_parameter(request.query.path) + try: + tree = repo_cache.ls_tree(repo, ref, path) + return { + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'tree': tree, + } + except Exception, e: + response.status = 404 + logging.debug('%s' % e) + @app.get('/bundles') def bundle(): repo = self._unescape_parameter(request.query.repo) diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index 3bb348ff..7061508d 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -81,6 +81,32 @@ class RepoCache(object): return self._cat_file(repo_dir, sha1, filename) + def ls_tree(self, repo_url, ref, path): + quoted_url = self._quote_url(repo_url) + repo_dir = os.path.join(self.repo_cache_dir, quoted_url) + if not self._is_valid_sha1(ref): + raise UnresolvedNamedReferenceError(repo_url, ref) + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) + + try: + sha1 = self._rev_list(repo_dir, ref).strip() + except: + raise InvalidReferenceError(repo_url, ref) + + lines = self._ls_tree(repo_dir, sha1, path).strip() + lines = lines.splitlines() + data = {} + for line in lines: + elements = line.split() + basename = elements[3] + data[basename] = { + 'mode': elements[0], + 'kind': elements[1], + 'sha1': elements[2], + } + return data + def get_bundle_filename(self, repo_url): quoted_url = self._quote_url(repo_url) return os.path.join(self.bundle_cache_dir, '%s.bndl' % quoted_url) @@ -102,6 +128,9 @@ class RepoCache(object): ['git', 'cat-file', 'blob', '%s:%s' % (sha1, filename)], cwd=repo_dir) + def _ls_tree(self, repo_dir, sha1, path): + return self.app.runcmd(['git', 'ls-tree', sha1, path], cwd=repo_dir) + def _is_valid_sha1(self, ref): valid_chars = 'abcdefABCDEF0123456789' return len(ref) == 40 and all([x in valid_chars for x in ref]) -- cgit v1.2.1 From e9e9d759805305af540c5890e626f2b91a70a2c7 Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Tue, 1 May 2012 11:41:59 +0100 Subject: Revert "Use the desired artifact filename as the download filename." This reverts commit f1fba299bd07510346082ef985ef08a494dca9d9. --- morph-cache-server | 1 - 1 file changed, 1 deletion(-) diff --git a/morph-cache-server b/morph-cache-server index 86a1fe26..3f72c186 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -116,7 +116,6 @@ class MorphCacheServer(cliapp.Application): if os.path.exists(filename): return static_file(basename, root=self.settings['artifact-dir'], - filename=basename, download=True) else: response.status = 404 -- cgit v1.2.1 From b53860b0aa27e5c004adc45552e7fead71de09e7 Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Mon, 3 Sep 2012 15:30:10 +0100 Subject: Add a .gitignore to ignore *.pyc To reduce the noise when I run 'git status' this gitignore will mean that git won't notify me of repocache.pyc and __init__.pyc --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..0d20b648 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc -- cgit v1.2.1 From cd00de30a0f4d2d422053692948ea9986960c43f Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Tue, 4 Sep 2012 10:49:35 +0100 Subject: A direct-mode for git cache access Direct-mode, when enabled, causes morph-cache-server to assume a more Trove-like structure for the repositories, rather than the morph-cache structure which it was originally written for. This means that for the workers, we can use the original code and for Trove, the direct mode. --- morph-cache-server | 5 ++++- morphcacheserver/repocache.py | 34 ++++++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index 3f72c186..bb84915a 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -48,13 +48,16 @@ class MorphCacheServer(cliapp.Application): 'path to the artifact cache directory', metavar='PATH', default=defaults['artifact-dir']) + self.settings.boolean(['direct-mode'], + 'cache directories are directly managed') def process_args(self, args): app = Bottle() repo_cache = RepoCache(self, self.settings['repo-dir'], - self.settings['bundle-dir']) + self.settings['bundle-dir'], + self.settings['direct-mode']) @app.get('/sha1s') def sha1(): diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index 7061508d..c226ef40 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -17,6 +17,7 @@ import cliapp import os import string +import urlparse class RepositoryNotFoundError(cliapp.AppException): @@ -44,19 +45,25 @@ class UnresolvedNamedReferenceError(cliapp.AppException): class RepoCache(object): - def __init__(self, app, repo_cache_dir, bundle_cache_dir): + def __init__(self, app, repo_cache_dir, bundle_cache_dir, direct_mode): self.app = app self.repo_cache_dir = repo_cache_dir self.bundle_cache_dir = bundle_cache_dir + self.direct_mode = direct_mode def resolve_ref(self, repo_url, ref): quoted_url = self._quote_url(repo_url) repo_dir = os.path.join(self.repo_cache_dir, quoted_url) if not os.path.exists(repo_dir): - raise RepositoryNotFoundError(repo_url) + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) try: refs = self._show_ref(repo_dir, ref).split('\n') - refs = [x.split() for x in refs if 'origin' in x] + if self.direct_mode: + refs = [x.split() for x in refs] + else: + refs = [x.split() for x in refs if 'origin' in x] return refs[0][0] except cliapp.AppException: pass @@ -70,6 +77,10 @@ class RepoCache(object): def cat_file(self, repo_url, ref, filename): quoted_url = self._quote_url(repo_url) repo_dir = os.path.join(self.repo_cache_dir, quoted_url) + if not os.path.exists(repo_dir): + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) if not self._is_valid_sha1(ref): raise UnresolvedNamedReferenceError(repo_url, ref) if not os.path.exists(repo_dir): @@ -84,6 +95,10 @@ class RepoCache(object): def ls_tree(self, repo_url, ref, path): quoted_url = self._quote_url(repo_url) repo_dir = os.path.join(self.repo_cache_dir, quoted_url) + if not os.path.exists(repo_dir): + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) if not self._is_valid_sha1(ref): raise UnresolvedNamedReferenceError(repo_url, ref) if not os.path.exists(repo_dir): @@ -108,13 +123,16 @@ class RepoCache(object): return data def get_bundle_filename(self, repo_url): - quoted_url = self._quote_url(repo_url) + quoted_url = self._quote_url(repo_url, True) return os.path.join(self.bundle_cache_dir, '%s.bndl' % quoted_url) - def _quote_url(self, url): - valid_chars = string.digits + string.letters + '%_' - transl = lambda x: x if x in valid_chars else '_' - return ''.join([transl(x) for x in url]) + def _quote_url(self, url, always_indirect=False): + if self.direct_mode and not always_indirect: + return urlparse.urlparse(url)[2] + else: + valid_chars = string.digits + string.letters + '%_' + transl = lambda x: x if x in valid_chars else '_' + return ''.join([transl(x) for x in url]) def _show_ref(self, repo_dir, ref): return self.app.runcmd(['git', 'show-ref', ref], cwd=repo_dir) -- cgit v1.2.1 From 2c04007fc74d5971b12f351a4c2076e403386997 Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Wed, 5 Sep 2012 17:49:21 +0100 Subject: Return tree SHA1 when looking for ref resolution. Morph now expects the tree SHA1 in addition when resolving references using the cache server. This is to better facilitate correct cache key computation since commits can be made which have no tree changes and thus nothing to usefully affect the build. (For example the morph branch and build features) --- morph-cache-server | 5 +++-- morphcacheserver/repocache.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index bb84915a..3a121d49 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -65,11 +65,12 @@ class MorphCacheServer(cliapp.Application): ref = self._unescape_parameter(request.query.ref) try: response.set_header('Cache-Control', 'no-cache') - sha1 = repo_cache.resolve_ref(repo, ref) + sha1, tree = repo_cache.resolve_ref(repo, ref) return { 'repo': '%s' % repo, 'ref': '%s' % ref, - 'sha1': '%s' % sha1 + 'sha1': '%s' % sha1, + 'tree': '%s' % tree } except Exception, e: response.status = 404 diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index c226ef40..b55692f2 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -64,16 +64,25 @@ class RepoCache(object): refs = [x.split() for x in refs] else: refs = [x.split() for x in refs if 'origin' in x] - return refs[0][0] + return refs[0][0], self._tree_from_commit(repo_dir, refs[0][0]) + except cliapp.AppException: pass + if not self._is_valid_sha1(ref): raise InvalidReferenceError(repo_url, ref) try: - return self._rev_list(ref).strip() + sha = self._rev_list(ref).strip() + return sha, self._tree_from_commit(repo_dir, sha) except: raise InvalidReferenceError(repo_url, ref) + def _tree_from_commit(self, repo_dir, commitsha): + commit_info = self.app.runcmd(['git', 'log', '-1', + '--format=format:%T', commitsha], + cwd=repo_dir) + return commit_info.strip() + def cat_file(self, repo_url, ref, filename): quoted_url = self._quote_url(repo_url) repo_dir = os.path.join(self.repo_cache_dir, quoted_url) -- cgit v1.2.1 From c2998750dbb3d79b7455a079aa3f3d243715a15f Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 7 Sep 2012 13:26:38 +0100 Subject: Support running on a different port from 8080 In order to allow multiple morph-cache-server instances to run on a single system, we need to support running on different ports. --- morph-cache-server | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/morph-cache-server b/morph-cache-server index 3a121d49..827da10a 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -30,12 +30,17 @@ defaults = { 'repo-dir': '/var/cache/morph-cache-server/gits', 'bundle-dir': '/var/cache/morph-cache-server/bundles', 'artifact-dir': '/var/cache/morph-cache-server/artifacts', + 'port': 8080, } class MorphCacheServer(cliapp.Application): def add_settings(self): + self.settings.integer(['port'], + 'port to listen on', + metavar='PORTNUM', + default=defaults['port']) self.settings.string(['repo-dir'], 'path to the repository cache directory', metavar='PATH', @@ -128,7 +133,7 @@ class MorphCacheServer(cliapp.Application): root = Bottle() root.mount(app, '/1.0') - run(root, host='0.0.0.0', port=8080, reloader=True) + run(root, host='0.0.0.0', port=self.settings['port'], reloader=True) def _unescape_parameter(self, param): return urllib.unquote(param) -- cgit v1.2.1 From 9c3279221262057c7eb8ebdcb29f366dc4de66d5 Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 7 Sep 2012 10:14:27 +0100 Subject: Add ability to have 'writable' cache servers. Since we need to be able to update the cache from builders, this patch introduces a --enable-writes argument to morph-cache-server and also adds a @writable decorator to the class ready for marking particular paths which are only available when --enable-writes is set. --- morph-cache-server | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/morph-cache-server b/morph-cache-server index 827da10a..ba5f0b2a 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -55,6 +55,8 @@ class MorphCacheServer(cliapp.Application): default=defaults['artifact-dir']) self.settings.boolean(['direct-mode'], 'cache directories are directly managed') + self.settings.boolean(['enable-writes'], + 'enable the write methods (fetch and delete)') def process_args(self, args): app = Bottle() @@ -64,6 +66,23 @@ class MorphCacheServer(cliapp.Application): self.settings['bundle-dir'], self.settings['direct-mode']) + def writable(prefix): + """Selectively enable bottle prefixes. + + prefix -- The path prefix we are enabling + + If the runtime configuration setting --enable-writes is provided + then we return the app.get() decorator for the given path prefix + otherwise we return a lambda which passes the function through + undecorated. + + This has the effect of being a runtime-enablable @app.get(...) + + """ + if self.settings['enable-writes']: + return app.get(prefix) + return lambda fn: fn + @app.get('/sha1s') def sha1(): repo = self._unescape_parameter(request.query.repo) -- cgit v1.2.1 From 465e830d1d6d2c51425e2418b8e802a95145b6ee Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 7 Sep 2012 10:14:27 +0100 Subject: Add a /list method When --enable-writes is set, we provide a /list target which produces a JSON dictionary of information about the state of the artifact cache. The dictionary is of the form: { "freespace": NBYTES_OF_SPACE, "files": { "artifact-filename": { "atime": ATIME_AS_NUMBER, "size": NBYTES_SIZE_OF_FILE, "used": NBYTES_USED_ON_DISK }, ... } } This allows a controller to decide which artifacts have not been requested in some time and also how big artifacts are, not only in terms of their 'byte' size, but also the space they consume on disk. System images in particular may differ in this respect since they should be sparsely stored. --- morph-cache-server | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/morph-cache-server b/morph-cache-server index ba5f0b2a..b726c1e5 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -83,6 +83,28 @@ class MorphCacheServer(cliapp.Application): return app.get(prefix) return lambda fn: fn + @writable('/list') + def list(): + response.set_header('Cache-Control', 'no-cache') + results = {} + files = {} + results["files"] = files + for artifactdir, __, filenames in \ + os.walk(self.settings['artifact-dir']): + fsstinfo = os.statvfs(artifactdir) + results["freespace"] = fsstinfo.f_bsize * fsstinfo.f_bavail + for fname in filenames: + try: + stinfo = os.stat("%s/%s" % (artifactdir, fname)) + files[fname] = { + "atime": stinfo.st_atime, + "size": stinfo.st_size, + "used": stinfo.st_blocks * 512, + } + except Exception, e: + print(e) + return results + @app.get('/sha1s') def sha1(): repo = self._unescape_parameter(request.query.repo) -- cgit v1.2.1 From 1a0e40d854d37f81e9cdaf8bb23e480790614d2a Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 7 Sep 2012 13:17:13 +0100 Subject: Support for fetching artifacts to the cache Rather than pushing artifacts to the cache, this method allows the caller to specify a host and artifact which the cache server will then fetch into its local cache. It takes the following arguments: host=hostname:port artifact=artifactname This is transformed into a fetch to: http://hostname:port/artifacts?basename=artifactname Which is then fetched into the cache under the given name. The return from this is a JSON object of the form: { "filename": artifactname, "size": NBYTES_SIZE_OF_FILE, "used": NBYTES_DISK_SPACE_USED } --- morph-cache-server | 50 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index b726c1e5..286e56db 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -20,6 +20,7 @@ import cliapp import logging import os import urllib +import shutil from bottle import Bottle, request, response, run, static_file @@ -94,17 +95,48 @@ class MorphCacheServer(cliapp.Application): fsstinfo = os.statvfs(artifactdir) results["freespace"] = fsstinfo.f_bsize * fsstinfo.f_bavail for fname in filenames: - try: - stinfo = os.stat("%s/%s" % (artifactdir, fname)) - files[fname] = { - "atime": stinfo.st_atime, - "size": stinfo.st_size, - "used": stinfo.st_blocks * 512, - } - except Exception, e: - print(e) + if not fname.startswith(".dl."): + try: + stinfo = os.stat("%s/%s" % (artifactdir, fname)) + files[fname] = { + "atime": stinfo.st_atime, + "size": stinfo.st_size, + "used": stinfo.st_blocks * 512, + } + except Exception, e: + print(e) return results + @writable('/fetch') + def fetch(): + host = self._unescape_parameter(request.query.host) + artifact = self._unescape_parameter(request.query.artifact) + try: + response.set_header('Cache-Control', 'no-cache') + in_fh = urllib.urlopen("http://%s/artifacts?basename=%s" % + (host, urllib.quote(artifact))) + tmpname = "%s/.dl.%s" % ( + self.settings['artifact-dir'], + artifact) + localtmp = open(tmpname, "w") + shutil.copyfileobj(in_fh, localtmp) + localtmp.close() + in_fh.close() + artifilename = "%s/%s" % (self.settings['artifact-dir'], + artifact) + os.rename(tmpname, artifilename) + stinfo = os.stat(artifilename) + ret = {} + ret[artifact] = { + "size": stinfo.st_size, + "used": stinfo.st_blocks * 512 + } + return ret + + except Exception, e: + response.status = 500 + logging.debug('%s' % e) + @app.get('/sha1s') def sha1(): repo = self._unescape_parameter(request.query.repo) -- cgit v1.2.1 From 5870c3581ac10c14d68337fc875000ead522e99d Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 7 Sep 2012 13:23:51 +0100 Subject: Add facility to delete artifacts In order to allow the artifact cache to be cleaned up, this patch allows for a /delete method which can remove artifacts from the cache. It takes the following arguments: artifact=artifactname The artifact will be deleted and a JSON object returned in the form: { "status": errno, "reason": strerror } Where errno is zero on success, 1 on EPERM, 2 on ENOENT etc. and reason is the strerror of the errno, in case the architectures differ between caller and cache. --- morph-cache-server | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/morph-cache-server b/morph-cache-server index 286e56db..b4f8fa1a 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -137,6 +137,19 @@ class MorphCacheServer(cliapp.Application): response.status = 500 logging.debug('%s' % e) + @writable('/delete') + def delete(): + artifact = self._unescape_parameter(request.query.artifact) + try: + os.unlink('%s/%s' % (self.settings['artifact-dir'], + artifact)) + return { "status": 0, "reason": "success" } + except OSError, ose: + return { "status": ose.errno, "reason": ose.strerror } + except Exception, e: + response.status = 500 + logging.debug('%s' % e) + @app.get('/sha1s') def sha1(): repo = self._unescape_parameter(request.query.repo) -- cgit v1.2.1 From 2ee9e745d46fd01f95cf598025ae10b88d1c051f Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Mon, 17 Sep 2012 13:49:21 +0100 Subject: Update /fetch API to latest definition --- morph-cache-server | 71 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index b4f8fa1a..04a5710c 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -20,6 +20,7 @@ import cliapp import logging import os import urllib +import urllib2 import shutil from bottle import Bottle, request, response, run, static_file @@ -59,6 +60,52 @@ class MorphCacheServer(cliapp.Application): self.settings.boolean(['enable-writes'], 'enable the write methods (fetch and delete)') + def _fetch_artifact(self, url, filename): + in_fh = None + try: + in_fh = urllib2.urlopen(url) + with open(filename, "w") as localtmp: + shutil.copyfileobj(in_fh, localtmp) + in_fh.close() + except Exception, e: + if in_fh is not None: + in_fh.close() + raise + else: + if in_fh is not None: + in_fh.close() + return os.stat(filename) + + def _fetch_artifacts(self, server, cacheid, artifacts): + ret = {} + try: + for artifact in artifacts: + artifact_name = "%s.%s" % (cacheid, artifact) + tmpname = os.path.join(self.settings['artifact-dir'], + ".dl.%s" % artifact_name) + url = "http://%s/1.0/artifacts?filename=%s" % ( + server, urllib.quote(artifact_name)) + stinfo = self._fetch_artifact(url, tmpname) + ret[artifact_name] = { + "size": stinfo.st_size, + "used": stinfo.st_blocks * 512, + } + except Exception, e: + for artifact in ret.iterkeys(): + os.unlink(os.path.join(self.settings['artifact-dir'], + ".dl.%s" % artifact)) + raise + + for artifact in ret.iterkeys(): + tmpname = os.path.join(self.settings['artifact-dir'], + ".dl.%s" % artifact) + artifilename = os.path.join(self.settings['artifact-dir'], + artifact) + os.rename(tmpname, artifilename) + + return ret + + def process_args(self, args): app = Bottle() @@ -110,28 +157,12 @@ class MorphCacheServer(cliapp.Application): @writable('/fetch') def fetch(): host = self._unescape_parameter(request.query.host) - artifact = self._unescape_parameter(request.query.artifact) + cacheid = self._unescape_parameter(request.query.cacheid) + artifacts = self._unescape_parameter(request.query.artifacts) try: response.set_header('Cache-Control', 'no-cache') - in_fh = urllib.urlopen("http://%s/artifacts?basename=%s" % - (host, urllib.quote(artifact))) - tmpname = "%s/.dl.%s" % ( - self.settings['artifact-dir'], - artifact) - localtmp = open(tmpname, "w") - shutil.copyfileobj(in_fh, localtmp) - localtmp.close() - in_fh.close() - artifilename = "%s/%s" % (self.settings['artifact-dir'], - artifact) - os.rename(tmpname, artifilename) - stinfo = os.stat(artifilename) - ret = {} - ret[artifact] = { - "size": stinfo.st_size, - "used": stinfo.st_blocks * 512 - } - return ret + artifacts = artifacts.split(",") + return self._fetch_artifacts(host, cacheid, artifacts) except Exception, e: response.status = 500 -- cgit v1.2.1 From f7b8001175f492b41d68269ff5294fde347f8f4d Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Tue, 25 Sep 2012 11:53:55 +0100 Subject: Trim leading slashes from URI element during direct-mode. rs=richardmaw --- morphcacheserver/repocache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index b55692f2..b7d46c35 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -137,7 +137,10 @@ class RepoCache(object): def _quote_url(self, url, always_indirect=False): if self.direct_mode and not always_indirect: - return urlparse.urlparse(url)[2] + quoted_url = urlparse.urlparse(url)[2] + while quoted_url.startswith("/"): + quoted_url = quoted_url[1:] + return quoted_url else: valid_chars = string.digits + string.letters + '%_' transl = lambda x: x if x in valid_chars else '_' -- cgit v1.2.1 From a1232696c7e57d98cb1bc3b093cde6f8c4eff5e6 Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Mon, 3 Dec 2012 14:05:48 +0000 Subject: Fix missing argument to rev_list(), breaking resolve_ref for SHA1s This bug has been present since the initial commit to the cache server. Due to the missing repo_dir argument to rev_list(), resolving SHA1s rather than symbolic refs via /1.0/sha1s fails. This feature, however, is absolutely required for morph to resolve petrified system branches. --- morphcacheserver/repocache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index b7d46c35..9675e04e 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -72,7 +72,7 @@ class RepoCache(object): if not self._is_valid_sha1(ref): raise InvalidReferenceError(repo_url, ref) try: - sha = self._rev_list(ref).strip() + sha = self._rev_list(repo_dir, ref).strip() return sha, self._tree_from_commit(repo_dir, sha) except: raise InvalidReferenceError(repo_url, ref) -- cgit v1.2.1 From 90e9be0128cae0a93d481c0a5a1bcb9fed6e2ee2 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 13 Dec 2012 17:37:47 +0000 Subject: Use 'git rev-parse --verify' to resolve refs 'git show-ref' returns multiple results where there are partial matches for the given ref, which creates the possibility that we might resolve a ref incorrectly. 'git rev-list' is also overkill for verifying that a SHA1 is valid. --- morphcacheserver/repocache.py | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index 9675e04e..388436b0 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -59,24 +59,14 @@ class RepoCache(object): if not os.path.exists(repo_dir): raise RepositoryNotFoundError(repo_url) try: - refs = self._show_ref(repo_dir, ref).split('\n') - if self.direct_mode: - refs = [x.split() for x in refs] - else: - refs = [x.split() for x in refs if 'origin' in x] - return refs[0][0], self._tree_from_commit(repo_dir, refs[0][0]) + if not self.direct_mode and not refs.startswith('refs/origin/'): + ref = 'refs/origin/' + ref + sha1 = self._rev_parse(repo_dir, ref) + return sha1, self._tree_from_commit(repo_dir, sha1) except cliapp.AppException: pass - if not self._is_valid_sha1(ref): - raise InvalidReferenceError(repo_url, ref) - try: - sha = self._rev_list(repo_dir, ref).strip() - return sha, self._tree_from_commit(repo_dir, sha) - except: - raise InvalidReferenceError(repo_url, ref) - def _tree_from_commit(self, repo_dir, commitsha): commit_info = self.app.runcmd(['git', 'log', '-1', '--format=format:%T', commitsha], @@ -95,7 +85,7 @@ class RepoCache(object): if not os.path.exists(repo_dir): raise RepositoryNotFoundError(repo_url) try: - sha1 = self._rev_list(repo_dir, ref).strip() + sha1 = self._rev_parse(repo_dir, ref) except: raise InvalidReferenceError(repo_url, ref) @@ -114,7 +104,7 @@ class RepoCache(object): raise RepositoryNotFoundError(repo_url) try: - sha1 = self._rev_list(repo_dir, ref).strip() + sha1 = self._rev_parse(repo_dir, ref) except: raise InvalidReferenceError(repo_url, ref) @@ -146,12 +136,9 @@ class RepoCache(object): transl = lambda x: x if x in valid_chars else '_' return ''.join([transl(x) for x in url]) - def _show_ref(self, repo_dir, ref): - return self.app.runcmd(['git', 'show-ref', ref], cwd=repo_dir) - - def _rev_list(self, repo_dir, ref): - return self.app.runcmd( - ['git', 'rev-list', '--no-walk', ref], cwd=repo_dir) + def _rev_parse(self, repo_dir, ref): + return self.app.runcmd(['git', 'rev-parse', '--verify', ref], + cwd=repo_dir)[0:40] def _cat_file(self, repo_dir, sha1, filename): return self.app.runcmd( -- cgit v1.2.1 From dcb9b9ce10c918f66bc9d239e72a805847247be8 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 13 Dec 2012 17:58:51 +0000 Subject: Log exceptions instead of ignoring them --- morphcacheserver/repocache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index 388436b0..fa7a515b 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -65,7 +65,7 @@ class RepoCache(object): return sha1, self._tree_from_commit(repo_dir, sha1) except cliapp.AppException: - pass + raise def _tree_from_commit(self, repo_dir, commitsha): commit_info = self.app.runcmd(['git', 'log', '-1', -- cgit v1.2.1 From 74df327db1727a26986cdf63294ae93cf3106081 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 13 Dec 2012 19:01:59 +0000 Subject: Fix misspelled variable name --- morphcacheserver/repocache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index fa7a515b..668d8fbb 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -59,7 +59,7 @@ class RepoCache(object): if not os.path.exists(repo_dir): raise RepositoryNotFoundError(repo_url) try: - if not self.direct_mode and not refs.startswith('refs/origin/'): + if not self.direct_mode and not ref.startswith('refs/origin/'): ref = 'refs/origin/' + ref sha1 = self._rev_parse(repo_dir, ref) return sha1, self._tree_from_commit(repo_dir, sha1) -- cgit v1.2.1 From 22d1bfbc91a46134dd6c9410b86a3cb3ba250887 Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Wed, 2 Jan 2013 16:28:28 +0000 Subject: Handle batch sha1/file queries using POST requests This commit adds support for resolving multiple refs into SHA1s and requesting multiple files at once, each using a single POST request. The (repo, ref) and (repo, ref, filename) parameters are passed to the POST requests as JSON dictionaries in a list. The response to both types of requests is a JSON list with the same dictionaries again, to which a "sha1" and "data" field are added, respectively. The file contents returned by "POST /1.0/files" requests are base64-encoded and need to be decoded at the receiver's end. This is because the contents may be binary or contain quotes and therefore cause JSON syntax errors. --- morph-cache-server | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/morph-cache-server b/morph-cache-server index 04a5710c..d3e42c62 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -16,7 +16,9 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import base64 import cliapp +import json import logging import os import urllib @@ -197,6 +199,31 @@ class MorphCacheServer(cliapp.Application): except Exception, e: response.status = 404 logging.debug('%s' % e) + + @app.post('/sha1s') + def sha1s(): + result = [] + for pair in request.json: + repo = pair['repo'] + ref = pair['ref'] + try: + sha1, tree = repo_cache.resolve_ref(repo, ref) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'sha1': '%s' % sha1, + 'tree': '%s' % tree + }) + except Exception, e: + logging.debug('%s' % e) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'error': '%s' % e + }) + response.set_header('Cache-Control', 'no-cache') + response.set_header('Content-Type', 'application/json') + return json.dumps(result) @app.get('/files') def file(): @@ -211,6 +238,32 @@ class MorphCacheServer(cliapp.Application): response.status = 404 logging.debug('%s' % e) + @app.post('/files') + def files(): + result = [] + for pair in request.json: + repo = pair['repo'] + ref = pair['ref'] + filename = pair['filename'] + try: + content = repo_cache.cat_file(repo, ref, filename) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'filename': '%s' % filename, + 'data': '%s' % base64.b64encode(content), + }) + except Exception, e: + logging.debug('%s' % e) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'filename': '%s' % filename, + 'error': '%s' % e + }) + response.set_header('Content-Type', 'application/json') + return json.dumps(result) + @app.get('/trees') def tree(): repo = self._unescape_parameter(request.query.repo) -- cgit v1.2.1 From 2b94817befc8538f85f454a8e112d7ae828ecf52 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Tue, 4 Jun 2013 11:32:00 +0100 Subject: Handle requesting a sha1 of a sha1 --- morphcacheserver/repocache.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index 668d8fbb..cd2eab56 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -16,6 +16,7 @@ import cliapp import os +import re import string import urlparse @@ -59,9 +60,13 @@ class RepoCache(object): if not os.path.exists(repo_dir): raise RepositoryNotFoundError(repo_url) try: - if not self.direct_mode and not ref.startswith('refs/origin/'): - ref = 'refs/origin/' + ref - sha1 = self._rev_parse(repo_dir, ref) + if re.match('^[0-9a-fA-F]{40}$', ref): + sha1 = ref + else: + if (not self.direct_mode and + not ref.startswith('refs/origin/')): + ref = 'refs/origin/' + ref + sha1 = self._rev_parse(repo_dir, ref) return sha1, self._tree_from_commit(repo_dir, sha1) except cliapp.AppException: -- cgit v1.2.1 From 5e4ad62d4d5b8ecabf4aa01b6de918fc322b8292 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Wed, 4 Sep 2013 16:29:50 +0100 Subject: Allow to use flup as the backend server for bottle. lighttpd does not support WSGI, so we need to be able to run the server in FCGI mode. The flup package provides one. Running with the default bottle server is still useful for testing purposes. --- morph-cache-server | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index d3e42c62..1c9a53de 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -26,7 +26,7 @@ import urllib2 import shutil from bottle import Bottle, request, response, run, static_file - +from flup.server.fcgi import WSGIServer from morphcacheserver.repocache import RepoCache @@ -61,6 +61,10 @@ class MorphCacheServer(cliapp.Application): 'cache directories are directly managed') self.settings.boolean(['enable-writes'], 'enable the write methods (fetch and delete)') + self.settings.boolean(['fcgi-server'], + 'runs a fcgi-server', + default=True) + def _fetch_artifact(self, url, filename): in_fh = None @@ -302,8 +306,12 @@ class MorphCacheServer(cliapp.Application): root = Bottle() root.mount(app, '/1.0') - - run(root, host='0.0.0.0', port=self.settings['port'], reloader=True) + + + if self.settings['fcgi-server']: + WSGIServer(root).run() + else: + run(root, host='0.0.0.0', port=self.settings['port'], reloader=True) def _unescape_parameter(self, param): return urllib.unquote(param) -- cgit v1.2.1 From cc5f95fa563c4817cdcffc428da18e263bd02ae2 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Mon, 16 Dec 2013 13:54:09 +0000 Subject: Update copyright notice --- README | 21 --------------------- morph-cache-server | 2 +- morphcacheserver/__init__.py | 2 +- morphcacheserver/repocache.py | 2 +- 4 files changed, 3 insertions(+), 24 deletions(-) delete mode 100644 README diff --git a/README b/README deleted file mode 100644 index a57e9a06..00000000 --- a/README +++ /dev/null @@ -1,21 +0,0 @@ -README for morph-cache-server -============================= - -Legalese --------- - -Copyright (C) 2012 Codethink Limited - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along -with this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - diff --git a/morph-cache-server b/morph-cache-server index 1c9a53de..dbf67856 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/morphcacheserver/__init__.py b/morphcacheserver/__init__.py index 9ad5a305..2c25ce28 100644 --- a/morphcacheserver/__init__.py +++ b/morphcacheserver/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index cd2eab56..0e4d909e 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From 15b7e4fad677c127d4011babc01f7e4a71f259d8 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 3 Apr 2014 14:41:31 +0100 Subject: Add post request for /artifacts With this we can request the state of a set of artifacts in a single request. Artifacts are sent as a json array. We check whether each artifact is in the cache or not and send our findings back to the client as a json object. --- morph-cache-server | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index dbf67856..a3c3c978 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013, 2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -303,12 +303,40 @@ class MorphCacheServer(cliapp.Application): else: response.status = 404 logging.debug('artifact %s does not exist' % basename) - + + @app.post('/artifacts') + def post_artifacts(): + if request.content_type != 'application/json': + logging.warning('Content-type is not json: ' + 'expecting a json post request') + + artifacts = json.load(request.body) + results = {} + + logging.debug('Received a POST request for /artifacts') + + for artifact in artifacts: + if artifact.startswith('/'): + response.status = 500 + logging.error("%s: artifact name cannot start with a '/'" + % artifact) + return + + filename = os.path.join(self.settings['artifact-dir'], artifact) + results[artifact] = os.path.exists(filename) + + if results[artifact]: + logging.debug('%s is in the cache', artifact) + else: + logging.debug('%s is NOT in the cache', artifact) + + return results + root = Bottle() root.mount(app, '/1.0') - if self.settings['fcgi-server']: + if self.settings['fcgi-server']: WSGIServer(root).run() else: run(root, host='0.0.0.0', port=self.settings['port'], reloader=True) -- cgit v1.2.1 From b228561897aecea8a60908c354cee79938cf4e31 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 24 Sep 2014 16:24:48 +0000 Subject: Add test shell for use in staging area tests This is intended to be statically linked, so it can be run in the staging area, without having to build a libc. You shouldn't generally statically link GLibc, because it dynamically links things like NSS modules or locale data, but we only need some very simple stuff, so we can get away with that. There's also potential licensing issues, as GLibc is LGPLv2, so distribution requires providing the ability to re-link against another library, but its use in the test-suite shouldn't count as distribution. There's a couple of commands, the only two needed by the yarn test suite are "copy files" which is like `cp -r . "$DESTDIR"`, and "false", which is for commands to deliberately fail. --- scripts/test-shell.c | 144 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 scripts/test-shell.c diff --git a/scripts/test-shell.c b/scripts/test-shell.c new file mode 100644 index 00000000..7975c188 --- /dev/null +++ b/scripts/test-shell.c @@ -0,0 +1,144 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +char *readlinka(char const *path){ + size_t buflen = BUFSIZ; + char *buf = malloc(buflen); + ssize_t read; + while ((read = readlink(path, buf, buflen - 1)) >= buflen - 1) { + char *newbuf = realloc(buf, buflen * 2); + if (newbuf == NULL) { + goto failure; + } + buf = newbuf; + buflen = buflen * 2; + } + buf[read] = '\0'; + return buf; +failure: + free(buf); + return NULL; +} + +int copy_file_paths(char const *source_file, char const *target_file) { + int source_fd; + int target_fd; + int ret = -1; + struct stat st; + if ((source_fd = open(source_file, O_RDONLY)) == -1) { + return ret; + } + if (fstat(source_fd, &st) == -1) { + perror("stat"); + ret = -2; + goto cleanup_in; + } + if ((target_fd = open(target_file, O_WRONLY|O_CREAT, st.st_mode)) == -1) { + ret = -3; + goto cleanup_in; + } + ssize_t read; + while ((read = sendfile(target_fd, source_fd, NULL, BUFSIZ)) > 0); + if (read < 0) { + perror("sendfile"); + ret = -4; + } + ret = 0; +cleanup_all: + close(target_fd); +cleanup_in: + close(source_fd); + return ret; +} + +int copy_entry(const char *fpath, const struct stat *sb, int typeflag, + struct FTW *ftwbuf) { + int ret = 0; + char *target_path = NULL; + if (asprintf(&target_path, "%s/%s", getenv("DESTDIR"), fpath) == -1) { + return -1; + } + switch (typeflag) { + case FTW_F: + /* Copy file */ + if ((ret = copy_file_paths(fpath, target_path)) < 0) { + perror("Copy file"); + ret = -1; + } + break; + case FTW_D: + case FTW_DNR: + /* Copy directory */ + if (mkdir(target_path, sb->st_mode)) { + if (errno != EEXIST) { + perror("mkdir"); + ret = -1; + } + } + break; + case FTW_NS: + case FTW_SL: + case FTW_SLN: { + /* Create symlink */ + char *link_target = readlinka(fpath); + if (link_target == NULL) { + perror("readlink"); + ret = -1; + } + if (symlink(link_target, target_path) == -1) { + perror("symlink"); + ret = -1; + } + break; + } + } +cleanup: + free(target_path); + return ret; +} + +int main(int argc, char *argv[]) { + int ret = 1; + if (argc != 3 || strcmp(argv[1], "-c") != 0) { + fprintf(stderr, "Usage: %s -c COMMAND\n", argv[0]); + return 1; + } + size_t cmdlen = strlen(argv[2]); + FILE *cmdstream = fmemopen(argv[2], cmdlen, "r"); + { + ssize_t read; + size_t len = 0; + char *line = NULL; + + ret = 0; + while ((read = getline(&line, &len, cmdstream)) != -1) { + if (line[read - 1] == '\n') line[read - 1] = '\0'; + if (strcmp(line, "copy files") == 0) { + /* Recursively copy contents of current dir to DESTDIR */ + if (nftw(".", copy_entry, 20, FTW_PHYS)) { + ret = 1; + break; + } + } else if (strcmp(line, "false") == 0 || + strstr(line, "false ") == line) { + ret = 1; + break; + } else { + ret = 127; + break; + } + } + free(line); + } + return ret; +} -- cgit v1.2.1 From 5ac9f90a64cdadaf4d9adb243c6be2f0f4d1764a Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 24 Sep 2014 16:36:34 +0000 Subject: Modify yarns to use test-shell instead of build-mode: test When all the tests that use build-mode: test are gone, the build mode can be removed. --- yarns/branches-workspaces.yarn | 11 +- yarns/implementations.yarn | 301 +++++++++++++++++++++-------------------- yarns/regression.yarn | 6 +- yarns/splitting.yarn | 27 ++-- 4 files changed, 178 insertions(+), 167 deletions(-) diff --git a/yarns/branches-workspaces.yarn b/yarns/branches-workspaces.yarn index 200f9508..34aa97e0 100644 --- a/yarns/branches-workspaces.yarn +++ b/yarns/branches-workspaces.yarn @@ -202,14 +202,14 @@ is a chunk repository created in the mocked git server, for testing purposes. WHEN the user edits the chunk test-chunk in branch foo - THEN in branch foo, stratum strata/test-stratum.morph refs test-chunk in foo + THEN in branch foo, stratum strata/core.morph refs test-chunk in foo AND the edited chunk test:test-chunk has git branch foo Editing a morphology should not cause it to start having repo or ref fields when referring to strata, when it didn't before. - AND in branch foo, system systems/test-system.morph refers to test-stratum without repo - AND in branch foo, system systems/test-system.morph refers to test-stratum without ref + AND in branch foo, system systems/test-system.morph refers to core without repo + AND in branch foo, system systems/test-system.morph refers to core without ref FINALLY the git server is shut down Temporary Build Branch behaviour @@ -398,14 +398,15 @@ Nor do we need temporary build branches for distributed builds. IMPLEMENTS WHEN the user makes changes to test-chunk in branch (\S+) chunkdir="$(slashify_colons "test:test-chunk")" cd "$DATADIR/workspace/$MATCH_1/$chunkdir" - sed -i -e 's/Hello/Goodbye/g' test-bin + sed -i -e 's/Hello/Goodbye/g' usr/libexec/test-bin IMPLEMENTS THEN the changes to test-chunk in branch (\S+) are included in the temporary build branch build_ref_prefix=baserock/builds/ chunkdir="$(slashify_colons "test:test-chunk")" cd "$DATADIR/workspace/$MATCH_1/$chunkdir" + testbin=usr/libexec/test-bin eval "$(git for-each-ref --count=1 --shell --sort=committerdate \ - --format='git cat-file -p %(refname):test-bin | diff test-bin -' \ + --format='git cat-file -p %(refname):$testbin | diff $testbin -' \ "$build_ref_prefix")" IMPLEMENTS WHEN the user commits changes to (\S+) in branch (\S+) diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 52eee01d..e99c1817 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -67,72 +67,47 @@ another to hold a chunk. # Create a directory for all the git repositories. mkdir "$DATADIR/gits" - # Create the chunk repository. - mkdir "$DATADIR/gits/test-chunk" - - run_in "$DATADIR/gits/test-chunk" git init . - cat > "$DATADIR/gits/test-chunk/test-bin" <<'EOF' - #!/bin/sh - echo Hello World - EOF - cat > "$DATADIR/gits/test-chunk/test.h" <<'EOF' - int foo(void); - EOF - cat > "$DATADIR/gits/test-chunk/test.pc" <<'EOF' - prefix=/usr - includedir=${prefix}/include - Name: test - Cflags: -I{includedir} + # Create the bootstrap chunk repositories + mkdir "$DATADIR/gits/bootstrap-chunk" + cd "$DATADIR/gits/bootstrap-chunk" + git init . + git checkout -b bootstrap + cp "$SRCDIR/scripts/test-shell.c" sh.c + install /dev/stdin <<'EOF' configure + #!/bin/true EOF - run_in "$DATADIR/gits/test-chunk" git add . - run_in "$DATADIR/gits/test-chunk" git commit --allow-empty -m Initial. + printf >Makefile ' + CFLAGS = -D_GNU_SOURCE -static - # Create a repo for the morphologies. + all: sh - mkdir "$DATADIR/gits/morphs" + install: sh + \tinstall -D -m755 sh $(DESTDIR)/bin/sh' + git add . + git commit -m "Add bootstrap shell" - arch=$(run_morph print-architecture) - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/systems/test-system.morph" - name: test-system - kind: system - arch: $arch - strata: - - name: test-stratum - morph: strata/test-stratum.morph - EOF + git checkout --orphan master HEAD + # Commit a pre-built test-shell, as a compiler is too heavy to bootstrap + make sh + mkdir bin + mv sh bin/sh + git rm -f Makefile sh.c configure + git add bin/sh + git commit -m "Build bootstrap shell with bootstrap shell" - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/strata/test-stratum.morph" - name: test-stratum - kind: stratum - chunks: - - name: test-chunk - repo: test:test-chunk - morph: test-chunk.morph - unpetrify-ref: master - ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) - build-mode: test - build-depends: [] - EOF + # Create the test chunk repository. + + mkdir "$DATADIR/gits/test-chunk" + cd "$DATADIR/gits/test-chunk" + git init . # To verify that chunk splitting works, we have a chunk that installs # dummy files in all the places that different kinds of files are # usually installed. e.g. executables in `/bin` and `/usr/bin` - install -m644 -D /dev/stdin << 'EOF' "$DATADIR/gits/morphs/test-chunk.morph" - name: test-chunk - kind: chunk - build-system: manual - - # `install-commands` is a list of shell commands to run. Commands - # may be on multiple lines, and indeed anything programmatic will - # benefit from doing so. Arguably we could have just one command, - # but it's split into multiple so that morph can inform us which - # command failed without us having to include a lot of status - # information in the command and look at the error message. - - install-commands: - + PREFIX=/usr + DESTDIR=. # It's important that we can test whether executables get # installed, so we install an empty script into `/usr/bin/test` and # `/usr/sbin/test`. @@ -144,10 +119,10 @@ another to hold a chunk. # that the file exists, rather than its contents, we can use /dev/null # as the source. - - | - for bindir in bin sbin; do - install -D /dev/null "$DESTDIR/$PREFIX/$bindir/test" - done + + for bindir in bin sbin; do + install -D /dev/null "$DESTDIR/$PREFIX/$bindir/test" + done # We need shared libraries too, sometimes they're libraries to support # the executables that a chunk provides, sometimes for other chunks. @@ -158,14 +133,13 @@ another to hold a chunk. # Shared libraries' file names start with lib and end with `.so` # for shared-object, with version numbers optionally suffixed. - - | - for libdir in lib lib32 lib64; do - dirpath="$DESTDIR/$PREFIX/$libdir" - install -D /dev/null "$dirpath/libtest.so" - ln -s libtest.so "$dirpath/libtest.so.0" - ln -s libtest.so.0 "$dirpath/libtest.so.0.0" - ln -s libtest.so.0.0 "$dirpath/libtest.so.0.0.0" - done + for libdir in lib lib32 lib64; do + dirpath="$DESTDIR/$PREFIX/$libdir" + install -D /dev/null "$dirpath/libtest.so" + ln -s libtest.so "$dirpath/libtest.so.0" + ln -s libtest.so.0 "$dirpath/libtest.so.0.0" + ln -s libtest.so.0.0 "$dirpath/libtest.so.0.0.0" + done # Shared objects aren't the only kind of library, some executable # binaries count as libraries, such as git's plumbing commands. @@ -173,8 +147,10 @@ another to hold a chunk. # In some distributions they go into /lib, in others, and the default # autotools configuration, they go into /libexec. - - | - install -D test-bin "$DESTDIR/$PREFIX/libexec/test-bin" + install -D /dev/stdin "$DESTDIR/$PREFIX/libexec/test-bin" <<'EOF' + #!/bin/sh + echo Hello World + EOF # As well as run-time libraries, there's development files. For C # this is headers, which describe the API of the libraries, which @@ -185,9 +161,9 @@ another to hold a chunk. # Header files go into `include` and end with `.h`. They are not # executable, so the install command changes the permissions with the # `-m` option. - - - | - install -D -m 644 test.h "$DESTDIR/$PREFIX/include/test.h" + install -D -m 644 /dev/stdin <<'EOF' "$DESTDIR/$PREFIX/include/test.h" + int foo(void); + EOF # `pkg-config` is a standard way to locate libraries and get the # compiler flags needed to build with the library. It's also used @@ -195,68 +171,138 @@ another to hold a chunk. # so as well as being found in `lib/pkgconfig`, it can be found in # `share/pkgconfig`, so we install dummy files to both. - - | - for pkgdir in lib lib32 lib64 share; do - install -D -m 644 test.pc \ - "$DESTDIR/$PREFIX/$pkgdir/pkgconfig/test.pc" - done + for pkgdir in lib lib32 lib64 share; do + install -D -m 644 /dev/stdin < "$DATADIR/gits/morphs/stage1-chunk.morph" - name: stage1-chunk - kind: chunk - build-system: dummy - EOF - - run_in "$DATADIR/gits/morphs" git add . - run_in "$DATADIR/gits/morphs" git commit -m "Add chunk for $MATCH_1" - - - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/strata/build-essential.morph" - name: build-essential - kind: stratum - chunks: - - name: stage1-chunk - morph: stage1-chunk.morph - repo: test:test-chunk - ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) - unpetrify-ref: master - build-mode: bootstrap - build-depends: [] - EOF - - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/strata/core.morph" - name: core - kind: stratum - build-depends: - - morph: strata/build-essential.morph - chunks: - - name: test-chunk - morph: test-chunk.morph - repo: test:test-chunk - unpetrify-ref: master - ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) - build-mode: test - build-depends: [] - EOF - name="$(basename "${MATCH_1%.*}")" cat << EOF > "$DATADIR/gits/morphs/$MATCH_1" arch: testarch @@ -380,7 +387,7 @@ We also need to verify that a system branch has been checked out. IMPLEMENTS THEN the system branch (\S+) is checked out is_dir "$DATADIR/workspace/$MATCH_1/test/morphs" is_file "$DATADIR/workspace/$MATCH_1/test/morphs/systems/test-system.morph" - is_file "$DATADIR/workspace/$MATCH_1/test/morphs/strata/test-stratum.morph" + is_file "$DATADIR/workspace/$MATCH_1/test/morphs/strata/core.morph" We can create a new branch, off master. @@ -487,8 +494,10 @@ print-architecture` to get a value appropriate for morph. kind: system name: $name strata: - - name: test-stratum - morph: strata/test-stratum.morph + - name: build-essential + morph: strata/build-essential.morph + - name: core + morph: strata/core.morph EOF Reporting status of checked out repositories: diff --git a/yarns/regression.yarn b/yarns/regression.yarn index 7991ab12..c424f437 100644 --- a/yarns/regression.yarn +++ b/yarns/regression.yarn @@ -58,15 +58,15 @@ source it depended on. SCENARIO changing the artifacts a system uses GIVEN a workspace AND a git server - AND system systems/test-system.morph uses test-stratum-runtime from test-stratum - AND stratum strata/test-stratum.morph has match rules: [{artifact: test-stratum-runtime, include: [.*-(bins|libs|locale)]}, {artifact: test-stratum-devel, include: [.*-(devel|doc|misc)]}] + AND system systems/test-system.morph uses core-runtime from core + AND stratum strata/core.morph has match rules: [{artifact: core-runtime, include: [.*-(bins|libs|locale)]}, {artifact: core-devel, include: [.*-(devel|doc|misc)]}] WHEN the user checks out the system branch called master GIVEN a cluster called test-cluster.morph in system branch master AND a system in cluster test-cluster.morph in branch master called test-system AND system test-system in cluster test-cluster.morph in branch master builds systems/test-system.morph AND system test-system in cluster test-cluster.morph in branch master has deployment type: tar WHEN the user builds the system systems/test-system.morph in branch master - GIVEN stratum strata/test-stratum.morph in system branch master has match rules: [{artifact: test-stratum-runtime, include: [.*-(bins|libs|misc)]}, {artifact: test-stratum-devel, include: [.*-(devel|doc|locale)]}] + GIVEN stratum strata/core.morph in system branch master has match rules: [{artifact: core-runtime, include: [.*-(bins|libs|misc)]}, {artifact: core-devel, include: [.*-(devel|doc|locale)]}] WHEN the user builds the system systems/test-system.morph in branch master AND the user deploys the cluster test-cluster.morph in branch master with options test-system.location="$DATADIR/test.tar" THEN tarball test.tar contains baserock/test-chunk-misc.meta diff --git a/yarns/splitting.yarn b/yarns/splitting.yarn index 9248f60c..2726d294 100644 --- a/yarns/splitting.yarn +++ b/yarns/splitting.yarn @@ -15,7 +15,7 @@ To test that all the fields are recognised, we set the new fields to their default values. AND chunk test-chunk includes the default splitting rules - AND stratum strata/test-stratum.morph includes the default splitting rules + AND stratum strata/core.morph includes the default splitting rules AND system systems/test-system.morph includes the default splitting rules The default rules produce a system that is identical to not providing @@ -41,7 +41,7 @@ libraries. The only change we need to make is to add a field to the system morphology to select which artifact to use in the system. - AND system systems/test-system.morph uses test-stratum-runtime from test-stratum + AND system systems/test-system.morph uses core-runtime from core WHEN the user checks out the system branch called master The best way to test that only using some stratum artifacts works is @@ -79,8 +79,8 @@ This GIVEN has a chunk in the stratum that never successfully builds, so we know that if the system successfully builds, then we only built chunks that were needed. - AND stratum strata/test-stratum.morph has chunks that aren't used in test-stratum-minimal - AND system systems/test-system.morph uses test-stratum-minimal from test-stratum + AND stratum strata/core.morph has chunks that aren't used in core-minimal + AND system systems/test-system.morph uses core-minimal from core WHEN the user checks out the system branch called master THEN morph build the system systems/test-system.morph of the branch master FINALLY the git server is shut down @@ -92,7 +92,7 @@ Implementations IMPLEMENTS GIVEN chunk (\S+) includes the default splitting rules # Append default products rules name="$(basename "${MATCH_1%.*}")" - cat <>"$DATADIR/gits/morphs/$MATCH_1" + cat <>"$DATADIR/gits/morphs/$MATCH_1.morph" products: - artifact: $name-bins include: [ "(usr/)?s?bin/.*" ] @@ -119,7 +119,7 @@ Implementations - artifact: $name-misc include: [ .* ] EOF - run_in "$DATADIR/gits/morphs" git add "$MATCH_1" + run_in "$DATADIR/gits/morphs" git add "$MATCH_1.morph" run_in "$DATADIR/gits/morphs" git commit -m 'Add default splitting rules' IMPLEMENTS GIVEN stratum (\S+) includes the default splitting rules @@ -127,6 +127,8 @@ Implementations cat <"$DATADIR/gits/morphs/$MATCH_1" name: $name kind: stratum + build-depends: + - morph: strata/build-essential.morph products: - artifact: $name-devel include: @@ -146,7 +148,6 @@ Implementations unpetrify-ref: master ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) morph: test-chunk.morph - build-mode: test build-depends: [] artifacts: test-chunk-bins: $name-runtime @@ -162,11 +163,13 @@ Implementations IMPLEMENTS GIVEN system (\S+) includes the default splitting rules cat << EOF >> "$DATADIR/gits/morphs/$MATCH_1" strata: - - name: test-stratum - morph: strata/test-stratum.morph + - name: build-essential + morph: strata/build-essential.morph + - name: core + morph: strata/core.morph artifacts: - - test-stratum-runtime - - test-stratum-devel + - core-runtime + - core-devel EOF run_in "$DATADIR/gits/morphs" git add "$MATCH_1" run_in "$DATADIR/gits/morphs" git commit -m 'Add default splitting rules' @@ -195,14 +198,12 @@ Implementations morph: test-chunk.morph unpetrify-ref: master ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) - build-mode: test build-depends: [] - name: unbuildable-chunk repo: test:test-chunk unpetrify-ref: refs/heads/master ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) morph: unbuildable-chunk.morph - build-mode: test build-depends: - test-chunk EOF -- cgit v1.2.1 From fd6e7adca086e86657efdcd0a10cb1d2b3c8ad1e Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 24 Sep 2014 16:55:06 +0000 Subject: Remove unused cmdtests tests.deploy is gone since we have yarn coverage instead. tests/setup had unnecessary setup in it, since the remaining tests require custom setup anyway. A bunch of tests were disabled anyway, so they're gone. Also, there was still code in ./check to run tests.merging, which have been gone for a while. --- check | 14 -- .../build-handles-stratum-build-depends.script | 50 ----- tests.as-root/build-with-external-strata.script | 62 ------ tests.as-root/build-with-push.script | 38 ---- ...iple-times-doesnt-generate-new-artifacts.script | 43 ---- ...system-branch-picks-up-committed-removes.script | 62 ------ .../building-a-system-branch-works-anywhere.script | 61 ------ tests.deploy/deploy-cluster.script | 60 ------ tests.deploy/deploy-cluster.stdout | 2 - .../deploy-rawdisk-without-disk-size-fails.script | 30 --- tests.deploy/deploy-rawdisk.script | 33 ---- tests.deploy/setup | 216 --------------------- tests.deploy/setup-build | 35 ---- tests/setup | 74 ------- 14 files changed, 780 deletions(-) delete mode 100755 tests.as-root/build-handles-stratum-build-depends.script delete mode 100755 tests.as-root/build-with-external-strata.script delete mode 100755 tests.as-root/build-with-push.script delete mode 100755 tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script delete mode 100755 tests.as-root/building-a-system-branch-picks-up-committed-removes.script delete mode 100755 tests.as-root/building-a-system-branch-works-anywhere.script delete mode 100755 tests.deploy/deploy-cluster.script delete mode 100644 tests.deploy/deploy-cluster.stdout delete mode 100755 tests.deploy/deploy-rawdisk-without-disk-size-fails.script delete mode 100755 tests.deploy/deploy-rawdisk.script delete mode 100755 tests.deploy/setup delete mode 100644 tests.deploy/setup-build diff --git a/check b/check index 82774119..a942ca90 100755 --- a/check +++ b/check @@ -153,20 +153,6 @@ else echo "NOT RUNNING test.branching" fi -if false && "$run_cmdtests" -then - cmdtest tests.merging -else - echo "NOT RUNNING test.merging" -fi - -if "$run_cmdtests" -then - cmdtest tests.deploy -else - echo "NOT RUNNING test.deploy" -fi - # Building systems requires the 'filter' parameter of tarfile.TarFile.add(): # this was introduced in Python 2.7 if ! "$run_cmdtests"; then diff --git a/tests.as-root/build-handles-stratum-build-depends.script b/tests.as-root/build-handles-stratum-build-depends.script deleted file mode 100755 index 6e6f82da..00000000 --- a/tests.as-root/build-handles-stratum-build-depends.script +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -## "morph build" should update the build-depends fields of strata correctly. - -# FIXME: The new "morph edit" seems to be changing build-dependencies -# (correctly) in a way that makes the old "morph build" fail, due to -# this error: -# -# Conflicting versions of stratum 'hello-stratum' appear in the build. -# -# I cannot find a way to fix the old "morph build", and so I'm disabling -# this test until it can be fixed. --liw -exit 0 - -set -eu - -. "$SRCDIR/tests.as-root/setup-build" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" branch test:morphs test/stratum-build-depends - -cd test/stratum-build-depends/test/morphs - -# 'linux-system' and the build-depends fields of 'linux-stratum' need to -# be updated here. Any build-depends of any altered strata also need to -# be altered, such as the 'tools-stratum' which depends on linux-stratum -# If they are not updated, the build command will fail. -"$SRCDIR/scripts/test-morph" edit linux-system hello-stratum - -# Likewise, this command must update build-depends or the 'repo' field will -# not be changed in the temporary build branch, leading to: -# -# ERROR: Ref test/stratum-build-depends is an invalid reference for -# repo file://TMP/morphs -# -"$SRCDIR/scripts/test-morph" build linux-system diff --git a/tests.as-root/build-with-external-strata.script b/tests.as-root/build-with-external-strata.script deleted file mode 100755 index be870053..00000000 --- a/tests.as-root/build-with-external-strata.script +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## "morph build" with strata outside the branch root repository. - -# FIXME: The new "morph edit" breaks this, for reasons unknown. Disabling -# it on the assumption that the new code (which changes refs to -# build-depends) is correct and the convoluted test and the old "morph -# build" code are wrong, but this needs to be revisited soon. --liw -exit 0 - -set -eu - -. "$SRCDIR/scripts/setup-3rd-party-strata" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" branch test:morphs branch1 - -# System will fail to build unless we add linux to it -- make the change but -# don't commit it, in one of the external strata, as a challenge for -# 'morph build'. -cd "branch1" -cd "test/external-strata" - -awk ' - /^chunks:/ { - print $0 - print "- name: linux" - print " repo: test:kernel-repo" - print " ref: master" - print " build-mode: test" - print " build-depends: []" - next - } - { print } -' stratum2.morph > temp -mv temp stratum2.morph - -# Ignore Morph's output for now because it gives us: -# 2012-11-07 16:26:12 Overlaps in system artifact hello-system-rootfs detected -# -# This is due to having a chunk named 'hello' in more than one stratum. It's -# a bug that this generates overlaps (the chunk's .meta file needs to be called -# $stratum.$chunk.meta rather than $chunk.meta to avoid the overlap) and the -# redirection should be removed once this bug is fixed. -"$SRCDIR/scripts/test-morph" build hello-system > /dev/null - -[ $("$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" | wc -l) -eq 23 ] diff --git a/tests.as-root/build-with-push.script b/tests.as-root/build-with-push.script deleted file mode 100755 index ead669ed..00000000 --- a/tests.as-root/build-with-push.script +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -# Test 'morph build' when build without push is disabled, i.e. everything is -# built from the remote repositories instead of the local checkouts. - -# FIXME: This seems to break because the new "morph edit" makes correct -# changes to build-dependencies, which breaks the old "morph build". -# Disable test now, re-enable it after "morph build" is fixed. --liw -exit 0 - - -set -eu - -source "$SRCDIR/tests.as-root/setup-build" - -cd "$DATADIR/workspace/branch1" -"$SRCDIR/scripts/test-morph" --push-build-branches build linux-system - -# Test that the chunk was built from test:kernel-repo and not a local branch -cd "$DATADIR/cache/artifacts" -tar xf *.chunk.linux baserock/linux.meta -grep -q "\"repo\": \"file://$DATADIR/kernel-repo\"" baserock/linux.meta - - diff --git a/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script b/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script deleted file mode 100755 index ac6cffec..00000000 --- a/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -## Make sure "morph build" works anywhere in a workspace or system branch -## and produces the same results every time. - -# FIXME: This seems to break because the new "morph edit" makes correct -# changes to build-dependencies, which breaks the old "morph build". -# Disable test now, re-enable it after "morph build" is fixed. --liw -exit 0 - -set -eu - -source "$SRCDIR/tests.as-root/setup-build" - -# Build once. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" build linux-system -ARTIFACT_COUNT="$(ls "$DATADIR/cache/artifacts" | wc -l)" - -# Build twice. -cd "$DATADIR/workspace/branch1" -"$SRCDIR/scripts/test-morph" build linux-system -[ "$ARTIFACT_COUNT" -eq $(ls "$DATADIR/cache/artifacts" | wc -l) ] - -# Build thrice, and that should be enough. -cd "$DATADIR/workspace/branch1/test/morphs" -"$SRCDIR/scripts/test-morph" build linux-system -[ "$ARTIFACT_COUNT" -eq $(ls "$DATADIR/cache/artifacts" | wc -l) ] diff --git a/tests.as-root/building-a-system-branch-picks-up-committed-removes.script b/tests.as-root/building-a-system-branch-picks-up-committed-removes.script deleted file mode 100755 index 64ae82c7..00000000 --- a/tests.as-root/building-a-system-branch-picks-up-committed-removes.script +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -# FIXME: This seems to break because the new "morph edit" makes correct -# changes to build-dependencies, which breaks the old "morph build". -# Disable test now, re-enable it after "morph build" is fixed. --liw -exit 0 - -set -e - -. "$SRCDIR/tests.as-root/setup-build" -KERNEL_BRANCH=baserock/builds/123456789/AABBCCDDE - -cd "$DATADIR/workspace/branch1/test/kernel-repo" -git checkout --quiet master -echo Use Morph >README -git add README -git commit --quiet -m 'Add README' - -# Build the linux system from the system branch. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" build linux-system - -# Make a change elsewhere to be pulled in -PEER_REPO="$DATADIR/peer-kernel" -git clone --quiet "file://$DATADIR/workspace/branch1/test/kernel-repo" \ - "$PEER_REPO" -cd "$PEER_REPO" -git checkout --quiet -b fix -# remove useless README -git rm --quiet README -git commit --quiet -m 'change stuff' -git checkout --quiet master -git merge --no-ff fix >/dev/null 2>&1 - -# Pull a commit in to the linux morphology. -cd "$DATADIR/workspace/branch1/test/kernel-repo" -git remote add peer "file://$PEER_REPO" -git remote update >/dev/null 2>&1 -git merge --quiet peer/master - -# Build the linux system again without comitting. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" build linux-system -cd branch1/test/kernel-repo - -# Check whether the new morphology exists in the temporary build ref -cd "$DATADIR/workspace/branch1/test/kernel-repo" -! git cat-file blob "$KERNEL_BRANCH:README" >/dev/null 2>&1 diff --git a/tests.as-root/building-a-system-branch-works-anywhere.script b/tests.as-root/building-a-system-branch-works-anywhere.script deleted file mode 100755 index cf946cd5..00000000 --- a/tests.as-root/building-a-system-branch-works-anywhere.script +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Make sure "morph build" works anywhere in a workspace or system branch -## and produces the same results every time. - -# FIXME: This seems to break because the new "morph edit" makes correct -# changes to build-dependencies, which breaks the old "morph build". -# Disable test now, re-enable it after "morph build" is fixed. --liw -exit 0 - -set -eu - -source "$SRCDIR/tests.as-root/setup-build" - -# Build from the workspace root. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" build linux-system -"$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" > "$DATADIR/output1" -rm -rf "$DATADIR/cache"/* - -# Build from the branch. -cd "$DATADIR/workspace/branch1" -"$SRCDIR/scripts/test-morph" build linux-system -"$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" > "$DATADIR/output2" -rm -rf "$DATADIR/cache/artifacts"/* - -# Build form the branch root repository. -cd "$DATADIR/workspace/branch1/test/morphs" -"$SRCDIR/scripts/test-morph" build linux-system -"$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" > "$DATADIR/output3" -rm -rf "$DATADIR/cache/artifacts"/* - -# Build from the linux directory. -cd "$DATADIR/workspace/branch1/test/kernel-repo" -"$SRCDIR/scripts/test-morph" build linux-system -"$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" > "$DATADIR/output4" -rm -rf "$DATADIR/cache/artifacts"/* - -# Verify that we build the right number of artifacts -[ $(wc < "$DATADIR/output1" -l) -eq 22 ] - -# List of files in the artifact cache should be identical after each build -diff "$DATADIR/output1" "$DATADIR/output2" -diff "$DATADIR/output2" "$DATADIR/output3" -diff "$DATADIR/output3" "$DATADIR/output4" diff --git a/tests.deploy/deploy-cluster.script b/tests.deploy/deploy-cluster.script deleted file mode 100755 index acc65a54..00000000 --- a/tests.deploy/deploy-cluster.script +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# Test "morph deploy" by deploying the systems in cluster morphology. - -set -eu - -. "$SRCDIR/tests.deploy/setup-build" - -cd "$DATADIR/workspace/branch1" - -"$SRCDIR/scripts/test-morph" build hello-system - -"$SRCDIR/scripts/test-morph" build linux-system - -GIT_DIR=test/morphs/.git git tag -a my-test-tag -m "Example tag" HEAD - -"$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ - deploy test_cluster \ - linux-system-2.EXAMPLE_PASSWORD="secret" \ - linux-system-2.HOSTNAME="baserock-rocks-even-more" \ - > /dev/null - -outputdir=test/morphs -test -e $outputdir/hello-system.img -test -e $outputdir/linux-system-1.tar -test -e $outputdir/linux-system-2.tar - -hostname1=$(tar -xf $outputdir/linux-system-1.tar ./etc/hostname -O) -hostname2=$(tar -xf $outputdir/linux-system-2.tar ./etc/hostname -O) - -[ "$hostname1" = baserock-rocks ] -[ "$hostname2" = baserock-rocks-even-more ] - -tar -xf $outputdir/linux-system-2.tar ./baserock/deployment.meta -metadata=baserock/deployment.meta - -# Check that 'git describe' of definitions repo was stored correctly -echo -n "definitions-version: " -"$SRCDIR/scripts/yaml-extract" $metadata definitions-version - -echo -n "configuration.HOSTNAME: " -"$SRCDIR/scripts/yaml-extract" $metadata configuration HOSTNAME - -! (grep -q "EXAMPLE_PASSWORD" $metadata) diff --git a/tests.deploy/deploy-cluster.stdout b/tests.deploy/deploy-cluster.stdout deleted file mode 100644 index 16b78015..00000000 --- a/tests.deploy/deploy-cluster.stdout +++ /dev/null @@ -1,2 +0,0 @@ -definitions-version: {'describe': 'my-test-tag-unreproducible'} -configuration.HOSTNAME: baserock-rocks-even-more diff --git a/tests.deploy/deploy-rawdisk-without-disk-size-fails.script b/tests.deploy/deploy-rawdisk-without-disk-size-fails.script deleted file mode 100755 index 035557dd..00000000 --- a/tests.deploy/deploy-rawdisk-without-disk-size-fails.script +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# Test "morph deploy" refuses to deploy if DISK_SIZE is unset. - -set -eu - - -. "$SRCDIR/tests.deploy/setup-build" -cd "$DATADIR/workspace/branch1" -"$SRCDIR/scripts/test-morph" build linux-system -! "$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ - deploy rawdisk_test_cluster_without_disk_size > /dev/null 2>&1 -test ! -e "$DATADIR/disk.img" - diff --git a/tests.deploy/deploy-rawdisk.script b/tests.deploy/deploy-rawdisk.script deleted file mode 100755 index 3489a198..00000000 --- a/tests.deploy/deploy-rawdisk.script +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# Test "morph deploy" by deploying to a raw disk image. - - -set -eu - - -. "$SRCDIR/tests.deploy/setup-build" -cd "$DATADIR/workspace/branch1" -"$SRCDIR/scripts/test-morph" build linux-system -"$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ - deploy rawdisk_test_cluster > /dev/null - -outputdir="test/morphs" -test -e $outputdir/disk.img - diff --git a/tests.deploy/setup b/tests.deploy/setup deleted file mode 100755 index 033598bc..00000000 --- a/tests.deploy/setup +++ /dev/null @@ -1,216 +0,0 @@ -#!/bin/bash -# -# Create git repositories for tests. The chunk repository will contain a -# simple "hello, world" C program, and two branches ("master", "farrokh"), -# with the master branch containing just a README. The two branches are there -# so that we can test building a branch that hasn't been checked out. -# The branches are different so that we know that if the wrong branch -# is uses, the build will fail. -# -# The stratum repository contains a single branch, "master", with a -# stratum and a system morphology that include the chunk above. -# -# Copyright (C) 2011, 2012, 2013, 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -set -eu - -source "$SRCDIR/scripts/fix-committer-info" - -# The $DATADIR should be empty at the beginnig of each test. -find "$DATADIR" -mindepth 1 -delete - -# Create an empty directory to be used as a morph workspace -mkdir "$DATADIR/workspace" - -# Create chunk repository. - -chunkrepo="$DATADIR/chunk-repo" -mkdir "$chunkrepo" -cd "$chunkrepo" -git init --quiet - -cat < README -This is a sample README. -EOF -git add README -git commit --quiet -m "add README" - -git checkout --quiet -b farrokh - -cat < hello.c -#include -int main(void) -{ - puts("hello, world"); - return 0; -} -EOF -git add hello.c - -cat < hello.morph -name: hello -kind: chunk -build-system: dummy -build-commands: - - gcc -o hello hello.c -install-commands: - - install -d "\$DESTDIR/etc" - - install -d "\$DESTDIR/bin" - - install hello "\$DESTDIR/bin/hello" -EOF -git add hello.morph - -git commit --quiet -m "add a hello world program and morph" - -git checkout --quiet master - - - -# Create morph repository. - -morphsrepo="$DATADIR/morphs" -mkdir "$morphsrepo" -cd "$morphsrepo" -git init --quiet - -cat < hello-stratum.morph -name: hello-stratum -kind: stratum -chunks: - - name: hello - repo: test:chunk-repo - ref: farrokh - build-depends: [] - build-mode: test -EOF -git add hello-stratum.morph - -cat < hello-system.morph -name: hello-system -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: hello-stratum -EOF -git add hello-system.morph - -cat < linux-stratum.morph -name: linux-stratum -kind: stratum -build-depends: - - morph: hello-stratum -chunks: - - name: linux - repo: test:kernel-repo - ref: master - build-depends: [] - build-mode: test -EOF -git add linux-stratum.morph - -cat < linux-system.morph -name: linux-system -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: hello-stratum - - morph: linux-stratum -configuration-extensions: - - set-hostname -EOF -git add linux-system.morph - - -cat < rawdisk_test_cluster.morph -name: rawdisk_test_cluster -kind: cluster -systems: - - morph: linux-system - deploy: - linux-system-1: - type: rawdisk - location: disk.img - DISK_SIZE: 1G -EOF -git add rawdisk_test_cluster.morph - -cat < rawdisk_test_cluster_without_disk_size.morph -name: rawdisk_test_cluster_without_disk_size -kind: cluster -systems: - - morph: linux-system - deploy: - linux-system-1: - type: rawdisk - location: disk.img -EOF -git add rawdisk_test_cluster_without_disk_size.morph - -cat < test_cluster.morph -name: test_cluster -kind: cluster -systems: - - morph: hello-system - deploy-defaults: - type: tar - deploy: - hello-system: - type: rawdisk - location: hello-system.img - DISK_SIZE: 1G - - morph: linux-system - deploy-defaults: - HOSTNAME: "baserock-rocks" - deploy: - linux-system-1: - type: tar - location: linux-system-1.tar - linux-system-2: - type: tar - location: linux-system-2.tar -EOF -git add test_cluster.morph - - -git commit --quiet -m "add morphs" - -# Make a dummy kernel chunk. -mkdir "$DATADIR/kernel-repo" -cat < "$DATADIR/kernel-repo/linux.morph" -name: linux -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/boot" - - touch "\$DESTDIR/extlinux.conf" - - touch "\$DESTDIR/boot/vmlinuz" - - touch "\$DESTDIR/boot/System.map" -EOF -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" init --quiet -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" add . -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" commit --quiet -m foo \ - > /dev/null - -# Create a morph configuration file. -cat < "$DATADIR/morph.conf" -[config] -repo-alias = test=file://$DATADIR/#file://$DATADIR/ -cachedir = $DATADIR/cache -log = $DATADIR/morph.log -no-distcc = true -quiet = true -log = /tmp/morph.log -EOF diff --git a/tests.deploy/setup-build b/tests.deploy/setup-build deleted file mode 100644 index c6b24da5..00000000 --- a/tests.deploy/setup-build +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Fixture for tests involving 'morph build' - -source "$SRCDIR/scripts/fix-committer-info" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" branch test:morphs branch1 -"$SRCDIR/scripts/test-morph" edit linux - -# Fix UUID's in the checked out repos to make build branch names deterministic -git config -f "$DATADIR/workspace/branch1/.morph-system-branch/config" \ - branch.uuid 123456789 -git config -f "$DATADIR/workspace/branch1/test/morphs/.git/config" \ - morph.uuid 987654321 -git config -f "$DATADIR/workspace/branch1/test/kernel-repo/.git/config" \ - morph.uuid AABBCCDDE - diff --git a/tests/setup b/tests/setup index 02ddb3af..6ebab880 100755 --- a/tests/setup +++ b/tests/setup @@ -31,80 +31,6 @@ set -eu # The $DATADIR should be empty at the beginnig of each test. find "$DATADIR" -mindepth 1 -delete -# Create chunk repository. - -chunkrepo="$DATADIR/chunk-repo" -mkdir "$chunkrepo" -cd "$chunkrepo" -git init --quiet - -cat < README -This is a sample README. -EOF -git add README -git commit --quiet -m "add README" - -git checkout --quiet -b farrokh - -cat < hello.c -#include -int main(void) -{ - puts("hello, world"); - return 0; -} -EOF -git add hello.c - -cat < hello.morph -name: hello -kind: chunk -build-system: dummy -build-commands: - - gcc -o hello hello.c -install-commands: - - install -d "\$DESTDIR/etc" - - install -d "\$DESTDIR/bin" - - install hello "\$DESTDIR/bin/hello" -EOF -git add hello.morph - -git commit --quiet -m "add a hello world program and morph" - -git checkout --quiet master - - - -# Create morph repository. - -morphsrepo="$DATADIR/morphs-repo" -mkdir "$morphsrepo" -cd "$morphsrepo" -git init --quiet - -cat < hello-stratum.morph -name: hello-stratum -kind: stratum -chunks: - - name: hello - repo: test:chunk-repo - ref: farrokh - build-mode: test - build-depends: [] -EOF -git add hello-stratum.morph - -cat < hello-system.morph -name: hello-system -kind: system -strata: - - morph: hello-stratum -EOF -git add hello-system.morph - -git commit --quiet -m "add morphs" - - # Create a morph configuration file. cat < "$DATADIR/morph.conf" [config] -- cgit v1.2.1 From 59859a4e6de1271dbb6cc3046df8be115e809489 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 24 Sep 2014 17:20:06 +0000 Subject: Stop tests.branching needing build-mode: test It doesn't do any actual building, so it doesn't matter that using build-mode: bootstrap everywhere means that it would end up with an empty system artifact. --- tests.branching/edit-updates-stratum.stdout | 4 ++-- tests.branching/setup | 2 +- tests.branching/setup-second-chunk | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests.branching/edit-updates-stratum.stdout b/tests.branching/edit-updates-stratum.stdout index a127f046..ee9510b5 100644 --- a/tests.branching/edit-updates-stratum.stdout +++ b/tests.branching/edit-updates-stratum.stdout @@ -1,5 +1,5 @@ diff --git a/hello-stratum.morph b/hello-stratum.morph -index e012b5f..cc0609a 100644 +index f335879..7bf9d37 100644 --- a/hello-stratum.morph +++ b/hello-stratum.morph @@ -3,6 +3,7 @@ kind: stratum @@ -10,4 +10,4 @@ index e012b5f..cc0609a 100644 + ref: newbranch + unpetrify-ref: master build-depends: [] - build-mode: test + build-mode: bootstrap diff --git a/tests.branching/setup b/tests.branching/setup index 22c51c24..a2d23090 100755 --- a/tests.branching/setup +++ b/tests.branching/setup @@ -63,7 +63,7 @@ chunks: repo: test:hello ref: master build-depends: [] - build-mode: test + build-mode: bootstrap EOF scripts/run-git-in "$DATADIR/morphs" init diff --git a/tests.branching/setup-second-chunk b/tests.branching/setup-second-chunk index 058fb222..24604ab8 100755 --- a/tests.branching/setup-second-chunk +++ b/tests.branching/setup-second-chunk @@ -49,12 +49,12 @@ chunks: repo: test:hello ref: master build-depends: [] - build-mode: test + build-mode: bootstrap - name: goodbye repo: test:goodbye ref: master build-depends: [] - build-mode: test + build-mode: bootstrap EOF git commit -q --all -m "Add goodbye to hello-stratum" -- cgit v1.2.1 From 6b28a2798c0ad938c4d365404ad2812b610b5b23 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 24 Sep 2014 19:08:39 +0000 Subject: Make show-dependencies not use build-mode: test --- tests/show-dependencies.setup | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/tests/show-dependencies.setup b/tests/show-dependencies.setup index c99e90a9..74d10c2b 100755 --- a/tests/show-dependencies.setup +++ b/tests/show-dependencies.setup @@ -64,40 +64,37 @@ chunks: - name: freetype repo: test:test-repo ref: master - build-mode: test build-depends: [] + build-mode: bootstrap - name: fontconfig repo: test:test-repo ref: master - build-mode: test build-depends: [] + build-mode: bootstrap - name: cairo repo: test:test-repo ref: master - build-mode: test build-depends: [] + build-mode: bootstrap - name: pango repo: test:test-repo ref: master - build-mode: test build-depends: - freetype - fontconfig - name: glib repo: test:test-repo ref: master - build-mode: test build-depends: [] + build-mode: bootstrap - name: gdk-pixbuf repo: test:test-repo ref: master - build-mode: test build-depends: - glib - name: gtk repo: test:test-repo ref: master - build-mode: test build-depends: - cairo - gdk-pixbuf @@ -106,12 +103,11 @@ chunks: - name: dbus repo: test:test-repo ref: master - build-mode: test build-depends: [] + build-mode: bootstrap - name: dbus-glib repo: test:test-repo ref: master - build-mode: test build-depends: - dbus - glib @@ -160,48 +156,40 @@ chunks: - name: libxfce4util repo: test:test-repo ref: master - build-mode: test build-depends: [] - name: xfconf repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4util - name: libxfce4ui repo: test:test-repo ref: master - build-mode: test build-depends: - xfconf - name: exo repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4util - name: garcon repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4util - name: thunar repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - exo - name: tumbler repo: test:test-repo ref: master - build-mode: test build-depends: [] - name: xfce4-panel repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - exo @@ -209,7 +197,6 @@ chunks: - name: xfce4-settings repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - exo @@ -217,7 +204,6 @@ chunks: - name: xfce4-session repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - exo @@ -225,21 +211,18 @@ chunks: - name: xfwm4 repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - xfconf - name: xfdesktop repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - xfconf - name: xfce4-appfinder repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - garcon @@ -247,7 +230,6 @@ chunks: - name: gtk-xfce-engine repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - garcon @@ -261,8 +243,8 @@ name: xfce-system kind: system arch: $("$SRCDIR/scripts/test-morph" print-architecture) strata: - - build-mode: test - morph: xfce-core + - morph: xfce-core + build-mode: bootstrap EOF git add xfce-system.morph git commit --quiet -m "add xfce-system" -- cgit v1.2.1 From 54c81674db48d5c9eb151199ef1bc434fd1bd4a6 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 24 Sep 2014 20:37:29 +0000 Subject: Move the branch-from-image test to a yarn --- tests.as-root/branch-from-image-works.script | 57 ---------------------------- tests.as-root/branch-from-image-works.setup | 1 - tests.as-root/branch-from-image-works.stdout | 1 - yarns/deployment.yarn | 45 ++++++++++++++++++++++ yarns/implementations.yarn | 4 ++ 5 files changed, 49 insertions(+), 59 deletions(-) delete mode 100755 tests.as-root/branch-from-image-works.script delete mode 120000 tests.as-root/branch-from-image-works.setup delete mode 100644 tests.as-root/branch-from-image-works.stdout diff --git a/tests.as-root/branch-from-image-works.script b/tests.as-root/branch-from-image-works.script deleted file mode 100755 index fb0b09c9..00000000 --- a/tests.as-root/branch-from-image-works.script +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# A branch created with branch-from-image has the right commits specified - -set -eu - -# Disable test on versions of Python before 2.7. -. "$SRCDIR/scripts/python-check" - -. "$SRCDIR/scripts/fix-committer-info" - -tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact \ - build-morphology test:morphs tarball hello-tarball) - -extracted="$DATADIR/extracted" -mkdir -p "$extracted" -tar -xf "$tar" -C "$extracted" -get_sha1(){ - sed -nre '/sha1/s/^.*([0-9a-f]{40}).*$/\1/p' "$1" -} -hello_chunk_commit=$(get_sha1 "$extracted/baserock/hello-bins.meta") - -# Make a commit so that petrifying from HEAD is detectable -chunkrepo="$DATADIR/chunk-repo" -cd "$chunkrepo" -git checkout --quiet farrokh -sed -i -e 's/hello, world/goodbye, world/g' hello.c -git add hello.c -git commit --quiet -m 'Make hello say goodbye' - - -workspace="$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init "$workspace" -cd "$workspace" -"$SRCDIR/scripts/test-morph" branch-from-image mybranch \ - --metadata-dir="$extracted/baserock" -cd mybranch/test/morphs -grep -qFe "$hello_chunk_commit" hello-stratum.morph -tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact build hello-tarball) -tar -xf "$tar" bin/hello -bin/hello diff --git a/tests.as-root/branch-from-image-works.setup b/tests.as-root/branch-from-image-works.setup deleted file mode 120000 index aac6926a..00000000 --- a/tests.as-root/branch-from-image-works.setup +++ /dev/null @@ -1 +0,0 @@ -metadata-includes-morph-version.setup \ No newline at end of file diff --git a/tests.as-root/branch-from-image-works.stdout b/tests.as-root/branch-from-image-works.stdout deleted file mode 100644 index 4b5fa637..00000000 --- a/tests.as-root/branch-from-image-works.stdout +++ /dev/null @@ -1 +0,0 @@ -hello, world diff --git a/yarns/deployment.yarn b/yarns/deployment.yarn index 40b50432..0782c7c1 100644 --- a/yarns/deployment.yarn +++ b/yarns/deployment.yarn @@ -283,3 +283,48 @@ deployment. WHEN the user attempts to deploy test-system.sysroot from cluster test-cluster.morph in branch master THEN morph failed FINALLY the git server is shut down + +Deploying branch-from-image produced systems +============================================ + +We have this nifty subcommand called branch-from-image, which can be +used to build the same thing as an existing image. + +There's no special requirements for making the image reproducible. + + SCENARIO reproducing systems + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user builds the system systems/test-system.morph in branch master + GIVEN a cluster called test-cluster.morph in system branch master + AND a system in cluster test-cluster.morph in branch master called test-system + AND system test-system in cluster test-cluster.morph in branch master builds systems/test-system.morph + AND system test-system in cluster test-cluster.morph in branch master has deployment type: sysroot + AND system test-system in cluster test-cluster.morph in branch master has deployment location: test-system + WHEN the user attempts to deploy the cluster test-cluster.morph in branch master + THEN morph succeeded + AND file workspace/master/test/morphs/test-system exists + +To reproduce an existing image, do a checkout with the extracted root +filesystem's /baserock directory as the `--metadata-dir` argument. + + WHEN the user attempts to check out the system branch from workspace/master/test/morphs/test-system called mybranch + THEN morph succeeded + AND the system branch mybranch is checked out + +After it is checked-out, the system can be rebuilt. + + WHEN the user attempts to build the system systems/test-system.morph in branch mybranch + THEN morph succeeded + +Once it is rebuilt, it can be deployed. + + GIVEN a cluster called test-cluster.morph in system branch mybranch + AND a system in cluster test-cluster.morph in branch mybranch called test-system + AND system test-system in cluster test-cluster.morph in branch mybranch builds systems/test-system.morph + AND system test-system in cluster test-cluster.morph in branch mybranch has deployment type: tar + AND system test-system in cluster test-cluster.morph in branch mybranch has deployment location: test-system.tar + WHEN the user attempts to deploy the cluster test-cluster.morph in branch mybranch + THEN morph succeeded + AND file workspace/mybranch/test/morphs/test-system.tar exists diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index e99c1817..9cf497fc 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -382,6 +382,10 @@ Attempt to check out a system branch from a root that has no systems. cd "$DATADIR/workspace" attempt_morph checkout test:test-chunk master + IMPLEMENTS WHEN the user attempts to check out the system branch from (\S+) called (\S+) + cd "$DATADIR/workspace" + attempt_morph branch-from-image --metadata-dir "$DATADIR/$MATCH_1/baserock" "$MATCH_2" + We also need to verify that a system branch has been checked out. IMPLEMENTS THEN the system branch (\S+) is checked out -- cgit v1.2.1 From 3c27e08f558bd55b9a39ee8a0a43a08881126f1c Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 24 Sep 2014 20:54:18 +0000 Subject: Move architecture existing test to yarn --- tests.as-root/archless-system-fails.exit | 1 - tests.as-root/archless-system-fails.script | 35 ------------------------------ tests.as-root/archless-system-fails.stderr | 1 - yarns/architecture.yarn | 12 ++++++++++ yarns/implementations.yarn | 19 ++++++++++++++++ 5 files changed, 31 insertions(+), 37 deletions(-) delete mode 100644 tests.as-root/archless-system-fails.exit delete mode 100755 tests.as-root/archless-system-fails.script delete mode 100644 tests.as-root/archless-system-fails.stderr diff --git a/tests.as-root/archless-system-fails.exit b/tests.as-root/archless-system-fails.exit deleted file mode 100644 index d00491fd..00000000 --- a/tests.as-root/archless-system-fails.exit +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests.as-root/archless-system-fails.script b/tests.as-root/archless-system-fails.script deleted file mode 100755 index e34e9ad6..00000000 --- a/tests.as-root/archless-system-fails.script +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Raise an error if system has no architecture defined. - -set -eu - -cd "$DATADIR/morphs" -git checkout --quiet -b archless master -cat <archless-system.morph -name: archless-system -kind: system -strata: - - morph: hello-stratum -EOF -git add archless-system.morph -git commit --quiet -m "add archless system" - -"$SRCDIR/scripts/test-morph" \ - build-morphology test:morphs archless archless-system diff --git a/tests.as-root/archless-system-fails.stderr b/tests.as-root/archless-system-fails.stderr deleted file mode 100644 index 71e7dbb0..00000000 --- a/tests.as-root/archless-system-fails.stderr +++ /dev/null @@ -1 +0,0 @@ -ERROR: Missing field arch from morphology string diff --git a/yarns/architecture.yarn b/yarns/architecture.yarn index d4de308e..07274ec3 100644 --- a/yarns/architecture.yarn +++ b/yarns/architecture.yarn @@ -22,3 +22,15 @@ Morph Cross-Bootstrap Tests WHEN the user checks out the system branch called master THEN the user cross-bootstraps the system base-system-testarch.morph in branch master of repo test:morphs to the arch testarch FINALLY the git server is shut down + +Architecture validation Tests +============================= + + SCENARIO building a system with no architecture + GIVEN a workspace + AND a git server + AND a system called base-system-noarch.morph with no architecture in the git server + WHEN the user checks out the system branch called master + AND the user attempts to build the system base-system-testarch.morph in branch master + THEN morph failed + FINALLY the git server is shut down diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 9cf497fc..86c3a9c4 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -363,6 +363,25 @@ have a morphology using the test architecture. run_in "$DATADIR/gits/morphs" git add "$MATCH_1" run_in "$DATADIR/gits/morphs" git commit -m "Added $MATCH_1 and strata morphologies." +You need an architecture to build a system, we don't default to the host architecture. + + IMPLEMENTS GIVEN a system called (\S+) with no architecture in the git server + name="$(basename "${MATCH_1%.*}")" + cat << EOF > "$DATADIR/gits/morphs/$MATCH_1" + configuration-extensions: [] + description: A system called $name for test architecture + kind: system + name: $name + strata: + - name: build-essential + morph: strata/build-essential.morph + - name: core + morph: strata/core.morph + EOF + + run_in "$DATADIR/gits/morphs" git add "$MATCH_1" + run_in "$DATADIR/gits/morphs" git commit -m "Added $MATCH_1." + Implementation sections for system branch operations ---------------------------------------------------- -- cgit v1.2.1 From 2db961da51842807f150d0a0c293f5c5b8bd2832 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 24 Sep 2014 20:57:20 +0000 Subject: Remove tarball image format tests from tests.as-root We're implicitly testing it in yarns already. Explicit tests would be better, but removing the old test makes it easier to get rid of build-mode: test. --- tests.as-root/tarball-image-is-sensible.script | 44 ---------------- tests.as-root/tarball-image-is-sensible.setup | 70 -------------------------- tests.as-root/tarball-image-is-sensible.stderr | 1 - tests.as-root/tarball-image-is-sensible.stdout | 42 ---------------- 4 files changed, 157 deletions(-) delete mode 100755 tests.as-root/tarball-image-is-sensible.script delete mode 100755 tests.as-root/tarball-image-is-sensible.setup delete mode 100644 tests.as-root/tarball-image-is-sensible.stderr delete mode 100644 tests.as-root/tarball-image-is-sensible.stdout diff --git a/tests.as-root/tarball-image-is-sensible.script b/tests.as-root/tarball-image-is-sensible.script deleted file mode 100755 index 0fb6d9b2..00000000 --- a/tests.as-root/tarball-image-is-sensible.script +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# A system kind of 'rootfs-tarball' should create a tarball containing -# all the required files. - -set -eu - -# Disable test on versions of Python before 2.7. -. "$SRCDIR/scripts/python-check" - -. "$SRCDIR/scripts/fix-committer-info" - -tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact \ - build-morphology test:morphs tarball-links hello-tarball) - -# Verify that the tar archive is not compressed. -if gunzip -t "$tar" > /dev/null -then - echo "ERROR: $tar is gzip'd!" 1>&2 - exit 1 -fi - -extracted="$DATADIR/extracted" -mkdir -p "$extracted" -cd "$extracted" -tar -xf "$tar" -find . -mindepth 1 | xargs ls -dF | LC_ALL=C sort -u -cat "$extracted/etc/os-release" diff --git a/tests.as-root/tarball-image-is-sensible.setup b/tests.as-root/tarball-image-is-sensible.setup deleted file mode 100755 index a687b691..00000000 --- a/tests.as-root/tarball-image-is-sensible.setup +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -set -eu - -source "$SRCDIR/scripts/fix-committer-info" - -chunkrepo="$DATADIR/chunk-repo" -cd "$chunkrepo" -git checkout -b tarball-links -cat >links.morph <<'EOF' -name: links -kind: chunk -build-system: manual -install-commands: - - mkdir -p "$DESTDIR/bin" - - touch "$DESTDIR/bin/true" - - cd "$DESTDIR/bin" && ln true true-hardlink - - cd "$DESTDIR/bin" && ln -s true true-symlink -EOF -git add links.morph -git commit --quiet -m 'Add link adding chunk' - -morphsrepo="$DATADIR/morphs" -cd "$morphsrepo" -git checkout -b tarball-links -cat < hello-tarball.morph -name: hello-tarball -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: link-stratum - - morph: hello-stratum - - morph: linux-stratum -EOF -git add hello-tarball.morph - -# Change build-depends ref of hello-stratum from master to tarball-links -sed -i linux-stratum.morph \ - -e '/build-depends:/,/chunks/ s/ref: master/ref: tarball-links/' - -git add linux-stratum.morph - -cat < link-stratum.morph -name: link-stratum -kind: stratum -chunks: - - name: links - repo: test:chunk-repo - ref: tarball-links - build-mode: test - build-depends: [] -EOF -git add link-stratum.morph -git commit --quiet -m "add morphs" diff --git a/tests.as-root/tarball-image-is-sensible.stderr b/tests.as-root/tarball-image-is-sensible.stderr deleted file mode 100644 index 9db65e02..00000000 --- a/tests.as-root/tarball-image-is-sensible.stderr +++ /dev/null @@ -1 +0,0 @@ -gunzip: invalid magic diff --git a/tests.as-root/tarball-image-is-sensible.stdout b/tests.as-root/tarball-image-is-sensible.stdout deleted file mode 100644 index cf74a1ec..00000000 --- a/tests.as-root/tarball-image-is-sensible.stdout +++ /dev/null @@ -1,42 +0,0 @@ -./baserock/ -./baserock/hello-bins.meta -./baserock/hello-devel.meta -./baserock/hello-doc.meta -./baserock/hello-libs.meta -./baserock/hello-locale.meta -./baserock/hello-misc.meta -./baserock/hello-stratum-devel.meta -./baserock/hello-stratum-runtime.meta -./baserock/hello-tarball-rootfs.meta -./baserock/link-stratum-devel.meta -./baserock/link-stratum-runtime.meta -./baserock/links-bins.meta -./baserock/links-devel.meta -./baserock/links-doc.meta -./baserock/links-libs.meta -./baserock/links-locale.meta -./baserock/links-misc.meta -./baserock/linux-bins.meta -./baserock/linux-devel.meta -./baserock/linux-doc.meta -./baserock/linux-libs.meta -./baserock/linux-locale.meta -./baserock/linux-misc.meta -./baserock/linux-stratum-devel.meta -./baserock/linux-stratum-runtime.meta -./bin/ -./bin/hello* -./bin/true -./bin/true-hardlink -./bin/true-symlink@ -./boot/ -./boot/System.map -./boot/vmlinuz -./etc/ -./etc/os-release -./extlinux.conf -NAME="Baserock" -ID=baserock -HOME_URL="http://wiki.baserock.org" -SUPPORT_URL="http://wiki.baserock.org/mailinglist" -BUG_REPORT_URL="http://wiki.baserock.org/mailinglist" -- cgit v1.2.1 From 037baabc2064700d2d2fa21cdacbf65058b3ce96 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 24 Sep 2014 21:10:25 +0000 Subject: Remove run-in-artifact subcommand It's easy enough to deploy the image. --- morphlib/plugins/artifact_inspection_plugin.py | 2 +- .../run-in-artifact-propagates-exit-code.exit | 1 - .../run-in-artifact-propagates-exit-code.script | 33 --------------- .../run-in-artifact-propagates-exit-code.stderr | 3 -- ...run-in-artifact-with-different-artifacts.script | 47 ---------------------- ...run-in-artifact-with-different-artifacts.stdout | 32 --------------- 6 files changed, 1 insertion(+), 117 deletions(-) delete mode 100644 tests.as-root/run-in-artifact-propagates-exit-code.exit delete mode 100755 tests.as-root/run-in-artifact-propagates-exit-code.script delete mode 100644 tests.as-root/run-in-artifact-propagates-exit-code.stderr delete mode 100755 tests.as-root/run-in-artifact-with-different-artifacts.script delete mode 100644 tests.as-root/run-in-artifact-with-different-artifacts.stdout diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index 6eeece77..d47de334 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.exit b/tests.as-root/run-in-artifact-propagates-exit-code.exit deleted file mode 100644 index d00491fd..00000000 --- a/tests.as-root/run-in-artifact-propagates-exit-code.exit +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.script b/tests.as-root/run-in-artifact-propagates-exit-code.script deleted file mode 100755 index d815c73d..00000000 --- a/tests.as-root/run-in-artifact-propagates-exit-code.script +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Test that 'run-in-artifact' propagates the exit code of its command. - -set -eu - -. "$SRCDIR/tests.as-root/lib" - -# Build first image. Remember the stratum. -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs master linux-system - -system=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*-rootfs') - -# Run 'run-in-artifact' with the system artifact. The command will fail -# and this should result in an exit code of 1 in the test. -"$SRCDIR/scripts/test-morph" run-in-artifact "$system" -- ls i-do-not-exist diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.stderr b/tests.as-root/run-in-artifact-propagates-exit-code.stderr deleted file mode 100644 index 98aa5450..00000000 --- a/tests.as-root/run-in-artifact-propagates-exit-code.stderr +++ /dev/null @@ -1,3 +0,0 @@ -ERROR: Command failed: ls i-do-not-exist -ls: i-do-not-exist: No such file or directory - diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.script b/tests.as-root/run-in-artifact-with-different-artifacts.script deleted file mode 100755 index 57d408e3..00000000 --- a/tests.as-root/run-in-artifact-with-different-artifacts.script +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Test the 'run-in-artifact' command with different types of artifacts. - -set -eu - -. "$SRCDIR/tests.as-root/lib" - -# Build first image. Remember the stratum. -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs master linux-system - -system=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*-rootfs') -chunk=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.chunk.linux-misc') -stratum=$(find "$DATADIR/cache/artifacts" -maxdepth 1 \ - -name '*.stratum.linux-stratum-runtime') - -# Run 'run-in-artifact' with the system artifact. -echo "System:" -"$SRCDIR/scripts/test-morph" run-in-artifact "$system" -- ls baserock/ -echo - -# Run 'run-in-artifact' with the chunk artifact. -echo "Chunk:" -"$SRCDIR/scripts/test-morph" run-in-artifact "$chunk" -- ls baserock/ -echo - -# Run 'run-in-artifact' with the statum artifact. -echo "Stratum:" -"$SRCDIR/scripts/test-morph" run-in-artifact "$stratum" -- ls baserock/ \ - 2>/dev/null || echo "Failed" diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.stdout b/tests.as-root/run-in-artifact-with-different-artifacts.stdout deleted file mode 100644 index 7473990b..00000000 --- a/tests.as-root/run-in-artifact-with-different-artifacts.stdout +++ /dev/null @@ -1,32 +0,0 @@ -System: -hello-bins.meta -hello-devel.meta -hello-doc.meta -hello-libs.meta -hello-locale.meta -hello-misc.meta -hello-stratum-devel.meta -hello-stratum-runtime.meta -linux-bins.meta -linux-devel.meta -linux-doc.meta -linux-libs.meta -linux-locale.meta -linux-misc.meta -linux-stratum-devel.meta -linux-stratum-runtime.meta -linux-system-rootfs.meta -tools-bins.meta -tools-devel.meta -tools-doc.meta -tools-libs.meta -tools-locale.meta -tools-misc.meta -tools-stratum-devel.meta -tools-stratum-runtime.meta - -Chunk: -linux-misc.meta - -Stratum: -Failed -- cgit v1.2.1 From bb43c48bd3ee9e18ce07d1a9b0649afe1676a3c1 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 24 Sep 2014 21:21:17 +0000 Subject: Remove overlap detection logic I've rarely needed to use it, and on those rare occasions, it would have been easy enough to calculate it. Let's get rid of this step, and save everyone some time in future. --- morphlib/builder2.py | 64 ------------ morphlib/plugins/artifact_inspection_plugin.py | 32 ------ scripts/list-overlaps | 45 --------- tests.as-root/system-overlap.script | 113 ---------------------- tests.as-root/system-overlap.stdout | 3 - tests.build/stratum-overlap-warns.script | 39 -------- tests.build/stratum-overlap-warns.setup | 102 ------------------- tests.build/stratum-overlap-writes-overlap.script | 35 ------- tests.build/stratum-overlap-writes-overlap.setup | 1 - tests.build/stratum-overlap-writes-overlap.stdout | 4 - 10 files changed, 438 deletions(-) delete mode 100755 scripts/list-overlaps delete mode 100755 tests.as-root/system-overlap.script delete mode 100644 tests.as-root/system-overlap.stdout delete mode 100755 tests.build/stratum-overlap-warns.script delete mode 100755 tests.build/stratum-overlap-warns.setup delete mode 100755 tests.build/stratum-overlap-writes-overlap.script delete mode 120000 tests.build/stratum-overlap-writes-overlap.setup delete mode 100644 tests.build/stratum-overlap-writes-overlap.stdout diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 20cae225..9cd3a074 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -162,46 +162,6 @@ def get_stratum_files(f, lac): # pragma: no cover cf.close() -def get_overlaps(source, constituents, lac): # pragma: no cover - # check whether strata overlap - installed = defaultdict(set) - for dep in constituents: - handle = lac.get(dep) - if source.morphology['kind'] == 'stratum': - for filename in get_chunk_files(handle): - installed[filename].add(dep) - elif source.morphology['kind'] == 'system': - for filename in get_stratum_files(handle, lac): - installed[filename].add(dep) - handle.close() - overlaps = defaultdict(set) - for filename, artifacts in installed.iteritems(): - if len(artifacts) > 1: - overlaps[frozenset(artifacts)].add(filename) - return overlaps - - -def log_overlaps(overlaps): # pragma: no cover - for overlapping, files in sorted(overlaps.iteritems()): - logging.warning(' Artifacts %s overlap with files:' % - ', '.join(sorted(a.name for a in overlapping))) - for filename in sorted(files): - logging.warning(' %s' % filename) - - -def write_overlap_metadata(artifact, overlaps, lac): # pragma: no cover - f = lac.put_artifact_metadata(artifact, 'overlaps') - # the big list comprehension is because json can't serialize - # artifacts, sets or dicts with non-string keys - json.dump( - [ - [ - [a.name for a in afs], list(files) - ] for afs, files in overlaps.iteritems() - ], f, indent=4, encoding='unicode-escape') - f.close() - - class BuilderBase(object): '''Base class for building artifacts.''' @@ -559,18 +519,6 @@ class StratumBuilder(BuilderBase): download_depends(constituents, self.local_artifact_cache, self.remote_artifact_cache) - # check for chunk overlaps - overlaps = get_overlaps(self.source, constituents, - self.local_artifact_cache) - if len(overlaps) > 0: - logging.warning( - 'Overlaps in stratum artifact %s detected' %a_name) - log_overlaps(overlaps) - self.app.status(msg='Overlaps in stratum artifact ' - '%(stratum_name)s detected', - stratum_name=a_name, error=True) - write_overlap_metadata(a, overlaps, - self.local_artifact_cache) with self.build_watch('create-chunk-list'): lac = self.local_artifact_cache @@ -674,18 +622,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover self.remote_artifact_cache) f.close() - # check whether the strata overlap - overlaps = get_overlaps(self.source, self.source.dependencies, - self.local_artifact_cache) - if len(overlaps) > 0: - self.app.status(msg='Overlaps in system artifact ' - '%(artifact_name)s detected', - artifact_name=a_name, - error=True) - log_overlaps(overlaps) - write_overlap_metadata(a, overlaps, - self.local_artifact_cache) - # unpack it from the local artifact cache for stratum_artifact in self.source.dependencies: self.unpack_one_stratum(stratum_artifact, path) diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index d47de334..74645f41 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -257,9 +257,6 @@ class ManifestGenerator(object): class ArtifactInspectionPlugin(cliapp.Plugin): def enable(self): - self.app.add_subcommand('run-in-artifact', - self.run_in_artifact, - arg_synopsis='ARTIFACT CMD') self.app.add_subcommand('generate-manifest', self.generate_manifest, arg_synopsis='SYSTEM-ARTIFACT') @@ -267,35 +264,6 @@ class ArtifactInspectionPlugin(cliapp.Plugin): def disable(self): pass - def run_in_artifact(self, args): - '''Run a command inside an extracted/mounted chunk or system. - - Command line arguments: - - * `ARTIFACT` is a filename for the artifact. - * `CMD` is the command to run. - - run-in-artifact unpacks an artifact, and runs a command in - the temporary directory it was unpacked to. - - The command must be given to Morph as a single argument, so - use shell quoting appropriately. - - ''' - - if len(args) < 2: - raise cliapp.AppException( - 'run-in-artifact requires arguments: a chunk or ' - 'system artifact and a a shell command') - - artifact, cmd = args[0], args[1:] - - def run_command_in_dir(dirname): - output = self.app.runcmd(cmd, cwd=dirname) - self.app.output.write(output) - - call_in_artifact_directory(self.app, artifact, run_command_in_dir) - def generate_manifest(self, args): '''Generate a content manifest for a system image. diff --git a/scripts/list-overlaps b/scripts/list-overlaps deleted file mode 100755 index d092ba75..00000000 --- a/scripts/list-overlaps +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -# This is a program to convert the json dump of the overlaps between artifacts -# in a format more suited to shell programs, or human reading - -import json - -import cliapp - - -class ListOverlaps(cliapp.Application): - - @staticmethod - def _load_overlap(filename): - data = json.load(open(filename), encoding='unicode-escape') - overlaps = dict((frozenset(pair[0]), set(pair[1])) for pair in data) - return overlaps - - def cmd_groups(self, args): - overlaps = ListOverlaps._load_overlap(args[0]) - for group in overlaps: - print(' '.join(sorted(group))) - - def cmd_list_files(self, args): - overlaps = self._load_overlap(args[0]) - group = frozenset(args[1:]) - for filename in overlaps[group]: - print filename - -ListOverlaps().run() diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script deleted file mode 100755 index 9be6df13..00000000 --- a/tests.as-root/system-overlap.script +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# If a system has multiple strata that have the same files in them, -# then this should be noted. - -set -eu - -cache="$DATADIR/cache/artifacts" -chunkrepo="$DATADIR/chunk-repo" -morphsrepo="$DATADIR/morphs" - -cd "$morphsrepo" -git checkout --quiet -b overlap master -cat <overlap-system.morph -name: overlap-system -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: foo-baz-stratum - - morph: foo-barqux-stratum -EOF -cat <foo-baz-stratum.morph -name: foo-baz-stratum -kind: stratum -chunks: - - name: overlap-foo-baz - repo: test:chunk-repo - ref: overlap - build-mode: test - build-depends: [] - - name: linux - repo: test:kernel-repo - ref: master - build-mode: test - build-depends: - - overlap-foo-baz -EOF -cat <foo-barqux-stratum.morph -name: foo-barqux-stratum -kind: stratum -chunks: - - name: overlap-foobar - repo: test:chunk-repo - ref: overlap - build-mode: test - build-depends: [] - - name: overlap-fooqux - repo: test:chunk-repo - ref: overlap - build-mode: test - build-depends: - - overlap-foobar -EOF -git add overlap-system.morph foo-baz-stratum.morph foo-barqux-stratum.morph -git commit --quiet -m "add overlapping system" - -cd "$chunkrepo" -git checkout --quiet -b overlap master -cat <overlap-foo-baz.morph -name: overlap-foo-baz -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR"/bin - - for f in foo bar baz; do echo echo \$f >"\$DESTDIR"/bin/\$f; done -EOF - -cat <overlap-foobar.morph -name: overlap-foobar -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR"/usr/bin "\$DESTDIR"/bin - - echo echo foobar >"\$DESTDIR"/usr/bin/foobar - - ln -s /usr/bin/foobar "\$DESTDIR"/bin/foo - - ln -s /usr/bin/foobar "\$DESTDIR"/bin/bar -EOF - -cat <overlap-fooqux.morph -name: overlap-fooqux -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR"/usr/bin "\$DESTDIR"/bin - - for f in qux fooqux; do echo echo \$f >"\$DESTDIR"/usr/bin/\$f; done - - ln -s /usr/bin/fooqux "\$DESTDIR"/bin/foo -EOF -git add overlap-*.morph - -git commit --quiet -m 'Add overlapping chunks' - -"$SRCDIR/scripts/test-morph" \ - build-morphology test:morphs overlap overlap-system > /dev/null -"$SRCDIR/scripts/list-overlaps" groups \ - "$cache"/*.system.overlap-system*.overlaps | -while IFS='\n' read overlaps; do - echo $overlaps - "$SRCDIR/scripts/list-overlaps" list-files \ - "$cache"/*.system.overlap-system*.overlaps $overlaps -done diff --git a/tests.as-root/system-overlap.stdout b/tests.as-root/system-overlap.stdout deleted file mode 100644 index f67d54c8..00000000 --- a/tests.as-root/system-overlap.stdout +++ /dev/null @@ -1,3 +0,0 @@ -foo-barqux-stratum-runtime foo-baz-stratum-runtime -bin/foo -bin/bar diff --git a/tests.build/stratum-overlap-warns.script b/tests.build/stratum-overlap-warns.script deleted file mode 100755 index 2a3b06e1..00000000 --- a/tests.build/stratum-overlap-warns.script +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## If a stratum has multiple chunks that have the same files in them, -## then this should be noted. - -set -eu - -log="$DATADIR/morph.log" -warnings="$DATADIR/warnings" -cache="$DATADIR/cache/artifacts" - -warning_mentions(){ - grep -F "$1" <"$warnings" >/dev/null 2>/dev/null -} - -"$SRCDIR/scripts/test-morph" build-morphology --log=$log \ - test:morphs-repo overlap hello-system > /dev/null -grep WARNING "$log" >"$warnings" -for str in hello-stratum \ - overlap-foo-baz overlap-foobar bin/bar \ - overlap-fooqux bin/foo; do - warning_mentions 'hello-stratum' || exit $? -done diff --git a/tests.build/stratum-overlap-warns.setup b/tests.build/stratum-overlap-warns.setup deleted file mode 100755 index b969822d..00000000 --- a/tests.build/stratum-overlap-warns.setup +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/sh -# -# If a stratum has multiple chunks that have the same files in them, -# then this should be notified -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -eu - -chunkrepo="$DATADIR/chunk-repo" -morphsrepo="$DATADIR/morphs-repo" - -cd "$morphsrepo" -git checkout --quiet -b overlap master -cat <hello-stratum.morph -name: hello-stratum -kind: stratum -chunks: - - name: dirs - repo: test:chunk-repo - ref: overlap - build-depends: [] - build-mode: test - - name: overlap-foobar - repo: test:chunk-repo - ref: overlap - build-depends: - - dirs - build-mode: test - - name: overlap-fooqux - repo: test:chunk-repo - ref: overlap - build-depends: - - overlap-foobar - build-mode: test - - name: overlap-foo-baz - repo: test:chunk-repo - ref: overlap - build-depends: - - overlap-fooqux - build-mode: test -EOF -sed -i 's/master/overlap/' hello-system.morph -git add hello-stratum.morph hello-system.morph -git commit --quiet -m "Make hello stratum contain overlaps" - -cd "$chunkrepo" -git checkout --quiet -b overlap master - -cat <dirs.morph -name: dirs -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/bin" - - ln -s .. "\$DESTDIR/usr" -EOF -git add dirs.morph - -cat <overlap-foo-baz.morph -name: overlap-foo-baz -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/bin" - - for f in foo bar baz; do echo echo \$f >"\$DESTDIR/bin/\$f"; done -EOF -git add overlap-foo-baz.morph - -cat <overlap-foobar.morph -name: overlap-foobar -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/usr/bin" "\$DESTDIR/bin" - - echo echo foobar >"\$DESTDIR/usr/bin/foobar" - - ln -s /usr/bin/foobar "\$DESTDIR/bin/foo" - - ln -s /usr/bin/foobar "\$DESTDIR/bin/bar" -EOF -git add overlap-foobar.morph - -cat <overlap-fooqux.morph -name: overlap-fooqux -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/usr/bin" "\$DESTDIR/bin" - - for f in qux fooqux; do echo echo \$f >"\$DESTDIR/usr/bin/\$f"; done - - ln -s /usr/bin/fooqux "\$DESTDIR/bin/foo" -EOF -git add overlap-fooqux.morph - -git commit --quiet -m 'Add overlapping chunks' diff --git a/tests.build/stratum-overlap-writes-overlap.script b/tests.build/stratum-overlap-writes-overlap.script deleted file mode 100755 index fe4ed4ee..00000000 --- a/tests.build/stratum-overlap-writes-overlap.script +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## If a stratum has multiple chunks that have the same files in them, -## then the overlaps must be written to the cache - -set -eu - -cache="$DATADIR/cache/artifacts" - - -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs-repo overlap hello-system > /dev/null -"$SRCDIR/scripts/list-overlaps" groups \ - "$cache"/*.stratum.hello-stratum-*.overlaps | -while IFS='\n' read overlaps; do - echo $overlaps - "$SRCDIR/scripts/list-overlaps" list-files \ - "$cache"/*.stratum.hello-stratum-*.overlaps $overlaps -done diff --git a/tests.build/stratum-overlap-writes-overlap.setup b/tests.build/stratum-overlap-writes-overlap.setup deleted file mode 120000 index 255e9a74..00000000 --- a/tests.build/stratum-overlap-writes-overlap.setup +++ /dev/null @@ -1 +0,0 @@ -stratum-overlap-warns.setup \ No newline at end of file diff --git a/tests.build/stratum-overlap-writes-overlap.stdout b/tests.build/stratum-overlap-writes-overlap.stdout deleted file mode 100644 index 1e36ca83..00000000 --- a/tests.build/stratum-overlap-writes-overlap.stdout +++ /dev/null @@ -1,4 +0,0 @@ -overlap-foo-baz-bins overlap-foobar-bins overlap-fooqux-bins -bin/foo -overlap-foo-baz-bins overlap-foobar-bins -bin/bar -- cgit v1.2.1 From 01ae89708994a7b3baa59683d2e264b5c6e3bd42 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 24 Sep 2014 21:30:55 +0000 Subject: Remove remaining test.as-root cmdtests The repo-alias is indirectly tested by branch-from-image, and checking the --version is part of the release process, so we're not going to miss these tests. --- check | 16 -- tests.as-root/lib | 36 ---- .../metadata-includes-morph-version.script | 53 ------ .../metadata-includes-morph-version.setup | 36 ---- tests.as-root/metadata-includes-repo-alias.script | 49 ------ tests.as-root/metadata-includes-repo-alias.setup | 36 ---- tests.as-root/setup | 194 --------------------- tests.as-root/setup-build | 35 ---- 8 files changed, 455 deletions(-) delete mode 100644 tests.as-root/lib delete mode 100755 tests.as-root/metadata-includes-morph-version.script delete mode 100755 tests.as-root/metadata-includes-morph-version.setup delete mode 100755 tests.as-root/metadata-includes-repo-alias.script delete mode 100755 tests.as-root/metadata-includes-repo-alias.setup delete mode 100755 tests.as-root/setup delete mode 100644 tests.as-root/setup-build diff --git a/check b/check index a942ca90..dc4f96bc 100755 --- a/check +++ b/check @@ -162,19 +162,3 @@ elif ! (python --version 2>&1 | grep -q '^Python 2\.[78]'); then else cmdtest tests.build fi - -# The as-root tests use YAML morphologies, so they require the PyYAML module. -if ! "$run_slow_cmdtests"; then - echo "NOT RUNNING tests.as-root" -elif [ $(whoami) != root ] || ! command -v mkfs.btrfs > /dev/null; then - echo "NOT RUNNING tests.as-root (no btrfs)" -elif ! python -c " -import morphlib, sys -if not morphlib.got_yaml: - sys.exit(1) -" > /dev/null 2>&1 -then - echo "NOT RUNNING tests.as-root (requires PyYAML)" -else - cmdtest tests.as-root -fi diff --git a/tests.as-root/lib b/tests.as-root/lib deleted file mode 100644 index d17d8e32..00000000 --- a/tests.as-root/lib +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Helper functions for as-root tests - -loopback_rootfs() { - # Find offset partition offset in a rootfs and mount it - ROOTFS="$1" - - PYTHONPATH="$SRCDIR" "$SRCDIR/scripts/sparse-gunzip" \ - <"$ROOTFS" >"$ROOTFS-unzipped" - - OFFSET=$(sfdisk -d "$ROOTFS-unzipped" | \ - grep -m 1 -o 'start=\s\+\([0-9]\+\)' | awk '{ print $2 }') - OFFSET=$(expr "$OFFSET" '*' 512) - - DEVICE=$(losetup --show -o "$OFFSET" -f "$ROOTFS-unzipped") - udevadm settle - - echo "$DEVICE" -} diff --git a/tests.as-root/metadata-includes-morph-version.script b/tests.as-root/metadata-includes-morph-version.script deleted file mode 100755 index 772d7d50..00000000 --- a/tests.as-root/metadata-includes-morph-version.script +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013 - 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# All the metadata files in /baserock of a system contain morph's version -# information - -set -eu - -# Disable test on versions of Python before 2.7. -. "$SRCDIR/scripts/python-check" - -. "$SRCDIR/scripts/fix-committer-info" - -# get git version info -cd "$SRCDIR" -description="$(git describe --abbrev=40 --always \ - --dirty=-unreproducible --match=DO-NOT-MATCH-ANY-TAGS)" -commit="$(git rev-parse 'HEAD^{commit}')" -tree="$(git rev-parse 'HEAD^{tree}')" -ref="$(git rev-parse --symbolic-full-name 'HEAD')" - -tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact \ - build-morphology test:morphs tarball hello-tarball) - -extracted="$DATADIR/extracted" -mkdir -p "$extracted" -tar -xf "$tar" -C "$extracted" -cd "$extracted/baserock" -for f in *.meta; do - # Check for git describe output - grep -q -F -e "$description" "$f" - # Check the Sha-1 commit is included - grep -q -F -e "$commit" "$f" - # Check the Sha-1 of the commit's tree is included - grep -q -F -e "$tree" "$f" - # Check the ref (e.g. branch) is included - grep -q -F -e "$ref" "$f" -done diff --git a/tests.as-root/metadata-includes-morph-version.setup b/tests.as-root/metadata-includes-morph-version.setup deleted file mode 100755 index e4557302..00000000 --- a/tests.as-root/metadata-includes-morph-version.setup +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -set -eu - -source "$SRCDIR/scripts/fix-committer-info" - -morphsrepo="$DATADIR/morphs" -cd "$morphsrepo" -git checkout -b tarball -cat < hello-tarball.morph -name: hello-tarball -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: hello-stratum - - morph: linux-stratum -EOF -git add hello-tarball.morph - -git commit --quiet -m "add tarball system" diff --git a/tests.as-root/metadata-includes-repo-alias.script b/tests.as-root/metadata-includes-repo-alias.script deleted file mode 100755 index 9e4a5d98..00000000 --- a/tests.as-root/metadata-includes-repo-alias.script +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# All the metadata files in /baserock of a system contain the repository -# alias they were built from. - -set -eu - -# Disable test on versions of Python before 2.7. -. "$SRCDIR/scripts/python-check" - -. "$SRCDIR/scripts/fix-committer-info" - -# get git version info -cd "$SRCDIR" - -tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact \ - build-morphology test:morphs tarball hello-tarball) - -extracted="$DATADIR/extracted" -mkdir -p "$extracted" -tar -xf "$tar" -C "$extracted" -cd "$extracted/baserock" - -# Check for test:morphs in System and Stratum -#grep -q -F -e test:morphs hello-tarball.meta # tarball bug -grep -q -F -e test:morphs hello-stratum-runtime.meta -grep -q -F -e test:morphs linux-stratum-runtime.meta - -# Check for test:kernel-repo in linux -grep -q -F -e test:kernel-repo linux-misc.meta - -# Check for test:chunk-repo in hello -grep -q -F -e test:chunk-repo hello-bins.meta diff --git a/tests.as-root/metadata-includes-repo-alias.setup b/tests.as-root/metadata-includes-repo-alias.setup deleted file mode 100755 index e4557302..00000000 --- a/tests.as-root/metadata-includes-repo-alias.setup +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -set -eu - -source "$SRCDIR/scripts/fix-committer-info" - -morphsrepo="$DATADIR/morphs" -cd "$morphsrepo" -git checkout -b tarball -cat < hello-tarball.morph -name: hello-tarball -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: hello-stratum - - morph: linux-stratum -EOF -git add hello-tarball.morph - -git commit --quiet -m "add tarball system" diff --git a/tests.as-root/setup b/tests.as-root/setup deleted file mode 100755 index 1cf9dd04..00000000 --- a/tests.as-root/setup +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash -# -# Create git repositories for tests. The chunk repository will contain a -# simple "hello, world" C program, and two branches ("master", "farrokh"), -# with the master branch containing just a README. The two branches are there -# so that we can test building a branch that hasn't been checked out. -# The branches are different so that we know that if the wrong branch -# is uses, the build will fail. -# -# The stratum repository contains a single branch, "master", with a -# stratum and a system morphology that include the chunk above. -# -# Copyright (C) 2011, 2012, 2013, 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -set -eu - -source "$SRCDIR/scripts/fix-committer-info" - -# The $DATADIR should be empty at the beginnig of each test. -find "$DATADIR" -mindepth 1 -delete - -# Create an empty directory to be used as a morph workspace -mkdir "$DATADIR/workspace" - -# Create chunk repository. - -chunkrepo="$DATADIR/chunk-repo" -mkdir "$chunkrepo" -cd "$chunkrepo" -git init --quiet - -cat < README -This is a sample README. -EOF -git add README -git commit --quiet -m "add README" - -git checkout --quiet -b farrokh - -cat < hello.c -#include -int main(void) -{ - puts("hello, world"); - return 0; -} -EOF -git add hello.c - -cat < hello.morph -name: hello -kind: chunk -build-system: dummy -build-commands: - - gcc -o hello hello.c -install-commands: - - install -d "\$DESTDIR"/etc - - install -d "\$DESTDIR"/bin - - install hello "\$DESTDIR"/bin/hello -EOF -git add hello.morph - -git commit --quiet -m "add a hello world program and morph" - -git checkout --quiet master - - -# Create tools repository, so there is an extra layer of build-depends -toolsrepo="$DATADIR/tools-repo" -mkdir -p "$toolsrepo" -cd "$toolsrepo" -git init --quiet -cat <<'EOF' >tools.morph -name: tools -kind: chunk -build-commands: - - echo 'echo dummy strace' >strace -install-commands: - - mkdir -p "$DESTDIR/bin" - - install strace "$DESTDIR/bin/strace" -EOF -git add tools.morph -git commit --quiet -m "add morphology" - -# Create morph repository. - -morphsrepo="$DATADIR/morphs" -mkdir "$morphsrepo" -cd "$morphsrepo" -git init --quiet - -cat < hello-stratum.morph -name: hello-stratum -kind: stratum -chunks: - - name: hello - repo: test:chunk-repo - ref: farrokh - build-mode: test - build-depends: [] -EOF -git add hello-stratum.morph - -cat < tools-stratum.morph -name: tools-stratum -kind: stratum -build-depends: - - morph: linux-stratum -chunks: - - name: tools - repo: test:tools-repo - ref: master - build-mode: test - build-depends: [] -EOF -git add tools-stratum.morph - -cat < hello-system.morph -name: hello-system -kind: system -arch: `"$SRCDIR/scripts/test-morph" print-architecture` -strata: - - morph: hello-stratum -EOF -git add hello-system.morph - -cat < linux-stratum.morph -name: linux-stratum -kind: stratum -build-depends: - - morph: hello-stratum -chunks: - - name: linux - repo: test:kernel-repo - ref: master - build-mode: test - build-depends: [] -EOF -git add linux-stratum.morph - -cat < linux-system.morph -name: linux-system -kind: system -arch: `"$SRCDIR/scripts/test-morph" print-architecture` -strata: - - morph: hello-stratum - - morph: linux-stratum - - morph: tools-stratum -EOF -git add linux-system.morph - -git commit --quiet -m "add morphs" - -# Make a dummy kernel chunk. -mkdir "$DATADIR/kernel-repo" -cat < "$DATADIR/kernel-repo/linux.morph" -name: linux -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/boot" - - touch "\$DESTDIR/extlinux.conf" - - touch "\$DESTDIR/boot/vmlinuz" - - touch "\$DESTDIR/boot/System.map" -EOF -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" init --quiet -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" add . -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" commit --quiet -m foo \ - > /dev/null - -# Create a morph configuration file. -cat < "$DATADIR/morph.conf" -[config] -repo-alias = test=file://$DATADIR/#file://$DATADIR/ -cachedir = $DATADIR/cache -log = $DATADIR/morph.log -no-distcc = true -quiet = true -log = /tmp/morph.log -EOF - diff --git a/tests.as-root/setup-build b/tests.as-root/setup-build deleted file mode 100644 index 1f6f1c39..00000000 --- a/tests.as-root/setup-build +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Fixture for tests involving 'morph build' - -source "$SRCDIR/scripts/fix-committer-info" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" branch test:morphs branch1 -"$SRCDIR/scripts/test-morph" edit linux-system linux-stratum linux - -# Fix UUID's in the checked out repos to make build branch names deterministic -git config -f "$DATADIR/workspace/branch1/.morph-system-branch/config" \ - branch.uuid 123456789 -git config -f "$DATADIR/workspace/branch1/test:morphs/.git/config" \ - morph.uuid 987654321 -git config -f "$DATADIR/workspace/branch1/test:kernel-repo/.git/config" \ - morph.uuid AABBCCDDE - -- cgit v1.2.1 From b40fb0e408c23277aa6f3c505baaf531d18ffcba Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 15:11:03 +0000 Subject: Allow default split rules to be overridden in-code This is needed for distbuild to deserialise based on the split rules on the node that did the graph calculation, rather than the node that does the building. --- morphlib/artifactsplitrule.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index cf0d1060..1511d694 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -230,7 +230,7 @@ DEFAULT_STRATUM_RULES = [ ] -def unify_chunk_matches(morphology): +def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES): '''Create split rules including defaults and per-chunk rules. With rules specified in the morphology's 'products' field and the @@ -246,7 +246,7 @@ def unify_chunk_matches(morphology): split_rules.add(ca_name, FileMatch(patterns)) name = morphology['name'] - for suffix, patterns in DEFAULT_CHUNK_RULES: + for suffix, patterns in default_rules: ca_name = name + suffix # Explicit rules override the default rules. This is an all-or-nothing # override: there is no way to extend the default split rules right now @@ -257,7 +257,7 @@ def unify_chunk_matches(morphology): return split_rules -def unify_stratum_matches(morphology): +def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES): '''Create split rules including defaults and per-stratum rules. With rules specified in the chunk spec's 'artifacts' fields, the @@ -284,7 +284,7 @@ def unify_stratum_matches(morphology): for d in morphology.get('products', {})): match_split_rules.add(sta_name, ArtifactMatch(patterns)) - for suffix, patterns in DEFAULT_STRATUM_RULES: + for suffix, patterns in default_rules: sta_name = morphology['name'] + suffix # Explicit rules override the default rules. This is an all-or-nothing # override: there is no way to extend the default split rules right now -- cgit v1.2.1 From ce2de6bd60db874f26502350a75de10faa1bf220 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 15:12:06 +0000 Subject: Fix and integrate distbuild unit tests --- distbuild/serialise.py | 139 ++++++++++++++++--------------------------- distbuild/serialise_tests.py | 101 +++++++++++++++++++------------ setup.py | 2 +- without-test-modules | 19 ++++++ 4 files changed, 133 insertions(+), 128 deletions(-) diff --git a/distbuild/serialise.py b/distbuild/serialise.py index d410b6cf..0a60b0c2 100644 --- a/distbuild/serialise.py +++ b/distbuild/serialise.py @@ -22,11 +22,6 @@ import morphlib import logging -morphology_attributes = [ - 'needs_artifact_metadata_cached', -] - - def serialise_artifact(artifact): '''Serialise an Artifact object and its dependencies into string form.''' @@ -34,24 +29,27 @@ def serialise_artifact(artifact): result = {} for key in morphology.keys(): result[key] = morphology[key] - for x in morphology_attributes: - result['__%s' % x] = getattr(morphology, x) return result def encode_source(source): source_dic = { + 'name': source.name, 'repo': None, 'repo_name': source.repo_name, 'original_ref': source.original_ref, 'sha1': source.sha1, 'tree': source.tree, - 'morphology': encode_morphology(source.morphology), + 'morphology': str(id(source.morphology)), 'filename': source.filename, # dict keys are converted to strings by json # so we encode the artifact ids as strings 'artifact_ids': [str(id(artifact)) for (_, artifact) in source.artifacts.iteritems()], + 'cache_id': source.cache_id, + 'cache_key': source.cache_key, + 'dependencies': [str(id(d)) + for d in source.dependencies], } if source.morphology['kind'] == 'chunk': @@ -59,68 +57,42 @@ def serialise_artifact(artifact): source_dic['prefix'] = source.prefix return source_dic - def encode_artifact(a, artifacts, source_id): - if artifact.source.morphology['kind'] == 'system': + def encode_artifact(a): + if artifact.source.morphology['kind'] == 'system': # pragma: no cover arch = artifact.source.morphology['arch'] else: arch = artifact.arch return { - 'source_id': source_id, + 'source_id': id(a.source), 'name': a.name, - 'cache_id': a.cache_id, - 'cache_key': a.cache_key, - 'dependencies': [str(id(artifacts[id(d)])) - for d in a.dependencies], 'arch': arch } - visited = set() - def traverse(a): - visited.add(a) - for dep in a.dependencies: - if dep in visited: - continue - for ret in traverse(dep): - yield ret - yield a - - - artifacts = {} encoded_artifacts = {} encoded_sources = {} + encoded_morphologies = {} - for a in traverse(artifact): + for a in artifact.walk(): if id(a.source) not in encoded_sources: - if a.source.morphology['kind'] == 'chunk': - for (_, sa) in a.source.artifacts.iteritems(): - if id(sa) not in artifacts: - artifacts[id(sa)] = sa - encoded_artifacts[id(sa)] = encode_artifact(sa, - artifacts, id(a.source)) - else: - # We create separate sources for strata and systems, - # this is a bit of a hack, but needed to allow - # us to build strata and systems independently - - s = a.source - t = morphlib.source.Source(s.repo_name, s.original_ref, - s.sha1, s.tree, s.morphology, s.filename) - - t.artifacts = {a.name: a} - a.source = t - + for (_, sa) in a.source.artifacts.iteritems(): + if id(sa) not in encoded_artifacts: + encoded_artifacts[id(sa)] = encode_artifact(sa) + encoded_morphologies[id(a.source.morphology)] = encode_morphology(a.source.morphology) encoded_sources[id(a.source)] = encode_source(a.source) - if id(a) not in artifacts: - artifacts[id(a)] = a - encoded_artifacts[id(a)] = encode_artifact(a, artifacts, - id(a.source)) - - encoded_artifacts['_root'] = str(id(artifact)) + if id(a) not in encoded_artifacts: # pragma: no cover + encoded_artifacts[id(a)] = encode_artifact(a) return json.dumps({'sources': encoded_sources, - 'artifacts': encoded_artifacts}) + 'artifacts': encoded_artifacts, + 'morphologies': encoded_morphologies, + 'root_artifact': str(id(artifact)), + 'default_split_rules': { + 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES, + 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES, + }, + }) def deserialise_artifact(encoded): @@ -141,38 +113,25 @@ def deserialise_artifact(encoded): ''' - class FakeMorphology(dict): - - def get_commands(self, which): - '''Get commands to run from a morphology or build system''' - if self[which] is None: - attr = '_'.join(which.split('-')) - bs = morphlib.buildsystem.lookup_build_system( - self['build-system']) - return getattr(bs, attr) - else: - return self[which] - - morphology = FakeMorphology(le_dict) - for x in morphology_attributes: - setattr(morphology, x, le_dict['__%s' % x]) - del morphology['__%s' % x] - return morphology - - def decode_source(le_dict): + return morphlib.morphology.Morphology(le_dict) + + def decode_source(le_dict, morphology, split_rules): '''Convert a dict into a Source object.''' - morphology = decode_morphology(le_dict['morphology']) - source = morphlib.source.Source(le_dict['repo_name'], + source = morphlib.source.Source(le_dict['name'], + le_dict['repo_name'], le_dict['original_ref'], le_dict['sha1'], le_dict['tree'], morphology, - le_dict['filename']) + le_dict['filename'], + split_rules) if morphology['kind'] == 'chunk': source.build_mode = le_dict['build_mode'] source.prefix = le_dict['prefix'] + source.cache_id = le_dict['cache_id'] + source.cache_key = le_dict['cache_key'] return source def decode_artifact(artifact_dict, source): @@ -183,8 +142,6 @@ def deserialise_artifact(encoded): ''' artifact = morphlib.artifact.Artifact(source, artifact_dict['name']) - artifact.cache_id = artifact_dict['cache_id'] - artifact.cache_key = artifact_dict['cache_key'] artifact.arch = artifact_dict['arch'] artifact.source = source @@ -193,18 +150,22 @@ def deserialise_artifact(encoded): le_dicts = json.loads(encoded) artifacts_dict = le_dicts['artifacts'] sources_dict = le_dicts['sources'] + morphologies_dict = le_dicts['morphologies'] + root_artifact = le_dicts['root_artifact'] - artifact_ids = ([artifacts_dict['_root']] + - filter(lambda k: k != '_root', artifacts_dict.keys())) - - source_ids = [sid for sid in sources_dict.keys()] + artifact_ids = ([root_artifact] + artifacts_dict.keys()) artifacts = {} sources = {} + morphologies = {id: decode_morphology(d) + for (id, d) in morphologies_dict.iteritems()} - for source_id in source_ids: - source_dict = sources_dict[source_id] - sources[source_id] = decode_source(source_dict) + for source_id, source_dict in sources_dict.iteritems(): + morphology = morphologies[source_dict['morphology']] + kind = morphology['kind'] + ruler = getattr(morphlib.artifactsplitrule, 'unify_%s_matches' % kind) + rules = ruler(morphology, le_dicts['default_split_rules'][kind]) + sources[source_id] = decode_source(source_dict, morphology, rules) # clear the source artifacts that get automatically generated # we want to add the ones that were sent to us @@ -222,9 +183,9 @@ def deserialise_artifact(encoded): sources[source_id].artifacts[key] = artifacts[artifact_id] # now add the dependencies - for artifact_id in artifact_ids: - artifact = artifacts[artifact_id] - artifact.dependencies = [artifacts[aid] for aid in - artifacts_dict[artifact_id]['dependencies']] + for source_id, source_dict in sources_dict.iteritems(): + source = sources[source_id] + source.dependencies = [artifacts[aid] + for aid in source_dict['dependencies']] - return artifacts[artifacts_dict['_root']] + return artifacts[root_artifact] diff --git a/distbuild/serialise_tests.py b/distbuild/serialise_tests.py index 2ad3a384..70973346 100644 --- a/distbuild/serialise_tests.py +++ b/distbuild/serialise_tests.py @@ -23,13 +23,22 @@ import distbuild class MockMorphology(object): - def __init__(self, name): + def __init__(self, name, kind): self.dict = { 'name': '%s.morphology.name' % name, - 'kind': '%s.morphology.kind' % name, + 'kind': kind, + 'chunks': [], + 'products': [ + { + 'artifact': name, + 'include': [r'.*'], + }, + ], } - self.needs_staging_area = None - self.needs_artifact_metadata_cached = None + + @property + def needs_artifact_metadata_cached(self): + return self.dict['kind'] == 'stratum' def keys(self): return self.dict.keys() @@ -40,36 +49,56 @@ class MockMorphology(object): class MockSource(object): - def __init__(self, name): + build_mode = 'staging' + prefix = '/usr' + def __init__(self, name, kind): + self.name = name self.repo = None self.repo_name = '%s.source.repo_name' % name self.original_ref = '%s.source.original_ref' % name self.sha1 = '%s.source.sha1' % name self.tree = '%s.source.tree' % name - self.morphology = MockMorphology(name) + self.morphology = MockMorphology(name, kind) self.filename = '%s.source.filename' % name - - -class MockArtifact(object): - - def __init__(self, name): - self.source = MockSource(name) - self.name = name + self.dependencies = [] self.cache_id = { 'blip': '%s.blip' % name, 'integer': 42, } self.cache_key = '%s.cache_key' % name - self.dependencies = [] + self.artifacts = {} + + +class MockArtifact(object): + + arch = 'testarch' + + def __init__(self, name, kind): + self.source = MockSource(name, kind) + self.source.artifacts = {name: self} + self.name = name + + def walk(self): # pragma: no cover + done = set() + + def depth_first(a): + if a not in done: + done.add(a) + for dep in a.source.dependencies: + for ret in depth_first(dep): + yield ret + yield a + + return list(depth_first(self)) class SerialisationTests(unittest.TestCase): def setUp(self): - self.art1 = MockArtifact('name1') - self.art2 = MockArtifact('name2') - self.art3 = MockArtifact('name3') - self.art4 = MockArtifact('name4') + self.art1 = MockArtifact('name1', 'stratum') + self.art2 = MockArtifact('name2', 'chunk') + self.art3 = MockArtifact('name3', 'chunk') + self.art4 = MockArtifact('name4', 'chunk') def assertEqualMorphologies(self, a, b): self.assertEqual(sorted(a.keys()), sorted(b.keys())) @@ -77,11 +106,8 @@ class SerialisationTests(unittest.TestCase): a_values = [a[k] for k in keys] b_values = [b[k] for k in keys] self.assertEqual(a_values, b_values) - self.assertEqual(a.needs_staging_area, b.needs_staging_area) self.assertEqual(a.needs_artifact_metadata_cached, b.needs_artifact_metadata_cached) - self.assertEqual(a.needs_staging_area, - b.needs_staging_area) def assertEqualSources(self, a, b): self.assertEqual(a.repo, b.repo) @@ -95,30 +121,29 @@ class SerialisationTests(unittest.TestCase): def assertEqualArtifacts(self, a, b): self.assertEqualSources(a.source, b.source) self.assertEqual(a.name, b.name) - self.assertEqual(a.cache_id, b.cache_id) - self.assertEqual(a.cache_key, b.cache_key) - self.assertEqual(len(a.dependencies), len(b.dependencies)) - for i in range(len(a.dependencies)): - self.assertEqualArtifacts(a.dependencies[i], b.dependencies[i]) + self.assertEqual(a.source.cache_id, b.source.cache_id) + self.assertEqual(a.source.cache_key, b.source.cache_key) + self.assertEqual(len(a.source.dependencies), + len(b.source.dependencies)) + for i in range(len(a.source.dependencies)): + self.assertEqualArtifacts(a.source.dependencies[i], + b.source.dependencies[i]) def verify_round_trip(self, artifact): encoded = distbuild.serialise_artifact(artifact) decoded = distbuild.deserialise_artifact(encoded) self.assertEqualArtifacts(artifact, decoded) - def key(a): - return a.cache_key - objs = {} queue = [decoded] while queue: obj = queue.pop() - k = key(obj) + k = obj.source.cache_key if k in objs: self.assertTrue(obj is objs[k]) else: objs[k] = obj - queue.extend(obj.dependencies) + queue.extend(obj.source.dependencies) def test_returns_string(self): encoded = distbuild.serialise_artifact(self.art1) @@ -128,21 +153,21 @@ class SerialisationTests(unittest.TestCase): self.verify_round_trip(self.art1) def test_works_with_single_dependency(self): - self.art1.dependencies = [self.art2] + self.art1.source.dependencies = [self.art2] self.verify_round_trip(self.art1) def test_works_with_two_dependencies(self): - self.art1.dependencies = [self.art2, self.art3] + self.art1.source.dependencies = [self.art2, self.art3] self.verify_round_trip(self.art1) def test_works_with_two_levels_of_dependencies(self): - self.art2.dependencies = [self.art4] - self.art1.dependencies = [self.art2, self.art3] + self.art2.source.dependencies = [self.art4] + self.art1.source.dependencies = [self.art2, self.art3] self.verify_round_trip(self.art1) def test_works_with_dag(self): - self.art2.dependencies = [self.art4] - self.art3.dependencies = [self.art4] - self.art1.dependencies = [self.art2, self.art3] + self.art2.source.dependencies = [self.art4] + self.art3.source.dependencies = [self.art4] + self.art1.source.dependencies = [self.art2, self.art3] self.verify_round_trip(self.art1) diff --git a/setup.py b/setup.py index 99c18ed4..8125788f 100644 --- a/setup.py +++ b/setup.py @@ -128,7 +128,7 @@ class Check(Command): def run(self): subprocess.check_call(['python', '-m', 'CoverageTestRunner', '--ignore-missing-from=without-test-modules', - 'morphlib']) + 'morphlib', 'distbuild']) os.remove('.coverage') diff --git a/without-test-modules b/without-test-modules index a3aedc45..55e5291d 100644 --- a/without-test-modules +++ b/without-test-modules @@ -31,5 +31,24 @@ morphlib/plugins/print_architecture_plugin.py morphlib/plugins/add_binary_plugin.py morphlib/plugins/push_pull_plugin.py morphlib/plugins/distbuild_plugin.py +distbuild/__init__.py +distbuild/build_controller.py +distbuild/connection_machine.py +distbuild/distbuild_socket.py +distbuild/eventsrc.py +distbuild/helper_router.py +distbuild/idgen.py +distbuild/initiator.py +distbuild/initiator_connection.py +distbuild/jm.py +distbuild/json_router.py +distbuild/mainloop.py +distbuild/protocol.py +distbuild/proxy_event_source.py +distbuild/sockbuf.py +distbuild/socketsrc.py +distbuild/sockserv.py +distbuild/timer_event_source.py +distbuild/worker_build_scheduler.py # Not unit tested, since it needs a full system branch morphlib/buildbranch.py -- cgit v1.2.1 From 01060c72158893863162acec85f576c442fc49ff Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 16:24:47 +0000 Subject: distbuild: allow daemons to bind to ephemeral ports You can bind to an ephemeral port by passing 0 as the port number. To work out which port you actually got, you need to call getsockname(). To facilitate being able to spawn multiple copies of the daemons for testing environments, you can pass a -file option, which will make the daemon write which port it actually bound to. If this path is a fifo, reading from it in the spawner process will allow synchronisation of only spawning services that require that port to be ready after it is. --- distbuild/sockserv.py | 7 ++++++- morphlib/plugins/distbuild_plugin.py | 24 ++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/distbuild/sockserv.py b/distbuild/sockserv.py index a5215e79..68991a93 100644 --- a/distbuild/sockserv.py +++ b/distbuild/sockserv.py @@ -26,15 +26,20 @@ class ListenServer(StateMachine): '''Listen for new connections on a port, send events for them.''' - def __init__(self, addr, port, machine, extra_args=None): + def __init__(self, addr, port, machine, extra_args=None, port_file=''): StateMachine.__init__(self, 'listening') self._addr = addr self._port = port self._machine = machine self._extra_args = extra_args or [] + self._port_file = port_file def setup(self): src = ListeningSocketEventSource(self._addr, self._port) + if self._port_file: + host, port = src.sock.getsockname() + with open(self._port_file, 'w') as f: + f.write(port) self.mainloop.add_event_source(src) spec = [ diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 50ab7eeb..7e8188dd 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -121,6 +121,11 @@ class WorkerDaemon(cliapp.Plugin): 'listen for connections on PORT', default=3434, group=group_distbuild) + self.app.settings.string( + ['worker-daemon-port-file'], + 'write port used by worker-daemon to FILE', + default='', + group=group_distbuild) self.app.add_subcommand( 'worker-daemon', self.worker_daemon, @@ -136,7 +141,9 @@ class WorkerDaemon(cliapp.Plugin): address = self.app.settings['worker-daemon-address'] port = self.app.settings['worker-daemon-port'] - router = distbuild.ListenServer(address, port, distbuild.JsonRouter) + port_file = self.app.settings['worker-daemon-port-file'] + router = distbuild.ListenServer(address, port, distbuild.JsonRouter, + port_file=port_file) loop = distbuild.MainLoop() loop.add_state_machine(router) loop.run() @@ -156,6 +163,11 @@ class ControllerDaemon(cliapp.Plugin): 'listen for initiator connections on PORT', default=7878, group=group_distbuild) + self.app.settings.string( + ['controller-initiator-port-file'], + 'write the port to listen for initiator connections to FILE', + default='', + group=group_distbuild) self.app.settings.string( ['controller-helper-address'], @@ -167,6 +179,11 @@ class ControllerDaemon(cliapp.Plugin): 'listen for helper connections on PORT', default=5656, group=group_distbuild) + self.app.settings.string( + ['controller-helper-port-file'], + 'write the port to listen for helper connections to FILE', + default='', + group=group_distbuild) self.app.settings.string_list( ['worker'], @@ -218,8 +235,10 @@ class ControllerDaemon(cliapp.Plugin): listener_specs = [ # address, port, class to initiate on connection, class init args ('controller-helper-address', 'controller-helper-port', + 'controller-helper-port-file', distbuild.HelperRouter, []), ('controller-initiator-address', 'controller-initiator-port', + 'controller-initiator-port-file', distbuild.InitiatorConnection, [artifact_cache_server, morph_instance]), ] @@ -229,9 +248,10 @@ class ControllerDaemon(cliapp.Plugin): queuer = distbuild.WorkerBuildQueuer() loop.add_state_machine(queuer) - for addr, port, sm, extra_args in listener_specs: + for addr, port, port_file, sm, extra_args in listener_specs: addr = self.app.settings[addr] port = self.app.settings[port] + port_file = self.app.settings[port_file] listener = distbuild.ListenServer( addr, port, sm, extra_args=extra_args) loop.add_state_machine(listener) -- cgit v1.2.1 From b760a2c4e1c22973baee8daf44a332b123ad1fdb Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 21:00:39 +0000 Subject: Allow ephemeral ports for distbuild services --- distbuild/sockserv.py | 2 +- morphlib/plugins/distbuild_plugin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distbuild/sockserv.py b/distbuild/sockserv.py index 68991a93..156394e2 100644 --- a/distbuild/sockserv.py +++ b/distbuild/sockserv.py @@ -39,7 +39,7 @@ class ListenServer(StateMachine): if self._port_file: host, port = src.sock.getsockname() with open(self._port_file, 'w') as f: - f.write(port) + f.write('%s\n' % port) self.mainloop.add_event_source(src) spec = [ diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 7e8188dd..26c26498 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -253,7 +253,7 @@ class ControllerDaemon(cliapp.Plugin): port = self.app.settings[port] port_file = self.app.settings[port_file] listener = distbuild.ListenServer( - addr, port, sm, extra_args=extra_args) + addr, port, sm, extra_args=extra_args, port_file=port_file) loop.add_state_machine(listener) for worker in self.app.settings['worker']: -- cgit v1.2.1 From 5ae8a2eff6f179bdb566bb1556626e708cfa1a8e Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 16:59:53 +0000 Subject: Allow morph-cache-server to be bound to an ephemeral port --- morph-cache-server | 30 ++++++++++++++++++++++++++++-- morphcacheserver/repocache.py | 6 +++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index a3c3c978..4af3cee3 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -45,6 +45,10 @@ class MorphCacheServer(cliapp.Application): 'port to listen on', metavar='PORTNUM', default=defaults['port']) + self.settings.string(['port-file'], + 'write port number to FILE', + metavar='FILE', + default='') self.settings.string(['repo-dir'], 'path to the repository cache directory', metavar='PATH', @@ -322,7 +326,8 @@ class MorphCacheServer(cliapp.Application): % artifact) return - filename = os.path.join(self.settings['artifact-dir'], artifact) + filename = os.path.join(self.settings['artifact-dir'], + artifact) results[artifact] = os.path.exists(filename) if results[artifact]: @@ -338,8 +343,29 @@ class MorphCacheServer(cliapp.Application): if self.settings['fcgi-server']: WSGIServer(root).run() + elif self.settings['port-file']: + import wsgiref.simple_server + + server_port_file = self.settings['port-file'] + class DebugServer(wsgiref.simple_server.WSGIServer): + '''WSGI-like server that uses an ephemeral port. + + Rather than use a specified port, or default, the + DebugServer binds to an ephemeral port on 127.0.0.1 + and writes its number to port-file, so a non-racy + temporary port can be used. + + ''' + + def __init__(self, (host, port), *args, **kwargs): + wsgiref.simple_server.WSGIServer.__init__( + self, ('127.0.0.1', 0), *args, **kwargs) + with open(server_port_file, 'w') as f: + f.write(str(self.server_port) + '\n') + run(root, server_class=DebugServer, debug=True) else: - run(root, host='0.0.0.0', port=self.settings['port'], reloader=True) + run(root, host='0.0.0.0', port=self.settings['port'], + reloader=True) def _unescape_parameter(self, param): return urllib.unquote(param) diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py index 0e4d909e..305c187c 100644 --- a/morphcacheserver/repocache.py +++ b/morphcacheserver/repocache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -91,7 +91,7 @@ class RepoCache(object): raise RepositoryNotFoundError(repo_url) try: sha1 = self._rev_parse(repo_dir, ref) - except: + except BaseException: raise InvalidReferenceError(repo_url, ref) return self._cat_file(repo_dir, sha1, filename) @@ -110,7 +110,7 @@ class RepoCache(object): try: sha1 = self._rev_parse(repo_dir, ref) - except: + except BaseException: raise InvalidReferenceError(repo_url, ref) lines = self._ls_tree(repo_dir, sha1, path).strip() -- cgit v1.2.1 From f814cf6c4a8cae6b5a28349897626d42f5fd9f9e Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 21:01:40 +0000 Subject: Add distbuilding yarn Deploying after distbuild currently broken --- yarns/building.yarn | 33 ++++++++++ yarns/implementations.yarn | 146 ++++++++++++++++++++++++++++++++++++++++++--- yarns/morph.shell-lib | 20 +++++++ 3 files changed, 192 insertions(+), 7 deletions(-) diff --git a/yarns/building.yarn b/yarns/building.yarn index c708b5bb..5f79aa3c 100644 --- a/yarns/building.yarn +++ b/yarns/building.yarn @@ -8,3 +8,36 @@ Morph Building Tests AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master THEN morph build the system systems/base-system.morph of the branch master FINALLY the git server is shut down + + SCENARIO distbuilding + GIVEN a workspace + AND a git server + AND a communal cache server + AND a distbuild worker + AND a distbuild controller + +Distbuilding works much the same way as regular building. + + WHEN the user checks out the system branch called master + AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master + THEN morph distbuild the system systems/base-system.morph of the branch master + +Distbuilt systems can be deployed locally, since the building process +lets you download the artifacts for local use. + +Note: Currently broken. It's currently complaining about not having +repos cached locally. + +> GIVEN a cluster called test-cluster.morph in system branch master +> AND a system in cluster test-cluster.morph in branch master called test-system +> AND system test-system in cluster test-cluster.morph in branch master builds systems/base-system.morph +> AND system test-system in cluster test-cluster.morph in branch master has deployment type: sysroot +> AND system test-system in cluster test-cluster.morph in branch master has deployment location: test-system +> WHEN the user attempts to deploy the cluster test-cluster.morph in branch master +> THEN morph succeeded +> AND file workspace/master/test/morphs/test-system exists + + FINALLY the distbuild controller is terminated + AND the distbuild worker is terminated + AND the communal cache server is terminated + AND the git server is shut down diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 86c3a9c4..71b8af60 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -683,10 +683,10 @@ Implementations for temporary build branch handling Implementation sections for building ==================================== - IMPLEMENTS WHEN the user (attempts to build|builds) the system (\S+) in branch (\S+) - cd "$DATADIR/workspace/$MATCH_3" - set build "$MATCH_2" - if [ $MATCH_1 == "builds" ]; then run_morph "$@" + IMPLEMENTS WHEN the user (attempts to )?((dist)?build)s? the system (\S+) in branch (\S+) + cd "$DATADIR/workspace/$MATCH_5" + set "$MATCH_2" "$MATCH_4" + if [ "$MATCH_1" != "attempts to " ]; then run_morph "$@" else attempt_morph "$@"; fi Implementation sections for cross-bootstraping @@ -872,9 +872,9 @@ variables in `$DATADIR/env`. We treat the value as a format string for Implementations for building systems ------------------------------------ - IMPLEMENTS THEN morph build the system (\S+) of the (branch|tag) (\S+) - cd "$DATADIR/workspace/$MATCH_3" - run_morph build "$MATCH_1" + IMPLEMENTS THEN morph ((dist)?build) the system (\S+) of the (branch|tag) (\S+) + cd "$DATADIR/workspace/$MATCH_5" + run_morph "$MATCH_1" "$MATCH_3" IMPLEMENTS WHEN the user builds (\S+) of the (\S+) (branch|tag) cd "$DATADIR/workspace/$MATCH_2" @@ -969,3 +969,135 @@ Altering morphologies in the workspace "$SRCDIR/scripts/edit-morph" cluster-system-set-deploy-variable \ "$DATADIR/workspace/$branch/test/morphs/$cluster" "$name" \ "$key" "$val" + + +Distbuild +========= + + IMPLEMENTS GIVEN a communal cache server + # The communal cache server has direct access to the git repositories + # and can have artifacts placed on it + artifact_dir="$DATADIR/shared-artifacts" + mkdir -p "$artifact_dir" + + read_cache_server_port_file="$DATADIR/read-cache-server-port" + read_cache_server_pid_file="$DATADIR/read-cache-server-pid" + start_cache_server "$read_cache_server_port_file" \ + "$read_cache_server_pid_file" \ + "$artifact_dir" + + write_cache_server_port_file="$DATADIR/write-cache-server-port" + write_cache_server_pid_file="$DATADIR/write-cache-server-pid" + start_cache_server "$write_cache_server_port_file" \ + "$write_cache_server_pid_file" \ + "$artifact_dir" --enable-writes + + IMPLEMENTS FINALLY the communal cache server is terminated + stop_daemon "$DATADIR/read-cache-server-pid" + stop_daemon "$DATADIR/write-cache-server-pid" + + IMPLEMENTS GIVEN a distbuild worker + # start worker cache server, so other workers can download results + worker_cachedir="$DATADIR/distbuild-worker-cache" + worker_artifacts="$worker_cachedir/artifacts" + mkdir -p "$worker_artifacts" + worker_cache_port_file="$DATADIR/worker-cache-server-port" + worker_cache_pid_file="$DATADIR/worker-cache-server-pid" + start_cache_server "$worker_cache_port_file" \ + "$worker_cache_pid_file" \ + "$worker_artifacts" + + # start worker daemon + worker_daemon_port_file="$DATADIR/worker-daemon-port" + worker_daemon_pid_file="$DATADIR/worker-daemon-pid" + mkfifo "$worker_daemon_port_file" + communal_cache_port="$(cat "$DATADIR/read-cache-server-port")" + start-stop-daemon --start --pidfile="$worker_daemon_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/morph" -- worker-daemon \ + --no-default-configs --config "$DATADIR/morph.conf" \ + --worker-daemon-port=0 \ + --worker-daemon-port-file="$worker_daemon_port_file" \ + --cachedir="$worker_cachedir" \ + --artifact-cache-server="http://localhost:$communal_cache_port/" \ + --git-resolve-cache-server="http://localhost:$communal_cache_port/" \ + --log="$DATADIR/worker-daemon-log" \ + >"$DATADIR/worker-daemon-out" 2>"$DATADIR/worker-daemon-err" + worker_daemon_port="$(cat "$worker_daemon_port_file")" + rm "$worker_daemon_port_file" + echo "$worker_daemon_port" >"$worker_daemon_port_file" + + # start worker helper + helper_pid_file="$DATADIR/worker-daemon-helper-pid" + start-stop-daemon --start --pidfile="$helper_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/distbuild-helper" -- \ + --no-default-configs \ + --parent-port="$worker_daemon_port" + + # set up builder config + install /dev/stdin <<'EOF' "$DATADIR/morph" + #!/bin/sh + exec "$SRCDIR/morph" --quiet \ + --cachedir-min-space=0 --tempdir-min-space=0 \ + --no-default-config --config "$DATADIR/morph.conf" \ + --cachedir "$DATADIR/distbuild-worker-cache" "$@" + EOF + + IMPLEMENTS FINALLY the distbuild worker is terminated + stop_daemon "$DATADIR/worker-cache-server-pid" + stop_daemon "$DATADIR/worker-daemon-pid" + stop_daemon "$DATADIR/worker-daemon-helper-pid" + + IMPLEMENTS GIVEN a distbuild controller + worker_cache_port_file="$DATADIR/worker-cache-server-port" + worker_cache_server_port="$(cat "$worker_cache_port_file")" + worker_daemon_port_file="$DATADIR/worker-daemon-port" + worker_daemon_port="$(cat "$worker_daemon_port_file")" + communal_cache_read_port="$(cat "$DATADIR/read-cache-server-port")" + communal_cache_write_port="$(cat "$DATADIR/write-cache-server-port")" + initiator_port_file="$DATADIR/controller-initiator-port" + mkfifo "$initiator_port_file" + helper_port_file="$DATADIR/controller-helper-port" + mkfifo "$helper_port_file" + controller_pid_file="$DATADIR/controller-pid" + start-stop-daemon --start --pidfile="$controller_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/morph" -- controller-daemon \ + --no-default-configs --config "$DATADIR/morph.conf" \ + --worker="localhost:$worker_daemon_port" \ + --worker-cache-server-port="$worker_cache_server_port" \ + --artifact-cache-server="http://localhost:$communal_cache_read_port/" \ + --git-resolve-cache-server="http://localhost:$communal_cache_read_port/" \ + --writeable-cache-server="http://localhost:$communal_cache_write_port/" \ + --controller-helper-port=0 \ + --controller-helper-port-file="$helper_port_file" \ + --controller-initiator-port=0 \ + --controller-initiator-port-file="$initiator_port_file" \ + --morph-instance="$DATADIR/morph" \ + --log="$DATADIR/controller-daemon-log" \ + >"$DATADIR/controller-daemon-out" 2>"$DATADIR/controller-daemon-err" + helper_port="$(cat "$helper_port_file")" + rm "$helper_port_file" + echo "$helper_port" >"$helper_port_file" + initiator_port="$(cat "$initiator_port_file")" + rm "$initiator_port_file" + echo "$initiator_port" >"$initiator_port_file" + + # start worker helper + helper_pid_file="$DATADIR/controller-helper-pid" + start-stop-daemon --start --pidfile="$helper_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/distbuild-helper" -- \ + --no-default-configs \ + --parent-port="$helper_port" + + # make builds use distbuild + echo "controller-initiator-port = $initiator_port" \ + >>"$DATADIR/morph.conf" + echo "controller-initiator-address = localhost" \ + >>"$DATADIR/morph.conf" + + IMPLEMENTS FINALLY the distbuild controller is terminated + stop_daemon "$DATADIR/controller-helper-pid" + stop_daemon "$DATADIR/controller-pid" diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index 9d67f2ab..9c13e449 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -184,3 +184,23 @@ slashify_colons() { echo "$1" | sed s,:,/,g } + +start_cache_server(){ + mkfifo "$1" + start-stop-daemon --start --pidfile="$2" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/morph-cache-server" -- \ + --port-file="$1" --no-fcgi \ + --repo-dir="$DATADIR/gits" --direct-mode \ + --bundle-dir="$DATADIR/bundles" \ + --artifact-dir="$3" "$@" + port="$(cat "$1")" + rm "$1" + echo "$port" >"$1" +} + +stop_daemon(){ + if [ -e "$1" ]; then + start-stop-daemon --stop --pidfile "$1" --oknodo + fi +} -- cgit v1.2.1 From 2689f61a305ec576f229f1d9f0929d7eb11d1acc Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 22:11:42 +0000 Subject: Fix issues with distbuild caused by moving to building per-source --- distbuild/build_controller.py | 11 ++++++----- distbuild/initiator_connection.py | 2 +- distbuild/serialise.py | 8 ++++++-- distbuild/worker_build_scheduler.py | 27 +++++++++++++-------------- morphlib/plugins/distbuild_plugin.py | 2 +- morphlib/plugins/list_artifacts_plugin.py | 6 ++++-- 6 files changed, 31 insertions(+), 25 deletions(-) diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index e8a8dc37..93f97fac 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -142,7 +142,7 @@ def map_build_graph(artifact, callback): a = queue.pop() if a not in done: result.append(callback(a)) - queue.extend(a.dependencies) + queue.extend(a.source.dependencies) done.add(a) return result @@ -388,7 +388,8 @@ class BuildController(distbuild.StateMachine): def _find_artifacts_that_are_ready_to_build(self): def is_ready_to_build(artifact): return (artifact.state == UNBUILT and - all(a.state == BUILT for a in artifact.dependencies)) + all(a.state == BUILT + for a in artifact.source.dependencies)) return [a for a in map_build_graph(self._artifact, lambda a: a) @@ -424,7 +425,7 @@ class BuildController(distbuild.StateMachine): logging.debug( 'Requesting worker-build of %s (%s)' % - (artifact.name, artifact.cache_key)) + (artifact.name, artifact.source.cache_key)) request = distbuild.WorkerBuildRequest(artifact, self._request['id']) self.mainloop.queue_event(distbuild.WorkerBuildQueuer, request) @@ -540,7 +541,7 @@ class BuildController(distbuild.StateMachine): def _find_artifact(self, cache_key): artifacts = map_build_graph(self._artifact, lambda a: a) - wanted = [a for a in artifacts if a.cache_key == cache_key] + wanted = [a for a in artifacts if a.source.cache_key == cache_key] if wanted: return wanted[0] else: @@ -637,7 +638,7 @@ class BuildController(distbuild.StateMachine): baseurl = urlparse.urljoin( self._artifact_cache_server, '/1.0/artifacts') filename = ('%s.%s.%s' % - (self._artifact.cache_key, + (self._artifact.source.cache_key, self._artifact.source.morphology['kind'], self._artifact.name)) url = '%s?filename=%s' % (baseurl, urllib.quote(filename)) diff --git a/distbuild/initiator_connection.py b/distbuild/initiator_connection.py index 0f009fcc..db982230 100644 --- a/distbuild/initiator_connection.py +++ b/distbuild/initiator_connection.py @@ -171,7 +171,7 @@ class InitiatorConnection(distbuild.StateMachine): 'name': distbuild.build_step_name(artifact), 'build-depends': [ distbuild.build_step_name(x) - for x in artifact.dependencies + for x in artifact.source.dependencies ] } diff --git a/distbuild/serialise.py b/distbuild/serialise.py index 0a60b0c2..2b39000e 100644 --- a/distbuild/serialise.py +++ b/distbuild/serialise.py @@ -78,7 +78,8 @@ def serialise_artifact(artifact): for (_, sa) in a.source.artifacts.iteritems(): if id(sa) not in encoded_artifacts: encoded_artifacts[id(sa)] = encode_artifact(sa) - encoded_morphologies[id(a.source.morphology)] = encode_morphology(a.source.morphology) + encoded_morphologies[id(a.source.morphology)] = \ + encode_morphology(a.source.morphology) encoded_sources[id(a.source)] = encode_source(a.source) if id(a) not in encoded_artifacts: # pragma: no cover @@ -164,7 +165,10 @@ def deserialise_artifact(encoded): morphology = morphologies[source_dict['morphology']] kind = morphology['kind'] ruler = getattr(morphlib.artifactsplitrule, 'unify_%s_matches' % kind) - rules = ruler(morphology, le_dicts['default_split_rules'][kind]) + if kind in ('chunk', 'stratum'): + rules = ruler(morphology, le_dicts['default_split_rules'][kind]) + else: # pragma: no cover + rules = ruler(morphology) sources[source_id] = decode_source(source_dict, morphology, rules) # clear the source artifacts that get automatically generated diff --git a/distbuild/worker_build_scheduler.py b/distbuild/worker_build_scheduler.py index 6cda5972..be732153 100644 --- a/distbuild/worker_build_scheduler.py +++ b/distbuild/worker_build_scheduler.py @@ -262,13 +262,13 @@ class WorkerBuildQueuer(distbuild.StateMachine): logging.debug('Worker build step already started: %s' % event.artifact.basename()) progress = WorkerBuildStepAlreadyStarted(event.initiator_id, - event.artifact.cache_key, job.who.name()) + event.artifact.source.cache_key, job.who.name()) else: logging.debug('Job created but not building yet ' '(waiting for a worker to become available): %s' % event.artifact.basename()) progress = WorkerBuildWaiting(event.initiator_id, - event.artifact.cache_key) + event.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, progress) else: @@ -279,7 +279,7 @@ class WorkerBuildQueuer(distbuild.StateMachine): self._give_job(job) else: progress = WorkerBuildWaiting(event.initiator_id, - event.artifact.cache_key) + event.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, progress) def _handle_cancel(self, event_source, event): @@ -483,7 +483,7 @@ class WorkerConnection(distbuild.StateMachine): % (self._worker_name, msg)) started = WorkerBuildStepStarted(self._job.initiators, - self._job.artifact.cache_key, self.name()) + self._job.artifact.source.cache_key, self.name()) self.mainloop.queue_event(WorkerConnection, _JobStarted(self._job)) self.mainloop.queue_event(WorkerConnection, started) @@ -510,7 +510,7 @@ class WorkerConnection(distbuild.StateMachine): logging.debug('WC: emitting: %s', repr(new)) self.mainloop.queue_event( WorkerConnection, - WorkerBuildOutput(new, self._job.artifact.cache_key)) + WorkerBuildOutput(new, self._job.artifact.source.cache_key)) def _handle_exec_response(self, msg): logging.debug('WC: finished building: %s' % self._job.artifact.name) @@ -522,7 +522,8 @@ class WorkerConnection(distbuild.StateMachine): if new['exit'] != 0: # Build failed. - new_event = WorkerBuildFailed(new, self._job.artifact.cache_key) + new_event = WorkerBuildFailed(new, + self._job.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(WorkerConnection, _JobFailed(self._job)) self.mainloop.queue_event(self, _BuildFailed()) @@ -556,10 +557,6 @@ class WorkerConnection(distbuild.StateMachine): if kind == 'stratum': suffixes.append(filename + '.meta') - elif kind == 'system': - # FIXME: This is a really ugly hack. - if filename.endswith('-rootfs'): - suffixes.append(filename[:-len('-rootfs')] + '-kernel') suffixes = [urllib.quote(x) for x in suffixes] suffixes = ','.join(suffixes) @@ -571,7 +568,7 @@ class WorkerConnection(distbuild.StateMachine): '/1.0/fetch?host=%s:%d&cacheid=%s&artifacts=%s' % (urllib.quote(worker_host), self._worker_cache_server_port, - urllib.quote(self._job.artifact.cache_key), + urllib.quote(self._job.artifact.source.cache_key), suffixes)) msg = distbuild.message( @@ -582,7 +579,7 @@ class WorkerConnection(distbuild.StateMachine): self.mainloop.queue_event(distbuild.HelperRouter, req) progress = WorkerBuildCaching(self._job.initiators, - self._job.artifact.cache_key) + self._job.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, progress) def _maybe_handle_helper_result(self, event_source, event): @@ -594,7 +591,8 @@ class WorkerConnection(distbuild.StateMachine): logging.debug('Shared artifact cache population done') new_event = WorkerBuildFinished( - self._exec_response_msg, self._job.artifact.cache_key) + self._exec_response_msg, + self._job.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(self, _Cached()) else: @@ -612,7 +610,8 @@ class WorkerConnection(distbuild.StateMachine): _JobFailed(self._job)) new_event = WorkerBuildFailed( - self._exec_response_msg, self._job.artifact.cache_key) + self._exec_response_msg, + self._job.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(self, _BuildFailed()) diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 26c26498..1858a9ba 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -103,7 +103,7 @@ class WorkerBuild(cliapp.Plugin): self.app.subcommands['gc']([]) arch = artifact.arch - bc.build_artifact(artifact, bc.new_build_env(arch)) + bc.build_source(artifact.source, bc.new_build_env(arch)) def is_system_artifact(self, filename): return re.match(r'^[0-9a-fA-F]{64}\.system\.', filename) diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py index 8074206b..61c8d160 100644 --- a/morphlib/plugins/list_artifacts_plugin.py +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -105,11 +105,13 @@ class ListArtifactsPlugin(cliapp.Plugin): self.app.settings, system_artifact.source.morphology['arch']) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) - artifact_files = set() - for artifact in system_artifact.walk(): + for source in set(a.source for a in system_artifact.walk()): artifact.cache_key = ckc.compute_key(artifact) artifact.cache_id = ckc.get_cache_id(artifact) + artifact_files = set() + for artifact in system_artifact.walk(): + artifact_files.add(artifact.basename()) if artifact.source.morphology.needs_artifact_metadata_cached: -- cgit v1.2.1 From fc2d9528f4e011559c5a34452a76fd5d5f8718e4 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Thu, 2 Oct 2014 10:38:06 +0000 Subject: Add echo to the test shell --- scripts/test-shell.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/test-shell.c b/scripts/test-shell.c index 7975c188..e3ef1ff1 100644 --- a/scripts/test-shell.c +++ b/scripts/test-shell.c @@ -133,6 +133,12 @@ int main(int argc, char *argv[]) { strstr(line, "false ") == line) { ret = 1; break; + } else if (strstr(line, "echo ") == line) { + if (puts(line + sizeof("echo ") - 1) == EOF){ + perror("echo"); + ret = 1; + break; + } } else { ret = 127; break; -- cgit v1.2.1 From 3f8ea43f0af9fd2c8a2cf2aa21b9963fa70043d6 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Thu, 2 Oct 2014 11:14:49 +0000 Subject: Test that morph can handle binary garbage output in builds --- yarns/implementations.yarn | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 71b8af60..0c0daf79 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -282,12 +282,22 @@ another to hold a chunk. name: test-chunk kind: chunk build-system: manual - # `install-commands` is a list of shell commands to run. Commands + # `build-commands` is a list of shell commands to run. Commands # may be on multiple lines, and indeed anything programmatic will # benefit from doing so. Arguably we could have just one command, # but it's split into multiple so that morph can inform us which # command failed without us having to include a lot of status # information in the command and look at the error message. + # `build-commands` are passed MAKEFLAGS to specify build parallelism, + # and are expected to generate files to be installed in `install-files` + build-commands: + # All of morph's building needs to handle binary output, so we can echo + # that out on the command line. + # Trust me, this gets decoded to "echo " then a bunch of binary output. + - !!binary | + ZWNobyBQ1+k661ol3khrsF4VO4HNcuYzwN0LYxEWS8mPmhQiQ7Vu8CME2+gsnKQaoIRIFuUEiLCI + vfIj1GTdXG6cVTJfNQ== + install-commands: - copy files EOF -- cgit v1.2.1 From 5aad0f5a8249f1f20f802aa094343fb9df1a1656 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Thu, 2 Oct 2014 11:26:01 +0000 Subject: Allow distbuilding morphologies with binary data embedded The horrible json.dumped, yaml dump is because we need it to be both binary safe (which yaml gives us) and one line per message (which json gives us). --- distbuild/serialise.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/distbuild/serialise.py b/distbuild/serialise.py index 2b39000e..a27a526b 100644 --- a/distbuild/serialise.py +++ b/distbuild/serialise.py @@ -17,6 +17,7 @@ import json +import yaml import morphlib import logging @@ -39,16 +40,14 @@ def serialise_artifact(artifact): 'original_ref': source.original_ref, 'sha1': source.sha1, 'tree': source.tree, - 'morphology': str(id(source.morphology)), + 'morphology': id(source.morphology), 'filename': source.filename, - # dict keys are converted to strings by json - # so we encode the artifact ids as strings - 'artifact_ids': [str(id(artifact)) for (_, artifact) + 'artifact_ids': [id(artifact) for (_, artifact) in source.artifacts.iteritems()], 'cache_id': source.cache_id, 'cache_key': source.cache_key, - 'dependencies': [str(id(d)) + 'dependencies': [id(d) for d in source.dependencies], } @@ -85,15 +84,17 @@ def serialise_artifact(artifact): if id(a) not in encoded_artifacts: # pragma: no cover encoded_artifacts[id(a)] = encode_artifact(a) - return json.dumps({'sources': encoded_sources, + content = { + 'sources': encoded_sources, 'artifacts': encoded_artifacts, 'morphologies': encoded_morphologies, - 'root_artifact': str(id(artifact)), + 'root_artifact': id(artifact), 'default_split_rules': { 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES, 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES, }, - }) + } + return json.dumps(yaml.dump(content)) def deserialise_artifact(encoded): @@ -148,7 +149,7 @@ def deserialise_artifact(encoded): return artifact - le_dicts = json.loads(encoded) + le_dicts = yaml.load(json.loads(encoded)) artifacts_dict = le_dicts['artifacts'] sources_dict = le_dicts['sources'] morphologies_dict = le_dicts['morphologies'] -- cgit v1.2.1 From 2f981e4a5908cdab10d2d0dc2801c5454c2f78d4 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Thu, 2 Oct 2014 12:00:47 +0000 Subject: distbuild: yaml-encode messages before json encoding JSON can only handle unicode strings, but commands can write anything to stdout/stderr, so we do the same trick as for the serialise, and json encode yaml. --- distbuild/jm.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/distbuild/jm.py b/distbuild/jm.py index 513c69fa..615100e4 100644 --- a/distbuild/jm.py +++ b/distbuild/jm.py @@ -22,6 +22,7 @@ import logging import os import socket import sys +import yaml from sm import StateMachine from stringbuffer import StringBuffer @@ -79,7 +80,12 @@ class JsonMachine(StateMachine): def send(self, msg): '''Send a message to the other side.''' - self.sockbuf.write('%s\n' % json.dumps(msg)) + if self.debug_json: + logging.debug('JsonMachine: Sending message %s' % repr(msg)) + s = json.dumps(yaml.safe_dump(msg)) + if self.debug_json: + logging.debug('JsonMachine: As %s' % repr(s)) + self.sockbuf.write('%s\n' % s) def close(self): '''Tell state machine it should shut down. @@ -103,7 +109,7 @@ class JsonMachine(StateMachine): line = line.rstrip() if self.debug_json: logging.debug('JsonMachine: line: %s' % repr(line)) - msg = json.loads(line) + msg = yaml.load(json.loads(line)) self.mainloop.queue_event(self, JsonNewMessage(msg)) def _send_eof(self, event_source, event): -- cgit v1.2.1 From 53f14b5061247e5d7bbd22417eb8e10c1470efb1 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Thu, 2 Oct 2014 12:23:20 +0000 Subject: Only attempt to run distbuild yarns if possible You need both bottle and flup for this to work, and it would wait forever for a cache-server that is never going to start unless you have bottle and flup. So we now check whether the cache server is functional before attempting distbuild tests. --- yarns/building.yarn | 1 + yarns/implementations.yarn | 3 +++ 2 files changed, 4 insertions(+) diff --git a/yarns/building.yarn b/yarns/building.yarn index 5f79aa3c..52742ac8 100644 --- a/yarns/building.yarn +++ b/yarns/building.yarn @@ -10,6 +10,7 @@ Morph Building Tests FINALLY the git server is shut down SCENARIO distbuilding + ASSUMING the morph-cache-server can be run GIVEN a workspace AND a git server AND a communal cache server diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 0c0daf79..6748e3d3 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -984,6 +984,9 @@ Altering morphologies in the workspace Distbuild ========= + IMPLEMENTS ASSUMING the morph-cache-server can be run + "$SRCDIR/morph-cache-server" --version + IMPLEMENTS GIVEN a communal cache server # The communal cache server has direct access to the git repositories # and can have artifacts placed on it -- cgit v1.2.1 From f3935d491f7b3890abaf8ee56e2b7fc7af63c558 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Thu, 2 Oct 2014 13:02:11 +0000 Subject: Allow distbuilds to choose where to put logs --- distbuild/initiator.py | 8 +++++++- morphlib/plugins/distbuild_plugin.py | 5 +++++ yarns/implementations.yarn | 2 ++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/distbuild/initiator.py b/distbuild/initiator.py index b60700fd..b0993aa3 100644 --- a/distbuild/initiator.py +++ b/distbuild/initiator.py @@ -18,6 +18,7 @@ import cliapp import logging +import os import random import sys @@ -48,6 +49,7 @@ class Initiator(distbuild.StateMachine): self._morphology = morphology self._steps = None self._step_outputs = {} + self._step_output_dir = app.settings['initiator-step-output-dir'] self.debug_transitions = False def setup(self): @@ -120,7 +122,11 @@ class Initiator(distbuild.StateMachine): def _open_output(self, msg): assert msg['step_name'] not in self._step_outputs - filename = 'build-step-%s.log' % msg['step_name'] + if self._step_output_dir: + filename = os.path.join(self._step_output_dir, + 'build-step-%s.log' % msg['step_name']) + else: + filename = '/dev/null' f = open(filename, 'a') self._step_outputs[msg['step_name']] = f diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 1858a9ba..653eeae8 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -168,6 +168,11 @@ class ControllerDaemon(cliapp.Plugin): 'write the port to listen for initiator connections to FILE', default='', group=group_distbuild) + self.app.settings.string( + ['initiator-step-output-dir'], + 'write build output to files in DIR', + default='.', + group=group_distbuild) self.app.settings.string( ['controller-helper-address'], diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 6748e3d3..c6d245d0 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -1110,6 +1110,8 @@ Distbuild >>"$DATADIR/morph.conf" echo "controller-initiator-address = localhost" \ >>"$DATADIR/morph.conf" + echo "initiator-step-output-dir = $DATADIR" \ + >>"$DATADIR/morph.conf" IMPLEMENTS FINALLY the distbuild controller is terminated stop_daemon "$DATADIR/controller-helper-pid" -- cgit v1.2.1 From 9034cd11ee2b9c050f2301c84627a3d7b2e67895 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 3 Oct 2014 14:42:03 +0000 Subject: distbuild: serialize dependent sources of graph --- distbuild/serialise.py | 69 +++++++++++++++++++++++++++++--------------- distbuild/serialise_tests.py | 1 + 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/distbuild/serialise.py b/distbuild/serialise.py index a27a526b..a7c6c4b9 100644 --- a/distbuild/serialise.py +++ b/distbuild/serialise.py @@ -32,7 +32,7 @@ def serialise_artifact(artifact): result[key] = morphology[key] return result - def encode_source(source): + def encode_source(source, prune_leaf=False): source_dic = { 'name': source.name, 'repo': None, @@ -42,14 +42,16 @@ def serialise_artifact(artifact): 'tree': source.tree, 'morphology': id(source.morphology), 'filename': source.filename, - - 'artifact_ids': [id(artifact) for (_, artifact) - in source.artifacts.iteritems()], + 'artifact_ids': [], 'cache_id': source.cache_id, 'cache_key': source.cache_key, - 'dependencies': [id(d) - for d in source.dependencies], + 'dependencies': [], } + if not prune_leaf: + source_dic['artifact_ids'].extend(id(artifact) for (_, artifact) + in source.artifacts.iteritems()) + source_dic['dependencies'].extend(id(d) + for d in source.dependencies) if source.morphology['kind'] == 'chunk': source_dic['build_mode'] = source.build_mode @@ -65,25 +67,40 @@ def serialise_artifact(artifact): return { 'source_id': id(a.source), 'name': a.name, - 'arch': arch + 'arch': arch, + 'dependents': [id(d) + for d in a.dependents], } encoded_artifacts = {} encoded_sources = {} encoded_morphologies = {} + visited_artifacts = {} for a in artifact.walk(): if id(a.source) not in encoded_sources: - for (_, sa) in a.source.artifacts.iteritems(): + for sa in a.source.artifacts.itervalues(): if id(sa) not in encoded_artifacts: + visited_artifacts[id(sa)] = sa encoded_artifacts[id(sa)] = encode_artifact(sa) encoded_morphologies[id(a.source.morphology)] = \ encode_morphology(a.source.morphology) encoded_sources[id(a.source)] = encode_source(a.source) if id(a) not in encoded_artifacts: # pragma: no cover + visited_artifacts[id(a)] = a encoded_artifacts[id(a)] = encode_artifact(a) + # Include one level of dependents above encoded artifacts, as we need + # them to be able to tell whether two sources are in the same stratum. + for a in visited_artifacts.itervalues(): + for source in a.dependents: # pragma: no cover + if id(source) not in encoded_sources: + encoded_morphologies[id(source.morphology)] = \ + encode_morphology(source.morphology) + encoded_sources[id(source)] = \ + encode_source(source, prune_leaf=True) + content = { 'sources': encoded_sources, 'artifacts': encoded_artifacts, @@ -154,14 +171,14 @@ def deserialise_artifact(encoded): sources_dict = le_dicts['sources'] morphologies_dict = le_dicts['morphologies'] root_artifact = le_dicts['root_artifact'] - - artifact_ids = ([root_artifact] + artifacts_dict.keys()) + assert root_artifact in artifacts_dict artifacts = {} sources = {} morphologies = {id: decode_morphology(d) for (id, d) in morphologies_dict.iteritems()} + # Decode sources for source_id, source_dict in sources_dict.iteritems(): morphology = morphologies[source_dict['morphology']] kind = morphology['kind'] @@ -172,25 +189,29 @@ def deserialise_artifact(encoded): rules = ruler(morphology) sources[source_id] = decode_source(source_dict, morphology, rules) - # clear the source artifacts that get automatically generated - # we want to add the ones that were sent to us - sources[source_id].artifacts = {} - source_artifacts = source_dict['artifact_ids'] - - for artifact_id in source_artifacts: - if artifact_id not in artifacts: - artifact_dict = artifacts_dict[artifact_id] - artifact = decode_artifact(artifact_dict, sources[source_id]) - - artifacts[artifact_id] = artifact + # decode artifacts + for artifact_id, artifact_dict in artifacts_dict.iteritems(): + source_id = artifact_dict['source_id'] + source = sources[source_id] + artifact = decode_artifact(artifact_dict, source) + artifacts[artifact_id] = artifact - key = artifacts[artifact_id].name - sources[source_id].artifacts[key] = artifacts[artifact_id] + # add source artifacts reference + for source_id, source in sources.iteritems(): + source_dict = sources_dict[source_id] + source.artifacts = {artifacts[a].name: artifacts[a] + for a in source_dict['artifact_ids']} - # now add the dependencies + # add source dependencies for source_id, source_dict in sources_dict.iteritems(): source = sources[source_id] source.dependencies = [artifacts[aid] for aid in source_dict['dependencies']] + # add artifact dependents + for artifact_id, artifact in artifacts.iteritems(): + artifact_dict = artifacts_dict[artifact_id] + artifact.dependents = [sources[sid] + for sid in artifact_dict['dependents']] + return artifacts[root_artifact] diff --git a/distbuild/serialise_tests.py b/distbuild/serialise_tests.py index 70973346..d80c3dd7 100644 --- a/distbuild/serialise_tests.py +++ b/distbuild/serialise_tests.py @@ -77,6 +77,7 @@ class MockArtifact(object): self.source = MockSource(name, kind) self.source.artifacts = {name: self} self.name = name + self.dependents = [] def walk(self): # pragma: no cover done = set() -- cgit v1.2.1