diff options
author | Richard Maw <richard.maw@gmail.com> | 2014-10-08 13:17:00 +0000 |
---|---|---|
committer | Richard Maw <richard.maw@gmail.com> | 2014-10-08 13:17:00 +0000 |
commit | 8b85d60e9294713097928d52b8ca4cfe5d5f801a (patch) | |
tree | 0de7707ca779b47f7c3882b5627cb3dedaf09a6a | |
parent | d3be16e282bfdf7a8db8538339719161725b5cad (diff) | |
parent | 9034cd11ee2b9c050f2301c84627a3d7b2e67895 (diff) | |
download | morph-8b85d60e9294713097928d52b8ca4cfe5d5f801a.tar.gz |
Merge branch 'baserock/richardmaw/fix-distbuild-v3'
Reviewed-by: Sam Thursfield
Reviewed-by: Richard Ipsum
Reviewed-by: Pedro Alvarez
74 files changed, 1436 insertions, 2431 deletions
@@ -153,20 +153,6 @@ else echo "NOT RUNNING test.branching" fi -if false && "$run_cmdtests" -then - cmdtest tests.merging -else - echo "NOT RUNNING test.merging" -fi - -if "$run_cmdtests" -then - cmdtest tests.deploy -else - echo "NOT RUNNING test.deploy" -fi - # Building systems requires the 'filter' parameter of tarfile.TarFile.add(): # this was introduced in Python 2.7 if ! "$run_cmdtests"; then @@ -176,19 +162,3 @@ elif ! (python --version 2>&1 | grep -q '^Python 2\.[78]'); then else cmdtest tests.build fi - -# The as-root tests use YAML morphologies, so they require the PyYAML module. -if ! "$run_slow_cmdtests"; then - echo "NOT RUNNING tests.as-root" -elif [ $(whoami) != root ] || ! command -v mkfs.btrfs > /dev/null; then - echo "NOT RUNNING tests.as-root (no btrfs)" -elif ! python -c " -import morphlib, sys -if not morphlib.got_yaml: - sys.exit(1) -" > /dev/null 2>&1 -then - echo "NOT RUNNING tests.as-root (requires PyYAML)" -else - cmdtest tests.as-root -fi diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index e8a8dc37..93f97fac 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -142,7 +142,7 @@ def map_build_graph(artifact, callback): a = queue.pop() if a not in done: result.append(callback(a)) - queue.extend(a.dependencies) + queue.extend(a.source.dependencies) done.add(a) return result @@ -388,7 +388,8 @@ class BuildController(distbuild.StateMachine): def _find_artifacts_that_are_ready_to_build(self): def is_ready_to_build(artifact): return (artifact.state == UNBUILT and - all(a.state == BUILT for a in artifact.dependencies)) + all(a.state == BUILT + for a in artifact.source.dependencies)) return [a for a in map_build_graph(self._artifact, lambda a: a) @@ -424,7 +425,7 @@ class BuildController(distbuild.StateMachine): logging.debug( 'Requesting worker-build of %s (%s)' % - (artifact.name, artifact.cache_key)) + (artifact.name, artifact.source.cache_key)) request = distbuild.WorkerBuildRequest(artifact, self._request['id']) self.mainloop.queue_event(distbuild.WorkerBuildQueuer, request) @@ -540,7 +541,7 @@ class BuildController(distbuild.StateMachine): def _find_artifact(self, cache_key): artifacts = map_build_graph(self._artifact, lambda a: a) - wanted = [a for a in artifacts if a.cache_key == cache_key] + wanted = [a for a in artifacts if a.source.cache_key == cache_key] if wanted: return wanted[0] else: @@ -637,7 +638,7 @@ class BuildController(distbuild.StateMachine): baseurl = urlparse.urljoin( self._artifact_cache_server, '/1.0/artifacts') filename = ('%s.%s.%s' % - (self._artifact.cache_key, + (self._artifact.source.cache_key, self._artifact.source.morphology['kind'], self._artifact.name)) url = '%s?filename=%s' % (baseurl, urllib.quote(filename)) diff --git a/distbuild/initiator.py b/distbuild/initiator.py index b60700fd..b0993aa3 100644 --- a/distbuild/initiator.py +++ b/distbuild/initiator.py @@ -18,6 +18,7 @@ import cliapp import logging +import os import random import sys @@ -48,6 +49,7 @@ class Initiator(distbuild.StateMachine): self._morphology = morphology self._steps = None self._step_outputs = {} + self._step_output_dir = app.settings['initiator-step-output-dir'] self.debug_transitions = False def setup(self): @@ -120,7 +122,11 @@ class Initiator(distbuild.StateMachine): def _open_output(self, msg): assert msg['step_name'] not in self._step_outputs - filename = 'build-step-%s.log' % msg['step_name'] + if self._step_output_dir: + filename = os.path.join(self._step_output_dir, + 'build-step-%s.log' % msg['step_name']) + else: + filename = '/dev/null' f = open(filename, 'a') self._step_outputs[msg['step_name']] = f diff --git a/distbuild/initiator_connection.py b/distbuild/initiator_connection.py index 0f009fcc..db982230 100644 --- a/distbuild/initiator_connection.py +++ b/distbuild/initiator_connection.py @@ -171,7 +171,7 @@ class InitiatorConnection(distbuild.StateMachine): 'name': distbuild.build_step_name(artifact), 'build-depends': [ distbuild.build_step_name(x) - for x in artifact.dependencies + for x in artifact.source.dependencies ] } diff --git a/distbuild/jm.py b/distbuild/jm.py index 513c69fa..615100e4 100644 --- a/distbuild/jm.py +++ b/distbuild/jm.py @@ -22,6 +22,7 @@ import logging import os import socket import sys +import yaml from sm import StateMachine from stringbuffer import StringBuffer @@ -79,7 +80,12 @@ class JsonMachine(StateMachine): def send(self, msg): '''Send a message to the other side.''' - self.sockbuf.write('%s\n' % json.dumps(msg)) + if self.debug_json: + logging.debug('JsonMachine: Sending message %s' % repr(msg)) + s = json.dumps(yaml.safe_dump(msg)) + if self.debug_json: + logging.debug('JsonMachine: As %s' % repr(s)) + self.sockbuf.write('%s\n' % s) def close(self): '''Tell state machine it should shut down. @@ -103,7 +109,7 @@ class JsonMachine(StateMachine): line = line.rstrip() if self.debug_json: logging.debug('JsonMachine: line: %s' % repr(line)) - msg = json.loads(line) + msg = yaml.load(json.loads(line)) self.mainloop.queue_event(self, JsonNewMessage(msg)) def _send_eof(self, event_source, event): diff --git a/distbuild/serialise.py b/distbuild/serialise.py index d410b6cf..a7c6c4b9 100644 --- a/distbuild/serialise.py +++ b/distbuild/serialise.py @@ -17,16 +17,12 @@ import json +import yaml import morphlib import logging -morphology_attributes = [ - 'needs_artifact_metadata_cached', -] - - def serialise_artifact(artifact): '''Serialise an Artifact object and its dependencies into string form.''' @@ -34,93 +30,88 @@ def serialise_artifact(artifact): result = {} for key in morphology.keys(): result[key] = morphology[key] - for x in morphology_attributes: - result['__%s' % x] = getattr(morphology, x) return result - def encode_source(source): + def encode_source(source, prune_leaf=False): source_dic = { + 'name': source.name, 'repo': None, 'repo_name': source.repo_name, 'original_ref': source.original_ref, 'sha1': source.sha1, 'tree': source.tree, - 'morphology': encode_morphology(source.morphology), + 'morphology': id(source.morphology), 'filename': source.filename, - - # dict keys are converted to strings by json - # so we encode the artifact ids as strings - 'artifact_ids': [str(id(artifact)) for (_, artifact) - in source.artifacts.iteritems()], + 'artifact_ids': [], + 'cache_id': source.cache_id, + 'cache_key': source.cache_key, + 'dependencies': [], } + if not prune_leaf: + source_dic['artifact_ids'].extend(id(artifact) for (_, artifact) + in source.artifacts.iteritems()) + source_dic['dependencies'].extend(id(d) + for d in source.dependencies) if source.morphology['kind'] == 'chunk': source_dic['build_mode'] = source.build_mode source_dic['prefix'] = source.prefix return source_dic - def encode_artifact(a, artifacts, source_id): - if artifact.source.morphology['kind'] == 'system': + def encode_artifact(a): + if artifact.source.morphology['kind'] == 'system': # pragma: no cover arch = artifact.source.morphology['arch'] else: arch = artifact.arch return { - 'source_id': source_id, + 'source_id': id(a.source), 'name': a.name, - 'cache_id': a.cache_id, - 'cache_key': a.cache_key, - 'dependencies': [str(id(artifacts[id(d)])) - for d in a.dependencies], - 'arch': arch + 'arch': arch, + 'dependents': [id(d) + for d in a.dependents], } - visited = set() - def traverse(a): - visited.add(a) - for dep in a.dependencies: - if dep in visited: - continue - for ret in traverse(dep): - yield ret - yield a - - - artifacts = {} encoded_artifacts = {} encoded_sources = {} + encoded_morphologies = {} + visited_artifacts = {} - for a in traverse(artifact): + for a in artifact.walk(): if id(a.source) not in encoded_sources: - if a.source.morphology['kind'] == 'chunk': - for (_, sa) in a.source.artifacts.iteritems(): - if id(sa) not in artifacts: - artifacts[id(sa)] = sa - encoded_artifacts[id(sa)] = encode_artifact(sa, - artifacts, id(a.source)) - else: - # We create separate sources for strata and systems, - # this is a bit of a hack, but needed to allow - # us to build strata and systems independently - - s = a.source - t = morphlib.source.Source(s.repo_name, s.original_ref, - s.sha1, s.tree, s.morphology, s.filename) - - t.artifacts = {a.name: a} - a.source = t - + for sa in a.source.artifacts.itervalues(): + if id(sa) not in encoded_artifacts: + visited_artifacts[id(sa)] = sa + encoded_artifacts[id(sa)] = encode_artifact(sa) + encoded_morphologies[id(a.source.morphology)] = \ + encode_morphology(a.source.morphology) encoded_sources[id(a.source)] = encode_source(a.source) - if id(a) not in artifacts: - artifacts[id(a)] = a - encoded_artifacts[id(a)] = encode_artifact(a, artifacts, - id(a.source)) - - encoded_artifacts['_root'] = str(id(artifact)) - - return json.dumps({'sources': encoded_sources, - 'artifacts': encoded_artifacts}) + if id(a) not in encoded_artifacts: # pragma: no cover + visited_artifacts[id(a)] = a + encoded_artifacts[id(a)] = encode_artifact(a) + + # Include one level of dependents above encoded artifacts, as we need + # them to be able to tell whether two sources are in the same stratum. + for a in visited_artifacts.itervalues(): + for source in a.dependents: # pragma: no cover + if id(source) not in encoded_sources: + encoded_morphologies[id(source.morphology)] = \ + encode_morphology(source.morphology) + encoded_sources[id(source)] = \ + encode_source(source, prune_leaf=True) + + content = { + 'sources': encoded_sources, + 'artifacts': encoded_artifacts, + 'morphologies': encoded_morphologies, + 'root_artifact': id(artifact), + 'default_split_rules': { + 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES, + 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES, + }, + } + return json.dumps(yaml.dump(content)) def deserialise_artifact(encoded): @@ -141,38 +132,25 @@ def deserialise_artifact(encoded): ''' - class FakeMorphology(dict): - - def get_commands(self, which): - '''Get commands to run from a morphology or build system''' - if self[which] is None: - attr = '_'.join(which.split('-')) - bs = morphlib.buildsystem.lookup_build_system( - self['build-system']) - return getattr(bs, attr) - else: - return self[which] - - morphology = FakeMorphology(le_dict) - for x in morphology_attributes: - setattr(morphology, x, le_dict['__%s' % x]) - del morphology['__%s' % x] - return morphology - - def decode_source(le_dict): + return morphlib.morphology.Morphology(le_dict) + + def decode_source(le_dict, morphology, split_rules): '''Convert a dict into a Source object.''' - morphology = decode_morphology(le_dict['morphology']) - source = morphlib.source.Source(le_dict['repo_name'], + source = morphlib.source.Source(le_dict['name'], + le_dict['repo_name'], le_dict['original_ref'], le_dict['sha1'], le_dict['tree'], morphology, - le_dict['filename']) + le_dict['filename'], + split_rules) if morphology['kind'] == 'chunk': source.build_mode = le_dict['build_mode'] source.prefix = le_dict['prefix'] + source.cache_id = le_dict['cache_id'] + source.cache_key = le_dict['cache_key'] return source def decode_artifact(artifact_dict, source): @@ -183,48 +161,57 @@ def deserialise_artifact(encoded): ''' artifact = morphlib.artifact.Artifact(source, artifact_dict['name']) - artifact.cache_id = artifact_dict['cache_id'] - artifact.cache_key = artifact_dict['cache_key'] artifact.arch = artifact_dict['arch'] artifact.source = source return artifact - le_dicts = json.loads(encoded) + le_dicts = yaml.load(json.loads(encoded)) artifacts_dict = le_dicts['artifacts'] sources_dict = le_dicts['sources'] - - artifact_ids = ([artifacts_dict['_root']] + - filter(lambda k: k != '_root', artifacts_dict.keys())) - - source_ids = [sid for sid in sources_dict.keys()] + morphologies_dict = le_dicts['morphologies'] + root_artifact = le_dicts['root_artifact'] + assert root_artifact in artifacts_dict artifacts = {} sources = {} - - for source_id in source_ids: + morphologies = {id: decode_morphology(d) + for (id, d) in morphologies_dict.iteritems()} + + # Decode sources + for source_id, source_dict in sources_dict.iteritems(): + morphology = morphologies[source_dict['morphology']] + kind = morphology['kind'] + ruler = getattr(morphlib.artifactsplitrule, 'unify_%s_matches' % kind) + if kind in ('chunk', 'stratum'): + rules = ruler(morphology, le_dicts['default_split_rules'][kind]) + else: # pragma: no cover + rules = ruler(morphology) + sources[source_id] = decode_source(source_dict, morphology, rules) + + # decode artifacts + for artifact_id, artifact_dict in artifacts_dict.iteritems(): + source_id = artifact_dict['source_id'] + source = sources[source_id] + artifact = decode_artifact(artifact_dict, source) + artifacts[artifact_id] = artifact + + # add source artifacts reference + for source_id, source in sources.iteritems(): source_dict = sources_dict[source_id] - sources[source_id] = decode_source(source_dict) - - # clear the source artifacts that get automatically generated - # we want to add the ones that were sent to us - sources[source_id].artifacts = {} - source_artifacts = source_dict['artifact_ids'] - - for artifact_id in source_artifacts: - if artifact_id not in artifacts: - artifact_dict = artifacts_dict[artifact_id] - artifact = decode_artifact(artifact_dict, sources[source_id]) - - artifacts[artifact_id] = artifact - - key = artifacts[artifact_id].name - sources[source_id].artifacts[key] = artifacts[artifact_id] - - # now add the dependencies - for artifact_id in artifact_ids: - artifact = artifacts[artifact_id] - artifact.dependencies = [artifacts[aid] for aid in - artifacts_dict[artifact_id]['dependencies']] - - return artifacts[artifacts_dict['_root']] + source.artifacts = {artifacts[a].name: artifacts[a] + for a in source_dict['artifact_ids']} + + # add source dependencies + for source_id, source_dict in sources_dict.iteritems(): + source = sources[source_id] + source.dependencies = [artifacts[aid] + for aid in source_dict['dependencies']] + + # add artifact dependents + for artifact_id, artifact in artifacts.iteritems(): + artifact_dict = artifacts_dict[artifact_id] + artifact.dependents = [sources[sid] + for sid in artifact_dict['dependents']] + + return artifacts[root_artifact] diff --git a/distbuild/serialise_tests.py b/distbuild/serialise_tests.py index 2ad3a384..d80c3dd7 100644 --- a/distbuild/serialise_tests.py +++ b/distbuild/serialise_tests.py @@ -23,13 +23,22 @@ import distbuild class MockMorphology(object): - def __init__(self, name): + def __init__(self, name, kind): self.dict = { 'name': '%s.morphology.name' % name, - 'kind': '%s.morphology.kind' % name, + 'kind': kind, + 'chunks': [], + 'products': [ + { + 'artifact': name, + 'include': [r'.*'], + }, + ], } - self.needs_staging_area = None - self.needs_artifact_metadata_cached = None + + @property + def needs_artifact_metadata_cached(self): + return self.dict['kind'] == 'stratum' def keys(self): return self.dict.keys() @@ -40,36 +49,57 @@ class MockMorphology(object): class MockSource(object): - def __init__(self, name): + build_mode = 'staging' + prefix = '/usr' + def __init__(self, name, kind): + self.name = name self.repo = None self.repo_name = '%s.source.repo_name' % name self.original_ref = '%s.source.original_ref' % name self.sha1 = '%s.source.sha1' % name self.tree = '%s.source.tree' % name - self.morphology = MockMorphology(name) + self.morphology = MockMorphology(name, kind) self.filename = '%s.source.filename' % name - - -class MockArtifact(object): - - def __init__(self, name): - self.source = MockSource(name) - self.name = name + self.dependencies = [] self.cache_id = { 'blip': '%s.blip' % name, 'integer': 42, } self.cache_key = '%s.cache_key' % name - self.dependencies = [] + self.artifacts = {} + + +class MockArtifact(object): + + arch = 'testarch' + + def __init__(self, name, kind): + self.source = MockSource(name, kind) + self.source.artifacts = {name: self} + self.name = name + self.dependents = [] + + def walk(self): # pragma: no cover + done = set() + + def depth_first(a): + if a not in done: + done.add(a) + for dep in a.source.dependencies: + for ret in depth_first(dep): + yield ret + yield a + + return list(depth_first(self)) class SerialisationTests(unittest.TestCase): def setUp(self): - self.art1 = MockArtifact('name1') - self.art2 = MockArtifact('name2') - self.art3 = MockArtifact('name3') - self.art4 = MockArtifact('name4') + self.art1 = MockArtifact('name1', 'stratum') + self.art2 = MockArtifact('name2', 'chunk') + self.art3 = MockArtifact('name3', 'chunk') + self.art4 = MockArtifact('name4', 'chunk') def assertEqualMorphologies(self, a, b): self.assertEqual(sorted(a.keys()), sorted(b.keys())) @@ -77,11 +107,8 @@ class SerialisationTests(unittest.TestCase): a_values = [a[k] for k in keys] b_values = [b[k] for k in keys] self.assertEqual(a_values, b_values) - self.assertEqual(a.needs_staging_area, b.needs_staging_area) self.assertEqual(a.needs_artifact_metadata_cached, b.needs_artifact_metadata_cached) - self.assertEqual(a.needs_staging_area, - b.needs_staging_area) def assertEqualSources(self, a, b): self.assertEqual(a.repo, b.repo) @@ -95,30 +122,29 @@ class SerialisationTests(unittest.TestCase): def assertEqualArtifacts(self, a, b): self.assertEqualSources(a.source, b.source) self.assertEqual(a.name, b.name) - self.assertEqual(a.cache_id, b.cache_id) - self.assertEqual(a.cache_key, b.cache_key) - self.assertEqual(len(a.dependencies), len(b.dependencies)) - for i in range(len(a.dependencies)): - self.assertEqualArtifacts(a.dependencies[i], b.dependencies[i]) + self.assertEqual(a.source.cache_id, b.source.cache_id) + self.assertEqual(a.source.cache_key, b.source.cache_key) + self.assertEqual(len(a.source.dependencies), + len(b.source.dependencies)) + for i in range(len(a.source.dependencies)): + self.assertEqualArtifacts(a.source.dependencies[i], + b.source.dependencies[i]) def verify_round_trip(self, artifact): encoded = distbuild.serialise_artifact(artifact) decoded = distbuild.deserialise_artifact(encoded) self.assertEqualArtifacts(artifact, decoded) - def key(a): - return a.cache_key - objs = {} queue = [decoded] while queue: obj = queue.pop() - k = key(obj) + k = obj.source.cache_key if k in objs: self.assertTrue(obj is objs[k]) else: objs[k] = obj - queue.extend(obj.dependencies) + queue.extend(obj.source.dependencies) def test_returns_string(self): encoded = distbuild.serialise_artifact(self.art1) @@ -128,21 +154,21 @@ class SerialisationTests(unittest.TestCase): self.verify_round_trip(self.art1) def test_works_with_single_dependency(self): - self.art1.dependencies = [self.art2] + self.art1.source.dependencies = [self.art2] self.verify_round_trip(self.art1) def test_works_with_two_dependencies(self): - self.art1.dependencies = [self.art2, self.art3] + self.art1.source.dependencies = [self.art2, self.art3] self.verify_round_trip(self.art1) def test_works_with_two_levels_of_dependencies(self): - self.art2.dependencies = [self.art4] - self.art1.dependencies = [self.art2, self.art3] + self.art2.source.dependencies = [self.art4] + self.art1.source.dependencies = [self.art2, self.art3] self.verify_round_trip(self.art1) def test_works_with_dag(self): - self.art2.dependencies = [self.art4] - self.art3.dependencies = [self.art4] - self.art1.dependencies = [self.art2, self.art3] + self.art2.source.dependencies = [self.art4] + self.art3.source.dependencies = [self.art4] + self.art1.source.dependencies = [self.art2, self.art3] self.verify_round_trip(self.art1) diff --git a/distbuild/sockserv.py b/distbuild/sockserv.py index a5215e79..156394e2 100644 --- a/distbuild/sockserv.py +++ b/distbuild/sockserv.py @@ -26,15 +26,20 @@ class ListenServer(StateMachine): '''Listen for new connections on a port, send events for them.''' - def __init__(self, addr, port, machine, extra_args=None): + def __init__(self, addr, port, machine, extra_args=None, port_file=''): StateMachine.__init__(self, 'listening') self._addr = addr self._port = port self._machine = machine self._extra_args = extra_args or [] + self._port_file = port_file def setup(self): src = ListeningSocketEventSource(self._addr, self._port) + if self._port_file: + host, port = src.sock.getsockname() + with open(self._port_file, 'w') as f: + f.write('%s\n' % port) self.mainloop.add_event_source(src) spec = [ diff --git a/distbuild/worker_build_scheduler.py b/distbuild/worker_build_scheduler.py index 6cda5972..be732153 100644 --- a/distbuild/worker_build_scheduler.py +++ b/distbuild/worker_build_scheduler.py @@ -262,13 +262,13 @@ class WorkerBuildQueuer(distbuild.StateMachine): logging.debug('Worker build step already started: %s' % event.artifact.basename()) progress = WorkerBuildStepAlreadyStarted(event.initiator_id, - event.artifact.cache_key, job.who.name()) + event.artifact.source.cache_key, job.who.name()) else: logging.debug('Job created but not building yet ' '(waiting for a worker to become available): %s' % event.artifact.basename()) progress = WorkerBuildWaiting(event.initiator_id, - event.artifact.cache_key) + event.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, progress) else: @@ -279,7 +279,7 @@ class WorkerBuildQueuer(distbuild.StateMachine): self._give_job(job) else: progress = WorkerBuildWaiting(event.initiator_id, - event.artifact.cache_key) + event.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, progress) def _handle_cancel(self, event_source, event): @@ -483,7 +483,7 @@ class WorkerConnection(distbuild.StateMachine): % (self._worker_name, msg)) started = WorkerBuildStepStarted(self._job.initiators, - self._job.artifact.cache_key, self.name()) + self._job.artifact.source.cache_key, self.name()) self.mainloop.queue_event(WorkerConnection, _JobStarted(self._job)) self.mainloop.queue_event(WorkerConnection, started) @@ -510,7 +510,7 @@ class WorkerConnection(distbuild.StateMachine): logging.debug('WC: emitting: %s', repr(new)) self.mainloop.queue_event( WorkerConnection, - WorkerBuildOutput(new, self._job.artifact.cache_key)) + WorkerBuildOutput(new, self._job.artifact.source.cache_key)) def _handle_exec_response(self, msg): logging.debug('WC: finished building: %s' % self._job.artifact.name) @@ -522,7 +522,8 @@ class WorkerConnection(distbuild.StateMachine): if new['exit'] != 0: # Build failed. - new_event = WorkerBuildFailed(new, self._job.artifact.cache_key) + new_event = WorkerBuildFailed(new, + self._job.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(WorkerConnection, _JobFailed(self._job)) self.mainloop.queue_event(self, _BuildFailed()) @@ -556,10 +557,6 @@ class WorkerConnection(distbuild.StateMachine): if kind == 'stratum': suffixes.append(filename + '.meta') - elif kind == 'system': - # FIXME: This is a really ugly hack. - if filename.endswith('-rootfs'): - suffixes.append(filename[:-len('-rootfs')] + '-kernel') suffixes = [urllib.quote(x) for x in suffixes] suffixes = ','.join(suffixes) @@ -571,7 +568,7 @@ class WorkerConnection(distbuild.StateMachine): '/1.0/fetch?host=%s:%d&cacheid=%s&artifacts=%s' % (urllib.quote(worker_host), self._worker_cache_server_port, - urllib.quote(self._job.artifact.cache_key), + urllib.quote(self._job.artifact.source.cache_key), suffixes)) msg = distbuild.message( @@ -582,7 +579,7 @@ class WorkerConnection(distbuild.StateMachine): self.mainloop.queue_event(distbuild.HelperRouter, req) progress = WorkerBuildCaching(self._job.initiators, - self._job.artifact.cache_key) + self._job.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, progress) def _maybe_handle_helper_result(self, event_source, event): @@ -594,7 +591,8 @@ class WorkerConnection(distbuild.StateMachine): logging.debug('Shared artifact cache population done') new_event = WorkerBuildFinished( - self._exec_response_msg, self._job.artifact.cache_key) + self._exec_response_msg, + self._job.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(self, _Cached()) else: @@ -612,7 +610,8 @@ class WorkerConnection(distbuild.StateMachine): _JobFailed(self._job)) new_event = WorkerBuildFailed( - self._exec_response_msg, self._job.artifact.cache_key) + self._exec_response_msg, + self._job.artifact.source.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(self, _BuildFailed()) diff --git a/morph-cache-server b/morph-cache-server new file mode 100755 index 00000000..4af3cee3 --- /dev/null +++ b/morph-cache-server @@ -0,0 +1,375 @@ +#!/usr/bin/env python +# +# Copyright (C) 2013, 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import base64 +import cliapp +import json +import logging +import os +import urllib +import urllib2 +import shutil + +from bottle import Bottle, request, response, run, static_file +from flup.server.fcgi import WSGIServer +from morphcacheserver.repocache import RepoCache + + +defaults = { + 'repo-dir': '/var/cache/morph-cache-server/gits', + 'bundle-dir': '/var/cache/morph-cache-server/bundles', + 'artifact-dir': '/var/cache/morph-cache-server/artifacts', + 'port': 8080, +} + + +class MorphCacheServer(cliapp.Application): + + def add_settings(self): + self.settings.integer(['port'], + 'port to listen on', + metavar='PORTNUM', + default=defaults['port']) + self.settings.string(['port-file'], + 'write port number to FILE', + metavar='FILE', + default='') + self.settings.string(['repo-dir'], + 'path to the repository cache directory', + metavar='PATH', + default=defaults['repo-dir']) + self.settings.string(['bundle-dir'], + 'path to the bundle cache directory', + metavar='PATH', + default=defaults['bundle-dir']) + self.settings.string(['artifact-dir'], + 'path to the artifact cache directory', + metavar='PATH', + default=defaults['artifact-dir']) + self.settings.boolean(['direct-mode'], + 'cache directories are directly managed') + self.settings.boolean(['enable-writes'], + 'enable the write methods (fetch and delete)') + self.settings.boolean(['fcgi-server'], + 'runs a fcgi-server', + default=True) + + + def _fetch_artifact(self, url, filename): + in_fh = None + try: + in_fh = urllib2.urlopen(url) + with open(filename, "w") as localtmp: + shutil.copyfileobj(in_fh, localtmp) + in_fh.close() + except Exception, e: + if in_fh is not None: + in_fh.close() + raise + else: + if in_fh is not None: + in_fh.close() + return os.stat(filename) + + def _fetch_artifacts(self, server, cacheid, artifacts): + ret = {} + try: + for artifact in artifacts: + artifact_name = "%s.%s" % (cacheid, artifact) + tmpname = os.path.join(self.settings['artifact-dir'], + ".dl.%s" % artifact_name) + url = "http://%s/1.0/artifacts?filename=%s" % ( + server, urllib.quote(artifact_name)) + stinfo = self._fetch_artifact(url, tmpname) + ret[artifact_name] = { + "size": stinfo.st_size, + "used": stinfo.st_blocks * 512, + } + except Exception, e: + for artifact in ret.iterkeys(): + os.unlink(os.path.join(self.settings['artifact-dir'], + ".dl.%s" % artifact)) + raise + + for artifact in ret.iterkeys(): + tmpname = os.path.join(self.settings['artifact-dir'], + ".dl.%s" % artifact) + artifilename = os.path.join(self.settings['artifact-dir'], + artifact) + os.rename(tmpname, artifilename) + + return ret + + + def process_args(self, args): + app = Bottle() + + repo_cache = RepoCache(self, + self.settings['repo-dir'], + self.settings['bundle-dir'], + self.settings['direct-mode']) + + def writable(prefix): + """Selectively enable bottle prefixes. + + prefix -- The path prefix we are enabling + + If the runtime configuration setting --enable-writes is provided + then we return the app.get() decorator for the given path prefix + otherwise we return a lambda which passes the function through + undecorated. + + This has the effect of being a runtime-enablable @app.get(...) + + """ + if self.settings['enable-writes']: + return app.get(prefix) + return lambda fn: fn + + @writable('/list') + def list(): + response.set_header('Cache-Control', 'no-cache') + results = {} + files = {} + results["files"] = files + for artifactdir, __, filenames in \ + os.walk(self.settings['artifact-dir']): + fsstinfo = os.statvfs(artifactdir) + results["freespace"] = fsstinfo.f_bsize * fsstinfo.f_bavail + for fname in filenames: + if not fname.startswith(".dl."): + try: + stinfo = os.stat("%s/%s" % (artifactdir, fname)) + files[fname] = { + "atime": stinfo.st_atime, + "size": stinfo.st_size, + "used": stinfo.st_blocks * 512, + } + except Exception, e: + print(e) + return results + + @writable('/fetch') + def fetch(): + host = self._unescape_parameter(request.query.host) + cacheid = self._unescape_parameter(request.query.cacheid) + artifacts = self._unescape_parameter(request.query.artifacts) + try: + response.set_header('Cache-Control', 'no-cache') + artifacts = artifacts.split(",") + return self._fetch_artifacts(host, cacheid, artifacts) + + except Exception, e: + response.status = 500 + logging.debug('%s' % e) + + @writable('/delete') + def delete(): + artifact = self._unescape_parameter(request.query.artifact) + try: + os.unlink('%s/%s' % (self.settings['artifact-dir'], + artifact)) + return { "status": 0, "reason": "success" } + except OSError, ose: + return { "status": ose.errno, "reason": ose.strerror } + except Exception, e: + response.status = 500 + logging.debug('%s' % e) + + @app.get('/sha1s') + def sha1(): + repo = self._unescape_parameter(request.query.repo) + ref = self._unescape_parameter(request.query.ref) + try: + response.set_header('Cache-Control', 'no-cache') + sha1, tree = repo_cache.resolve_ref(repo, ref) + return { + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'sha1': '%s' % sha1, + 'tree': '%s' % tree + } + except Exception, e: + response.status = 404 + logging.debug('%s' % e) + + @app.post('/sha1s') + def sha1s(): + result = [] + for pair in request.json: + repo = pair['repo'] + ref = pair['ref'] + try: + sha1, tree = repo_cache.resolve_ref(repo, ref) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'sha1': '%s' % sha1, + 'tree': '%s' % tree + }) + except Exception, e: + logging.debug('%s' % e) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'error': '%s' % e + }) + response.set_header('Cache-Control', 'no-cache') + response.set_header('Content-Type', 'application/json') + return json.dumps(result) + + @app.get('/files') + def file(): + repo = self._unescape_parameter(request.query.repo) + ref = self._unescape_parameter(request.query.ref) + filename = self._unescape_parameter(request.query.filename) + try: + content = repo_cache.cat_file(repo, ref, filename) + response.set_header('Content-Type', 'application/octet-stream') + return content + except Exception, e: + response.status = 404 + logging.debug('%s' % e) + + @app.post('/files') + def files(): + result = [] + for pair in request.json: + repo = pair['repo'] + ref = pair['ref'] + filename = pair['filename'] + try: + content = repo_cache.cat_file(repo, ref, filename) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'filename': '%s' % filename, + 'data': '%s' % base64.b64encode(content), + }) + except Exception, e: + logging.debug('%s' % e) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'filename': '%s' % filename, + 'error': '%s' % e + }) + response.set_header('Content-Type', 'application/json') + return json.dumps(result) + + @app.get('/trees') + def tree(): + repo = self._unescape_parameter(request.query.repo) + ref = self._unescape_parameter(request.query.ref) + path = self._unescape_parameter(request.query.path) + try: + tree = repo_cache.ls_tree(repo, ref, path) + return { + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'tree': tree, + } + except Exception, e: + response.status = 404 + logging.debug('%s' % e) + + @app.get('/bundles') + def bundle(): + repo = self._unescape_parameter(request.query.repo) + filename = repo_cache.get_bundle_filename(repo) + dirname = os.path.dirname(filename) + basename = os.path.basename(filename) + return static_file(basename, root=dirname, download=True) + + @app.get('/artifacts') + def artifact(): + basename = self._unescape_parameter(request.query.filename) + filename = os.path.join(self.settings['artifact-dir'], basename) + if os.path.exists(filename): + return static_file(basename, + root=self.settings['artifact-dir'], + download=True) + else: + response.status = 404 + logging.debug('artifact %s does not exist' % basename) + + @app.post('/artifacts') + def post_artifacts(): + if request.content_type != 'application/json': + logging.warning('Content-type is not json: ' + 'expecting a json post request') + + artifacts = json.load(request.body) + results = {} + + logging.debug('Received a POST request for /artifacts') + + for artifact in artifacts: + if artifact.startswith('/'): + response.status = 500 + logging.error("%s: artifact name cannot start with a '/'" + % artifact) + return + + filename = os.path.join(self.settings['artifact-dir'], + artifact) + results[artifact] = os.path.exists(filename) + + if results[artifact]: + logging.debug('%s is in the cache', artifact) + else: + logging.debug('%s is NOT in the cache', artifact) + + return results + + root = Bottle() + root.mount(app, '/1.0') + + + if self.settings['fcgi-server']: + WSGIServer(root).run() + elif self.settings['port-file']: + import wsgiref.simple_server + + server_port_file = self.settings['port-file'] + class DebugServer(wsgiref.simple_server.WSGIServer): + '''WSGI-like server that uses an ephemeral port. + + Rather than use a specified port, or default, the + DebugServer binds to an ephemeral port on 127.0.0.1 + and writes its number to port-file, so a non-racy + temporary port can be used. + + ''' + + def __init__(self, (host, port), *args, **kwargs): + wsgiref.simple_server.WSGIServer.__init__( + self, ('127.0.0.1', 0), *args, **kwargs) + with open(server_port_file, 'w') as f: + f.write(str(self.server_port) + '\n') + run(root, server_class=DebugServer, debug=True) + else: + run(root, host='0.0.0.0', port=self.settings['port'], + reloader=True) + + def _unescape_parameter(self, param): + return urllib.unquote(param) + + +if __name__ == '__main__': + MorphCacheServer().run() diff --git a/tests.deploy/deploy-rawdisk-without-disk-size-fails.script b/morphcacheserver/__init__.py index 035557dd..2c25ce28 100755..100644 --- a/tests.deploy/deploy-rawdisk-without-disk-size-fails.script +++ b/morphcacheserver/__init__.py @@ -1,30 +1,17 @@ -#!/bin/bash -# # Copyright (C) 2013 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# Test "morph deploy" refuses to deploy if DISK_SIZE is unset. - -set -eu - - -. "$SRCDIR/tests.deploy/setup-build" -cd "$DATADIR/workspace/branch1" -"$SRCDIR/scripts/test-morph" build linux-system -! "$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ - deploy rawdisk_test_cluster_without_disk_size > /dev/null 2>&1 -test ! -e "$DATADIR/disk.img" - +import repocache diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py new file mode 100644 index 00000000..305c187c --- /dev/null +++ b/morphcacheserver/repocache.py @@ -0,0 +1,158 @@ +# Copyright (C) 2013,2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import os +import re +import string +import urlparse + + +class RepositoryNotFoundError(cliapp.AppException): + + def __init__(self, repo): + cliapp.AppException.__init__( + self, 'Repository %s does not exist in the cache' % repo) + + +class InvalidReferenceError(cliapp.AppException): + + def __init__(self, repo, ref): + cliapp.AppException.__init__( + self, 'Ref %s is an invalid reference for repo %s' % + (ref, repo)) + + +class UnresolvedNamedReferenceError(cliapp.AppException): + + def __init__(self, repo, ref): + cliapp.AppException.__init__( + self, 'Ref %s is not a SHA1 ref for repo %s' % + (ref, repo)) + + +class RepoCache(object): + + def __init__(self, app, repo_cache_dir, bundle_cache_dir, direct_mode): + self.app = app + self.repo_cache_dir = repo_cache_dir + self.bundle_cache_dir = bundle_cache_dir + self.direct_mode = direct_mode + + def resolve_ref(self, repo_url, ref): + quoted_url = self._quote_url(repo_url) + repo_dir = os.path.join(self.repo_cache_dir, quoted_url) + if not os.path.exists(repo_dir): + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) + try: + if re.match('^[0-9a-fA-F]{40}$', ref): + sha1 = ref + else: + if (not self.direct_mode and + not ref.startswith('refs/origin/')): + ref = 'refs/origin/' + ref + sha1 = self._rev_parse(repo_dir, ref) + return sha1, self._tree_from_commit(repo_dir, sha1) + + except cliapp.AppException: + raise + + def _tree_from_commit(self, repo_dir, commitsha): + commit_info = self.app.runcmd(['git', 'log', '-1', + '--format=format:%T', commitsha], + cwd=repo_dir) + return commit_info.strip() + + def cat_file(self, repo_url, ref, filename): + quoted_url = self._quote_url(repo_url) + repo_dir = os.path.join(self.repo_cache_dir, quoted_url) + if not os.path.exists(repo_dir): + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) + if not self._is_valid_sha1(ref): + raise UnresolvedNamedReferenceError(repo_url, ref) + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) + try: + sha1 = self._rev_parse(repo_dir, ref) + except BaseException: + raise InvalidReferenceError(repo_url, ref) + + return self._cat_file(repo_dir, sha1, filename) + + def ls_tree(self, repo_url, ref, path): + quoted_url = self._quote_url(repo_url) + repo_dir = os.path.join(self.repo_cache_dir, quoted_url) + if not os.path.exists(repo_dir): + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) + if not self._is_valid_sha1(ref): + raise UnresolvedNamedReferenceError(repo_url, ref) + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) + + try: + sha1 = self._rev_parse(repo_dir, ref) + except BaseException: + raise InvalidReferenceError(repo_url, ref) + + lines = self._ls_tree(repo_dir, sha1, path).strip() + lines = lines.splitlines() + data = {} + for line in lines: + elements = line.split() + basename = elements[3] + data[basename] = { + 'mode': elements[0], + 'kind': elements[1], + 'sha1': elements[2], + } + return data + + def get_bundle_filename(self, repo_url): + quoted_url = self._quote_url(repo_url, True) + return os.path.join(self.bundle_cache_dir, '%s.bndl' % quoted_url) + + def _quote_url(self, url, always_indirect=False): + if self.direct_mode and not always_indirect: + quoted_url = urlparse.urlparse(url)[2] + while quoted_url.startswith("/"): + quoted_url = quoted_url[1:] + return quoted_url + else: + valid_chars = string.digits + string.letters + '%_' + transl = lambda x: x if x in valid_chars else '_' + return ''.join([transl(x) for x in url]) + + def _rev_parse(self, repo_dir, ref): + return self.app.runcmd(['git', 'rev-parse', '--verify', ref], + cwd=repo_dir)[0:40] + + def _cat_file(self, repo_dir, sha1, filename): + return self.app.runcmd( + ['git', 'cat-file', 'blob', '%s:%s' % (sha1, filename)], + cwd=repo_dir) + + def _ls_tree(self, repo_dir, sha1, path): + return self.app.runcmd(['git', 'ls-tree', sha1, path], cwd=repo_dir) + + def _is_valid_sha1(self, ref): + valid_chars = 'abcdefABCDEF0123456789' + return len(ref) == 40 and all([x in valid_chars for x in ref]) diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index cf0d1060..1511d694 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -230,7 +230,7 @@ DEFAULT_STRATUM_RULES = [ ] -def unify_chunk_matches(morphology): +def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES): '''Create split rules including defaults and per-chunk rules. With rules specified in the morphology's 'products' field and the @@ -246,7 +246,7 @@ def unify_chunk_matches(morphology): split_rules.add(ca_name, FileMatch(patterns)) name = morphology['name'] - for suffix, patterns in DEFAULT_CHUNK_RULES: + for suffix, patterns in default_rules: ca_name = name + suffix # Explicit rules override the default rules. This is an all-or-nothing # override: there is no way to extend the default split rules right now @@ -257,7 +257,7 @@ def unify_chunk_matches(morphology): return split_rules -def unify_stratum_matches(morphology): +def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES): '''Create split rules including defaults and per-stratum rules. With rules specified in the chunk spec's 'artifacts' fields, the @@ -284,7 +284,7 @@ def unify_stratum_matches(morphology): for d in morphology.get('products', {})): match_split_rules.add(sta_name, ArtifactMatch(patterns)) - for suffix, patterns in DEFAULT_STRATUM_RULES: + for suffix, patterns in default_rules: sta_name = morphology['name'] + suffix # Explicit rules override the default rules. This is an all-or-nothing # override: there is no way to extend the default split rules right now diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 20cae225..9cd3a074 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -162,46 +162,6 @@ def get_stratum_files(f, lac): # pragma: no cover cf.close() -def get_overlaps(source, constituents, lac): # pragma: no cover - # check whether strata overlap - installed = defaultdict(set) - for dep in constituents: - handle = lac.get(dep) - if source.morphology['kind'] == 'stratum': - for filename in get_chunk_files(handle): - installed[filename].add(dep) - elif source.morphology['kind'] == 'system': - for filename in get_stratum_files(handle, lac): - installed[filename].add(dep) - handle.close() - overlaps = defaultdict(set) - for filename, artifacts in installed.iteritems(): - if len(artifacts) > 1: - overlaps[frozenset(artifacts)].add(filename) - return overlaps - - -def log_overlaps(overlaps): # pragma: no cover - for overlapping, files in sorted(overlaps.iteritems()): - logging.warning(' Artifacts %s overlap with files:' % - ', '.join(sorted(a.name for a in overlapping))) - for filename in sorted(files): - logging.warning(' %s' % filename) - - -def write_overlap_metadata(artifact, overlaps, lac): # pragma: no cover - f = lac.put_artifact_metadata(artifact, 'overlaps') - # the big list comprehension is because json can't serialize - # artifacts, sets or dicts with non-string keys - json.dump( - [ - [ - [a.name for a in afs], list(files) - ] for afs, files in overlaps.iteritems() - ], f, indent=4, encoding='unicode-escape') - f.close() - - class BuilderBase(object): '''Base class for building artifacts.''' @@ -559,18 +519,6 @@ class StratumBuilder(BuilderBase): download_depends(constituents, self.local_artifact_cache, self.remote_artifact_cache) - # check for chunk overlaps - overlaps = get_overlaps(self.source, constituents, - self.local_artifact_cache) - if len(overlaps) > 0: - logging.warning( - 'Overlaps in stratum artifact %s detected' %a_name) - log_overlaps(overlaps) - self.app.status(msg='Overlaps in stratum artifact ' - '%(stratum_name)s detected', - stratum_name=a_name, error=True) - write_overlap_metadata(a, overlaps, - self.local_artifact_cache) with self.build_watch('create-chunk-list'): lac = self.local_artifact_cache @@ -674,18 +622,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover self.remote_artifact_cache) f.close() - # check whether the strata overlap - overlaps = get_overlaps(self.source, self.source.dependencies, - self.local_artifact_cache) - if len(overlaps) > 0: - self.app.status(msg='Overlaps in system artifact ' - '%(artifact_name)s detected', - artifact_name=a_name, - error=True) - log_overlaps(overlaps) - write_overlap_metadata(a, overlaps, - self.local_artifact_cache) - # unpack it from the local artifact cache for stratum_artifact in self.source.dependencies: self.unpack_one_stratum(stratum_artifact, path) diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index 6eeece77..74645f41 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -257,9 +257,6 @@ class ManifestGenerator(object): class ArtifactInspectionPlugin(cliapp.Plugin): def enable(self): - self.app.add_subcommand('run-in-artifact', - self.run_in_artifact, - arg_synopsis='ARTIFACT CMD') self.app.add_subcommand('generate-manifest', self.generate_manifest, arg_synopsis='SYSTEM-ARTIFACT') @@ -267,35 +264,6 @@ class ArtifactInspectionPlugin(cliapp.Plugin): def disable(self): pass - def run_in_artifact(self, args): - '''Run a command inside an extracted/mounted chunk or system. - - Command line arguments: - - * `ARTIFACT` is a filename for the artifact. - * `CMD` is the command to run. - - run-in-artifact unpacks an artifact, and runs a command in - the temporary directory it was unpacked to. - - The command must be given to Morph as a single argument, so - use shell quoting appropriately. - - ''' - - if len(args) < 2: - raise cliapp.AppException( - 'run-in-artifact requires arguments: a chunk or ' - 'system artifact and a a shell command') - - artifact, cmd = args[0], args[1:] - - def run_command_in_dir(dirname): - output = self.app.runcmd(cmd, cwd=dirname) - self.app.output.write(output) - - call_in_artifact_directory(self.app, artifact, run_command_in_dir) - def generate_manifest(self, args): '''Generate a content manifest for a system image. diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 50ab7eeb..653eeae8 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -103,7 +103,7 @@ class WorkerBuild(cliapp.Plugin): self.app.subcommands['gc']([]) arch = artifact.arch - bc.build_artifact(artifact, bc.new_build_env(arch)) + bc.build_source(artifact.source, bc.new_build_env(arch)) def is_system_artifact(self, filename): return re.match(r'^[0-9a-fA-F]{64}\.system\.', filename) @@ -121,6 +121,11 @@ class WorkerDaemon(cliapp.Plugin): 'listen for connections on PORT', default=3434, group=group_distbuild) + self.app.settings.string( + ['worker-daemon-port-file'], + 'write port used by worker-daemon to FILE', + default='', + group=group_distbuild) self.app.add_subcommand( 'worker-daemon', self.worker_daemon, @@ -136,7 +141,9 @@ class WorkerDaemon(cliapp.Plugin): address = self.app.settings['worker-daemon-address'] port = self.app.settings['worker-daemon-port'] - router = distbuild.ListenServer(address, port, distbuild.JsonRouter) + port_file = self.app.settings['worker-daemon-port-file'] + router = distbuild.ListenServer(address, port, distbuild.JsonRouter, + port_file=port_file) loop = distbuild.MainLoop() loop.add_state_machine(router) loop.run() @@ -156,6 +163,16 @@ class ControllerDaemon(cliapp.Plugin): 'listen for initiator connections on PORT', default=7878, group=group_distbuild) + self.app.settings.string( + ['controller-initiator-port-file'], + 'write the port to listen for initiator connections to FILE', + default='', + group=group_distbuild) + self.app.settings.string( + ['initiator-step-output-dir'], + 'write build output to files in DIR', + default='.', + group=group_distbuild) self.app.settings.string( ['controller-helper-address'], @@ -167,6 +184,11 @@ class ControllerDaemon(cliapp.Plugin): 'listen for helper connections on PORT', default=5656, group=group_distbuild) + self.app.settings.string( + ['controller-helper-port-file'], + 'write the port to listen for helper connections to FILE', + default='', + group=group_distbuild) self.app.settings.string_list( ['worker'], @@ -218,8 +240,10 @@ class ControllerDaemon(cliapp.Plugin): listener_specs = [ # address, port, class to initiate on connection, class init args ('controller-helper-address', 'controller-helper-port', + 'controller-helper-port-file', distbuild.HelperRouter, []), ('controller-initiator-address', 'controller-initiator-port', + 'controller-initiator-port-file', distbuild.InitiatorConnection, [artifact_cache_server, morph_instance]), ] @@ -229,11 +253,12 @@ class ControllerDaemon(cliapp.Plugin): queuer = distbuild.WorkerBuildQueuer() loop.add_state_machine(queuer) - for addr, port, sm, extra_args in listener_specs: + for addr, port, port_file, sm, extra_args in listener_specs: addr = self.app.settings[addr] port = self.app.settings[port] + port_file = self.app.settings[port_file] listener = distbuild.ListenServer( - addr, port, sm, extra_args=extra_args) + addr, port, sm, extra_args=extra_args, port_file=port_file) loop.add_state_machine(listener) for worker in self.app.settings['worker']: diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py index 8074206b..61c8d160 100644 --- a/morphlib/plugins/list_artifacts_plugin.py +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -105,11 +105,13 @@ class ListArtifactsPlugin(cliapp.Plugin): self.app.settings, system_artifact.source.morphology['arch']) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) - artifact_files = set() - for artifact in system_artifact.walk(): + for source in set(a.source for a in system_artifact.walk()): artifact.cache_key = ckc.compute_key(artifact) artifact.cache_id = ckc.get_cache_id(artifact) + artifact_files = set() + for artifact in system_artifact.walk(): + artifact_files.add(artifact.basename()) if artifact.source.morphology.needs_artifact_metadata_cached: diff --git a/scripts/list-overlaps b/scripts/list-overlaps deleted file mode 100755 index d092ba75..00000000 --- a/scripts/list-overlaps +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -# This is a program to convert the json dump of the overlaps between artifacts -# in a format more suited to shell programs, or human reading - -import json - -import cliapp - - -class ListOverlaps(cliapp.Application): - - @staticmethod - def _load_overlap(filename): - data = json.load(open(filename), encoding='unicode-escape') - overlaps = dict((frozenset(pair[0]), set(pair[1])) for pair in data) - return overlaps - - def cmd_groups(self, args): - overlaps = ListOverlaps._load_overlap(args[0]) - for group in overlaps: - print(' '.join(sorted(group))) - - def cmd_list_files(self, args): - overlaps = self._load_overlap(args[0]) - group = frozenset(args[1:]) - for filename in overlaps[group]: - print filename - -ListOverlaps().run() diff --git a/scripts/test-shell.c b/scripts/test-shell.c new file mode 100644 index 00000000..e3ef1ff1 --- /dev/null +++ b/scripts/test-shell.c @@ -0,0 +1,150 @@ +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <getopt.h> + +#include <stdint.h> +#include <ftw.h> +#include <errno.h> + +char *readlinka(char const *path){ + size_t buflen = BUFSIZ; + char *buf = malloc(buflen); + ssize_t read; + while ((read = readlink(path, buf, buflen - 1)) >= buflen - 1) { + char *newbuf = realloc(buf, buflen * 2); + if (newbuf == NULL) { + goto failure; + } + buf = newbuf; + buflen = buflen * 2; + } + buf[read] = '\0'; + return buf; +failure: + free(buf); + return NULL; +} + +int copy_file_paths(char const *source_file, char const *target_file) { + int source_fd; + int target_fd; + int ret = -1; + struct stat st; + if ((source_fd = open(source_file, O_RDONLY)) == -1) { + return ret; + } + if (fstat(source_fd, &st) == -1) { + perror("stat"); + ret = -2; + goto cleanup_in; + } + if ((target_fd = open(target_file, O_WRONLY|O_CREAT, st.st_mode)) == -1) { + ret = -3; + goto cleanup_in; + } + ssize_t read; + while ((read = sendfile(target_fd, source_fd, NULL, BUFSIZ)) > 0); + if (read < 0) { + perror("sendfile"); + ret = -4; + } + ret = 0; +cleanup_all: + close(target_fd); +cleanup_in: + close(source_fd); + return ret; +} + +int copy_entry(const char *fpath, const struct stat *sb, int typeflag, + struct FTW *ftwbuf) { + int ret = 0; + char *target_path = NULL; + if (asprintf(&target_path, "%s/%s", getenv("DESTDIR"), fpath) == -1) { + return -1; + } + switch (typeflag) { + case FTW_F: + /* Copy file */ + if ((ret = copy_file_paths(fpath, target_path)) < 0) { + perror("Copy file"); + ret = -1; + } + break; + case FTW_D: + case FTW_DNR: + /* Copy directory */ + if (mkdir(target_path, sb->st_mode)) { + if (errno != EEXIST) { + perror("mkdir"); + ret = -1; + } + } + break; + case FTW_NS: + case FTW_SL: + case FTW_SLN: { + /* Create symlink */ + char *link_target = readlinka(fpath); + if (link_target == NULL) { + perror("readlink"); + ret = -1; + } + if (symlink(link_target, target_path) == -1) { + perror("symlink"); + ret = -1; + } + break; + } + } +cleanup: + free(target_path); + return ret; +} + +int main(int argc, char *argv[]) { + int ret = 1; + if (argc != 3 || strcmp(argv[1], "-c") != 0) { + fprintf(stderr, "Usage: %s -c COMMAND\n", argv[0]); + return 1; + } + size_t cmdlen = strlen(argv[2]); + FILE *cmdstream = fmemopen(argv[2], cmdlen, "r"); + { + ssize_t read; + size_t len = 0; + char *line = NULL; + + ret = 0; + while ((read = getline(&line, &len, cmdstream)) != -1) { + if (line[read - 1] == '\n') line[read - 1] = '\0'; + if (strcmp(line, "copy files") == 0) { + /* Recursively copy contents of current dir to DESTDIR */ + if (nftw(".", copy_entry, 20, FTW_PHYS)) { + ret = 1; + break; + } + } else if (strcmp(line, "false") == 0 || + strstr(line, "false ") == line) { + ret = 1; + break; + } else if (strstr(line, "echo ") == line) { + if (puts(line + sizeof("echo ") - 1) == EOF){ + perror("echo"); + ret = 1; + break; + } + } else { + ret = 127; + break; + } + } + free(line); + } + return ret; +} @@ -128,15 +128,11 @@ class Check(Command): def run(self): subprocess.check_call(['python', '-m', 'CoverageTestRunner', '--ignore-missing-from=without-test-modules', - 'morphlib']) + 'morphlib', 'distbuild']) os.remove('.coverage') setup(name='morph', - description='FIXME', - long_description='''\ -FIXME -''', classifiers=[ 'Development Status :: 2 - Pre-Alpha', 'Environment :: Console', @@ -152,8 +148,9 @@ FIXME author='Codethink Limited', author_email='baserock-dev@baserock.org', url='http://www.baserock.org/', - scripts=['morph', 'distbuild-helper'], - packages=['morphlib', 'morphlib.plugins', 'distbuild'], + scripts=['morph', 'distbuild-helper', 'morph-cache-server'], + packages=['morphlib', 'morphlib.plugins', 'distbuild', + 'morphcacheserver'], package_data={ 'morphlib': [ 'xfer-hole', diff --git a/tests.as-root/archless-system-fails.exit b/tests.as-root/archless-system-fails.exit deleted file mode 100644 index d00491fd..00000000 --- a/tests.as-root/archless-system-fails.exit +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests.as-root/archless-system-fails.script b/tests.as-root/archless-system-fails.script deleted file mode 100755 index e34e9ad6..00000000 --- a/tests.as-root/archless-system-fails.script +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Raise an error if system has no architecture defined. - -set -eu - -cd "$DATADIR/morphs" -git checkout --quiet -b archless master -cat <<EOF >archless-system.morph -name: archless-system -kind: system -strata: - - morph: hello-stratum -EOF -git add archless-system.morph -git commit --quiet -m "add archless system" - -"$SRCDIR/scripts/test-morph" \ - build-morphology test:morphs archless archless-system diff --git a/tests.as-root/archless-system-fails.stderr b/tests.as-root/archless-system-fails.stderr deleted file mode 100644 index 71e7dbb0..00000000 --- a/tests.as-root/archless-system-fails.stderr +++ /dev/null @@ -1 +0,0 @@ -ERROR: Missing field arch from morphology string diff --git a/tests.as-root/branch-from-image-works.script b/tests.as-root/branch-from-image-works.script deleted file mode 100755 index fb0b09c9..00000000 --- a/tests.as-root/branch-from-image-works.script +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# A branch created with branch-from-image has the right commits specified - -set -eu - -# Disable test on versions of Python before 2.7. -. "$SRCDIR/scripts/python-check" - -. "$SRCDIR/scripts/fix-committer-info" - -tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact \ - build-morphology test:morphs tarball hello-tarball) - -extracted="$DATADIR/extracted" -mkdir -p "$extracted" -tar -xf "$tar" -C "$extracted" -get_sha1(){ - sed -nre '/sha1/s/^.*([0-9a-f]{40}).*$/\1/p' "$1" -} -hello_chunk_commit=$(get_sha1 "$extracted/baserock/hello-bins.meta") - -# Make a commit so that petrifying from HEAD is detectable -chunkrepo="$DATADIR/chunk-repo" -cd "$chunkrepo" -git checkout --quiet farrokh -sed -i -e 's/hello, world/goodbye, world/g' hello.c -git add hello.c -git commit --quiet -m 'Make hello say goodbye' - - -workspace="$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init "$workspace" -cd "$workspace" -"$SRCDIR/scripts/test-morph" branch-from-image mybranch \ - --metadata-dir="$extracted/baserock" -cd mybranch/test/morphs -grep -qFe "$hello_chunk_commit" hello-stratum.morph -tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact build hello-tarball) -tar -xf "$tar" bin/hello -bin/hello diff --git a/tests.as-root/branch-from-image-works.setup b/tests.as-root/branch-from-image-works.setup deleted file mode 120000 index aac6926a..00000000 --- a/tests.as-root/branch-from-image-works.setup +++ /dev/null @@ -1 +0,0 @@ -metadata-includes-morph-version.setup
\ No newline at end of file diff --git a/tests.as-root/branch-from-image-works.stdout b/tests.as-root/branch-from-image-works.stdout deleted file mode 100644 index 4b5fa637..00000000 --- a/tests.as-root/branch-from-image-works.stdout +++ /dev/null @@ -1 +0,0 @@ -hello, world diff --git a/tests.as-root/build-handles-stratum-build-depends.script b/tests.as-root/build-handles-stratum-build-depends.script deleted file mode 100755 index 6e6f82da..00000000 --- a/tests.as-root/build-handles-stratum-build-depends.script +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -## "morph build" should update the build-depends fields of strata correctly. - -# FIXME: The new "morph edit" seems to be changing build-dependencies -# (correctly) in a way that makes the old "morph build" fail, due to -# this error: -# -# Conflicting versions of stratum 'hello-stratum' appear in the build. -# -# I cannot find a way to fix the old "morph build", and so I'm disabling -# this test until it can be fixed. --liw -exit 0 - -set -eu - -. "$SRCDIR/tests.as-root/setup-build" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" branch test:morphs test/stratum-build-depends - -cd test/stratum-build-depends/test/morphs - -# 'linux-system' and the build-depends fields of 'linux-stratum' need to -# be updated here. Any build-depends of any altered strata also need to -# be altered, such as the 'tools-stratum' which depends on linux-stratum -# If they are not updated, the build command will fail. -"$SRCDIR/scripts/test-morph" edit linux-system hello-stratum - -# Likewise, this command must update build-depends or the 'repo' field will -# not be changed in the temporary build branch, leading to: -# -# ERROR: Ref test/stratum-build-depends is an invalid reference for -# repo file://TMP/morphs -# -"$SRCDIR/scripts/test-morph" build linux-system diff --git a/tests.as-root/build-with-external-strata.script b/tests.as-root/build-with-external-strata.script deleted file mode 100755 index be870053..00000000 --- a/tests.as-root/build-with-external-strata.script +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## "morph build" with strata outside the branch root repository. - -# FIXME: The new "morph edit" breaks this, for reasons unknown. Disabling -# it on the assumption that the new code (which changes refs to -# build-depends) is correct and the convoluted test and the old "morph -# build" code are wrong, but this needs to be revisited soon. --liw -exit 0 - -set -eu - -. "$SRCDIR/scripts/setup-3rd-party-strata" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" branch test:morphs branch1 - -# System will fail to build unless we add linux to it -- make the change but -# don't commit it, in one of the external strata, as a challenge for -# 'morph build'. -cd "branch1" -cd "test/external-strata" - -awk ' - /^chunks:/ { - print $0 - print "- name: linux" - print " repo: test:kernel-repo" - print " ref: master" - print " build-mode: test" - print " build-depends: []" - next - } - { print } -' stratum2.morph > temp -mv temp stratum2.morph - -# Ignore Morph's output for now because it gives us: -# 2012-11-07 16:26:12 Overlaps in system artifact hello-system-rootfs detected -# -# This is due to having a chunk named 'hello' in more than one stratum. It's -# a bug that this generates overlaps (the chunk's .meta file needs to be called -# $stratum.$chunk.meta rather than $chunk.meta to avoid the overlap) and the -# redirection should be removed once this bug is fixed. -"$SRCDIR/scripts/test-morph" build hello-system > /dev/null - -[ $("$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" | wc -l) -eq 23 ] diff --git a/tests.as-root/build-with-push.script b/tests.as-root/build-with-push.script deleted file mode 100755 index ead669ed..00000000 --- a/tests.as-root/build-with-push.script +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -# Test 'morph build' when build without push is disabled, i.e. everything is -# built from the remote repositories instead of the local checkouts. - -# FIXME: This seems to break because the new "morph edit" makes correct -# changes to build-dependencies, which breaks the old "morph build". -# Disable test now, re-enable it after "morph build" is fixed. --liw -exit 0 - - -set -eu - -source "$SRCDIR/tests.as-root/setup-build" - -cd "$DATADIR/workspace/branch1" -"$SRCDIR/scripts/test-morph" --push-build-branches build linux-system - -# Test that the chunk was built from test:kernel-repo and not a local branch -cd "$DATADIR/cache/artifacts" -tar xf *.chunk.linux baserock/linux.meta -grep -q "\"repo\": \"file://$DATADIR/kernel-repo\"" baserock/linux.meta - - diff --git a/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script b/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script deleted file mode 100755 index ac6cffec..00000000 --- a/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -## Make sure "morph build" works anywhere in a workspace or system branch -## and produces the same results every time. - -# FIXME: This seems to break because the new "morph edit" makes correct -# changes to build-dependencies, which breaks the old "morph build". -# Disable test now, re-enable it after "morph build" is fixed. --liw -exit 0 - -set -eu - -source "$SRCDIR/tests.as-root/setup-build" - -# Build once. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" build linux-system -ARTIFACT_COUNT="$(ls "$DATADIR/cache/artifacts" | wc -l)" - -# Build twice. -cd "$DATADIR/workspace/branch1" -"$SRCDIR/scripts/test-morph" build linux-system -[ "$ARTIFACT_COUNT" -eq $(ls "$DATADIR/cache/artifacts" | wc -l) ] - -# Build thrice, and that should be enough. -cd "$DATADIR/workspace/branch1/test/morphs" -"$SRCDIR/scripts/test-morph" build linux-system -[ "$ARTIFACT_COUNT" -eq $(ls "$DATADIR/cache/artifacts" | wc -l) ] diff --git a/tests.as-root/building-a-system-branch-picks-up-committed-removes.script b/tests.as-root/building-a-system-branch-picks-up-committed-removes.script deleted file mode 100755 index 64ae82c7..00000000 --- a/tests.as-root/building-a-system-branch-picks-up-committed-removes.script +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -# FIXME: This seems to break because the new "morph edit" makes correct -# changes to build-dependencies, which breaks the old "morph build". -# Disable test now, re-enable it after "morph build" is fixed. --liw -exit 0 - -set -e - -. "$SRCDIR/tests.as-root/setup-build" -KERNEL_BRANCH=baserock/builds/123456789/AABBCCDDE - -cd "$DATADIR/workspace/branch1/test/kernel-repo" -git checkout --quiet master -echo Use Morph >README -git add README -git commit --quiet -m 'Add README' - -# Build the linux system from the system branch. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" build linux-system - -# Make a change elsewhere to be pulled in -PEER_REPO="$DATADIR/peer-kernel" -git clone --quiet "file://$DATADIR/workspace/branch1/test/kernel-repo" \ - "$PEER_REPO" -cd "$PEER_REPO" -git checkout --quiet -b fix -# remove useless README -git rm --quiet README -git commit --quiet -m 'change stuff' -git checkout --quiet master -git merge --no-ff fix >/dev/null 2>&1 - -# Pull a commit in to the linux morphology. -cd "$DATADIR/workspace/branch1/test/kernel-repo" -git remote add peer "file://$PEER_REPO" -git remote update >/dev/null 2>&1 -git merge --quiet peer/master - -# Build the linux system again without comitting. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" build linux-system -cd branch1/test/kernel-repo - -# Check whether the new morphology exists in the temporary build ref -cd "$DATADIR/workspace/branch1/test/kernel-repo" -! git cat-file blob "$KERNEL_BRANCH:README" >/dev/null 2>&1 diff --git a/tests.as-root/building-a-system-branch-works-anywhere.script b/tests.as-root/building-a-system-branch-works-anywhere.script deleted file mode 100755 index cf946cd5..00000000 --- a/tests.as-root/building-a-system-branch-works-anywhere.script +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Make sure "morph build" works anywhere in a workspace or system branch -## and produces the same results every time. - -# FIXME: This seems to break because the new "morph edit" makes correct -# changes to build-dependencies, which breaks the old "morph build". -# Disable test now, re-enable it after "morph build" is fixed. --liw -exit 0 - -set -eu - -source "$SRCDIR/tests.as-root/setup-build" - -# Build from the workspace root. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" build linux-system -"$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" > "$DATADIR/output1" -rm -rf "$DATADIR/cache"/* - -# Build from the branch. -cd "$DATADIR/workspace/branch1" -"$SRCDIR/scripts/test-morph" build linux-system -"$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" > "$DATADIR/output2" -rm -rf "$DATADIR/cache/artifacts"/* - -# Build form the branch root repository. -cd "$DATADIR/workspace/branch1/test/morphs" -"$SRCDIR/scripts/test-morph" build linux-system -"$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" > "$DATADIR/output3" -rm -rf "$DATADIR/cache/artifacts"/* - -# Build from the linux directory. -cd "$DATADIR/workspace/branch1/test/kernel-repo" -"$SRCDIR/scripts/test-morph" build linux-system -"$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" > "$DATADIR/output4" -rm -rf "$DATADIR/cache/artifacts"/* - -# Verify that we build the right number of artifacts -[ $(wc < "$DATADIR/output1" -l) -eq 22 ] - -# List of files in the artifact cache should be identical after each build -diff "$DATADIR/output1" "$DATADIR/output2" -diff "$DATADIR/output2" "$DATADIR/output3" -diff "$DATADIR/output3" "$DATADIR/output4" diff --git a/tests.as-root/lib b/tests.as-root/lib deleted file mode 100644 index d17d8e32..00000000 --- a/tests.as-root/lib +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Helper functions for as-root tests - -loopback_rootfs() { - # Find offset partition offset in a rootfs and mount it - ROOTFS="$1" - - PYTHONPATH="$SRCDIR" "$SRCDIR/scripts/sparse-gunzip" \ - <"$ROOTFS" >"$ROOTFS-unzipped" - - OFFSET=$(sfdisk -d "$ROOTFS-unzipped" | \ - grep -m 1 -o 'start=\s\+\([0-9]\+\)' | awk '{ print $2 }') - OFFSET=$(expr "$OFFSET" '*' 512) - - DEVICE=$(losetup --show -o "$OFFSET" -f "$ROOTFS-unzipped") - udevadm settle - - echo "$DEVICE" -} diff --git a/tests.as-root/metadata-includes-morph-version.script b/tests.as-root/metadata-includes-morph-version.script deleted file mode 100755 index 772d7d50..00000000 --- a/tests.as-root/metadata-includes-morph-version.script +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013 - 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# All the metadata files in /baserock of a system contain morph's version -# information - -set -eu - -# Disable test on versions of Python before 2.7. -. "$SRCDIR/scripts/python-check" - -. "$SRCDIR/scripts/fix-committer-info" - -# get git version info -cd "$SRCDIR" -description="$(git describe --abbrev=40 --always \ - --dirty=-unreproducible --match=DO-NOT-MATCH-ANY-TAGS)" -commit="$(git rev-parse 'HEAD^{commit}')" -tree="$(git rev-parse 'HEAD^{tree}')" -ref="$(git rev-parse --symbolic-full-name 'HEAD')" - -tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact \ - build-morphology test:morphs tarball hello-tarball) - -extracted="$DATADIR/extracted" -mkdir -p "$extracted" -tar -xf "$tar" -C "$extracted" -cd "$extracted/baserock" -for f in *.meta; do - # Check for git describe output - grep -q -F -e "$description" "$f" - # Check the Sha-1 commit is included - grep -q -F -e "$commit" "$f" - # Check the Sha-1 of the commit's tree is included - grep -q -F -e "$tree" "$f" - # Check the ref (e.g. branch) is included - grep -q -F -e "$ref" "$f" -done diff --git a/tests.as-root/metadata-includes-morph-version.setup b/tests.as-root/metadata-includes-morph-version.setup deleted file mode 100755 index e4557302..00000000 --- a/tests.as-root/metadata-includes-morph-version.setup +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -set -eu - -source "$SRCDIR/scripts/fix-committer-info" - -morphsrepo="$DATADIR/morphs" -cd "$morphsrepo" -git checkout -b tarball -cat <<EOF > hello-tarball.morph -name: hello-tarball -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: hello-stratum - - morph: linux-stratum -EOF -git add hello-tarball.morph - -git commit --quiet -m "add tarball system" diff --git a/tests.as-root/metadata-includes-repo-alias.script b/tests.as-root/metadata-includes-repo-alias.script deleted file mode 100755 index 9e4a5d98..00000000 --- a/tests.as-root/metadata-includes-repo-alias.script +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# All the metadata files in /baserock of a system contain the repository -# alias they were built from. - -set -eu - -# Disable test on versions of Python before 2.7. -. "$SRCDIR/scripts/python-check" - -. "$SRCDIR/scripts/fix-committer-info" - -# get git version info -cd "$SRCDIR" - -tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact \ - build-morphology test:morphs tarball hello-tarball) - -extracted="$DATADIR/extracted" -mkdir -p "$extracted" -tar -xf "$tar" -C "$extracted" -cd "$extracted/baserock" - -# Check for test:morphs in System and Stratum -#grep -q -F -e test:morphs hello-tarball.meta # tarball bug -grep -q -F -e test:morphs hello-stratum-runtime.meta -grep -q -F -e test:morphs linux-stratum-runtime.meta - -# Check for test:kernel-repo in linux -grep -q -F -e test:kernel-repo linux-misc.meta - -# Check for test:chunk-repo in hello -grep -q -F -e test:chunk-repo hello-bins.meta diff --git a/tests.as-root/metadata-includes-repo-alias.setup b/tests.as-root/metadata-includes-repo-alias.setup deleted file mode 100755 index e4557302..00000000 --- a/tests.as-root/metadata-includes-repo-alias.setup +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -set -eu - -source "$SRCDIR/scripts/fix-committer-info" - -morphsrepo="$DATADIR/morphs" -cd "$morphsrepo" -git checkout -b tarball -cat <<EOF > hello-tarball.morph -name: hello-tarball -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: hello-stratum - - morph: linux-stratum -EOF -git add hello-tarball.morph - -git commit --quiet -m "add tarball system" diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.exit b/tests.as-root/run-in-artifact-propagates-exit-code.exit deleted file mode 100644 index d00491fd..00000000 --- a/tests.as-root/run-in-artifact-propagates-exit-code.exit +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.script b/tests.as-root/run-in-artifact-propagates-exit-code.script deleted file mode 100755 index d815c73d..00000000 --- a/tests.as-root/run-in-artifact-propagates-exit-code.script +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Test that 'run-in-artifact' propagates the exit code of its command. - -set -eu - -. "$SRCDIR/tests.as-root/lib" - -# Build first image. Remember the stratum. -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs master linux-system - -system=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*-rootfs') - -# Run 'run-in-artifact' with the system artifact. The command will fail -# and this should result in an exit code of 1 in the test. -"$SRCDIR/scripts/test-morph" run-in-artifact "$system" -- ls i-do-not-exist diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.stderr b/tests.as-root/run-in-artifact-propagates-exit-code.stderr deleted file mode 100644 index 98aa5450..00000000 --- a/tests.as-root/run-in-artifact-propagates-exit-code.stderr +++ /dev/null @@ -1,3 +0,0 @@ -ERROR: Command failed: ls i-do-not-exist -ls: i-do-not-exist: No such file or directory - diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.script b/tests.as-root/run-in-artifact-with-different-artifacts.script deleted file mode 100755 index 57d408e3..00000000 --- a/tests.as-root/run-in-artifact-with-different-artifacts.script +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Test the 'run-in-artifact' command with different types of artifacts. - -set -eu - -. "$SRCDIR/tests.as-root/lib" - -# Build first image. Remember the stratum. -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs master linux-system - -system=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*-rootfs') -chunk=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.chunk.linux-misc') -stratum=$(find "$DATADIR/cache/artifacts" -maxdepth 1 \ - -name '*.stratum.linux-stratum-runtime') - -# Run 'run-in-artifact' with the system artifact. -echo "System:" -"$SRCDIR/scripts/test-morph" run-in-artifact "$system" -- ls baserock/ -echo - -# Run 'run-in-artifact' with the chunk artifact. -echo "Chunk:" -"$SRCDIR/scripts/test-morph" run-in-artifact "$chunk" -- ls baserock/ -echo - -# Run 'run-in-artifact' with the statum artifact. -echo "Stratum:" -"$SRCDIR/scripts/test-morph" run-in-artifact "$stratum" -- ls baserock/ \ - 2>/dev/null || echo "Failed" diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.stdout b/tests.as-root/run-in-artifact-with-different-artifacts.stdout deleted file mode 100644 index 7473990b..00000000 --- a/tests.as-root/run-in-artifact-with-different-artifacts.stdout +++ /dev/null @@ -1,32 +0,0 @@ -System: -hello-bins.meta -hello-devel.meta -hello-doc.meta -hello-libs.meta -hello-locale.meta -hello-misc.meta -hello-stratum-devel.meta -hello-stratum-runtime.meta -linux-bins.meta -linux-devel.meta -linux-doc.meta -linux-libs.meta -linux-locale.meta -linux-misc.meta -linux-stratum-devel.meta -linux-stratum-runtime.meta -linux-system-rootfs.meta -tools-bins.meta -tools-devel.meta -tools-doc.meta -tools-libs.meta -tools-locale.meta -tools-misc.meta -tools-stratum-devel.meta -tools-stratum-runtime.meta - -Chunk: -linux-misc.meta - -Stratum: -Failed diff --git a/tests.as-root/setup b/tests.as-root/setup deleted file mode 100755 index 1cf9dd04..00000000 --- a/tests.as-root/setup +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash -# -# Create git repositories for tests. The chunk repository will contain a -# simple "hello, world" C program, and two branches ("master", "farrokh"), -# with the master branch containing just a README. The two branches are there -# so that we can test building a branch that hasn't been checked out. -# The branches are different so that we know that if the wrong branch -# is uses, the build will fail. -# -# The stratum repository contains a single branch, "master", with a -# stratum and a system morphology that include the chunk above. -# -# Copyright (C) 2011, 2012, 2013, 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -set -eu - -source "$SRCDIR/scripts/fix-committer-info" - -# The $DATADIR should be empty at the beginnig of each test. -find "$DATADIR" -mindepth 1 -delete - -# Create an empty directory to be used as a morph workspace -mkdir "$DATADIR/workspace" - -# Create chunk repository. - -chunkrepo="$DATADIR/chunk-repo" -mkdir "$chunkrepo" -cd "$chunkrepo" -git init --quiet - -cat <<EOF > README -This is a sample README. -EOF -git add README -git commit --quiet -m "add README" - -git checkout --quiet -b farrokh - -cat <<EOF > hello.c -#include <stdio.h> -int main(void) -{ - puts("hello, world"); - return 0; -} -EOF -git add hello.c - -cat <<EOF > hello.morph -name: hello -kind: chunk -build-system: dummy -build-commands: - - gcc -o hello hello.c -install-commands: - - install -d "\$DESTDIR"/etc - - install -d "\$DESTDIR"/bin - - install hello "\$DESTDIR"/bin/hello -EOF -git add hello.morph - -git commit --quiet -m "add a hello world program and morph" - -git checkout --quiet master - - -# Create tools repository, so there is an extra layer of build-depends -toolsrepo="$DATADIR/tools-repo" -mkdir -p "$toolsrepo" -cd "$toolsrepo" -git init --quiet -cat <<'EOF' >tools.morph -name: tools -kind: chunk -build-commands: - - echo 'echo dummy strace' >strace -install-commands: - - mkdir -p "$DESTDIR/bin" - - install strace "$DESTDIR/bin/strace" -EOF -git add tools.morph -git commit --quiet -m "add morphology" - -# Create morph repository. - -morphsrepo="$DATADIR/morphs" -mkdir "$morphsrepo" -cd "$morphsrepo" -git init --quiet - -cat <<EOF > hello-stratum.morph -name: hello-stratum -kind: stratum -chunks: - - name: hello - repo: test:chunk-repo - ref: farrokh - build-mode: test - build-depends: [] -EOF -git add hello-stratum.morph - -cat <<EOF > tools-stratum.morph -name: tools-stratum -kind: stratum -build-depends: - - morph: linux-stratum -chunks: - - name: tools - repo: test:tools-repo - ref: master - build-mode: test - build-depends: [] -EOF -git add tools-stratum.morph - -cat <<EOF > hello-system.morph -name: hello-system -kind: system -arch: `"$SRCDIR/scripts/test-morph" print-architecture` -strata: - - morph: hello-stratum -EOF -git add hello-system.morph - -cat <<EOF > linux-stratum.morph -name: linux-stratum -kind: stratum -build-depends: - - morph: hello-stratum -chunks: - - name: linux - repo: test:kernel-repo - ref: master - build-mode: test - build-depends: [] -EOF -git add linux-stratum.morph - -cat <<EOF > linux-system.morph -name: linux-system -kind: system -arch: `"$SRCDIR/scripts/test-morph" print-architecture` -strata: - - morph: hello-stratum - - morph: linux-stratum - - morph: tools-stratum -EOF -git add linux-system.morph - -git commit --quiet -m "add morphs" - -# Make a dummy kernel chunk. -mkdir "$DATADIR/kernel-repo" -cat <<EOF > "$DATADIR/kernel-repo/linux.morph" -name: linux -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/boot" - - touch "\$DESTDIR/extlinux.conf" - - touch "\$DESTDIR/boot/vmlinuz" - - touch "\$DESTDIR/boot/System.map" -EOF -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" init --quiet -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" add . -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" commit --quiet -m foo \ - > /dev/null - -# Create a morph configuration file. -cat <<EOF > "$DATADIR/morph.conf" -[config] -repo-alias = test=file://$DATADIR/#file://$DATADIR/ -cachedir = $DATADIR/cache -log = $DATADIR/morph.log -no-distcc = true -quiet = true -log = /tmp/morph.log -EOF - diff --git a/tests.as-root/setup-build b/tests.as-root/setup-build deleted file mode 100644 index 1f6f1c39..00000000 --- a/tests.as-root/setup-build +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Fixture for tests involving 'morph build' - -source "$SRCDIR/scripts/fix-committer-info" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" branch test:morphs branch1 -"$SRCDIR/scripts/test-morph" edit linux-system linux-stratum linux - -# Fix UUID's in the checked out repos to make build branch names deterministic -git config -f "$DATADIR/workspace/branch1/.morph-system-branch/config" \ - branch.uuid 123456789 -git config -f "$DATADIR/workspace/branch1/test:morphs/.git/config" \ - morph.uuid 987654321 -git config -f "$DATADIR/workspace/branch1/test:kernel-repo/.git/config" \ - morph.uuid AABBCCDDE - diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script deleted file mode 100755 index 9be6df13..00000000 --- a/tests.as-root/system-overlap.script +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# If a system has multiple strata that have the same files in them, -# then this should be noted. - -set -eu - -cache="$DATADIR/cache/artifacts" -chunkrepo="$DATADIR/chunk-repo" -morphsrepo="$DATADIR/morphs" - -cd "$morphsrepo" -git checkout --quiet -b overlap master -cat <<EOF >overlap-system.morph -name: overlap-system -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: foo-baz-stratum - - morph: foo-barqux-stratum -EOF -cat <<EOF >foo-baz-stratum.morph -name: foo-baz-stratum -kind: stratum -chunks: - - name: overlap-foo-baz - repo: test:chunk-repo - ref: overlap - build-mode: test - build-depends: [] - - name: linux - repo: test:kernel-repo - ref: master - build-mode: test - build-depends: - - overlap-foo-baz -EOF -cat <<EOF >foo-barqux-stratum.morph -name: foo-barqux-stratum -kind: stratum -chunks: - - name: overlap-foobar - repo: test:chunk-repo - ref: overlap - build-mode: test - build-depends: [] - - name: overlap-fooqux - repo: test:chunk-repo - ref: overlap - build-mode: test - build-depends: - - overlap-foobar -EOF -git add overlap-system.morph foo-baz-stratum.morph foo-barqux-stratum.morph -git commit --quiet -m "add overlapping system" - -cd "$chunkrepo" -git checkout --quiet -b overlap master -cat <<EOF >overlap-foo-baz.morph -name: overlap-foo-baz -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR"/bin - - for f in foo bar baz; do echo echo \$f >"\$DESTDIR"/bin/\$f; done -EOF - -cat <<EOF >overlap-foobar.morph -name: overlap-foobar -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR"/usr/bin "\$DESTDIR"/bin - - echo echo foobar >"\$DESTDIR"/usr/bin/foobar - - ln -s /usr/bin/foobar "\$DESTDIR"/bin/foo - - ln -s /usr/bin/foobar "\$DESTDIR"/bin/bar -EOF - -cat <<EOF >overlap-fooqux.morph -name: overlap-fooqux -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR"/usr/bin "\$DESTDIR"/bin - - for f in qux fooqux; do echo echo \$f >"\$DESTDIR"/usr/bin/\$f; done - - ln -s /usr/bin/fooqux "\$DESTDIR"/bin/foo -EOF -git add overlap-*.morph - -git commit --quiet -m 'Add overlapping chunks' - -"$SRCDIR/scripts/test-morph" \ - build-morphology test:morphs overlap overlap-system > /dev/null -"$SRCDIR/scripts/list-overlaps" groups \ - "$cache"/*.system.overlap-system*.overlaps | -while IFS='\n' read overlaps; do - echo $overlaps - "$SRCDIR/scripts/list-overlaps" list-files \ - "$cache"/*.system.overlap-system*.overlaps $overlaps -done diff --git a/tests.as-root/system-overlap.stdout b/tests.as-root/system-overlap.stdout deleted file mode 100644 index f67d54c8..00000000 --- a/tests.as-root/system-overlap.stdout +++ /dev/null @@ -1,3 +0,0 @@ -foo-barqux-stratum-runtime foo-baz-stratum-runtime -bin/foo -bin/bar diff --git a/tests.as-root/tarball-image-is-sensible.script b/tests.as-root/tarball-image-is-sensible.script deleted file mode 100755 index 0fb6d9b2..00000000 --- a/tests.as-root/tarball-image-is-sensible.script +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# A system kind of 'rootfs-tarball' should create a tarball containing -# all the required files. - -set -eu - -# Disable test on versions of Python before 2.7. -. "$SRCDIR/scripts/python-check" - -. "$SRCDIR/scripts/fix-committer-info" - -tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact \ - build-morphology test:morphs tarball-links hello-tarball) - -# Verify that the tar archive is not compressed. -if gunzip -t "$tar" > /dev/null -then - echo "ERROR: $tar is gzip'd!" 1>&2 - exit 1 -fi - -extracted="$DATADIR/extracted" -mkdir -p "$extracted" -cd "$extracted" -tar -xf "$tar" -find . -mindepth 1 | xargs ls -dF | LC_ALL=C sort -u -cat "$extracted/etc/os-release" diff --git a/tests.as-root/tarball-image-is-sensible.setup b/tests.as-root/tarball-image-is-sensible.setup deleted file mode 100755 index a687b691..00000000 --- a/tests.as-root/tarball-image-is-sensible.setup +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -set -eu - -source "$SRCDIR/scripts/fix-committer-info" - -chunkrepo="$DATADIR/chunk-repo" -cd "$chunkrepo" -git checkout -b tarball-links -cat >links.morph <<'EOF' -name: links -kind: chunk -build-system: manual -install-commands: - - mkdir -p "$DESTDIR/bin" - - touch "$DESTDIR/bin/true" - - cd "$DESTDIR/bin" && ln true true-hardlink - - cd "$DESTDIR/bin" && ln -s true true-symlink -EOF -git add links.morph -git commit --quiet -m 'Add link adding chunk' - -morphsrepo="$DATADIR/morphs" -cd "$morphsrepo" -git checkout -b tarball-links -cat <<EOF > hello-tarball.morph -name: hello-tarball -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: link-stratum - - morph: hello-stratum - - morph: linux-stratum -EOF -git add hello-tarball.morph - -# Change build-depends ref of hello-stratum from master to tarball-links -sed -i linux-stratum.morph \ - -e '/build-depends:/,/chunks/ s/ref: master/ref: tarball-links/' - -git add linux-stratum.morph - -cat <<EOF > link-stratum.morph -name: link-stratum -kind: stratum -chunks: - - name: links - repo: test:chunk-repo - ref: tarball-links - build-mode: test - build-depends: [] -EOF -git add link-stratum.morph -git commit --quiet -m "add morphs" diff --git a/tests.as-root/tarball-image-is-sensible.stderr b/tests.as-root/tarball-image-is-sensible.stderr deleted file mode 100644 index 9db65e02..00000000 --- a/tests.as-root/tarball-image-is-sensible.stderr +++ /dev/null @@ -1 +0,0 @@ -gunzip: invalid magic diff --git a/tests.as-root/tarball-image-is-sensible.stdout b/tests.as-root/tarball-image-is-sensible.stdout deleted file mode 100644 index cf74a1ec..00000000 --- a/tests.as-root/tarball-image-is-sensible.stdout +++ /dev/null @@ -1,42 +0,0 @@ -./baserock/ -./baserock/hello-bins.meta -./baserock/hello-devel.meta -./baserock/hello-doc.meta -./baserock/hello-libs.meta -./baserock/hello-locale.meta -./baserock/hello-misc.meta -./baserock/hello-stratum-devel.meta -./baserock/hello-stratum-runtime.meta -./baserock/hello-tarball-rootfs.meta -./baserock/link-stratum-devel.meta -./baserock/link-stratum-runtime.meta -./baserock/links-bins.meta -./baserock/links-devel.meta -./baserock/links-doc.meta -./baserock/links-libs.meta -./baserock/links-locale.meta -./baserock/links-misc.meta -./baserock/linux-bins.meta -./baserock/linux-devel.meta -./baserock/linux-doc.meta -./baserock/linux-libs.meta -./baserock/linux-locale.meta -./baserock/linux-misc.meta -./baserock/linux-stratum-devel.meta -./baserock/linux-stratum-runtime.meta -./bin/ -./bin/hello* -./bin/true -./bin/true-hardlink -./bin/true-symlink@ -./boot/ -./boot/System.map -./boot/vmlinuz -./etc/ -./etc/os-release -./extlinux.conf -NAME="Baserock" -ID=baserock -HOME_URL="http://wiki.baserock.org" -SUPPORT_URL="http://wiki.baserock.org/mailinglist" -BUG_REPORT_URL="http://wiki.baserock.org/mailinglist" diff --git a/tests.branching/edit-updates-stratum.stdout b/tests.branching/edit-updates-stratum.stdout index a127f046..ee9510b5 100644 --- a/tests.branching/edit-updates-stratum.stdout +++ b/tests.branching/edit-updates-stratum.stdout @@ -1,5 +1,5 @@ diff --git a/hello-stratum.morph b/hello-stratum.morph -index e012b5f..cc0609a 100644 +index f335879..7bf9d37 100644 --- a/hello-stratum.morph +++ b/hello-stratum.morph @@ -3,6 +3,7 @@ kind: stratum @@ -10,4 +10,4 @@ index e012b5f..cc0609a 100644 + ref: newbranch + unpetrify-ref: master build-depends: [] - build-mode: test + build-mode: bootstrap diff --git a/tests.branching/setup b/tests.branching/setup index 22c51c24..a2d23090 100755 --- a/tests.branching/setup +++ b/tests.branching/setup @@ -63,7 +63,7 @@ chunks: repo: test:hello ref: master build-depends: [] - build-mode: test + build-mode: bootstrap EOF scripts/run-git-in "$DATADIR/morphs" init diff --git a/tests.branching/setup-second-chunk b/tests.branching/setup-second-chunk index 058fb222..24604ab8 100755 --- a/tests.branching/setup-second-chunk +++ b/tests.branching/setup-second-chunk @@ -49,12 +49,12 @@ chunks: repo: test:hello ref: master build-depends: [] - build-mode: test + build-mode: bootstrap - name: goodbye repo: test:goodbye ref: master build-depends: [] - build-mode: test + build-mode: bootstrap EOF git commit -q --all -m "Add goodbye to hello-stratum" diff --git a/tests.build/stratum-overlap-warns.script b/tests.build/stratum-overlap-warns.script deleted file mode 100755 index 2a3b06e1..00000000 --- a/tests.build/stratum-overlap-warns.script +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## If a stratum has multiple chunks that have the same files in them, -## then this should be noted. - -set -eu - -log="$DATADIR/morph.log" -warnings="$DATADIR/warnings" -cache="$DATADIR/cache/artifacts" - -warning_mentions(){ - grep -F "$1" <"$warnings" >/dev/null 2>/dev/null -} - -"$SRCDIR/scripts/test-morph" build-morphology --log=$log \ - test:morphs-repo overlap hello-system > /dev/null -grep WARNING "$log" >"$warnings" -for str in hello-stratum \ - overlap-foo-baz overlap-foobar bin/bar \ - overlap-fooqux bin/foo; do - warning_mentions 'hello-stratum' || exit $? -done diff --git a/tests.build/stratum-overlap-warns.setup b/tests.build/stratum-overlap-warns.setup deleted file mode 100755 index b969822d..00000000 --- a/tests.build/stratum-overlap-warns.setup +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/sh -# -# If a stratum has multiple chunks that have the same files in them, -# then this should be notified -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -eu - -chunkrepo="$DATADIR/chunk-repo" -morphsrepo="$DATADIR/morphs-repo" - -cd "$morphsrepo" -git checkout --quiet -b overlap master -cat <<EOF >hello-stratum.morph -name: hello-stratum -kind: stratum -chunks: - - name: dirs - repo: test:chunk-repo - ref: overlap - build-depends: [] - build-mode: test - - name: overlap-foobar - repo: test:chunk-repo - ref: overlap - build-depends: - - dirs - build-mode: test - - name: overlap-fooqux - repo: test:chunk-repo - ref: overlap - build-depends: - - overlap-foobar - build-mode: test - - name: overlap-foo-baz - repo: test:chunk-repo - ref: overlap - build-depends: - - overlap-fooqux - build-mode: test -EOF -sed -i 's/master/overlap/' hello-system.morph -git add hello-stratum.morph hello-system.morph -git commit --quiet -m "Make hello stratum contain overlaps" - -cd "$chunkrepo" -git checkout --quiet -b overlap master - -cat <<EOF >dirs.morph -name: dirs -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/bin" - - ln -s .. "\$DESTDIR/usr" -EOF -git add dirs.morph - -cat <<EOF >overlap-foo-baz.morph -name: overlap-foo-baz -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/bin" - - for f in foo bar baz; do echo echo \$f >"\$DESTDIR/bin/\$f"; done -EOF -git add overlap-foo-baz.morph - -cat <<EOF >overlap-foobar.morph -name: overlap-foobar -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/usr/bin" "\$DESTDIR/bin" - - echo echo foobar >"\$DESTDIR/usr/bin/foobar" - - ln -s /usr/bin/foobar "\$DESTDIR/bin/foo" - - ln -s /usr/bin/foobar "\$DESTDIR/bin/bar" -EOF -git add overlap-foobar.morph - -cat <<EOF >overlap-fooqux.morph -name: overlap-fooqux -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/usr/bin" "\$DESTDIR/bin" - - for f in qux fooqux; do echo echo \$f >"\$DESTDIR/usr/bin/\$f"; done - - ln -s /usr/bin/fooqux "\$DESTDIR/bin/foo" -EOF -git add overlap-fooqux.morph - -git commit --quiet -m 'Add overlapping chunks' diff --git a/tests.build/stratum-overlap-writes-overlap.script b/tests.build/stratum-overlap-writes-overlap.script deleted file mode 100755 index fe4ed4ee..00000000 --- a/tests.build/stratum-overlap-writes-overlap.script +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## If a stratum has multiple chunks that have the same files in them, -## then the overlaps must be written to the cache - -set -eu - -cache="$DATADIR/cache/artifacts" - - -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs-repo overlap hello-system > /dev/null -"$SRCDIR/scripts/list-overlaps" groups \ - "$cache"/*.stratum.hello-stratum-*.overlaps | -while IFS='\n' read overlaps; do - echo $overlaps - "$SRCDIR/scripts/list-overlaps" list-files \ - "$cache"/*.stratum.hello-stratum-*.overlaps $overlaps -done diff --git a/tests.build/stratum-overlap-writes-overlap.setup b/tests.build/stratum-overlap-writes-overlap.setup deleted file mode 120000 index 255e9a74..00000000 --- a/tests.build/stratum-overlap-writes-overlap.setup +++ /dev/null @@ -1 +0,0 @@ -stratum-overlap-warns.setup
\ No newline at end of file diff --git a/tests.build/stratum-overlap-writes-overlap.stdout b/tests.build/stratum-overlap-writes-overlap.stdout deleted file mode 100644 index 1e36ca83..00000000 --- a/tests.build/stratum-overlap-writes-overlap.stdout +++ /dev/null @@ -1,4 +0,0 @@ -overlap-foo-baz-bins overlap-foobar-bins overlap-fooqux-bins -bin/foo -overlap-foo-baz-bins overlap-foobar-bins -bin/bar diff --git a/tests.deploy/deploy-cluster.script b/tests.deploy/deploy-cluster.script deleted file mode 100755 index acc65a54..00000000 --- a/tests.deploy/deploy-cluster.script +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# Test "morph deploy" by deploying the systems in cluster morphology. - -set -eu - -. "$SRCDIR/tests.deploy/setup-build" - -cd "$DATADIR/workspace/branch1" - -"$SRCDIR/scripts/test-morph" build hello-system - -"$SRCDIR/scripts/test-morph" build linux-system - -GIT_DIR=test/morphs/.git git tag -a my-test-tag -m "Example tag" HEAD - -"$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ - deploy test_cluster \ - linux-system-2.EXAMPLE_PASSWORD="secret" \ - linux-system-2.HOSTNAME="baserock-rocks-even-more" \ - > /dev/null - -outputdir=test/morphs -test -e $outputdir/hello-system.img -test -e $outputdir/linux-system-1.tar -test -e $outputdir/linux-system-2.tar - -hostname1=$(tar -xf $outputdir/linux-system-1.tar ./etc/hostname -O) -hostname2=$(tar -xf $outputdir/linux-system-2.tar ./etc/hostname -O) - -[ "$hostname1" = baserock-rocks ] -[ "$hostname2" = baserock-rocks-even-more ] - -tar -xf $outputdir/linux-system-2.tar ./baserock/deployment.meta -metadata=baserock/deployment.meta - -# Check that 'git describe' of definitions repo was stored correctly -echo -n "definitions-version: " -"$SRCDIR/scripts/yaml-extract" $metadata definitions-version - -echo -n "configuration.HOSTNAME: " -"$SRCDIR/scripts/yaml-extract" $metadata configuration HOSTNAME - -! (grep -q "EXAMPLE_PASSWORD" $metadata) diff --git a/tests.deploy/deploy-cluster.stdout b/tests.deploy/deploy-cluster.stdout deleted file mode 100644 index 16b78015..00000000 --- a/tests.deploy/deploy-cluster.stdout +++ /dev/null @@ -1,2 +0,0 @@ -definitions-version: {'describe': 'my-test-tag-unreproducible'} -configuration.HOSTNAME: baserock-rocks-even-more diff --git a/tests.deploy/deploy-rawdisk.script b/tests.deploy/deploy-rawdisk.script deleted file mode 100755 index 3489a198..00000000 --- a/tests.deploy/deploy-rawdisk.script +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2013-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# Test "morph deploy" by deploying to a raw disk image. - - -set -eu - - -. "$SRCDIR/tests.deploy/setup-build" -cd "$DATADIR/workspace/branch1" -"$SRCDIR/scripts/test-morph" build linux-system -"$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ - deploy rawdisk_test_cluster > /dev/null - -outputdir="test/morphs" -test -e $outputdir/disk.img - diff --git a/tests.deploy/setup b/tests.deploy/setup deleted file mode 100755 index 033598bc..00000000 --- a/tests.deploy/setup +++ /dev/null @@ -1,216 +0,0 @@ -#!/bin/bash -# -# Create git repositories for tests. The chunk repository will contain a -# simple "hello, world" C program, and two branches ("master", "farrokh"), -# with the master branch containing just a README. The two branches are there -# so that we can test building a branch that hasn't been checked out. -# The branches are different so that we know that if the wrong branch -# is uses, the build will fail. -# -# The stratum repository contains a single branch, "master", with a -# stratum and a system morphology that include the chunk above. -# -# Copyright (C) 2011, 2012, 2013, 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -set -eu - -source "$SRCDIR/scripts/fix-committer-info" - -# The $DATADIR should be empty at the beginnig of each test. -find "$DATADIR" -mindepth 1 -delete - -# Create an empty directory to be used as a morph workspace -mkdir "$DATADIR/workspace" - -# Create chunk repository. - -chunkrepo="$DATADIR/chunk-repo" -mkdir "$chunkrepo" -cd "$chunkrepo" -git init --quiet - -cat <<EOF > README -This is a sample README. -EOF -git add README -git commit --quiet -m "add README" - -git checkout --quiet -b farrokh - -cat <<EOF > hello.c -#include <stdio.h> -int main(void) -{ - puts("hello, world"); - return 0; -} -EOF -git add hello.c - -cat <<EOF > hello.morph -name: hello -kind: chunk -build-system: dummy -build-commands: - - gcc -o hello hello.c -install-commands: - - install -d "\$DESTDIR/etc" - - install -d "\$DESTDIR/bin" - - install hello "\$DESTDIR/bin/hello" -EOF -git add hello.morph - -git commit --quiet -m "add a hello world program and morph" - -git checkout --quiet master - - - -# Create morph repository. - -morphsrepo="$DATADIR/morphs" -mkdir "$morphsrepo" -cd "$morphsrepo" -git init --quiet - -cat <<EOF > hello-stratum.morph -name: hello-stratum -kind: stratum -chunks: - - name: hello - repo: test:chunk-repo - ref: farrokh - build-depends: [] - build-mode: test -EOF -git add hello-stratum.morph - -cat <<EOF > hello-system.morph -name: hello-system -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: hello-stratum -EOF -git add hello-system.morph - -cat <<EOF > linux-stratum.morph -name: linux-stratum -kind: stratum -build-depends: - - morph: hello-stratum -chunks: - - name: linux - repo: test:kernel-repo - ref: master - build-depends: [] - build-mode: test -EOF -git add linux-stratum.morph - -cat <<EOF > linux-system.morph -name: linux-system -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: hello-stratum - - morph: linux-stratum -configuration-extensions: - - set-hostname -EOF -git add linux-system.morph - - -cat <<EOF > rawdisk_test_cluster.morph -name: rawdisk_test_cluster -kind: cluster -systems: - - morph: linux-system - deploy: - linux-system-1: - type: rawdisk - location: disk.img - DISK_SIZE: 1G -EOF -git add rawdisk_test_cluster.morph - -cat <<EOF > rawdisk_test_cluster_without_disk_size.morph -name: rawdisk_test_cluster_without_disk_size -kind: cluster -systems: - - morph: linux-system - deploy: - linux-system-1: - type: rawdisk - location: disk.img -EOF -git add rawdisk_test_cluster_without_disk_size.morph - -cat <<EOF > test_cluster.morph -name: test_cluster -kind: cluster -systems: - - morph: hello-system - deploy-defaults: - type: tar - deploy: - hello-system: - type: rawdisk - location: hello-system.img - DISK_SIZE: 1G - - morph: linux-system - deploy-defaults: - HOSTNAME: "baserock-rocks" - deploy: - linux-system-1: - type: tar - location: linux-system-1.tar - linux-system-2: - type: tar - location: linux-system-2.tar -EOF -git add test_cluster.morph - - -git commit --quiet -m "add morphs" - -# Make a dummy kernel chunk. -mkdir "$DATADIR/kernel-repo" -cat <<EOF > "$DATADIR/kernel-repo/linux.morph" -name: linux -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/boot" - - touch "\$DESTDIR/extlinux.conf" - - touch "\$DESTDIR/boot/vmlinuz" - - touch "\$DESTDIR/boot/System.map" -EOF -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" init --quiet -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" add . -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" commit --quiet -m foo \ - > /dev/null - -# Create a morph configuration file. -cat <<EOF > "$DATADIR/morph.conf" -[config] -repo-alias = test=file://$DATADIR/#file://$DATADIR/ -cachedir = $DATADIR/cache -log = $DATADIR/morph.log -no-distcc = true -quiet = true -log = /tmp/morph.log -EOF diff --git a/tests.deploy/setup-build b/tests.deploy/setup-build deleted file mode 100644 index c6b24da5..00000000 --- a/tests.deploy/setup-build +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Fixture for tests involving 'morph build' - -source "$SRCDIR/scripts/fix-committer-info" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" branch test:morphs branch1 -"$SRCDIR/scripts/test-morph" edit linux - -# Fix UUID's in the checked out repos to make build branch names deterministic -git config -f "$DATADIR/workspace/branch1/.morph-system-branch/config" \ - branch.uuid 123456789 -git config -f "$DATADIR/workspace/branch1/test/morphs/.git/config" \ - morph.uuid 987654321 -git config -f "$DATADIR/workspace/branch1/test/kernel-repo/.git/config" \ - morph.uuid AABBCCDDE - diff --git a/tests/setup b/tests/setup index 02ddb3af..6ebab880 100755 --- a/tests/setup +++ b/tests/setup @@ -31,80 +31,6 @@ set -eu # The $DATADIR should be empty at the beginnig of each test. find "$DATADIR" -mindepth 1 -delete -# Create chunk repository. - -chunkrepo="$DATADIR/chunk-repo" -mkdir "$chunkrepo" -cd "$chunkrepo" -git init --quiet - -cat <<EOF > README -This is a sample README. -EOF -git add README -git commit --quiet -m "add README" - -git checkout --quiet -b farrokh - -cat <<EOF > hello.c -#include <stdio.h> -int main(void) -{ - puts("hello, world"); - return 0; -} -EOF -git add hello.c - -cat <<EOF > hello.morph -name: hello -kind: chunk -build-system: dummy -build-commands: - - gcc -o hello hello.c -install-commands: - - install -d "\$DESTDIR/etc" - - install -d "\$DESTDIR/bin" - - install hello "\$DESTDIR/bin/hello" -EOF -git add hello.morph - -git commit --quiet -m "add a hello world program and morph" - -git checkout --quiet master - - - -# Create morph repository. - -morphsrepo="$DATADIR/morphs-repo" -mkdir "$morphsrepo" -cd "$morphsrepo" -git init --quiet - -cat <<EOF > hello-stratum.morph -name: hello-stratum -kind: stratum -chunks: - - name: hello - repo: test:chunk-repo - ref: farrokh - build-mode: test - build-depends: [] -EOF -git add hello-stratum.morph - -cat <<EOF > hello-system.morph -name: hello-system -kind: system -strata: - - morph: hello-stratum -EOF -git add hello-system.morph - -git commit --quiet -m "add morphs" - - # Create a morph configuration file. cat <<EOF > "$DATADIR/morph.conf" [config] diff --git a/tests/show-dependencies.setup b/tests/show-dependencies.setup index c99e90a9..74d10c2b 100755 --- a/tests/show-dependencies.setup +++ b/tests/show-dependencies.setup @@ -64,40 +64,37 @@ chunks: - name: freetype repo: test:test-repo ref: master - build-mode: test build-depends: [] + build-mode: bootstrap - name: fontconfig repo: test:test-repo ref: master - build-mode: test build-depends: [] + build-mode: bootstrap - name: cairo repo: test:test-repo ref: master - build-mode: test build-depends: [] + build-mode: bootstrap - name: pango repo: test:test-repo ref: master - build-mode: test build-depends: - freetype - fontconfig - name: glib repo: test:test-repo ref: master - build-mode: test build-depends: [] + build-mode: bootstrap - name: gdk-pixbuf repo: test:test-repo ref: master - build-mode: test build-depends: - glib - name: gtk repo: test:test-repo ref: master - build-mode: test build-depends: - cairo - gdk-pixbuf @@ -106,12 +103,11 @@ chunks: - name: dbus repo: test:test-repo ref: master - build-mode: test build-depends: [] + build-mode: bootstrap - name: dbus-glib repo: test:test-repo ref: master - build-mode: test build-depends: - dbus - glib @@ -160,48 +156,40 @@ chunks: - name: libxfce4util repo: test:test-repo ref: master - build-mode: test build-depends: [] - name: xfconf repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4util - name: libxfce4ui repo: test:test-repo ref: master - build-mode: test build-depends: - xfconf - name: exo repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4util - name: garcon repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4util - name: thunar repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - exo - name: tumbler repo: test:test-repo ref: master - build-mode: test build-depends: [] - name: xfce4-panel repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - exo @@ -209,7 +197,6 @@ chunks: - name: xfce4-settings repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - exo @@ -217,7 +204,6 @@ chunks: - name: xfce4-session repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - exo @@ -225,21 +211,18 @@ chunks: - name: xfwm4 repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - xfconf - name: xfdesktop repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - xfconf - name: xfce4-appfinder repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - garcon @@ -247,7 +230,6 @@ chunks: - name: gtk-xfce-engine repo: test:test-repo ref: master - build-mode: test build-depends: - libxfce4ui - garcon @@ -261,8 +243,8 @@ name: xfce-system kind: system arch: $("$SRCDIR/scripts/test-morph" print-architecture) strata: - - build-mode: test - morph: xfce-core + - morph: xfce-core + build-mode: bootstrap EOF git add xfce-system.morph git commit --quiet -m "add xfce-system" diff --git a/without-test-modules b/without-test-modules index a3aedc45..55e5291d 100644 --- a/without-test-modules +++ b/without-test-modules @@ -31,5 +31,24 @@ morphlib/plugins/print_architecture_plugin.py morphlib/plugins/add_binary_plugin.py morphlib/plugins/push_pull_plugin.py morphlib/plugins/distbuild_plugin.py +distbuild/__init__.py +distbuild/build_controller.py +distbuild/connection_machine.py +distbuild/distbuild_socket.py +distbuild/eventsrc.py +distbuild/helper_router.py +distbuild/idgen.py +distbuild/initiator.py +distbuild/initiator_connection.py +distbuild/jm.py +distbuild/json_router.py +distbuild/mainloop.py +distbuild/protocol.py +distbuild/proxy_event_source.py +distbuild/sockbuf.py +distbuild/socketsrc.py +distbuild/sockserv.py +distbuild/timer_event_source.py +distbuild/worker_build_scheduler.py # Not unit tested, since it needs a full system branch morphlib/buildbranch.py diff --git a/yarns/architecture.yarn b/yarns/architecture.yarn index d4de308e..07274ec3 100644 --- a/yarns/architecture.yarn +++ b/yarns/architecture.yarn @@ -22,3 +22,15 @@ Morph Cross-Bootstrap Tests WHEN the user checks out the system branch called master THEN the user cross-bootstraps the system base-system-testarch.morph in branch master of repo test:morphs to the arch testarch FINALLY the git server is shut down + +Architecture validation Tests +============================= + + SCENARIO building a system with no architecture + GIVEN a workspace + AND a git server + AND a system called base-system-noarch.morph with no architecture in the git server + WHEN the user checks out the system branch called master + AND the user attempts to build the system base-system-testarch.morph in branch master + THEN morph failed + FINALLY the git server is shut down diff --git a/yarns/branches-workspaces.yarn b/yarns/branches-workspaces.yarn index 200f9508..34aa97e0 100644 --- a/yarns/branches-workspaces.yarn +++ b/yarns/branches-workspaces.yarn @@ -202,14 +202,14 @@ is a chunk repository created in the mocked git server, for testing purposes. WHEN the user edits the chunk test-chunk in branch foo - THEN in branch foo, stratum strata/test-stratum.morph refs test-chunk in foo + THEN in branch foo, stratum strata/core.morph refs test-chunk in foo AND the edited chunk test:test-chunk has git branch foo Editing a morphology should not cause it to start having repo or ref fields when referring to strata, when it didn't before. - AND in branch foo, system systems/test-system.morph refers to test-stratum without repo - AND in branch foo, system systems/test-system.morph refers to test-stratum without ref + AND in branch foo, system systems/test-system.morph refers to core without repo + AND in branch foo, system systems/test-system.morph refers to core without ref FINALLY the git server is shut down Temporary Build Branch behaviour @@ -398,14 +398,15 @@ Nor do we need temporary build branches for distributed builds. IMPLEMENTS WHEN the user makes changes to test-chunk in branch (\S+) chunkdir="$(slashify_colons "test:test-chunk")" cd "$DATADIR/workspace/$MATCH_1/$chunkdir" - sed -i -e 's/Hello/Goodbye/g' test-bin + sed -i -e 's/Hello/Goodbye/g' usr/libexec/test-bin IMPLEMENTS THEN the changes to test-chunk in branch (\S+) are included in the temporary build branch build_ref_prefix=baserock/builds/ chunkdir="$(slashify_colons "test:test-chunk")" cd "$DATADIR/workspace/$MATCH_1/$chunkdir" + testbin=usr/libexec/test-bin eval "$(git for-each-ref --count=1 --shell --sort=committerdate \ - --format='git cat-file -p %(refname):test-bin | diff test-bin -' \ + --format='git cat-file -p %(refname):$testbin | diff $testbin -' \ "$build_ref_prefix")" IMPLEMENTS WHEN the user commits changes to (\S+) in branch (\S+) diff --git a/yarns/building.yarn b/yarns/building.yarn index c708b5bb..52742ac8 100644 --- a/yarns/building.yarn +++ b/yarns/building.yarn @@ -8,3 +8,37 @@ Morph Building Tests AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master THEN morph build the system systems/base-system.morph of the branch master FINALLY the git server is shut down + + SCENARIO distbuilding + ASSUMING the morph-cache-server can be run + GIVEN a workspace + AND a git server + AND a communal cache server + AND a distbuild worker + AND a distbuild controller + +Distbuilding works much the same way as regular building. + + WHEN the user checks out the system branch called master + AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master + THEN morph distbuild the system systems/base-system.morph of the branch master + +Distbuilt systems can be deployed locally, since the building process +lets you download the artifacts for local use. + +Note: Currently broken. It's currently complaining about not having +repos cached locally. + +> GIVEN a cluster called test-cluster.morph in system branch master +> AND a system in cluster test-cluster.morph in branch master called test-system +> AND system test-system in cluster test-cluster.morph in branch master builds systems/base-system.morph +> AND system test-system in cluster test-cluster.morph in branch master has deployment type: sysroot +> AND system test-system in cluster test-cluster.morph in branch master has deployment location: test-system +> WHEN the user attempts to deploy the cluster test-cluster.morph in branch master +> THEN morph succeeded +> AND file workspace/master/test/morphs/test-system exists + + FINALLY the distbuild controller is terminated + AND the distbuild worker is terminated + AND the communal cache server is terminated + AND the git server is shut down diff --git a/yarns/deployment.yarn b/yarns/deployment.yarn index 40b50432..0782c7c1 100644 --- a/yarns/deployment.yarn +++ b/yarns/deployment.yarn @@ -283,3 +283,48 @@ deployment. WHEN the user attempts to deploy test-system.sysroot from cluster test-cluster.morph in branch master THEN morph failed FINALLY the git server is shut down + +Deploying branch-from-image produced systems +============================================ + +We have this nifty subcommand called branch-from-image, which can be +used to build the same thing as an existing image. + +There's no special requirements for making the image reproducible. + + SCENARIO reproducing systems + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user builds the system systems/test-system.morph in branch master + GIVEN a cluster called test-cluster.morph in system branch master + AND a system in cluster test-cluster.morph in branch master called test-system + AND system test-system in cluster test-cluster.morph in branch master builds systems/test-system.morph + AND system test-system in cluster test-cluster.morph in branch master has deployment type: sysroot + AND system test-system in cluster test-cluster.morph in branch master has deployment location: test-system + WHEN the user attempts to deploy the cluster test-cluster.morph in branch master + THEN morph succeeded + AND file workspace/master/test/morphs/test-system exists + +To reproduce an existing image, do a checkout with the extracted root +filesystem's /baserock directory as the `--metadata-dir` argument. + + WHEN the user attempts to check out the system branch from workspace/master/test/morphs/test-system called mybranch + THEN morph succeeded + AND the system branch mybranch is checked out + +After it is checked-out, the system can be rebuilt. + + WHEN the user attempts to build the system systems/test-system.morph in branch mybranch + THEN morph succeeded + +Once it is rebuilt, it can be deployed. + + GIVEN a cluster called test-cluster.morph in system branch mybranch + AND a system in cluster test-cluster.morph in branch mybranch called test-system + AND system test-system in cluster test-cluster.morph in branch mybranch builds systems/test-system.morph + AND system test-system in cluster test-cluster.morph in branch mybranch has deployment type: tar + AND system test-system in cluster test-cluster.morph in branch mybranch has deployment location: test-system.tar + WHEN the user attempts to deploy the cluster test-cluster.morph in branch mybranch + THEN morph succeeded + AND file workspace/mybranch/test/morphs/test-system.tar exists diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 52eee01d..c6d245d0 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -67,72 +67,47 @@ another to hold a chunk. # Create a directory for all the git repositories. mkdir "$DATADIR/gits" - # Create the chunk repository. - mkdir "$DATADIR/gits/test-chunk" - - run_in "$DATADIR/gits/test-chunk" git init . - cat > "$DATADIR/gits/test-chunk/test-bin" <<'EOF' - #!/bin/sh - echo Hello World - EOF - cat > "$DATADIR/gits/test-chunk/test.h" <<'EOF' - int foo(void); + # Create the bootstrap chunk repositories + mkdir "$DATADIR/gits/bootstrap-chunk" + cd "$DATADIR/gits/bootstrap-chunk" + git init . + git checkout -b bootstrap + cp "$SRCDIR/scripts/test-shell.c" sh.c + install /dev/stdin <<'EOF' configure + #!/bin/true EOF - cat > "$DATADIR/gits/test-chunk/test.pc" <<'EOF' - prefix=/usr - includedir=${prefix}/include - Name: test - Cflags: -I{includedir} - EOF - run_in "$DATADIR/gits/test-chunk" git add . - run_in "$DATADIR/gits/test-chunk" git commit --allow-empty -m Initial. + printf >Makefile ' + CFLAGS = -D_GNU_SOURCE -static - # Create a repo for the morphologies. + all: sh - mkdir "$DATADIR/gits/morphs" + install: sh + \tinstall -D -m755 sh $(DESTDIR)/bin/sh' + git add . + git commit -m "Add bootstrap shell" - arch=$(run_morph print-architecture) - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/systems/test-system.morph" - name: test-system - kind: system - arch: $arch - strata: - - name: test-stratum - morph: strata/test-stratum.morph - EOF + git checkout --orphan master HEAD + # Commit a pre-built test-shell, as a compiler is too heavy to bootstrap + make sh + mkdir bin + mv sh bin/sh + git rm -f Makefile sh.c configure + git add bin/sh + git commit -m "Build bootstrap shell with bootstrap shell" - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/strata/test-stratum.morph" - name: test-stratum - kind: stratum - chunks: - - name: test-chunk - repo: test:test-chunk - morph: test-chunk.morph - unpetrify-ref: master - ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) - build-mode: test - build-depends: [] - EOF + # Create the test chunk repository. + + mkdir "$DATADIR/gits/test-chunk" + cd "$DATADIR/gits/test-chunk" + git init . # To verify that chunk splitting works, we have a chunk that installs # dummy files in all the places that different kinds of files are # usually installed. e.g. executables in `/bin` and `/usr/bin` - install -m644 -D /dev/stdin << 'EOF' "$DATADIR/gits/morphs/test-chunk.morph" - name: test-chunk - kind: chunk - build-system: manual - - # `install-commands` is a list of shell commands to run. Commands - # may be on multiple lines, and indeed anything programmatic will - # benefit from doing so. Arguably we could have just one command, - # but it's split into multiple so that morph can inform us which - # command failed without us having to include a lot of status - # information in the command and look at the error message. - - install-commands: - + PREFIX=/usr + DESTDIR=. # It's important that we can test whether executables get # installed, so we install an empty script into `/usr/bin/test` and # `/usr/sbin/test`. @@ -144,10 +119,10 @@ another to hold a chunk. # that the file exists, rather than its contents, we can use /dev/null # as the source. - - | - for bindir in bin sbin; do - install -D /dev/null "$DESTDIR/$PREFIX/$bindir/test" - done + + for bindir in bin sbin; do + install -D /dev/null "$DESTDIR/$PREFIX/$bindir/test" + done # We need shared libraries too, sometimes they're libraries to support # the executables that a chunk provides, sometimes for other chunks. @@ -158,14 +133,13 @@ another to hold a chunk. # Shared libraries' file names start with lib and end with `.so` # for shared-object, with version numbers optionally suffixed. - - | - for libdir in lib lib32 lib64; do - dirpath="$DESTDIR/$PREFIX/$libdir" - install -D /dev/null "$dirpath/libtest.so" - ln -s libtest.so "$dirpath/libtest.so.0" - ln -s libtest.so.0 "$dirpath/libtest.so.0.0" - ln -s libtest.so.0.0 "$dirpath/libtest.so.0.0.0" - done + for libdir in lib lib32 lib64; do + dirpath="$DESTDIR/$PREFIX/$libdir" + install -D /dev/null "$dirpath/libtest.so" + ln -s libtest.so "$dirpath/libtest.so.0" + ln -s libtest.so.0 "$dirpath/libtest.so.0.0" + ln -s libtest.so.0.0 "$dirpath/libtest.so.0.0.0" + done # Shared objects aren't the only kind of library, some executable # binaries count as libraries, such as git's plumbing commands. @@ -173,8 +147,10 @@ another to hold a chunk. # In some distributions they go into /lib, in others, and the default # autotools configuration, they go into /libexec. - - | - install -D test-bin "$DESTDIR/$PREFIX/libexec/test-bin" + install -D /dev/stdin "$DESTDIR/$PREFIX/libexec/test-bin" <<'EOF' + #!/bin/sh + echo Hello World + EOF # As well as run-time libraries, there's development files. For C # this is headers, which describe the API of the libraries, which @@ -185,9 +161,9 @@ another to hold a chunk. # Header files go into `include` and end with `.h`. They are not # executable, so the install command changes the permissions with the # `-m` option. - - - | - install -D -m 644 test.h "$DESTDIR/$PREFIX/include/test.h" + install -D -m 644 /dev/stdin <<'EOF' "$DESTDIR/$PREFIX/include/test.h" + int foo(void); + EOF # `pkg-config` is a standard way to locate libraries and get the # compiler flags needed to build with the library. It's also used @@ -195,68 +171,148 @@ another to hold a chunk. # so as well as being found in `lib/pkgconfig`, it can be found in # `share/pkgconfig`, so we install dummy files to both. - - | - for pkgdir in lib lib32 lib64 share; do - install -D -m 644 test.pc \ - "$DESTDIR/$PREFIX/$pkgdir/pkgconfig/test.pc" - done + for pkgdir in lib lib32 lib64 share; do + install -D -m 644 /dev/stdin <<EOF \ + "$DESTDIR/$PREFIX/$pkgdir/pkgconfig/test.pc" + prefix=$PREFIX + includedir=\${prefix}/include + Name: test + Cflags: -I{includedir} + EOF + done # Static libraries can be used to build static binaries, which don't # require their dependencies to be installed. They are typically in # the form of `.a` archive and `.la` libtool archives. - - | - for libdir in lib lib32 lib64; do - for libname in libtest.a libtest.la; do - install -D -m 644 /dev/null "$DESTDIR/$PREFIX/$libdir/$libname" - done - done + for libdir in lib lib32 lib64; do + for libname in libtest.a libtest.la; do + install -D -m 644 /dev/null "$DESTDIR/$PREFIX/$libdir/$libname" + done + done # Packages may also install documentation, this comes in a variety # of formats, but info pages, man pages and html documentation are # the most common. - - | - for docfile in info/test.info.gz man/man3/test.3.gz doc/test/doc.html; do - install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/$docfile" - done + for docfile in info/test.info.gz man/man3/test.3.gz doc/test/doc.html; do + install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/$docfile" + done # Locale covers translations, timezones, keyboard layouts etc. in # all manner of strange file formats and locations. # Locale provides various translations for specific messages. - - | - install -D -m 644 /dev/null \ - "$DESTDIR/$PREFIX/share/locale/en_GB/LC_MESSAGES/test.mo" + install -D -m 644 /dev/null \ + "$DESTDIR/$PREFIX/share/locale/en_GB/LC_MESSAGES/test.mo" # Internationalisation (i18n) includes character maps and other data # such as currency. - - | - for localefile in i18n/locales/en_GB charmaps/UTF-8.gz; do - install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/$localefile" - done + for localefile in i18n/locales/en_GB charmaps/UTF-8.gz; do + install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/$localefile" + done # Timezones are another kind of localisation. - - | - install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/zoneinfo/UTC" + install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/zoneinfo/UTC" # We also need a catch rule for everything that doesn't fit into # the above categories, so to test that, we create some files that # don't belong in one. - - | - for cfgfile in test.conf README; do - install -D -m 644 /dev/null "$DESTDIR/etc/test.d/$cfgfile" - done + for cfgfile in test.conf README; do + install -D -m 644 /dev/null "$DESTDIR/etc/test.d/$cfgfile" + done + + git add . + git commit --allow-empty -m Initial. + + # Create a repo for the morphologies. + + mkdir "$DATADIR/gits/morphs" + cd "$DATADIR/gits/morphs" + git init . + arch=$(run_morph print-architecture) + install -m644 -D /dev/stdin << EOF "systems/test-system.morph" + name: test-system + kind: system + arch: $arch + strata: + - name: build-essential + morph: strata/build-essential.morph + - name: core + morph: strata/core.morph EOF - run_in "$DATADIR/gits/morphs" git init . - run_in "$DATADIR/gits/morphs" git add . - run_in "$DATADIR/gits/morphs" git commit -m Initial. - run_in "$DATADIR/gits/morphs" git tag -a "test-tag" -m "Tagging test-tag" + install -m644 -D /dev/stdin << EOF "strata/build-essential.morph" + name: build-essential + kind: stratum + chunks: + - name: stage1-chunk + repo: test:bootstrap-chunk + ref: $(run_in "$DATADIR/gits/bootstrap-chunk" git rev-parse bootstrap) + unpetrify-ref: nootstrap + build-mode: bootstrap + build-depends: [] + - name: stage2-chunk + morph: stage2-chunk.morph + repo: test:bootstrap-chunk + ref: $(run_in "$DATADIR/gits/bootstrap-chunk" git rev-parse master) + unpetrify-ref: master + build-depends: + - stage1-chunk + EOF + install -m644 -D /dev/stdin << EOF "strata/core.morph" + name: core + kind: stratum + build-depends: + - morph: strata/build-essential.morph + chunks: + - name: test-chunk + morph: test-chunk.morph + repo: test:test-chunk + unpetrify-ref: master + ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) + build-depends: [] + EOF + + install -m644 -D /dev/stdin << 'EOF' "test-chunk.morph" + name: test-chunk + kind: chunk + build-system: manual + # `build-commands` is a list of shell commands to run. Commands + # may be on multiple lines, and indeed anything programmatic will + # benefit from doing so. Arguably we could have just one command, + # but it's split into multiple so that morph can inform us which + # command failed without us having to include a lot of status + # information in the command and look at the error message. + # `build-commands` are passed MAKEFLAGS to specify build parallelism, + # and are expected to generate files to be installed in `install-files` + build-commands: + # All of morph's building needs to handle binary output, so we can echo + # that out on the command line. + # Trust me, this gets decoded to "echo " then a bunch of binary output. + - !!binary | + ZWNobyBQ1+k661ol3khrsF4VO4HNcuYzwN0LYxEWS8mPmhQiQ7Vu8CME2+gsnKQaoIRIFuUEiLCI + vfIj1GTdXG6cVTJfNQ== + + install-commands: + - copy files + EOF + + install -m644 -D /dev/stdin << 'EOF' "stage2-chunk.morph" + name: test-chunk + kind: chunk + build-system: manual + install-commands: + - copy files + EOF + + git add . + git commit -m Initial. + git tag -a "test-tag" -m "Tagging test-tag" # Start a git daemon to serve our git repositories port_file="$DATADIR/git-daemon-port" @@ -298,45 +354,6 @@ We need a consistent value for the architecture in some tests, so we have a morphology using the test architecture. IMPLEMENTS GIVEN a system called (\S+) for the test architecture in the git server - - cat << EOF > "$DATADIR/gits/morphs/stage1-chunk.morph" - name: stage1-chunk - kind: chunk - build-system: dummy - EOF - - run_in "$DATADIR/gits/morphs" git add . - run_in "$DATADIR/gits/morphs" git commit -m "Add chunk for $MATCH_1" - - - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/strata/build-essential.morph" - name: build-essential - kind: stratum - chunks: - - name: stage1-chunk - morph: stage1-chunk.morph - repo: test:test-chunk - ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) - unpetrify-ref: master - build-mode: bootstrap - build-depends: [] - EOF - - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/strata/core.morph" - name: core - kind: stratum - build-depends: - - morph: strata/build-essential.morph - chunks: - - name: test-chunk - morph: test-chunk.morph - repo: test:test-chunk - unpetrify-ref: master - ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) - build-mode: test - build-depends: [] - EOF - name="$(basename "${MATCH_1%.*}")" cat << EOF > "$DATADIR/gits/morphs/$MATCH_1" arch: testarch @@ -356,6 +373,25 @@ have a morphology using the test architecture. run_in "$DATADIR/gits/morphs" git add "$MATCH_1" run_in "$DATADIR/gits/morphs" git commit -m "Added $MATCH_1 and strata morphologies." +You need an architecture to build a system, we don't default to the host architecture. + + IMPLEMENTS GIVEN a system called (\S+) with no architecture in the git server + name="$(basename "${MATCH_1%.*}")" + cat << EOF > "$DATADIR/gits/morphs/$MATCH_1" + configuration-extensions: [] + description: A system called $name for test architecture + kind: system + name: $name + strata: + - name: build-essential + morph: strata/build-essential.morph + - name: core + morph: strata/core.morph + EOF + + run_in "$DATADIR/gits/morphs" git add "$MATCH_1" + run_in "$DATADIR/gits/morphs" git commit -m "Added $MATCH_1." + Implementation sections for system branch operations ---------------------------------------------------- @@ -375,12 +411,16 @@ Attempt to check out a system branch from a root that has no systems. cd "$DATADIR/workspace" attempt_morph checkout test:test-chunk master + IMPLEMENTS WHEN the user attempts to check out the system branch from (\S+) called (\S+) + cd "$DATADIR/workspace" + attempt_morph branch-from-image --metadata-dir "$DATADIR/$MATCH_1/baserock" "$MATCH_2" + We also need to verify that a system branch has been checked out. IMPLEMENTS THEN the system branch (\S+) is checked out is_dir "$DATADIR/workspace/$MATCH_1/test/morphs" is_file "$DATADIR/workspace/$MATCH_1/test/morphs/systems/test-system.morph" - is_file "$DATADIR/workspace/$MATCH_1/test/morphs/strata/test-stratum.morph" + is_file "$DATADIR/workspace/$MATCH_1/test/morphs/strata/core.morph" We can create a new branch, off master. @@ -487,8 +527,10 @@ print-architecture` to get a value appropriate for morph. kind: system name: $name strata: - - name: test-stratum - morph: strata/test-stratum.morph + - name: build-essential + morph: strata/build-essential.morph + - name: core + morph: strata/core.morph EOF Reporting status of checked out repositories: @@ -651,10 +693,10 @@ Implementations for temporary build branch handling Implementation sections for building ==================================== - IMPLEMENTS WHEN the user (attempts to build|builds) the system (\S+) in branch (\S+) - cd "$DATADIR/workspace/$MATCH_3" - set build "$MATCH_2" - if [ $MATCH_1 == "builds" ]; then run_morph "$@" + IMPLEMENTS WHEN the user (attempts to )?((dist)?build)s? the system (\S+) in branch (\S+) + cd "$DATADIR/workspace/$MATCH_5" + set "$MATCH_2" "$MATCH_4" + if [ "$MATCH_1" != "attempts to " ]; then run_morph "$@" else attempt_morph "$@"; fi Implementation sections for cross-bootstraping @@ -840,9 +882,9 @@ variables in `$DATADIR/env`. We treat the value as a format string for Implementations for building systems ------------------------------------ - IMPLEMENTS THEN morph build the system (\S+) of the (branch|tag) (\S+) - cd "$DATADIR/workspace/$MATCH_3" - run_morph build "$MATCH_1" + IMPLEMENTS THEN morph ((dist)?build) the system (\S+) of the (branch|tag) (\S+) + cd "$DATADIR/workspace/$MATCH_5" + run_morph "$MATCH_1" "$MATCH_3" IMPLEMENTS WHEN the user builds (\S+) of the (\S+) (branch|tag) cd "$DATADIR/workspace/$MATCH_2" @@ -937,3 +979,140 @@ Altering morphologies in the workspace "$SRCDIR/scripts/edit-morph" cluster-system-set-deploy-variable \ "$DATADIR/workspace/$branch/test/morphs/$cluster" "$name" \ "$key" "$val" + + +Distbuild +========= + + IMPLEMENTS ASSUMING the morph-cache-server can be run + "$SRCDIR/morph-cache-server" --version + + IMPLEMENTS GIVEN a communal cache server + # The communal cache server has direct access to the git repositories + # and can have artifacts placed on it + artifact_dir="$DATADIR/shared-artifacts" + mkdir -p "$artifact_dir" + + read_cache_server_port_file="$DATADIR/read-cache-server-port" + read_cache_server_pid_file="$DATADIR/read-cache-server-pid" + start_cache_server "$read_cache_server_port_file" \ + "$read_cache_server_pid_file" \ + "$artifact_dir" + + write_cache_server_port_file="$DATADIR/write-cache-server-port" + write_cache_server_pid_file="$DATADIR/write-cache-server-pid" + start_cache_server "$write_cache_server_port_file" \ + "$write_cache_server_pid_file" \ + "$artifact_dir" --enable-writes + + IMPLEMENTS FINALLY the communal cache server is terminated + stop_daemon "$DATADIR/read-cache-server-pid" + stop_daemon "$DATADIR/write-cache-server-pid" + + IMPLEMENTS GIVEN a distbuild worker + # start worker cache server, so other workers can download results + worker_cachedir="$DATADIR/distbuild-worker-cache" + worker_artifacts="$worker_cachedir/artifacts" + mkdir -p "$worker_artifacts" + worker_cache_port_file="$DATADIR/worker-cache-server-port" + worker_cache_pid_file="$DATADIR/worker-cache-server-pid" + start_cache_server "$worker_cache_port_file" \ + "$worker_cache_pid_file" \ + "$worker_artifacts" + + # start worker daemon + worker_daemon_port_file="$DATADIR/worker-daemon-port" + worker_daemon_pid_file="$DATADIR/worker-daemon-pid" + mkfifo "$worker_daemon_port_file" + communal_cache_port="$(cat "$DATADIR/read-cache-server-port")" + start-stop-daemon --start --pidfile="$worker_daemon_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/morph" -- worker-daemon \ + --no-default-configs --config "$DATADIR/morph.conf" \ + --worker-daemon-port=0 \ + --worker-daemon-port-file="$worker_daemon_port_file" \ + --cachedir="$worker_cachedir" \ + --artifact-cache-server="http://localhost:$communal_cache_port/" \ + --git-resolve-cache-server="http://localhost:$communal_cache_port/" \ + --log="$DATADIR/worker-daemon-log" \ + >"$DATADIR/worker-daemon-out" 2>"$DATADIR/worker-daemon-err" + worker_daemon_port="$(cat "$worker_daemon_port_file")" + rm "$worker_daemon_port_file" + echo "$worker_daemon_port" >"$worker_daemon_port_file" + + # start worker helper + helper_pid_file="$DATADIR/worker-daemon-helper-pid" + start-stop-daemon --start --pidfile="$helper_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/distbuild-helper" -- \ + --no-default-configs \ + --parent-port="$worker_daemon_port" + + # set up builder config + install /dev/stdin <<'EOF' "$DATADIR/morph" + #!/bin/sh + exec "$SRCDIR/morph" --quiet \ + --cachedir-min-space=0 --tempdir-min-space=0 \ + --no-default-config --config "$DATADIR/morph.conf" \ + --cachedir "$DATADIR/distbuild-worker-cache" "$@" + EOF + + IMPLEMENTS FINALLY the distbuild worker is terminated + stop_daemon "$DATADIR/worker-cache-server-pid" + stop_daemon "$DATADIR/worker-daemon-pid" + stop_daemon "$DATADIR/worker-daemon-helper-pid" + + IMPLEMENTS GIVEN a distbuild controller + worker_cache_port_file="$DATADIR/worker-cache-server-port" + worker_cache_server_port="$(cat "$worker_cache_port_file")" + worker_daemon_port_file="$DATADIR/worker-daemon-port" + worker_daemon_port="$(cat "$worker_daemon_port_file")" + communal_cache_read_port="$(cat "$DATADIR/read-cache-server-port")" + communal_cache_write_port="$(cat "$DATADIR/write-cache-server-port")" + initiator_port_file="$DATADIR/controller-initiator-port" + mkfifo "$initiator_port_file" + helper_port_file="$DATADIR/controller-helper-port" + mkfifo "$helper_port_file" + controller_pid_file="$DATADIR/controller-pid" + start-stop-daemon --start --pidfile="$controller_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/morph" -- controller-daemon \ + --no-default-configs --config "$DATADIR/morph.conf" \ + --worker="localhost:$worker_daemon_port" \ + --worker-cache-server-port="$worker_cache_server_port" \ + --artifact-cache-server="http://localhost:$communal_cache_read_port/" \ + --git-resolve-cache-server="http://localhost:$communal_cache_read_port/" \ + --writeable-cache-server="http://localhost:$communal_cache_write_port/" \ + --controller-helper-port=0 \ + --controller-helper-port-file="$helper_port_file" \ + --controller-initiator-port=0 \ + --controller-initiator-port-file="$initiator_port_file" \ + --morph-instance="$DATADIR/morph" \ + --log="$DATADIR/controller-daemon-log" \ + >"$DATADIR/controller-daemon-out" 2>"$DATADIR/controller-daemon-err" + helper_port="$(cat "$helper_port_file")" + rm "$helper_port_file" + echo "$helper_port" >"$helper_port_file" + initiator_port="$(cat "$initiator_port_file")" + rm "$initiator_port_file" + echo "$initiator_port" >"$initiator_port_file" + + # start worker helper + helper_pid_file="$DATADIR/controller-helper-pid" + start-stop-daemon --start --pidfile="$helper_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/distbuild-helper" -- \ + --no-default-configs \ + --parent-port="$helper_port" + + # make builds use distbuild + echo "controller-initiator-port = $initiator_port" \ + >>"$DATADIR/morph.conf" + echo "controller-initiator-address = localhost" \ + >>"$DATADIR/morph.conf" + echo "initiator-step-output-dir = $DATADIR" \ + >>"$DATADIR/morph.conf" + + IMPLEMENTS FINALLY the distbuild controller is terminated + stop_daemon "$DATADIR/controller-helper-pid" + stop_daemon "$DATADIR/controller-pid" diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index 9d67f2ab..9c13e449 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -184,3 +184,23 @@ slashify_colons() { echo "$1" | sed s,:,/,g } + +start_cache_server(){ + mkfifo "$1" + start-stop-daemon --start --pidfile="$2" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/morph-cache-server" -- \ + --port-file="$1" --no-fcgi \ + --repo-dir="$DATADIR/gits" --direct-mode \ + --bundle-dir="$DATADIR/bundles" \ + --artifact-dir="$3" "$@" + port="$(cat "$1")" + rm "$1" + echo "$port" >"$1" +} + +stop_daemon(){ + if [ -e "$1" ]; then + start-stop-daemon --stop --pidfile "$1" --oknodo + fi +} diff --git a/yarns/regression.yarn b/yarns/regression.yarn index 7991ab12..c424f437 100644 --- a/yarns/regression.yarn +++ b/yarns/regression.yarn @@ -58,15 +58,15 @@ source it depended on. SCENARIO changing the artifacts a system uses GIVEN a workspace AND a git server - AND system systems/test-system.morph uses test-stratum-runtime from test-stratum - AND stratum strata/test-stratum.morph has match rules: [{artifact: test-stratum-runtime, include: [.*-(bins|libs|locale)]}, {artifact: test-stratum-devel, include: [.*-(devel|doc|misc)]}] + AND system systems/test-system.morph uses core-runtime from core + AND stratum strata/core.morph has match rules: [{artifact: core-runtime, include: [.*-(bins|libs|locale)]}, {artifact: core-devel, include: [.*-(devel|doc|misc)]}] WHEN the user checks out the system branch called master GIVEN a cluster called test-cluster.morph in system branch master AND a system in cluster test-cluster.morph in branch master called test-system AND system test-system in cluster test-cluster.morph in branch master builds systems/test-system.morph AND system test-system in cluster test-cluster.morph in branch master has deployment type: tar WHEN the user builds the system systems/test-system.morph in branch master - GIVEN stratum strata/test-stratum.morph in system branch master has match rules: [{artifact: test-stratum-runtime, include: [.*-(bins|libs|misc)]}, {artifact: test-stratum-devel, include: [.*-(devel|doc|locale)]}] + GIVEN stratum strata/core.morph in system branch master has match rules: [{artifact: core-runtime, include: [.*-(bins|libs|misc)]}, {artifact: core-devel, include: [.*-(devel|doc|locale)]}] WHEN the user builds the system systems/test-system.morph in branch master AND the user deploys the cluster test-cluster.morph in branch master with options test-system.location="$DATADIR/test.tar" THEN tarball test.tar contains baserock/test-chunk-misc.meta diff --git a/yarns/splitting.yarn b/yarns/splitting.yarn index 9248f60c..2726d294 100644 --- a/yarns/splitting.yarn +++ b/yarns/splitting.yarn @@ -15,7 +15,7 @@ To test that all the fields are recognised, we set the new fields to their default values. AND chunk test-chunk includes the default splitting rules - AND stratum strata/test-stratum.morph includes the default splitting rules + AND stratum strata/core.morph includes the default splitting rules AND system systems/test-system.morph includes the default splitting rules The default rules produce a system that is identical to not providing @@ -41,7 +41,7 @@ libraries. The only change we need to make is to add a field to the system morphology to select which artifact to use in the system. - AND system systems/test-system.morph uses test-stratum-runtime from test-stratum + AND system systems/test-system.morph uses core-runtime from core WHEN the user checks out the system branch called master The best way to test that only using some stratum artifacts works is @@ -79,8 +79,8 @@ This GIVEN has a chunk in the stratum that never successfully builds, so we know that if the system successfully builds, then we only built chunks that were needed. - AND stratum strata/test-stratum.morph has chunks that aren't used in test-stratum-minimal - AND system systems/test-system.morph uses test-stratum-minimal from test-stratum + AND stratum strata/core.morph has chunks that aren't used in core-minimal + AND system systems/test-system.morph uses core-minimal from core WHEN the user checks out the system branch called master THEN morph build the system systems/test-system.morph of the branch master FINALLY the git server is shut down @@ -92,7 +92,7 @@ Implementations IMPLEMENTS GIVEN chunk (\S+) includes the default splitting rules # Append default products rules name="$(basename "${MATCH_1%.*}")" - cat <<EOF >>"$DATADIR/gits/morphs/$MATCH_1" + cat <<EOF >>"$DATADIR/gits/morphs/$MATCH_1.morph" products: - artifact: $name-bins include: [ "(usr/)?s?bin/.*" ] @@ -119,7 +119,7 @@ Implementations - artifact: $name-misc include: [ .* ] EOF - run_in "$DATADIR/gits/morphs" git add "$MATCH_1" + run_in "$DATADIR/gits/morphs" git add "$MATCH_1.morph" run_in "$DATADIR/gits/morphs" git commit -m 'Add default splitting rules' IMPLEMENTS GIVEN stratum (\S+) includes the default splitting rules @@ -127,6 +127,8 @@ Implementations cat <<EOF >"$DATADIR/gits/morphs/$MATCH_1" name: $name kind: stratum + build-depends: + - morph: strata/build-essential.morph products: - artifact: $name-devel include: @@ -146,7 +148,6 @@ Implementations unpetrify-ref: master ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) morph: test-chunk.morph - build-mode: test build-depends: [] artifacts: test-chunk-bins: $name-runtime @@ -162,11 +163,13 @@ Implementations IMPLEMENTS GIVEN system (\S+) includes the default splitting rules cat << EOF >> "$DATADIR/gits/morphs/$MATCH_1" strata: - - name: test-stratum - morph: strata/test-stratum.morph + - name: build-essential + morph: strata/build-essential.morph + - name: core + morph: strata/core.morph artifacts: - - test-stratum-runtime - - test-stratum-devel + - core-runtime + - core-devel EOF run_in "$DATADIR/gits/morphs" git add "$MATCH_1" run_in "$DATADIR/gits/morphs" git commit -m 'Add default splitting rules' @@ -195,14 +198,12 @@ Implementations morph: test-chunk.morph unpetrify-ref: master ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) - build-mode: test build-depends: [] - name: unbuildable-chunk repo: test:test-chunk unpetrify-ref: refs/heads/master ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) morph: unbuildable-chunk.morph - build-mode: test build-depends: - test-chunk EOF |