diff options
author | Adam Coldrick <adam.coldrick@codethink.co.uk> | 2015-03-16 15:10:42 +0000 |
---|---|---|
committer | Adam Coldrick <adam.coldrick@codethink.co.uk> | 2015-03-20 17:13:34 +0000 |
commit | 9500e44ceaf6c36ecd933680901737a7b322e8d7 (patch) | |
tree | c5c44a6565e092fa78079b9fc083c0ebb682d05b | |
parent | 88a378f9223c10fbd97a744196ec012140795d6c (diff) | |
download | morph-9500e44ceaf6c36ecd933680901737a7b322e8d7.tar.gz |
Implement partial distbuilds
In addition to partial builds we also want to be able to do partial
distbuilds, and distbuild uses a different codepath.
This commit updates the distbuild code to know what to do if a partial
build is requested. It only builds up to the latest chunk/stratum that
was requested, and displays where to find the artifacts for each of
the chunks/strata requested upon completion of the build.
The usage is the same as for local builds.
Change-Id: Id34d91115c268797d3b9871371cf4c7280933f83
-rw-r--r-- | distbuild/build_controller.py | 89 | ||||
-rw-r--r-- | distbuild/initiator.py | 8 | ||||
-rw-r--r-- | distbuild/protocol.py | 6 | ||||
-rw-r--r-- | morphlib/buildcommand.py | 6 | ||||
-rw-r--r-- | morphlib/plugins/build_plugin.py | 14 |
5 files changed, 95 insertions, 28 deletions
diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index d6f3398f..183dd155 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -116,19 +116,32 @@ def build_step_name(artifact): return artifact.source.name -def map_build_graph(artifact, callback): +def map_build_graph(artifact, callback, components=[]): result = [] + mapped_components = [] done = set() - queue = [artifact] + if components: + queue = list(components) + else: + queue = [artifact] while queue: a = queue.pop() if a not in done: result.append(callback(a)) queue.extend(a.source.dependencies) done.add(a) - return result + if a in components: + mapped_components.append(a) + return result, mapped_components +def find_artifacts(components, artifact): + found = [] + for a in artifact.walk(): + if a.source.filename in components: + found.append(a) + return found + class BuildController(distbuild.StateMachine): '''Control one build-request fulfillment. @@ -314,6 +327,17 @@ class BuildController(distbuild.StateMachine): distbuild.crash_point() self._artifact = event.artifact + filenames = self._request['component_filenames'] + self._components = find_artifacts(filenames, self._artifact) + failed = False + for component in self._components: + if component.source.filename not in filenames: + logging.debug('Failed to find %s in build graph' + % component.filename) + failed = True + if failed: + self.fail('Failed to find all components in %s' + % self._artifact.name) self._helper_id = self._idgen.next() artifact_names = [] @@ -321,7 +345,9 @@ class BuildController(distbuild.StateMachine): artifact.state = UNKNOWN artifact_names.append(artifact.basename()) - map_build_graph(self._artifact, set_state_and_append) + _, self._components = map_build_graph(self._artifact, + set_state_and_append, + self._components) url = urlparse.urljoin(self._artifact_cache_server, '/1.0/artifacts') msg = distbuild.message('http-request', @@ -355,11 +381,13 @@ class BuildController(distbuild.StateMachine): return cache_state = json.loads(event.msg['body']) - map_build_graph(self._artifact, set_status) + _, self._components = map_build_graph(self._artifact, set_status, + self._components) self.mainloop.queue_event(self, _Annotated()) unbuilt = len([a for a in self._artifact.walk() if a.state == UNBUILT]) total = len([a for _ in self._artifact.walk()]) + progress = BuildProgress( self._request['id'], 'Need to build %d artifacts, of %d total' % (unbuilt, total)) @@ -375,22 +403,30 @@ class BuildController(distbuild.StateMachine): all(a.state == BUILT for a in artifact.source.dependencies)) - return [a - for a in map_build_graph(self._artifact, lambda a: a) - if is_ready_to_build(a)] + artifacts, _ = map_build_graph(self._artifact, lambda a: a, + self._components) + return [a for a in artifacts if is_ready_to_build(a)] def _queue_worker_builds(self, event_source, event): distbuild.crash_point() - if self._artifact.state == BUILT: - logging.info('Requested artifact is built') - self.mainloop.queue_event(self, _Built()) - return + if not self._components: + if self._artifact.state == BUILT: + logging.info('Requested artifact is built') + self.mainloop.queue_event(self, _Built()) + return + + else: + if not any(c.state != BUILT for c in self._components): + logging.info('Requested components are built') + self.mainloop.queue_event(self, _Built()) + return logging.debug('Queuing more worker-builds to run') if self.debug_graph_state: logging.debug('Current state of build graph nodes:') - for a in map_build_graph(self._artifact, lambda a: a): + for a, _ in map_build_graph(self._artifact, + lambda a: a, self._components): logging.debug(' %s state is %s' % (a.name, a.state)) if a.state != BUILT: for dep in a.dependencies: @@ -524,7 +560,8 @@ class BuildController(distbuild.StateMachine): self.mainloop.queue_event(BuildController, progress) def _find_artifact(self, cache_key): - artifacts = map_build_graph(self._artifact, lambda a: a) + artifacts, _ = map_build_graph(self._artifact, lambda a: a, + self._components) wanted = [a for a in artifacts if a.source.cache_key == cache_key] if wanted: return wanted[0] @@ -559,7 +596,8 @@ class BuildController(distbuild.StateMachine): # yields all chunk artifacts for the given source # so we set the state of this source's artifacts # to BUILT - map_build_graph(self._artifact, set_state) + _, self._components = map_build_graph(self._artifact, set_state, + self._components) self._queue_worker_builds(None, event) @@ -610,10 +648,19 @@ class BuildController(distbuild.StateMachine): logging.debug('Notifying initiator of successful build') baseurl = urlparse.urljoin( self._artifact_cache_server, '/1.0/artifacts') - filename = ('%s.%s.%s' % - (self._artifact.source.cache_key, - self._artifact.source.morphology['kind'], - self._artifact.name)) - url = '%s?filename=%s' % (baseurl, urllib.quote(filename)) - finished = BuildFinished(self._request['id'], [url]) + urls = [] + for c in self._components: + name = ('%s.%s.%s' % + (c.source.cache_key, + c.source.morphology['kind'], + c.name)) + urls.append('%s?filename=%s' % (baseurl, urllib.quote(name))) + if not self._components: + name = ('%s.%s.%s' % + (self._artifact.source.cache_key, + self._artifact.source.morphology['kind'], + self._artifact.name)) + urls.append('%s?filename=%s' % (baseurl, urllib.quote(name))) + + finished = BuildFinished(self._request['id'], urls) self.mainloop.queue_event(BuildController, finished) diff --git a/distbuild/initiator.py b/distbuild/initiator.py index eef4c9ec..eec52763 100644 --- a/distbuild/initiator.py +++ b/distbuild/initiator.py @@ -54,7 +54,7 @@ def create_build_directory(prefix='build'): class Initiator(distbuild.StateMachine): def __init__(self, cm, conn, app, repo_name, ref, morphology, - original_ref): + original_ref, component_filenames): distbuild.StateMachine.__init__(self, 'waiting') self._cm = cm self._conn = conn @@ -63,6 +63,10 @@ class Initiator(distbuild.StateMachine): self._ref = ref self._morphology = morphology self._original_ref = original_ref + self._component_filenames = component_filenames + self._partial = False + if self._component_filenames: + self._partial = True self._step_outputs = {} self.debug_transitions = False @@ -101,6 +105,8 @@ class Initiator(distbuild.StateMachine): ref=self._ref, morphology=self._morphology, original_ref=self._original_ref, + component_filenames=self._component_filenames, + partial=self._partial, protocol_version=distbuild.protocol.VERSION ) self._jm.send(msg) diff --git a/distbuild/protocol.py b/distbuild/protocol.py index 73d72d1d..042a06bd 100644 --- a/distbuild/protocol.py +++ b/distbuild/protocol.py @@ -22,7 +22,7 @@ # time a change is introduced that would break server/initiator compatibility -VERSION = 1 +VERSION = 2 _required_fields = { @@ -31,6 +31,7 @@ _required_fields = { 'repo', 'ref', 'morphology', + 'partial', 'protocol_version', ], 'build-progress': [ @@ -89,7 +90,8 @@ _required_fields = { _optional_fields = { 'build-request': [ - 'original_ref' + 'original_ref', + 'component_filenames' ] } diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 40510560..2153990e 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -523,7 +523,8 @@ class InitiatorBuildCommand(BuildCommand): self.app.settings['push-build-branches'] = True super(InitiatorBuildCommand, self).__init__(app) - def build(self, repo_name, ref, filename, original_ref=None): + def build(self, repo_name, ref, filename, original_ref=None, + component_filenames=[]): '''Initiate a distributed build on a controller''' distbuild.add_crash_conditions(self.app.settings['crash-condition']) @@ -534,7 +535,8 @@ class InitiatorBuildCommand(BuildCommand): self.app.status(msg='Starting distributed build') loop = distbuild.MainLoop() - args = [repo_name, ref, filename, original_ref or ref] + args = [repo_name, ref, filename, original_ref or ref, + component_filenames] cm = distbuild.InitiatorConnectionMachine(self.app, self.addr, self.port, diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 06f6ced7..0d783ecf 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -64,10 +64,15 @@ class BuildPlugin(cliapp.Plugin): addr = self.app.settings['controller-initiator-address'] port = self.app.settings['controller-initiator-port'] + self.use_distbuild = True build_command = morphlib.buildcommand.InitiatorBuildCommand( self.app, addr, port) - for repo_name, ref, filename in self.app.itertriplets(args): - build_command.build(repo_name, ref, filename) + repo, ref, filename = args[0:3] + filename = morphlib.util.sanitise_morphology_path(filename) + component_filenames = [morphlib.util.sanitise_morphology_path(name) + for name in args[3:]] + self.start_build(repo, ref, build_command, filename, + component_filenames) def distbuild(self, args): '''Distbuild a system image in the current system branch @@ -307,6 +312,11 @@ class BuildPlugin(cliapp.Plugin): these in and build each one in turn. ''' + if self.use_distbuild: + bc.build(repo, commit, system_filename, + original_ref=original_ref, + component_filenames=component_filenames) + return if not self.app.settings['partial']: if original_ref: bc.build(repo, commit, system_filename, |