From 19cb5098dcec8e4dd4152aeff95b8e5a7fe1c69a Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Mon, 16 Mar 2015 15:10:42 +0000 Subject: Implement partial distbuilds In addition to partial builds we also want to be able to do partial distbuilds, and distbuild uses a different codepath. This commit updates the distbuild code to know what to do if a partial build is requested. It only builds up to the latest chunk/stratum that was requested, and displays where to find the artifacts for each of the chunks/strata requested upon completion of the build. The usage is the same as for local builds. Change-Id: I0537f74e2e65c7aefe5e71795f17999e2415fce5 --- distbuild/build_controller.py | 108 ++++++++++++++++++++++++++++++--------- distbuild/initiator.py | 8 ++- distbuild/protocol.py | 6 ++- morphlib/buildcommand.py | 6 ++- morphlib/plugins/build_plugin.py | 24 +++++++-- 5 files changed, 120 insertions(+), 32 deletions(-) diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index d6f3398f..5b55dff3 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -116,18 +116,39 @@ def build_step_name(artifact): return artifact.source.name -def map_build_graph(artifact, callback): +def map_build_graph(artifact, callback, components=[]): + """Run callback on each artifact in the build graph and return result. + + If components is given, then only look at the components given and + their dependencies. Also, return a list of the components after they + have had callback called on them. + + """ result = [] + mapped_components = [] done = set() - queue = [artifact] + if components: + queue = list(components) + else: + queue = [artifact] while queue: a = queue.pop() if a not in done: result.append(callback(a)) queue.extend(a.source.dependencies) done.add(a) - return result + if a in components: + mapped_components.append(a) + return result, mapped_components + +def find_artifacts(components, artifact): + found = [] + for a in artifact.walk(): + name = a.source.morphology['name'] + if name in components: + found.append(a) + return found class BuildController(distbuild.StateMachine): @@ -314,6 +335,17 @@ class BuildController(distbuild.StateMachine): distbuild.crash_point() self._artifact = event.artifact + names = self._request['component_names'] + self._components = find_artifacts(names, self._artifact) + failed = False + for component in self._components: + if component.source.morphology['name'] not in names: + logging.debug('Failed to find %s in build graph' + % component.filename) + failed = True + if failed: + self.fail('Failed to find all components in %s' + % self._artifact.name) self._helper_id = self._idgen.next() artifact_names = [] @@ -321,7 +353,9 @@ class BuildController(distbuild.StateMachine): artifact.state = UNKNOWN artifact_names.append(artifact.basename()) - map_build_graph(self._artifact, set_state_and_append) + _, self._components = map_build_graph(self._artifact, + set_state_and_append, + self._components) url = urlparse.urljoin(self._artifact_cache_server, '/1.0/artifacts') msg = distbuild.message('http-request', @@ -355,11 +389,20 @@ class BuildController(distbuild.StateMachine): return cache_state = json.loads(event.msg['body']) - map_build_graph(self._artifact, set_status) + _, self._components = map_build_graph(self._artifact, set_status, + self._components) self.mainloop.queue_event(self, _Annotated()) - unbuilt = len([a for a in self._artifact.walk() if a.state == UNBUILT]) - total = len([a for _ in self._artifact.walk()]) + unbuilt = set() + for c in self._components: + unbuilt.update([a for a in c.walk() if a.state == UNBUILT]) + unbuilt = len(unbuilt) or len([a for a in self._artifact.walk() + if a.state == UNBUILT]) + total = set() + for c in self._components: + total.update([a for a in c.walk()]) + total = len(total) or len([a for _ in self._artifact.walk()]) + progress = BuildProgress( self._request['id'], 'Need to build %d artifacts, of %d total' % (unbuilt, total)) @@ -375,22 +418,30 @@ class BuildController(distbuild.StateMachine): all(a.state == BUILT for a in artifact.source.dependencies)) - return [a - for a in map_build_graph(self._artifact, lambda a: a) - if is_ready_to_build(a)] + artifacts, _ = map_build_graph(self._artifact, lambda a: a, + self._components) + return [a for a in artifacts if is_ready_to_build(a)] def _queue_worker_builds(self, event_source, event): distbuild.crash_point() - if self._artifact.state == BUILT: - logging.info('Requested artifact is built') - self.mainloop.queue_event(self, _Built()) - return + if not self._components: + if self._artifact.state == BUILT: + logging.info('Requested artifact is built') + self.mainloop.queue_event(self, _Built()) + return + + else: + if not any(c.state != BUILT for c in self._components): + logging.info('Requested components are built') + self.mainloop.queue_event(self, _Built()) + return logging.debug('Queuing more worker-builds to run') if self.debug_graph_state: logging.debug('Current state of build graph nodes:') - for a in map_build_graph(self._artifact, lambda a: a): + for a, _ in map_build_graph(self._artifact, + lambda a: a, self._components): logging.debug(' %s state is %s' % (a.name, a.state)) if a.state != BUILT: for dep in a.dependencies: @@ -524,7 +575,8 @@ class BuildController(distbuild.StateMachine): self.mainloop.queue_event(BuildController, progress) def _find_artifact(self, cache_key): - artifacts = map_build_graph(self._artifact, lambda a: a) + artifacts, _ = map_build_graph(self._artifact, lambda a: a, + self._components) wanted = [a for a in artifacts if a.source.cache_key == cache_key] if wanted: return wanted[0] @@ -559,7 +611,8 @@ class BuildController(distbuild.StateMachine): # yields all chunk artifacts for the given source # so we set the state of this source's artifacts # to BUILT - map_build_graph(self._artifact, set_state) + _, self._components = map_build_graph(self._artifact, set_state, + self._components) self._queue_worker_builds(None, event) @@ -610,10 +663,19 @@ class BuildController(distbuild.StateMachine): logging.debug('Notifying initiator of successful build') baseurl = urlparse.urljoin( self._artifact_cache_server, '/1.0/artifacts') - filename = ('%s.%s.%s' % - (self._artifact.source.cache_key, - self._artifact.source.morphology['kind'], - self._artifact.name)) - url = '%s?filename=%s' % (baseurl, urllib.quote(filename)) - finished = BuildFinished(self._request['id'], [url]) + urls = [] + for c in self._components: + name = ('%s.%s.%s' % + (c.source.cache_key, + c.source.morphology['kind'], + c.name)) + urls.append('%s?filename=%s' % (baseurl, urllib.quote(name))) + if not self._components: + name = ('%s.%s.%s' % + (self._artifact.source.cache_key, + self._artifact.source.morphology['kind'], + self._artifact.name)) + urls.append('%s?filename=%s' % (baseurl, urllib.quote(name))) + + finished = BuildFinished(self._request['id'], urls) self.mainloop.queue_event(BuildController, finished) diff --git a/distbuild/initiator.py b/distbuild/initiator.py index eef4c9ec..48299a3d 100644 --- a/distbuild/initiator.py +++ b/distbuild/initiator.py @@ -54,7 +54,7 @@ def create_build_directory(prefix='build'): class Initiator(distbuild.StateMachine): def __init__(self, cm, conn, app, repo_name, ref, morphology, - original_ref): + original_ref, component_names): distbuild.StateMachine.__init__(self, 'waiting') self._cm = cm self._conn = conn @@ -63,6 +63,10 @@ class Initiator(distbuild.StateMachine): self._ref = ref self._morphology = morphology self._original_ref = original_ref + self._component_names = component_names + self._partial = False + if self._component_names: + self._partial = True self._step_outputs = {} self.debug_transitions = False @@ -101,6 +105,8 @@ class Initiator(distbuild.StateMachine): ref=self._ref, morphology=self._morphology, original_ref=self._original_ref, + component_names=self._component_names, + partial=self._partial, protocol_version=distbuild.protocol.VERSION ) self._jm.send(msg) diff --git a/distbuild/protocol.py b/distbuild/protocol.py index 73d72d1d..268dcbf6 100644 --- a/distbuild/protocol.py +++ b/distbuild/protocol.py @@ -22,7 +22,7 @@ # time a change is introduced that would break server/initiator compatibility -VERSION = 1 +VERSION = 2 _required_fields = { @@ -31,6 +31,7 @@ _required_fields = { 'repo', 'ref', 'morphology', + 'partial', 'protocol_version', ], 'build-progress': [ @@ -89,7 +90,8 @@ _required_fields = { _optional_fields = { 'build-request': [ - 'original_ref' + 'original_ref', + 'component_names' ] } diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 874f8d6f..fd5acdf5 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -526,7 +526,8 @@ class InitiatorBuildCommand(BuildCommand): self.app.settings['push-build-branches'] = True super(InitiatorBuildCommand, self).__init__(app) - def build(self, repo_name, ref, filename, original_ref=None): + def build(self, repo_name, ref, filename, original_ref=None, + component_names=[]): '''Initiate a distributed build on a controller''' distbuild.add_crash_conditions(self.app.settings['crash-condition']) @@ -537,7 +538,8 @@ class InitiatorBuildCommand(BuildCommand): self.app.status(msg='Starting distributed build') loop = distbuild.MainLoop() - args = [repo_name, ref, filename, original_ref or ref] + args = [repo_name, ref, filename, original_ref or ref, + component_names] cm = distbuild.InitiatorConnectionMachine(self.app, self.addr, self.port, diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index f6372aed..e5b35853 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -42,9 +42,10 @@ class BuildPlugin(cliapp.Plugin): arg_synopsis='SYSTEM [COMPONENT...]') self.app.add_subcommand('distbuild-morphology', self.distbuild_morphology, - arg_synopsis='SYSTEM') + arg_synopsis='REPO REF FILENAME ' + '[COMPONENT...]') self.app.add_subcommand('distbuild', self.distbuild, - arg_synopsis='SYSTEM') + arg_synopsis='SYSTEM [COMPONENT...]') self.use_distbuild = False def disable(self): @@ -58,6 +59,8 @@ class BuildPlugin(cliapp.Plugin): * `REPO` is a git repository URL. * `REF` is a branch or other commit reference in that repository. * `FILENAME` is a morphology filename at that ref. + * `COMPONENT...` is the names of one or more chunks or strata to + build. If none are given the the system at FILENAME is built. See 'help distbuild' and 'help build-morphology' for more information. @@ -66,10 +69,15 @@ class BuildPlugin(cliapp.Plugin): addr = self.app.settings['controller-initiator-address'] port = self.app.settings['controller-initiator-port'] + self.use_distbuild = True build_command = morphlib.buildcommand.InitiatorBuildCommand( self.app, addr, port) - for repo_name, ref, filename in self.app.itertriplets(args): - build_command.build(repo_name, ref, filename) + repo, ref, filename = args[0:3] + filename = morphlib.util.sanitise_morphology_path(filename) + component_names = [morphlib.util.sanitise_morphology_path(name) + for name in args[3:]] + self.start_build(repo, ref, build_command, filename, + component_names) def distbuild(self, args): '''Distbuild a system image in the current system branch @@ -77,6 +85,8 @@ class BuildPlugin(cliapp.Plugin): Command line arguments: * `SYSTEM` is the name of the system to build. + * `COMPONENT...` is the names of one or more chunks or strata to + build. If none are given then SYSTEM is built. This command launches a distributed build, to use this command you must first set up a distbuild cluster. @@ -297,6 +307,12 @@ class BuildPlugin(cliapp.Plugin): build the whole system. ''' + if self.use_distbuild: + bc.build(repo, commit, system_filename, + original_ref=original_ref, + component_names=component_names) + return + self.app.status(msg='Deciding on task order') srcpool = bc.create_source_pool(repo, commit, system_filename) bc.validate_sources(srcpool) -- cgit v1.2.1