summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAdam Coldrick <adam.coldrick@codethink.co.uk>2015-03-16 15:10:42 +0000
committerAdam Coldrick <adam.coldrick@codethink.co.uk>2015-04-02 08:07:09 +0000
commit19cb5098dcec8e4dd4152aeff95b8e5a7fe1c69a (patch)
tree281fa9424c6423ebc9cd86f61184de605a22cf62
parent128a8a927ce3c0ba4dd8a1d3b7a83dae45d8e0a3 (diff)
downloadmorph-19cb5098dcec8e4dd4152aeff95b8e5a7fe1c69a.tar.gz
Implement partial distbuilds
In addition to partial builds we also want to be able to do partial distbuilds, and distbuild uses a different codepath. This commit updates the distbuild code to know what to do if a partial build is requested. It only builds up to the latest chunk/stratum that was requested, and displays where to find the artifacts for each of the chunks/strata requested upon completion of the build. The usage is the same as for local builds. Change-Id: I0537f74e2e65c7aefe5e71795f17999e2415fce5
-rw-r--r--distbuild/build_controller.py108
-rw-r--r--distbuild/initiator.py8
-rw-r--r--distbuild/protocol.py6
-rw-r--r--morphlib/buildcommand.py6
-rw-r--r--morphlib/plugins/build_plugin.py24
5 files changed, 120 insertions, 32 deletions
diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py
index d6f3398f..5b55dff3 100644
--- a/distbuild/build_controller.py
+++ b/distbuild/build_controller.py
@@ -116,18 +116,39 @@ def build_step_name(artifact):
return artifact.source.name
-def map_build_graph(artifact, callback):
+def map_build_graph(artifact, callback, components=[]):
+ """Run callback on each artifact in the build graph and return result.
+
+ If components is given, then only look at the components given and
+ their dependencies. Also, return a list of the components after they
+ have had callback called on them.
+
+ """
result = []
+ mapped_components = []
done = set()
- queue = [artifact]
+ if components:
+ queue = list(components)
+ else:
+ queue = [artifact]
while queue:
a = queue.pop()
if a not in done:
result.append(callback(a))
queue.extend(a.source.dependencies)
done.add(a)
- return result
+ if a in components:
+ mapped_components.append(a)
+ return result, mapped_components
+
+def find_artifacts(components, artifact):
+ found = []
+ for a in artifact.walk():
+ name = a.source.morphology['name']
+ if name in components:
+ found.append(a)
+ return found
class BuildController(distbuild.StateMachine):
@@ -314,6 +335,17 @@ class BuildController(distbuild.StateMachine):
distbuild.crash_point()
self._artifact = event.artifact
+ names = self._request['component_names']
+ self._components = find_artifacts(names, self._artifact)
+ failed = False
+ for component in self._components:
+ if component.source.morphology['name'] not in names:
+ logging.debug('Failed to find %s in build graph'
+ % component.filename)
+ failed = True
+ if failed:
+ self.fail('Failed to find all components in %s'
+ % self._artifact.name)
self._helper_id = self._idgen.next()
artifact_names = []
@@ -321,7 +353,9 @@ class BuildController(distbuild.StateMachine):
artifact.state = UNKNOWN
artifact_names.append(artifact.basename())
- map_build_graph(self._artifact, set_state_and_append)
+ _, self._components = map_build_graph(self._artifact,
+ set_state_and_append,
+ self._components)
url = urlparse.urljoin(self._artifact_cache_server, '/1.0/artifacts')
msg = distbuild.message('http-request',
@@ -355,11 +389,20 @@ class BuildController(distbuild.StateMachine):
return
cache_state = json.loads(event.msg['body'])
- map_build_graph(self._artifact, set_status)
+ _, self._components = map_build_graph(self._artifact, set_status,
+ self._components)
self.mainloop.queue_event(self, _Annotated())
- unbuilt = len([a for a in self._artifact.walk() if a.state == UNBUILT])
- total = len([a for _ in self._artifact.walk()])
+ unbuilt = set()
+ for c in self._components:
+ unbuilt.update([a for a in c.walk() if a.state == UNBUILT])
+ unbuilt = len(unbuilt) or len([a for a in self._artifact.walk()
+ if a.state == UNBUILT])
+ total = set()
+ for c in self._components:
+ total.update([a for a in c.walk()])
+ total = len(total) or len([a for _ in self._artifact.walk()])
+
progress = BuildProgress(
self._request['id'],
'Need to build %d artifacts, of %d total' % (unbuilt, total))
@@ -375,22 +418,30 @@ class BuildController(distbuild.StateMachine):
all(a.state == BUILT
for a in artifact.source.dependencies))
- return [a
- for a in map_build_graph(self._artifact, lambda a: a)
- if is_ready_to_build(a)]
+ artifacts, _ = map_build_graph(self._artifact, lambda a: a,
+ self._components)
+ return [a for a in artifacts if is_ready_to_build(a)]
def _queue_worker_builds(self, event_source, event):
distbuild.crash_point()
- if self._artifact.state == BUILT:
- logging.info('Requested artifact is built')
- self.mainloop.queue_event(self, _Built())
- return
+ if not self._components:
+ if self._artifact.state == BUILT:
+ logging.info('Requested artifact is built')
+ self.mainloop.queue_event(self, _Built())
+ return
+
+ else:
+ if not any(c.state != BUILT for c in self._components):
+ logging.info('Requested components are built')
+ self.mainloop.queue_event(self, _Built())
+ return
logging.debug('Queuing more worker-builds to run')
if self.debug_graph_state:
logging.debug('Current state of build graph nodes:')
- for a in map_build_graph(self._artifact, lambda a: a):
+ for a, _ in map_build_graph(self._artifact,
+ lambda a: a, self._components):
logging.debug(' %s state is %s' % (a.name, a.state))
if a.state != BUILT:
for dep in a.dependencies:
@@ -524,7 +575,8 @@ class BuildController(distbuild.StateMachine):
self.mainloop.queue_event(BuildController, progress)
def _find_artifact(self, cache_key):
- artifacts = map_build_graph(self._artifact, lambda a: a)
+ artifacts, _ = map_build_graph(self._artifact, lambda a: a,
+ self._components)
wanted = [a for a in artifacts if a.source.cache_key == cache_key]
if wanted:
return wanted[0]
@@ -559,7 +611,8 @@ class BuildController(distbuild.StateMachine):
# yields all chunk artifacts for the given source
# so we set the state of this source's artifacts
# to BUILT
- map_build_graph(self._artifact, set_state)
+ _, self._components = map_build_graph(self._artifact, set_state,
+ self._components)
self._queue_worker_builds(None, event)
@@ -610,10 +663,19 @@ class BuildController(distbuild.StateMachine):
logging.debug('Notifying initiator of successful build')
baseurl = urlparse.urljoin(
self._artifact_cache_server, '/1.0/artifacts')
- filename = ('%s.%s.%s' %
- (self._artifact.source.cache_key,
- self._artifact.source.morphology['kind'],
- self._artifact.name))
- url = '%s?filename=%s' % (baseurl, urllib.quote(filename))
- finished = BuildFinished(self._request['id'], [url])
+ urls = []
+ for c in self._components:
+ name = ('%s.%s.%s' %
+ (c.source.cache_key,
+ c.source.morphology['kind'],
+ c.name))
+ urls.append('%s?filename=%s' % (baseurl, urllib.quote(name)))
+ if not self._components:
+ name = ('%s.%s.%s' %
+ (self._artifact.source.cache_key,
+ self._artifact.source.morphology['kind'],
+ self._artifact.name))
+ urls.append('%s?filename=%s' % (baseurl, urllib.quote(name)))
+
+ finished = BuildFinished(self._request['id'], urls)
self.mainloop.queue_event(BuildController, finished)
diff --git a/distbuild/initiator.py b/distbuild/initiator.py
index eef4c9ec..48299a3d 100644
--- a/distbuild/initiator.py
+++ b/distbuild/initiator.py
@@ -54,7 +54,7 @@ def create_build_directory(prefix='build'):
class Initiator(distbuild.StateMachine):
def __init__(self, cm, conn, app, repo_name, ref, morphology,
- original_ref):
+ original_ref, component_names):
distbuild.StateMachine.__init__(self, 'waiting')
self._cm = cm
self._conn = conn
@@ -63,6 +63,10 @@ class Initiator(distbuild.StateMachine):
self._ref = ref
self._morphology = morphology
self._original_ref = original_ref
+ self._component_names = component_names
+ self._partial = False
+ if self._component_names:
+ self._partial = True
self._step_outputs = {}
self.debug_transitions = False
@@ -101,6 +105,8 @@ class Initiator(distbuild.StateMachine):
ref=self._ref,
morphology=self._morphology,
original_ref=self._original_ref,
+ component_names=self._component_names,
+ partial=self._partial,
protocol_version=distbuild.protocol.VERSION
)
self._jm.send(msg)
diff --git a/distbuild/protocol.py b/distbuild/protocol.py
index 73d72d1d..268dcbf6 100644
--- a/distbuild/protocol.py
+++ b/distbuild/protocol.py
@@ -22,7 +22,7 @@
# time a change is introduced that would break server/initiator compatibility
-VERSION = 1
+VERSION = 2
_required_fields = {
@@ -31,6 +31,7 @@ _required_fields = {
'repo',
'ref',
'morphology',
+ 'partial',
'protocol_version',
],
'build-progress': [
@@ -89,7 +90,8 @@ _required_fields = {
_optional_fields = {
'build-request': [
- 'original_ref'
+ 'original_ref',
+ 'component_names'
]
}
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 874f8d6f..fd5acdf5 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -526,7 +526,8 @@ class InitiatorBuildCommand(BuildCommand):
self.app.settings['push-build-branches'] = True
super(InitiatorBuildCommand, self).__init__(app)
- def build(self, repo_name, ref, filename, original_ref=None):
+ def build(self, repo_name, ref, filename, original_ref=None,
+ component_names=[]):
'''Initiate a distributed build on a controller'''
distbuild.add_crash_conditions(self.app.settings['crash-condition'])
@@ -537,7 +538,8 @@ class InitiatorBuildCommand(BuildCommand):
self.app.status(msg='Starting distributed build')
loop = distbuild.MainLoop()
- args = [repo_name, ref, filename, original_ref or ref]
+ args = [repo_name, ref, filename, original_ref or ref,
+ component_names]
cm = distbuild.InitiatorConnectionMachine(self.app,
self.addr,
self.port,
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index f6372aed..e5b35853 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -42,9 +42,10 @@ class BuildPlugin(cliapp.Plugin):
arg_synopsis='SYSTEM [COMPONENT...]')
self.app.add_subcommand('distbuild-morphology',
self.distbuild_morphology,
- arg_synopsis='SYSTEM')
+ arg_synopsis='REPO REF FILENAME '
+ '[COMPONENT...]')
self.app.add_subcommand('distbuild', self.distbuild,
- arg_synopsis='SYSTEM')
+ arg_synopsis='SYSTEM [COMPONENT...]')
self.use_distbuild = False
def disable(self):
@@ -58,6 +59,8 @@ class BuildPlugin(cliapp.Plugin):
* `REPO` is a git repository URL.
* `REF` is a branch or other commit reference in that repository.
* `FILENAME` is a morphology filename at that ref.
+ * `COMPONENT...` is the names of one or more chunks or strata to
+ build. If none are given the the system at FILENAME is built.
See 'help distbuild' and 'help build-morphology' for more information.
@@ -66,10 +69,15 @@ class BuildPlugin(cliapp.Plugin):
addr = self.app.settings['controller-initiator-address']
port = self.app.settings['controller-initiator-port']
+ self.use_distbuild = True
build_command = morphlib.buildcommand.InitiatorBuildCommand(
self.app, addr, port)
- for repo_name, ref, filename in self.app.itertriplets(args):
- build_command.build(repo_name, ref, filename)
+ repo, ref, filename = args[0:3]
+ filename = morphlib.util.sanitise_morphology_path(filename)
+ component_names = [morphlib.util.sanitise_morphology_path(name)
+ for name in args[3:]]
+ self.start_build(repo, ref, build_command, filename,
+ component_names)
def distbuild(self, args):
'''Distbuild a system image in the current system branch
@@ -77,6 +85,8 @@ class BuildPlugin(cliapp.Plugin):
Command line arguments:
* `SYSTEM` is the name of the system to build.
+ * `COMPONENT...` is the names of one or more chunks or strata to
+ build. If none are given then SYSTEM is built.
This command launches a distributed build, to use this command
you must first set up a distbuild cluster.
@@ -297,6 +307,12 @@ class BuildPlugin(cliapp.Plugin):
build the whole system.
'''
+ if self.use_distbuild:
+ bc.build(repo, commit, system_filename,
+ original_ref=original_ref,
+ component_names=component_names)
+ return
+
self.app.status(msg='Deciding on task order')
srcpool = bc.create_source_pool(repo, commit, system_filename)
bc.validate_sources(srcpool)