From 01060c72158893863162acec85f576c442fc49ff Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 16:24:47 +0000 Subject: distbuild: allow daemons to bind to ephemeral ports You can bind to an ephemeral port by passing 0 as the port number. To work out which port you actually got, you need to call getsockname(). To facilitate being able to spawn multiple copies of the daemons for testing environments, you can pass a -file option, which will make the daemon write which port it actually bound to. If this path is a fifo, reading from it in the spawner process will allow synchronisation of only spawning services that require that port to be ready after it is. --- morphlib/plugins/distbuild_plugin.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'morphlib/plugins/distbuild_plugin.py') diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 50ab7eeb..7e8188dd 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -121,6 +121,11 @@ class WorkerDaemon(cliapp.Plugin): 'listen for connections on PORT', default=3434, group=group_distbuild) + self.app.settings.string( + ['worker-daemon-port-file'], + 'write port used by worker-daemon to FILE', + default='', + group=group_distbuild) self.app.add_subcommand( 'worker-daemon', self.worker_daemon, @@ -136,7 +141,9 @@ class WorkerDaemon(cliapp.Plugin): address = self.app.settings['worker-daemon-address'] port = self.app.settings['worker-daemon-port'] - router = distbuild.ListenServer(address, port, distbuild.JsonRouter) + port_file = self.app.settings['worker-daemon-port-file'] + router = distbuild.ListenServer(address, port, distbuild.JsonRouter, + port_file=port_file) loop = distbuild.MainLoop() loop.add_state_machine(router) loop.run() @@ -156,6 +163,11 @@ class ControllerDaemon(cliapp.Plugin): 'listen for initiator connections on PORT', default=7878, group=group_distbuild) + self.app.settings.string( + ['controller-initiator-port-file'], + 'write the port to listen for initiator connections to FILE', + default='', + group=group_distbuild) self.app.settings.string( ['controller-helper-address'], @@ -167,6 +179,11 @@ class ControllerDaemon(cliapp.Plugin): 'listen for helper connections on PORT', default=5656, group=group_distbuild) + self.app.settings.string( + ['controller-helper-port-file'], + 'write the port to listen for helper connections to FILE', + default='', + group=group_distbuild) self.app.settings.string_list( ['worker'], @@ -218,8 +235,10 @@ class ControllerDaemon(cliapp.Plugin): listener_specs = [ # address, port, class to initiate on connection, class init args ('controller-helper-address', 'controller-helper-port', + 'controller-helper-port-file', distbuild.HelperRouter, []), ('controller-initiator-address', 'controller-initiator-port', + 'controller-initiator-port-file', distbuild.InitiatorConnection, [artifact_cache_server, morph_instance]), ] @@ -229,9 +248,10 @@ class ControllerDaemon(cliapp.Plugin): queuer = distbuild.WorkerBuildQueuer() loop.add_state_machine(queuer) - for addr, port, sm, extra_args in listener_specs: + for addr, port, port_file, sm, extra_args in listener_specs: addr = self.app.settings[addr] port = self.app.settings[port] + port_file = self.app.settings[port_file] listener = distbuild.ListenServer( addr, port, sm, extra_args=extra_args) loop.add_state_machine(listener) -- cgit v1.2.1 From b760a2c4e1c22973baee8daf44a332b123ad1fdb Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 21:00:39 +0000 Subject: Allow ephemeral ports for distbuild services --- morphlib/plugins/distbuild_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'morphlib/plugins/distbuild_plugin.py') diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 7e8188dd..26c26498 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -253,7 +253,7 @@ class ControllerDaemon(cliapp.Plugin): port = self.app.settings[port] port_file = self.app.settings[port_file] listener = distbuild.ListenServer( - addr, port, sm, extra_args=extra_args) + addr, port, sm, extra_args=extra_args, port_file=port_file) loop.add_state_machine(listener) for worker in self.app.settings['worker']: -- cgit v1.2.1 From 2689f61a305ec576f229f1d9f0929d7eb11d1acc Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 22:11:42 +0000 Subject: Fix issues with distbuild caused by moving to building per-source --- morphlib/plugins/distbuild_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'morphlib/plugins/distbuild_plugin.py') diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 26c26498..1858a9ba 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -103,7 +103,7 @@ class WorkerBuild(cliapp.Plugin): self.app.subcommands['gc']([]) arch = artifact.arch - bc.build_artifact(artifact, bc.new_build_env(arch)) + bc.build_source(artifact.source, bc.new_build_env(arch)) def is_system_artifact(self, filename): return re.match(r'^[0-9a-fA-F]{64}\.system\.', filename) -- cgit v1.2.1 From f3935d491f7b3890abaf8ee56e2b7fc7af63c558 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Thu, 2 Oct 2014 13:02:11 +0000 Subject: Allow distbuilds to choose where to put logs --- morphlib/plugins/distbuild_plugin.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'morphlib/plugins/distbuild_plugin.py') diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 1858a9ba..653eeae8 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -168,6 +168,11 @@ class ControllerDaemon(cliapp.Plugin): 'write the port to listen for initiator connections to FILE', default='', group=group_distbuild) + self.app.settings.string( + ['initiator-step-output-dir'], + 'write build output to files in DIR', + default='.', + group=group_distbuild) self.app.settings.string( ['controller-helper-address'], -- cgit v1.2.1