documentation

4 files changed, 525 insertions, 51 deletions

diff --git a/lib/net/ssh/multi.rb b/lib/net/ssh/multi.rb
index ca6928e..fe60802 100644
--- a/lib/net/ssh/multi.rb
+++ b/lib/net/ssh/multi.rb
@@ -1,7 +1,57 @@
 require 'net/ssh/multi/session'
 
 module Net; module SSH
+  # Net::SSH::Multi is a library for controlling multiple Net::SSH
+  # connections via a single interface. It exposes an API similar to that of
+  # Net::SSH::Connection::Session and Net::SSH::Connection::Channel, making it
+  # simpler to adapt programs designed for single connections to be used with
+  # multiple connections.
+  #
+  # This library is particularly useful for automating repetitive tasks that
+  # must be performed on multiple machines. It executes the commands in
+  # parallel, and allows commands to be executed on subsets of servers
+  # (defined by groups).
+  #
+  #   require 'net/ssh/multi'
+  # 
+  #   Net::SSH::Multi.start do |session|
+  #     # access servers via a gateway
+  #     session.via 'gateway', 'gateway-user'
+  # 
+  #     # define the servers we want to use
+  #     session.use 'host1', 'user1'
+  #     session.use 'host2', 'user2'
+  # 
+  #     # define servers in groups for more granular access
+  #     session.group :app do
+  #       session.use 'app1', 'user'
+  #       session.use 'app2', 'user'
+  #     end
+  # 
+  #     # execute commands
+  #     session.exec "uptime"
+  # 
+  #     # execute commands on a subset of servers
+  #     session.with(:app) { session.exec "hostname" }
+  # 
+  #     # run the aggregated event loop
+  #     session.loop
+  #   end
+  # 
+  # See Net::SSH::Multi::Session for more documentation.
   module Multi
+    # This is a convenience method for instantiating a new
+    # Net::SSH::Multi::Session. If a block is given, the session will be
+    # yielded to the block automatically closed (see Net::SSH::Multi::Session#close)
+    # when the block finishes. Otherwise, the new session will be returned.
+    #
+    #   Net::SSH::Multi.start do |session|
+    #     # ...
+    #   end
+    #
+    #   session = Net::SSH::Multi.start
+    #   # ...
+    #   session.close
     def self.start
       session = Session.new
 
diff --git a/lib/net/ssh/multi/channel.rb b/lib/net/ssh/multi/channel.rb
index 8d33fa3..4c1f80c 100644
--- a/lib/net/ssh/multi/channel.rb
+++ b/lib/net/ssh/multi/channel.rb
@@ -1,93 +1,213 @@
 module Net; module SSH; module Multi
+  # Net::SSH::Multi::Channel encapsulates a collection of Net::SSH::Connection::Channel
+  # instances from multiple different connections. It allows for operations to
+  # be performed on all contained channels, simultaneously, using an interface
+  # mostly identical to Net::SSH::Connection::Channel itself.
+  #
+  # You typically obtain a Net::SSH::Multi::Channel instance via
+  # Net::SSH::Multi::Session#open_channel or Net::SSH::Multi::Session#exec,
+  # though there is nothing stopping you from instantiating one yourself with
+  # a handful of Net::SSH::Connection::Channel objects (though they should be
+  # associated with connections managed by a Net::SSH::Multi::Session object
+  # for consistent behavior).
+  #
+  #   channel = session.open_channel do |ch|
+  #     # ...
+  #   end
+  #
+  #   channel.wait
   class Channel
     include Enumerable
 
+    # The Net::SSH::Multi::Session instance that controls this channel collection.
     attr_reader :connection
+
+    # The collection of Net::SSH::Connection::Channel instances that this multi-channel aggregates.
     attr_reader :channels
+
+    # A Hash of custom properties that may be set and queried on this object.
     attr_reader :properties
 
+    # Instantiate a new Net::SSH::Multi::Channel instance, controlled by the
+    # given +connection+ (a Net::SSH::Multi::Session object) and wrapping the
+    # given +channels+ (Net::SSH::Connection::Channel instances).
+    #
+    # You will typically never call this directly; rather, you'll get your
+    # multi-channel references via Net::SSH::Multi::Session#open_channel and
+    # friends.
     def initialize(connection, channels)
       @connection = connection
       @channels = channels
       @properties = {}
     end
 
+    # Iterate over each component channel object, yielding each in order to the
+    # associated block.
     def each
       @channels.each { |channel| yield channel }
     end
 
+    # Retrieve the property (see #properties) with the given +key+.
+    #
+    #   host = channel[:host]
     def [](key)
       @properties[key]
     end
 
+    # Set the property (see #properties) with the given +key+ to the given
+    # +value+.
+    #
+    #   channel[:visited] = true
     def []=(key, value)
       @properties[key] = value
     end
 
+    # Perform an +exec+ command on all component channels. The block, if given,
+    # is passed to each component channel, so it will (potentially) be invoked
+    # once for every channel in the collection. The block will receive two
+    # parameters: the specific channel object being operated on, and a boolean
+    # indicating whether the exec succeeded or not.
+    #
+    #   channel.exec "ls -l" do |ch, success|
+    #     # ...
+    #   end
+    #
+    # See the documentation in Net::SSH for Net::SSH::Connection::Channel#exec
+    # for more information on how to work with the callback.
     def exec(command, &block)
       channels.each { |channel| channel.exec(command, &block) }
       self
     end
 
-    def subsystem(subsystem, &block)
-      channels.each { |channel| channel.subsystem(subsystem, &block) }
-      self
-    end
-
+    # Perform a +request_pty+ command on all component channels. The block, if
+    # given, is passed to each component channel, so it will (potentially) be
+    # invoked once for every channel in the collection. The block will
+    # receive two parameters: the specific channel object being operated on,
+    # and a boolean indicating whether the pty request succeeded or not.
+    #
+    #   channel.request_pty do |ch, success|
+    #     # ...
+    #   end
+    #
+    # See the documentation in Net::SSH for
+    # Net::SSH::Connection::Channel#request_pty for more information on how to
+    # work with the callback.
     def request_pty(opts={}, &block)
       channels.each { |channel| channel.request_pty(opts, &block) }
       self
     end
 
+    # Send the given +data+ to each component channel. It will be sent to the
+    # remote process, typically being received on the process' +stdin+ stream.
+    #
+    #   channel.send_data "password\n"
     def send_data(data)
       channels.each { |channel| channel.send_data(data) }
       self
     end
 
+    # Returns true as long as any of the component channels are active.
+    #
+    #   connection.loop { channel.active? }
     def active?
       channels.any? { |channel| channel.active? }
     end
 
+    # Runs the connection's event loop until the channel is no longer active
+    # (see #active?).
+    #
+    #   channel.exec "something"
+    #   channel.wait
     def wait
       connection.loop { active? }
       self
     end
 
+    # Closes all component channels.
     def close
       channels.each { |channel| channel.close }
       self
     end
 
+    # Tells the remote process for each component channel not to expect any
+    # further data from this end of the channel.
     def eof!
       channels.each { |channel| channel.eof! }
       self
     end
 
+    # Registers a callback on all component channels, to be invoked when the
+    # remote process emits data (usually on its +stdout+ stream). The block
+    # will be invoked with two arguments: the specific channel object, and the
+    # data that was received.
+    #
+    #   channel.on_data do |ch, data|
+    #     puts "got data: #{data}"
+    #   end
     def on_data(&block)
       channels.each { |channel| channel.on_data(&block) }
       self
     end
 
+    # Registers a callback on all component channels, to be invoked when the
+    # remote process emits "extended" data (typically on its +stderr+ stream).
+    # The block will be invoked with three arguments: the specific channel
+    # object, an integer describing the data type (usually a 1 for +stderr+)
+    # and the data that was received.
+    #
+    #   channel.on_extended_data do |ch, type, data|
+    #     puts "got extended data: #{data}"
+    #   end
     def on_extended_data(&block)
       channels.each { |channel| channel.on_extended_data(&block) }
       self
     end
 
+    # Registers a callback on all component channels, to be invoked during the
+    # idle portion of the connection event loop. The callback will be invoked
+    # with one argument: the specific channel object being processed.
+    #
+    #   channel.on_process do |ch|
+    #     # ...
+    #   end
     def on_process(&block)
       channels.each { |channel| channel.on_process(&block) }
       self
     end
 
+    # Registers a callback on all component channels, to be invoked when the
+    # remote server terminates the channel. The callback will be invoked
+    # with one argument: the specific channel object being closed.
+    #
+    #   channel.on_close do |ch|
+    #     # ...
+    #   end
     def on_close(&block)
       channels.each { |channel| channel.on_close(&block) }
       self
     end
 
+    # Registers a callback on all component channels, to be invoked when the
+    # remote server has no further data to send. The callback will be invoked
+    # with one argument: the specific channel object being marked EOF.
+    #
+    #   channel.on_eof do |ch|
+    #     # ...
+    #   end
     def on_eof(&block)
       channels.each { |channel| channel.on_eof(&block) }
       self
     end
 
+    # Registers a callback on all component channels, to be invoked when the
+    # remote server sends a channel request of the given +type+. The callback
+    # will be invoked with two arguments: the specific channel object receiving
+    # the request, and a Net::SSH::Buffer instance containing the request-specific
+    # data.
+    #
+    #   channel.on_request("exit-status") do |ch, data|
+    #     puts "exited with #{data.read_long}"
+    #   end
     def on_request(type, &block)
       channels.each { |channel| channel.on_request(type, &block) }
       self
diff --git a/lib/net/ssh/multi/server.rb b/lib/net/ssh/multi/server.rb
index 338a870..65967ab 100644
--- a/lib/net/ssh/multi/server.rb
+++ b/lib/net/ssh/multi/server.rb
@@ -1,12 +1,31 @@
 require 'net/ssh'
 
 module Net; module SSH; module Multi
+  # Encapsulates the connection information for a single remote server, as well
+  # as the Net::SSH session corresponding to that information. You'll rarely
+  # need to instantiate one of these directly: instead, you should use
+  # Net::SSH::Multi::Session#use.
   class Server
+    # The host name (or IP address) of the server to connect to.
     attr_reader :host
+
+    # The user name to use when connecting to this server.
     attr_reader :user
+
+    # The Hash of additional options to pass to Net::SSH when connecting
+    # (including things like :password, and so forth).
     attr_reader :options
+
+    # The Net::SSH::Gateway instance to use to establish the connection. Will
+    # be +nil+ if the connection should be established without a gateway.
     attr_reader :gateway
 
+    # Creates a new Server instance with the given connection information. The
+    # +options+ hash must conform to the options described for Net::SSH::start,
+    # with one addition:
+    #
+    # * :via => a Net::SSH::Gateway instance to use when establishing a
+    #   connection to this server.
     def initialize(host, user, options={})
       @host = host
       @user = user
@@ -14,14 +33,20 @@ module Net; module SSH; module Multi
       @gateway = @options.delete(:via)
     end
 
+    # Returns the value of the server property with the given +key+. Server
+    # properties are described via the +:properties+ key in the options hash
+    # when defining the Server.
     def [](key)
-      (@options[:properties] || {})[key]
+      (options[:properties] || {})[key]
     end
 
+    # Returns the port number to use for this connection.
     def port
       options[:port] || 22
     end
 
+    # Compares the given +server+ to this instance, and returns true if they
+    # have the same host, user, and port.
     def eql?(server)
       host == server.host &&
       user == server.user &&
@@ -30,10 +55,15 @@ module Net; module SSH; module Multi
 
     alias :== :eql?
 
+    # Generates a +Fixnum+ hash value for this object. This function has the
+    # property that +a.eql?(b)+ implies +a.hash == b.hash+. The
+    # hash value is used by class +Hash+. Any hash value that exceeds the
+    # capacity of a +Fixnum+ will be truncated before being used.
     def hash
       @hash ||= [host, user, port].hash
     end
 
+    # Returns a human-readable representation of this server instance.
     def to_s
       @to_s ||= begin
         s = "#{user}@#{host}"
@@ -42,10 +72,28 @@ module Net; module SSH; module Multi
       end
     end
 
+    # Returns a human-readable representation of this server instance.
     def inspect
       @inspect ||= "#<%s:0x%x %s>" % [self.class.name, object_id, to_s]
     end
 
+    # Returns the Net::SSH session object for this server. If +ensure_open+
+    # is false and the session has not previously been created, this will
+    # return +nil+. If +ensure_open+ is true, the session will be instantiated
+    # if it has not already been instantiated, via the +gateway+ if one is
+    # given, or directly (via Net::SSH::start) otherwise.
+    #
+    #   if server.session.nil?
+    #     puts "connecting..."
+    #     server.session(true)
+    #   end
+    #
+    # Note that the sessions returned by this are "enhanced" slightly, to make
+    # them easier to deal with in a multi-session environment: they have a
+    # :server property automatically set on them, that refers to this object
+    # (the Server instance that spawned them).
+    #
+    #   assert_equal server, server.session[:server]
     def session(ensure_open=false)
       return @session if @session || !ensure_open
       @session ||= begin 
@@ -62,38 +110,54 @@ module Net; module SSH; module Multi
       raise Net::SSH::AuthenticationFailed.new("#{error.message}@#{host}")
     end
 
-    def close_channels
-      session.channels.each { |id, channel| channel.close } if session
-    end
-
-    def close
-      session.transport.close if session
-    end
-
+    # Returns +true+ if the session has been opened, and the session is currently
+    # busy (as defined by Net::SSH::Connection::Session#busy?).
     def busy?(include_invisible=false)
       session && session.busy?(include_invisible)
     end
 
-    def preprocess
-      session.preprocess if session
-    end
+    public # but not published, e.g., these are used internally only...
+    
+      # Closes all open channels on this server's session. If the session has
+      # not yet been opened, this does nothing.
+      def close_channels #:nodoc:
+        session.channels.each { |id, channel| channel.close } if session
+      end
 
-    def readers
-      return [] unless session
-      session.listeners.keys
-    end
+      # Closes this server's session's transport layer. If the session has not
+      # yet been opened, this does nothing.
+      def close #:nodoc:
+        session.transport.close if session
+      end
 
-    def writers
-      return [] unless session
-      session.listeners.keys.select do |io|
-        io.respond_to?(:pending_write?) && io.pending_write?
+      # Runs the session's preprocess action, if the session has been opened.
+      def preprocess #:nodoc:
+        session.preprocess if session
       end
-    end
 
-    def postprocess(readers, writers)
-      return true unless session
-      listeners = session.listeners.keys
-      session.postprocess(listeners & readers, listeners & writers)
-    end
+      # Returns all registered readers on the session, or an empty array if the
+      # session is not open.
+      def readers #:nodoc:
+        return [] unless session
+        session.listeners.keys
+      end
+
+      # Returns all registered and pending writers on the session, or an empty
+      # array if the session is not open.
+      def writers #:nodoc:
+        return [] unless session
+        session.listeners.keys.select do |io|
+          io.respond_to?(:pending_write?) && io.pending_write?
+        end
+      end
+
+      # Runs the post-process action on the session, if the session has been
+      # opened. Only the +readers+ and +writers+ that actually belong to this
+      # session will be postprocessed by this server.
+      def postprocess(readers, writers) #:nodoc:
+        return true unless session
+        listeners = session.listeners.keys
+        session.postprocess(listeners & readers, listeners & writers)
+      end
   end
 end; end; end
\ No newline at end of file
diff --git a/lib/net/ssh/multi/session.rb b/lib/net/ssh/multi/session.rb
index f94028b..cffe972 100644
--- a/lib/net/ssh/multi/session.rb
+++ b/lib/net/ssh/multi/session.rb
@@ -3,14 +3,64 @@ require 'net/ssh/multi/server'
 require 'net/ssh/multi/channel'
 
 module Net; module SSH; module Multi
+  # Represents a collection of connections to various servers. It provides an
+  # interface for organizing the connections (#group), as well as a way to
+  # scope commands to a subset of all connections (#with). You can also provide
+  # a default gateway connection that servers should use when connecting
+  # (#via). It exposes an interface similar to Net::SSH::Connection::Session
+  # for opening SSH channels and executing commands, allowing for these
+  # operations to be done in parallel across multiple connections.
+  #
+  #   Net::SSH::Multi.start do |session|
+  #     # access servers via a gateway
+  #     session.via 'gateway', 'gateway-user'
+  # 
+  #     # define the servers we want to use
+  #     session.use 'host1', 'user1'
+  #     session.use 'host2', 'user2'
+  # 
+  #     # define servers in groups for more granular access
+  #     session.group :app do
+  #       session.use 'app1', 'user'
+  #       session.use 'app2', 'user'
+  #     end
+  # 
+  #     # execute commands
+  #     session.exec "uptime"
+  # 
+  #     # execute commands on a subset of servers
+  #     session.with(:app) { session.exec "hostname" }
+  # 
+  #     # run the aggregated event loop
+  #     session.loop
+  #   end
+  #
+  # Note that connections are established lazily, as soon as they are needed.
+  # You can force the connections to be opened immediately, though, using the
+  # #connect! method.
   class Session
+    # The list of Net::SSH::Multi::Server definitions managed by this session.
     attr_reader :servers
+
+    # The default Net::SSH::Gateway instance to use to connect to the servers.
+    # If +nil+, no default gateway will be used.
     attr_reader :default_gateway
+
+    # The hash of group definitions, mapping each group name to the list of
+    # corresponding Net::SSH::Multi::Server definitions.
     attr_reader :groups
 
-    attr_reader :open_groups
-    attr_reader :active_groups
+    # The list of "open" groups, which will receive subsequent server definitions.
+    # See #use and #group.
+    attr_reader :open_groups #:nodoc:
+
+    # The list of "active" groups, which will be used to restrict subsequent
+    # commands. This is actually a Hash, mapping group names to their corresponding
+    # constraints (see #with).
+    attr_reader :active_groups #:nodoc:
 
+    # Creates a new Net::SSH::Multi::Session instance. Initially, it contains
+    # no server definitions, no group definitions, and no default gateway.
     def initialize
       @servers = []
       @groups = {}
@@ -19,6 +69,32 @@ module Net; module SSH; module Multi
       @open_groups = []
     end
 
+    # At its simplest, this associates a named group with a server definition.
+    # It can be used in either of two ways:
+    #
+    # First, you can use it to associate a group (or array of groups) with a
+    # server definition (or array of server definitions). The server definitions
+    # must already exist in the #servers array (typically by calling #use):
+    #
+    #   server1 = session.use('host1', 'user1')
+    #   server2 = session.use('host2', 'user2')
+    #   session.group :app => server1, :web => server2
+    #   session.group :staging => [server1, server2]
+    #   session.group %w(xen linux) => server2
+    #   session.group %w(rackspace backup) => [server1, server2]
+    #
+    # Secondly, instead of a mapping of groups to servers, you can just
+    # provide a list of group names, and then a block. Inside the block, any
+    # calls to #use will automatically associate the new server definition with
+    # those groups. You can nest #group calls, too, which will aggregate the
+    # group definitions.
+    #
+    #   session.group :rackspace, :backup do
+    #     session.use 'host1', 'user1'
+    #     session.group :xen do
+    #       session.use 'host2', 'user2'
+    #     end
+    #   end
     def group(*args)
       mapping = args.last.is_a?(Hash) ? args.pop : {}
 
@@ -41,11 +117,32 @@ module Net; module SSH; module Multi
       end
     end
 
+    # Sets up a default gateway to use when establishing connections to servers.
+    # Note that any servers defined prior to this invocation will not use the
+    # default gateway; it only affects servers defined subsequently.
+    #
+    #   session.via 'gateway.host', 'user'
+    #
+    # You may override the default gateway on a per-server basis by passing the
+    # :via key to the #use method; see #use for details.
     def via(host, user, options={})
       @default_gateway = Net::SSH::Gateway.new(host, user, options)
       self
     end
 
+    # Defines a new server definition, to be managed by this session. The
+    # server is at the given +host+, and will be connected to as the given
+    # +user+. The other options are passed as-is to the Net::SSH session
+    # constructor.
+    #
+    # If a default gateway has been specified previously (with #via) it will
+    # be passed to the new server definition. You can override this by passing
+    # a different Net::SSH::Gateway instance (or +nil+) with the :via key in
+    # the +options+.
+    #
+    #   session.use 'host', 'user'
+    #   session.use 'host2', 'user2', :via => nil
+    #   session.use 'host3', 'user3', :via => Net::SSH::Gateway.new('gateway.host', 'user')
     def use(host, user, options={})
       server = Server.new(host, user, {:via => default_gateway}.merge(options))
       exists = servers.index(server)
@@ -58,6 +155,49 @@ module Net; module SSH; module Multi
       server
     end
 
+    # Restricts the set of servers that will be targeted by commands within
+    # the associated block. It can be used in either of two ways (or both ways
+    # used together).
+    #
+    # First, you can simply specify a list of group names. All servers in all
+    # named groups will be the target of the commands. (Nested calls to #with
+    # are cumulative.)
+    #
+    #   # execute 'hostname' on all servers in the :app group, and 'uptime'
+    #   # on all servers in either :app or :db.
+    #   session.with(:app) do
+    #     session.exec('hostname')
+    #     session.with(:db) do
+    #       session.exec('uptime')
+    #     end
+    #   end
+    #
+    # Secondly, you can specify a hash with group names as keys, and property
+    # constraints as the values. These property constraints are either "only"
+    # constraints (which restrict the set of servers to "only" those that match
+    # the given properties) or "except" constraints (which restrict the set of
+    # servers to those whose properties do _not_ match). Properties are described
+    # when the server is defined (via the :properties key):
+    #
+    #   session.group :db do
+    #     session.use 'dbmain', 'user', :properties => { :primary => true }
+    #     session.use 'dbslave', 'user2'
+    #     session.use 'dbslve2', 'user2'
+    #   end
+    #
+    #   # execute the given rake task ONLY on the servers in the :db group
+    #   # which have the :primary property set to true.
+    #   session.with :db => { :only => { :primary => true } } do
+    #     session.exec "rake db:migrate"
+    #   end
+    #
+    # You can, naturally, combine these methods:
+    #
+    #   # all servers in :app and :web, and all servers in :db with the
+    #   # :primary property set to true
+    #   session.with :app, :web, :db => { :only => { :primary => true } } do
+    #     # ...
+    #   end
     def with(*groups)
       saved_groups = active_groups.dup
 
@@ -81,6 +221,18 @@ module Net; module SSH; module Multi
       active_groups.replace(saved_groups)
     end
 
+    # Works as #with, but for specific servers rather than groups. In other
+    # words, you can use this to restrict actions within the block to only
+    # a specific list of servers. It works by creating an ad-hoc group, adding
+    # the servers to that group, and then making that group the only active
+    # group. (Note that because of this, you cannot nest #on within #with,
+    # though you could nest #with inside of #on.)
+    #
+    #   srv = session.use('host', 'user')
+    #   # ...
+    #   session.on(srv) do
+    #     session.exec('hostname')
+    #   end
     def on(*servers)
       adhoc_group = "adhoc_group_#{servers.hash}_#{rand(0xffffffff)}".to_sym
       group(adhoc_group => servers)
@@ -92,6 +244,10 @@ module Net; module SSH; module Multi
       groups.delete(adhoc_group)
     end
 
+    # Returns the list of Net::SSH sessions for all servers that match the
+    # current scope (e.g., the groups or servers named in the outer #with or
+    # #on calls). If any servers have not yet been connected to, this will
+    # block until the connections have been made.
     def active_sessions
       list = if active_groups.empty?
         servers
@@ -105,14 +261,25 @@ module Net; module SSH; module Multi
         end
       end
 
-      sessions_for(list.uniq)
+      list.uniq!
+      threads = list.map { |server| Thread.new { server.session(true) } if server.session.nil? }
+      threads.each { |thread| thread.join if thread }
+
+      list.map { |server| server.session }
     end
 
+    # Connections are normally established lazily, as soon as they are needed.
+    # This method forces all servers selected by the current scope to connect,
+    # if they have not yet been connected.
     def connect!
       active_sessions
       self
     end
 
+    # Closes the multi-session by shutting down all open server sessions, and
+    # the default gateway (if one was specified using #via). Note that other
+    # gateway connections (e.g., those passed to #use directly) will _not_ be
+    # closed by this method, and must be managed externally.
     def close
       servers.each { |server| server.close_channels }
       loop(0) { busy?(true) }
@@ -120,28 +287,31 @@ module Net; module SSH; module Multi
       default_gateway.shutdown! if default_gateway
     end
 
+    # Returns +true+ if any server has an open SSH session that is currently
+    # processing any channels. If +include_invisible+ is +false+ (the default)
+    # then invisible channels (such as those created by port forwarding) will
+    # not be counted; otherwise, they will be.
     def busy?(include_invisible=false)
       servers.any? { |server| server.busy?(include_invisible) }
     end
 
     alias :loop_forever :loop
 
+    # Run the aggregated event loop for all open server sessions, until the given
+    # block returns +false+. If no block is given, the loop will run for as
+    # long as #busy? returns +true+ (in other words, for as long as there are
+    # any (non-invisible) channels open).
     def loop(wait=nil, &block)
       running = block || Proc.new { |c| busy? }
       loop_forever { break unless process(wait, &running) }
     end
 
-    def preprocess(&block)
-      return false if block && !block[self]
-      servers.each { |server| server.preprocess }
-      block.nil? || block[self]
-    end
-
-    def postprocess(readers, writers)
-      servers.each { |server| server.postprocess(readers, writers) }
-      true
-    end
-
+    # Run a single iteration of the aggregated event loop for all open server
+    # sessions. The +wait+ parameter indicates how long to wait for an event
+    # to appear on any of the different sessions; +nil+ (the default) means
+    # "wait forever". If the block is given, then it will be used to determine
+    # whether #process returns +true+ (the block did not return +false+), or
+    # +false+ (the block returned +false+).
     def process(wait=nil, &block)
       return false unless preprocess(&block)
 
@@ -153,11 +323,45 @@ module Net; module SSH; module Multi
       return postprocess(readers, writers)
     end
 
+    # Sends a global request to all active sessions (see #active_sessions).
+    # This can be used to (e.g.) ping the remote servers to prevent them from
+    # timing out.
+    #
+    #   session.send_global_request("keep-alive@openssh.com")
+    #
+    # If a block is given, it will be invoked when the server responds, with
+    # two arguments: the Net::SSH connection that is responding, and a boolean
+    # indicating whether the request succeeded or not.
     def send_global_request(type, *extra, &callback)
       active_sessions.each { |ssh| ssh.send_global_request(type, *extra, &callback) }
       self
     end
 
+    # Asks all active sessions (see #active_sessions) to open a new channel.
+    # When each server responds, the +on_confirm+ block will be invoked with
+    # a single argument, the channel object for that server. This means that
+    # the block will be invoked one time for each active session.
+    #
+    # All new channels will be collected and returned, aggregated into a new
+    # Net::SSH::Multi::Channel instance.
+    #
+    # Note that the channels are "enhanced" slightly--they have two properties
+    # set on them automatically, to make dealing with them in a multi-session
+    # environment slightly easier:
+    #
+    # * :server => the Net::SSH::Multi::Server instance that spawned the channel
+    # * :host => the host name of the server
+    #
+    # Having access to these things lets you more easily report which host
+    # (e.g.) data was received from:
+    #
+    #   session.open_channel do |channel|
+    #     channel.exec "command" do |ch, success|
+    #       ch.on_data do |ch, data|
+    #         puts "got data #{data} from #{ch[:host]}"
+    #       end
+    #     end
+    #   end
     def open_channel(type="session", *extra, &on_confirm)
       channels = active_sessions.map do |ssh|
         ssh.open_channel(type, *extra) do |c|
@@ -169,6 +373,34 @@ module Net; module SSH; module Multi
       Multi::Channel.new(self, channels)
     end
 
+    # A convenience method for executing a command on multiple hosts and
+    # either displaying or capturing the output. It opens a channel on all
+    # active sessions (see #open_channel and #active_sessions), and then
+    # executes a command on each channel (Net::SSH::Connection::Channel#exec).
+    #
+    # If a block is given, it will be invoked whenever data is received across
+    # the channel, with three arguments: the channel object, a symbol identifying
+    # which output stream the data was received on (+:stdout+ or +:stderr+)
+    # and a string containing the data that was received:
+    #
+    #   session.exec("command") do |ch, stream, data|
+    #     puts "[#{ch[:host]} : #{stream}] #{data}"
+    #   end
+    #
+    # If no block is given, all output will be written to +$stdout+ or
+    # +$stderr+, as appropriate.
+    #
+    # Note that #exec will also capture the exit status of the process in the
+    # +:exit_status+ property of each channel. Since #exec returns all of the
+    # channels in a Net::SSH::Multi::Channel object, you can check for the
+    # exit status like this:
+    #
+    #   channel = session.exec("command") { ... }
+    #   channel.wait
+    #
+    #   if channel.any? { |c| c[:exit_status] != 0 }
+    #     puts "executing failed on at least one host!"
+    #   end
     def exec(command, &block)
       open_channel do |channel|
         channel.exec(command) do |ch, success|
@@ -201,12 +433,20 @@ module Net; module SSH; module Multi
       end
     end
 
-    private
+    # Runs the preprocess stage on all servers. Returns false if the block
+    # returns false, and true if there either is no block, or it returns true.
+    # This is called as part of the #process method.
+    def preprocess(&block) #:nodoc:
+      return false if block && !block[self]
+      servers.each { |server| server.preprocess }
+      block.nil? || block[self]
+    end
 
-      def sessions_for(servers)
-        threads = servers.map { |server| Thread.new { server.session(true) } if server.session.nil? }
-        threads.each { |thread| thread.join if thread }
-        servers.map { |server| server.session }
-      end
+    # Runs the postprocess stage on all servers. Always returns true. This is
+    # called as part of the #process method.
+    def postprocess(readers, writers) #:nodoc:
+      servers.each { |server| server.postprocess(readers, writers) }
+      true
+    end
   end
 end; end; end
\ No newline at end of file