Add setup.py and move exts/ inside baserockimport package

It's slightly annoying during development, but the exts/ must be inside
the package or it would be installed somewhere silly like
/usr/lib/python2.7/site-packages/exts.

10 files changed, 1157 insertions, 0 deletions

diff --git a/baserockimport/exts/importer_base.py b/baserockimport/exts/importer_base.py
new file mode 100644
index 0000000..5e75f65
--- /dev/null
+++ b/baserockimport/exts/importer_base.py
@@ -0,0 +1,77 @@
+# Base class for import tools written in Python.
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import logging
+import os
+import sys
+
+
+class ImportException(Exception):
+    pass
+
+
+class ImportExtension(object):
+    '''A base class for import extensions.
+
+    A subclass should subclass this class, and add a ``process_args`` method.
+
+    Note that it is not necessary to subclass this class for import extensions.
+    This class is here just to collect common code.
+
+    '''
+
+    def __init__(self):
+        self.setup_logging()
+
+    def setup_logging(self):
+        '''Direct all logging output to MORPH_LOG_FD, if set.
+
+        This file descriptor is read by Morph and written into its own log
+        file.
+
+        This overrides cliapp's usual configurable logging setup.
+
+        '''
+        log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0))
+
+        if log_write_fd == 0:
+            return
+
+        formatter = logging.Formatter('%(message)s')
+
+        handler = logging.StreamHandler(os.fdopen(log_write_fd, 'w'))
+        handler.setFormatter(formatter)
+
+        logger = logging.getLogger()
+        logger.addHandler(handler)
+        logger.setLevel(logging.DEBUG)
+
+    def process_args(self, args):
+        raise NotImplementedError()
+
+    def local_data_path(self, filename):
+        '''Return path to 'file' inside the package data/ directory. '''
+        script_dir = os.path.dirname(os.path.realpath(__file__))
+        return os.path.join(script_dir, '..', 'data', filename)
+
+    def run(self):
+        try:
+            self.process_args(sys.argv[1:])
+        except ImportException as e:
+            sys.stderr.write('ERROR: %s\n' % e.message)
+            sys.exit(1)
diff --git a/baserockimport/exts/importer_base.rb b/baserockimport/exts/importer_base.rb
new file mode 100644
index 0000000..98eb05c
--- /dev/null
+++ b/baserockimport/exts/importer_base.rb
@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+#
+# Base class for importers written in Ruby
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'json'
+require 'logger'
+require 'optparse'
+require 'yaml'
+
+module Importer
+  class Base
+    private
+
+    def create_option_parser(banner, description)
+      opts = OptionParser.new
+
+      opts.banner = banner
+
+      opts.on('-?', '--help', 'print this help') do
+        puts opts
+        print "\n", description
+        exit 255
+      end
+    end
+
+    def log
+      @logger ||= create_logger
+    end
+
+    def error(message)
+      log.error(message)
+      STDERR.puts(message)
+    end
+
+    def local_data_path(file)
+      # Return the path to 'file' relative to the currently running program.
+      # Used as a simple mechanism of finding local data files.
+      script_dir = File.dirname(__FILE__)
+      File.join(script_dir, '..', 'data', file)
+    end
+
+    def write_lorry(file, lorry)
+      format_options = { :indent => '    ' }
+      file.puts(JSON.pretty_generate(lorry, format_options))
+    end
+
+    def write_morph(file, morph)
+      file.write(YAML.dump(morph))
+    end
+
+    def write_dependencies(file, dependencies)
+      format_options = { :indent => '    ' }
+      file.puts(JSON.pretty_generate(dependencies, format_options))
+    end
+
+    def create_logger
+      # Use the logger that was passed in from the 'main' import process, if
+      # detected.
+      log_fd = ENV['MORPH_LOG_FD']
+      if log_fd
+        log_stream = IO.new(Integer(log_fd), 'w')
+        logger = Logger.new(log_stream)
+        logger.level = Logger::DEBUG
+        logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" }
+      else
+        logger = Logger.new('/dev/null')
+      end
+      logger
+    end
+  end
+end
diff --git a/baserockimport/exts/importer_bundler_extensions.rb b/baserockimport/exts/importer_bundler_extensions.rb
new file mode 100644
index 0000000..034b3c2
--- /dev/null
+++ b/baserockimport/exts/importer_bundler_extensions.rb
@@ -0,0 +1,87 @@
+#!/usr/bin/env ruby
+#
+# Extensions to Bundler library which allow using it in importers.
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'bundler'
+
+class << Bundler
+  def default_gemfile
+    # This is a hack to make things not crash when there's no Gemfile
+    Pathname.new('.')
+  end
+end
+
+module Importer
+  module BundlerExtensions
+    def create_bundler_definition_for_gemspec(gem_name)
+      # Using the real Gemfile doesn't get great results, because people can put
+      # lots of stuff in there that is handy for developers to have but
+      # irrelevant if you just want to produce a .gem. Also, there is only one
+      # Gemfile per repo, but a repo may include multiple .gemspecs that we want
+      # to process individually. Also, some projects don't use Bundler and may
+      # not have a Gemfile at all.
+      #
+      # Instead of reading the real Gemfile, invent one that simply includes the
+      # chosen .gemspec. If present, the Gemfile.lock will be honoured.
+      fake_gemfile = Bundler::Dsl.new
+      fake_gemfile.source('https://rubygems.org')
+      begin
+        fake_gemfile.gemspec({:name => gem_name})
+      rescue Bundler::InvalidOption
+        error "Did not find #{gem_name}.gemspec in current directory."
+        exit 1
+      end
+
+      fake_gemfile.to_definition('Gemfile.lock', true)
+    end
+
+    def get_spec_for_gem(specs, gem_name)
+      found = specs[gem_name].select {|s| Gem::Platform.match(s.platform)}
+      if found.empty?
+        raise "No Gemspecs found matching '#{gem_name}'"
+      elsif found.length != 1
+        raise "Unsure which Gem to use for #{gem_name}, got #{found}"
+      end
+      found[0]
+    end
+
+    def spec_is_from_current_source_tree(spec, source_dir)
+      Dir.chdir(source_dir) do
+        spec.source.instance_of? Bundler::Source::Path and
+          File.identical?(spec.source.path, '.')
+      end
+    end
+
+    def validate_spec(spec, source_dir_name, expected_version)
+      if not spec_is_from_current_source_tree(spec, source_dir_name)
+        error "Specified gem '#{spec.name}' doesn't live in the source in " \
+              "'#{source_dir_name}'"
+        log.debug "SPEC: #{spec.inspect} #{spec.source}"
+        exit 1
+      end
+
+      if expected_version != nil && spec.version != expected_version
+        # This check is brought to you by Coderay, which changes its version
+        # number based on an environment variable. Other Gems may do this too.
+        error "Source in #{source_dir_name} produces #{spec.full_name}, but " \
+              "the expected version was #{expected_version}."
+        exit 1
+      end
+    end
+  end
+end
diff --git a/baserockimport/exts/importer_omnibus_extensions.rb b/baserockimport/exts/importer_omnibus_extensions.rb
new file mode 100644
index 0000000..2286d35
--- /dev/null
+++ b/baserockimport/exts/importer_omnibus_extensions.rb
@@ -0,0 +1,92 @@
+# Extensions for the Omnibus tool that allow using it to generate morphologies.
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'omnibus'
+
+require 'optparse'
+require 'rubygems/commands/build_command'
+require 'rubygems/commands/install_command'
+require 'shellwords'
+
+class Omnibus::Builder
+  # It's possible to use `gem install` in build commands, which is a great
+  # way of subverting the dependency tracking Omnibus provides. It's done
+  # in `omnibus-chef/config/software/chefdk.rb`, for example.
+  #
+  # To handle this, here we extend the class that executes the build commands
+  # to detect when `gem install` is run. It uses the Gem library to turn the
+  # commandline back into a Bundler::Dependency object that we can use.
+  #
+  # We also trap `gem build` so we know when a software component is a RubyGem
+  # that should be handled by 'rubygems.to_chunk'.
+
+  class GemBuildCommandParser < Gem::Commands::BuildCommand
+    def gemspec_path(args)
+      handle_options args
+      if options[:args].length != 1
+        raise "Invalid `gem build` commandline: 1 argument expected, got " \
+               "#{options[:args]}."
+      end
+      options[:args][0]
+    end
+  end
+
+  class GemInstallCommandParser < Gem::Commands::InstallCommand
+    def dependency_list_from_commandline(args)
+      handle_options args
+
+      # `gem install foo*` is sometimes used when installing a locally built
+      # Gem, to avoid needing to know the exact version number that was built.
+      # We only care about remote Gems being installed, so anything with a '*'
+      # in its name can be ignored.
+      gem_names = options[:args].delete_if { |name| name.include?('*') }
+
+      gem_names.collect do |gem_name|
+        Bundler::Dependency.new(gem_name, options[:version])
+      end
+    end
+  end
+
+  def gem(command, options = {})
+    # This function re-implements the 'gem' function in the build-commands DSL.
+    if command.start_with? 'build'
+      parser = GemBuildCommandParser.new
+      args = Shellwords.split(command).drop(1)
+      if built_gemspec != nil
+        raise "More than one `gem build` command was run as part f the build " \
+              "process. The 'rubygems.to_chunk' program currently supports " \
+              "only one .gemspec build per chunk, so this can't be " \
+              "processed automatically."
+      end
+      @built_gemspec = parser.gemspec_path(args)
+    elsif command.start_with? 'install'
+      parser = GemInstallCommandParser.new
+      args = Shellwords.split(command).drop(1)
+      args_without_build_flags = args.take_while { |item| item != '--' }
+      gems = parser.dependency_list_from_commandline(args_without_build_flags)
+      manually_installed_rubygems.concat gems
+    end
+  end
+
+  def built_gemspec
+    @built_gemspec
+  end
+
+  def manually_installed_rubygems
+    @manually_installed_rubygems ||= []
+  end
+end
diff --git a/baserockimport/exts/omnibus.find_deps b/baserockimport/exts/omnibus.find_deps
new file mode 100755
index 0000000..8fea31d
--- /dev/null
+++ b/baserockimport/exts/omnibus.find_deps
@@ -0,0 +1,138 @@
+#!/usr/bin/env ruby
+#
+# Find dependencies for an Omnibus software component.
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'bundler'
+
+require_relative 'importer_base'
+require_relative 'importer_omnibus_extensions'
+
+BANNER = "Usage: omnibus.find_deps PROJECT_DIR PROJECT_NAME SOURCE_DIR SOFTWARE_NAME"
+
+DESCRIPTION = <<-END
+Calculate dependencies for a given Omnibus software component.
+END
+
+class OmnibusDependencyFinder < Importer::Base
+  def initialize
+    local_data = YAML.load_file(local_data_path("omnibus.yaml"))
+    @dependency_blacklist = local_data['dependency-blacklist']
+  end
+
+  def parse_options(arguments)
+    opts = create_option_parser(BANNER, DESCRIPTION)
+
+    parsed_arguments = opts.parse!(arguments)
+
+    if parsed_arguments.length != 4 and parsed_arguments.length != 5
+      STDERR.puts "Expected 4 or 5 arguments, got #{parsed_arguments}."
+      opts.parse(['-?'])
+      exit 255
+    end
+
+    project_dir, project_name, source_dir, software_name, expected_version = \
+      parsed_arguments
+    # Not yet implemented
+    #if expected_version != nil
+    #  expected_version = Gem::Version.new(expected_version)
+    #end
+    [project_dir, project_name, source_dir, software_name, expected_version]
+  end
+
+  def resolve_rubygems_deps(requirements)
+    return {} if requirements.empty?
+
+    log.info('Resolving RubyGem requirements with Bundler')
+
+    fake_gemfile = Bundler::Dsl.new
+    fake_gemfile.source('https://rubygems.org')
+
+    requirements.each do |dep|
+      fake_gemfile.gem(dep.name, dep.requirement)
+    end
+
+    definition = fake_gemfile.to_definition('Gemfile.lock', true)
+    resolved_specs = definition.resolve_remotely!
+
+    Hash[resolved_specs.collect { |spec| [spec.name, spec.version.to_s]}]
+  end
+
+  def calculate_dependencies_for_software(project, software, source_dir)
+    omnibus_deps = {}
+    rubygems_deps = {}
+
+    software.dependencies.each do |name|
+      software = Omnibus::Software.load(project, name)
+      if @dependency_blacklist.member? name
+        log.info(
+          "Not adding #{name} as a dependency as it is marked to be ignored.")
+      elsif software.fetcher.instance_of?(Omnibus::PathFetcher)
+        log.info(
+          "Not adding #{name} as a dependency: it's installed from " +
+          "a path which probably means that it is package configuration, not " +
+          "a 3rd-party component to be imported.")
+      elsif software.fetcher.instance_of?(Omnibus::NullFetcher)
+        if software.builder.built_gemspec
+          log.info(
+            "Adding #{name} as a RubyGem dependency because it builds " +
+            "#{software.builder.built_gemspec}")
+          rubygems_deps[name] = software.version
+        else
+          log.info(
+            "Not adding #{name} as a dependency: no sources listed.")
+        end
+      else
+        omnibus_deps[name] = software.version
+      end
+    end
+
+    gem_requirements = software.builder.manually_installed_rubygems
+    rubygems_deps = resolve_rubygems_deps(gem_requirements)
+
+    {
+      "omnibus" => {
+        # FIXME: are these build or runtime dependencies? We'll assume both.
+        "build-dependencies" => omnibus_deps,
+        "runtime-dependencies" => omnibus_deps,
+      },
+      "rubygems" => {
+        "build-dependencies" => {},
+        "runtime-dependencies" => rubygems_deps,
+      }
+    }
+  end
+
+  def run
+    project_dir, project_name, source_dir, software_name = parse_options(ARGV)
+
+    log.info("Calculating dependencies for #{software_name} from project " +
+             "#{project_name}, defined in #{project_dir}")
+
+    Dir.chdir(project_dir)
+
+    project = Omnibus::Project.load(project_name)
+
+    software = Omnibus::Software.load(@project, software_name)
+
+    dependencies = calculate_dependencies_for_software(
+      project, software, source_dir)
+    write_dependencies(STDOUT, dependencies)
+  end
+end
+
+OmnibusDependencyFinder.new.run
diff --git a/baserockimport/exts/omnibus.to_chunk b/baserockimport/exts/omnibus.to_chunk
new file mode 100755
index 0000000..5e527a9
--- /dev/null
+++ b/baserockimport/exts/omnibus.to_chunk
@@ -0,0 +1,134 @@
+#!/usr/bin/env ruby
+#
+# Create a chunk morphology to build Omnibus software in Baserock
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require_relative 'importer_base'
+require_relative 'importer_omnibus_extensions'
+
+BANNER = "Usage: omnibus.to_chunk PROJECT_DIR PROJECT_NAME SOURCE_DIR SOFTWARE_NAME"
+
+DESCRIPTION = <<-END
+Generate a .morph file for a given Omnibus software component.
+END
+
+class OmnibusChunkMorphologyGenerator < Importer::Base
+  def parse_options(arguments)
+    opts = create_option_parser(BANNER, DESCRIPTION)
+
+    parsed_arguments = opts.parse!(arguments)
+
+    if parsed_arguments.length != 4 and parsed_arguments.length != 5
+      STDERR.puts "Expected 4 or 5 arguments, got #{parsed_arguments}."
+      opts.parse(['-?'])
+      exit 255
+    end
+
+    project_dir, project_name, source_dir, software_name, expected_version = \
+      parsed_arguments
+    # Not yet implemented
+    #if expected_version != nil
+    #  expected_version = Gem::Version.new(expected_version)
+    #end
+    [project_dir, project_name, source_dir, software_name, expected_version]
+  end
+
+  class SubprocessError < RuntimeError
+  end
+
+  def run_tool_capture_output(tool_name, *args)
+    scripts_dir = local_data_path('.')
+    tool_path = local_data_path(tool_name)
+
+    # FIXME: something breaks when we try to share this FD, it's not
+    # ideal that the subprocess doesn't log anything, though.
+    env_changes = {'MORPH_LOG_FD' => nil}
+
+    command = [[tool_path, tool_name], *args]
+    log.info("Running #{command.join(' ')} in #{scripts_dir}")
+
+    text = IO.popen(
+      env_changes, command, :chdir => scripts_dir, :err => [:child, :out]
+    ) do |io|
+      io.read
+    end
+
+    if $? == 0
+      text
+    else
+      raise SubprocessError, text
+    end
+  end
+
+  def generate_chunk_morph_for_rubygems_software(software, source_dir)
+    # This is a better heuristic for getting the name of the Gem
+    # than the software name, it seems ...
+    gem_name = software.relative_path
+
+    text = run_tool_capture_output('rubygems.to_chunk', source_dir, gem_name)
+    log.debug("Text from output: #{text}, result #{$?}")
+
+    morphology = YAML::load(text)
+    return morphology
+  rescue SubprocessError => e
+    error "Tried to import #{software.name} as a RubyGem, got the " \
+          "following error from rubygems.to_chunk: #{e.message}"
+    exit 1
+  end
+
+  def generate_chunk_morph_for_software(project, software, source_dir)
+    if software.builder.built_gemspec != nil
+      morphology = generate_chunk_morph_for_rubygems_software(software,
+                                                              source_dir)
+    else
+      morphology = {
+        "name" => software.name,
+        "kind" => "chunk",
+        "description" => "Automatically generated by omnibus.to_chunk"
+      }
+    end
+
+    # Possibly this tool should look at software.build and
+    # generate suitable configure, build and install-commands.
+    # For now: don't bother!
+
+    if software.description
+      morphology['description'] = software.description + '\n\n' +
+        morphology['description']
+    end
+
+    morphology
+  end
+
+  def run
+    project_dir, project_name, source_dir, software_name = parse_options(ARGV)
+
+    log.info("Creating chunk morph for #{software_name} from project " +
+             "#{project_name}, defined in #{project_dir}")
+
+    Dir.chdir(project_dir)
+
+    project = Omnibus::Project.load(project_name)
+
+    software = Omnibus::Software.load(@project, software_name)
+
+    morph = generate_chunk_morph_for_software(project, software, source_dir)
+    write_morph(STDOUT, morph)
+  end
+end
+
+OmnibusChunkMorphologyGenerator.new.run
diff --git a/baserockimport/exts/omnibus.to_lorry b/baserockimport/exts/omnibus.to_lorry
new file mode 100755
index 0000000..256f924
--- /dev/null
+++ b/baserockimport/exts/omnibus.to_lorry
@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+#
+# Create a Baserock .lorry file for a given Omnibus software component
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'bundler'
+require 'omnibus'
+
+require 'optparse'
+require 'rubygems/commands/install_command'
+require 'shellwords'
+
+require_relative 'importer_base'
+
+BANNER = "Usage: omnibus.to_lorry PROJECT_DIR PROJECT_NAME SOFTWARE_NAME"
+
+DESCRIPTION = <<-END
+Generate a .lorry file for a given Omnibus software component.
+END
+
+class OmnibusLorryGenerator < Importer::Base
+  def parse_options(arguments)
+    opts = create_option_parser(BANNER, DESCRIPTION)
+
+    parsed_arguments = opts.parse!(arguments)
+
+    if parsed_arguments.length != 3
+      STDERR.puts "Expected 3 arguments, got #{parsed_arguments}."
+      opts.parse(['-?'])
+      exit 255
+    end
+
+    project_dir, project_name, software_name = parsed_arguments
+    [project_dir, project_name, software_name]
+  end
+
+  def generate_lorry_for_software(software)
+    lorry_body = {
+      'x-products-omnibus' => [software.name]
+    }
+
+    if software.source and software.source.member? :git
+      lorry_body.update({
+        'type' => 'git',
+        'url' => software.source[:git],
+      })
+    elsif software.source and software.source.member? :url
+      lorry_body.update({
+        'type' => 'tarball',
+        'url' => software.source[:url],
+        # lorry doesn't validate the checksum right now, but maybe it should.
+      'x-md5' => software.source[:md5],
+      })
+    else
+      error "Couldn't generate lorry file from source '#{software.source.inspect}'"
+      exit 1
+    end
+
+    { software.name => lorry_body }
+  end
+
+  def run
+    project_dir, project_name, software_name = parse_options(ARGV)
+
+    log.info("Creating lorry for #{software_name} from project " +
+             "#{project_name}, defined in #{project_dir}")
+
+    Dir.chdir(project_dir)
+
+    project = Omnibus::Project.load(project_name)
+
+    software = Omnibus::Software.load(project, software_name)
+
+    lorry = generate_lorry_for_software(software)
+
+    write_lorry(STDOUT, lorry)
+  end
+end
+
+OmnibusLorryGenerator.new.run
diff --git a/baserockimport/exts/rubygems.find_deps b/baserockimport/exts/rubygems.find_deps
new file mode 100755
index 0000000..228c88b
--- /dev/null
+++ b/baserockimport/exts/rubygems.find_deps
@@ -0,0 +1,114 @@
+#!/usr/bin/env ruby
+#
+# Find dependencies for a RubyGem.
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require_relative 'importer_base'
+require_relative 'importer_bundler_extensions'
+
+BANNER = "Usage: rubygems.find_deps SOURCE_DIR GEM_NAME [VERSION]"
+
+DESCRIPTION = <<-END
+This tool looks for a .gemspec file for GEM_NAME in SOURCE_DIR, and outputs the
+set of RubyGems dependencies required to build it. It will honour a
+Gemfile.lock file if one is present.
+
+It is intended for use with the `baserock-import` tool.
+END
+
+class RubyGemDependencyFinder < Importer::Base
+  include Importer::BundlerExtensions
+
+  def initialize
+    local_data = YAML.load_file(local_data_path("rubygems.yaml"))
+    @build_dependency_whitelist = local_data['build-dependency-whitelist']
+  end
+
+  def parse_options(arguments)
+    opts = create_option_parser(BANNER, DESCRIPTION)
+
+    parsed_arguments = opts.parse!(arguments)
+
+    if parsed_arguments.length != 2 && parsed_arguments.length != 3
+      STDERR.puts "Expected 2 or 3 arguments, got #{parsed_arguments}."
+      opts.parse(['-?'])
+      exit 255
+    end
+
+    source_dir, gem_name, expected_version = parsed_arguments
+    source_dir = File.absolute_path(source_dir)
+    if expected_version != nil
+      expected_version = Gem::Version.new(expected_version.dup)
+    end
+    [source_dir, gem_name, expected_version]
+  end
+
+  def build_deps_for_gem(spec)
+    deps = spec.dependencies.select do |d|
+      d.type == :development && @build_dependency_whitelist.member?(d.name)
+    end
+  end
+
+  def runtime_deps_for_gem(spec)
+    spec.dependencies.select {|d| d.type == :runtime}
+  end
+
+  def run
+    source_dir_name, gem_name, expected_version = parse_options(ARGV)
+
+    log.info("Finding dependencies for #{gem_name} based on source code in " \
+             "#{source_dir_name}")
+
+    resolved_specs = Dir.chdir(source_dir_name) do
+      definition = create_bundler_definition_for_gemspec(gem_name)
+      definition.resolve_remotely!
+    end
+
+    spec = get_spec_for_gem(resolved_specs, gem_name)
+    validate_spec(spec, source_dir_name, expected_version)
+
+    # One might think that you could use the Bundler::Dependency.groups
+    # field to filter but it doesn't seem to be useful. Instead we go back to
+    # the Gem::Specification of the target Gem and use the dependencies fild
+    # there. We look up each dependency in the resolved_specset to find out
+    # what version Bundler has chosen of it.
+
+    def format_deps(specset, dep_list)
+      info = dep_list.collect do |dep|
+        spec = specset[dep][0]
+        [spec.name, spec.version.to_s]
+      end
+      Hash[info]
+    end
+
+    build_deps = format_deps(
+      resolved_specs, build_deps_for_gem(spec))
+    runtime_deps = format_deps(
+      resolved_specs, runtime_deps_for_gem(spec))
+
+    deps = {
+      'rubygems' => {
+        'build-dependencies' => build_deps,
+        'runtime-dependencies' => runtime_deps,
+      }
+    }
+
+    write_dependencies(STDOUT, deps)
+  end
+end
+
+RubyGemDependencyFinder.new.run
diff --git a/baserockimport/exts/rubygems.to_chunk b/baserockimport/exts/rubygems.to_chunk
new file mode 100755
index 0000000..c1a3e7c
--- /dev/null
+++ b/baserockimport/exts/rubygems.to_chunk
@@ -0,0 +1,171 @@
+#!/usr/bin/env ruby
+#
+# Create a chunk morphology to build a RubyGem in Baserock
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'bundler'
+
+require_relative 'importer_base'
+require_relative 'importer_bundler_extensions'
+
+BANNER = "Usage: rubygems.to_chunk SOURCE_DIR GEM_NAME [VERSION]"
+
+DESCRIPTION = <<-END
+This tool looks in SOURCE_DIR to generate a chunk morphology with build
+instructions for GEM_NAME. If VERSION is supplied, it is used to check that the
+build instructions will produce the expected version of the Gem.
+
+It is intended for use with the `baserock-import` tool.
+END
+
+class RubyGemChunkMorphologyGenerator < Importer::Base
+  include Importer::BundlerExtensions
+
+  def parse_options(arguments)
+    opts = create_option_parser(BANNER, DESCRIPTION)
+
+    parsed_arguments = opts.parse!(arguments)
+
+    if parsed_arguments.length != 2 && parsed_arguments.length != 3
+      STDERR.puts "Expected 2 or 3 arguments, got #{parsed_arguments}."
+      opts.parse(['-?'])
+      exit 255
+    end
+
+    source_dir, gem_name, expected_version = parsed_arguments
+    source_dir = File.absolute_path(source_dir)
+    if expected_version != nil
+      expected_version = Gem::Version.new(expected_version.dup)
+    end
+    [source_dir, gem_name, expected_version]
+  end
+
+  def chunk_name_for_gemspec(spec)
+    # Chunk names are the Gem's "full name" (name + version number), so
+    # that we don't break in the rare but possible case that two different
+    # versions of the same Gem are required for something to work. It'd be
+    # nicer to only use the full_name if we detect such a conflict.
+    spec.full_name
+  end
+
+  def is_signed_gem(spec)
+    spec.signing_key != nil
+  end
+
+  def generate_chunk_morph_for_gem(spec)
+    description = 'Automatically generated by rubygems.to_chunk'
+
+    bin_dir = "\"$DESTDIR/$PREFIX/bin\""
+    gem_dir = "\"$DESTDIR/$(gem environment home)\""
+
+    # There's more splitting to be done, but putting the docs in the
+    # correct artifact is the single biggest win for enabling smaller
+    # system images.
+    #
+    # Adding this to Morph's default ruleset is painful, because:
+    #   - Changing the default split rules triggers a rebuild of everything.
+    #   - The whole split rule code needs reworking to prevent overlaps and to
+    #     make it possible to extend rules without creating overlaps. It's
+    #     otherwise impossible to reason about.
+
+    split_rules = [
+      {
+        'artifact' => "#{spec.full_name}-doc",
+        'include' => [
+          'usr/lib/ruby/gems/\d[\w.]*/doc/.*'
+        ]
+      }
+    ]
+
+    # It'd be rather tricky to include these build instructions as a
+    # BuildSystem implementation in Morph. The problem is that there's no
+    # way for the default commands to know what .gemspec file they should
+    # be building. It doesn't help that the .gemspec may be in a subdirectory
+    # (as in Rails, for example).
+    #
+    # Note that `gem help build` says the following:
+    #
+    #   The best way to build a gem is to use a Rakefile and the
+    #   Gem::PackageTask which ships with RubyGems.
+    #
+    # It's often possible to run `rake gem`, but this may require Hoe,
+    # rake-compiler, Jeweler or other assistance tools to be present at Gem
+    # construction time. It seems that many Ruby projects that use these tools
+    # also maintain an up-to-date generated .gemspec file, which means that we
+    # can get away with using `gem build` just fine in many cases.
+    #
+    # Were we to use `setup.rb install` or `rake install`, programs that loaded
+    # with the 'rubygems' library would complain that required Gems were not
+    # installed. We must have the Gem metadata available, and `gem build; gem
+    # install` seems the easiest way to achieve that.
+
+    configure_commands = []
+
+    if is_signed_gem(spec)
+      # This is a best-guess hack for allowing unsigned builds of Gems that are
+      # normally built signed. There's no value in building signed Gems when we
+      # control the build and deployment environment, and we obviously can't
+      # provide the private key of the Gem's maintainer.
+      configure_commands <<
+        "sed -e '/cert_chain\\s*=/d' -e '/signing_key\\s*=/d' -i " \
+        "#{spec.name}.gemspec"
+    end
+
+    build_commands = [
+      "gem build #{spec.name}.gemspec",
+    ]
+
+    install_commands = [
+      "mkdir -p #{gem_dir}",
+      "gem install --install-dir #{gem_dir} --bindir #{bin_dir} " \
+        "--ignore-dependencies --local ./#{spec.full_name}.gem"
+    ]
+
+    {
+      'name' => chunk_name_for_gemspec(spec),
+      'kind' => 'chunk',
+      'description' => description,
+      'build-system' => 'manual',
+      'products' => split_rules,
+      'configure-commands' => configure_commands,
+      'build-commands' => build_commands,
+      'install-commands' => install_commands,
+    }
+  end
+
+  def run
+    source_dir_name, gem_name, expected_version = parse_options(ARGV)
+
+    log.info("Creating chunk morph for #{gem_name} based on " \
+             "source code in #{source_dir_name}")
+
+    resolved_specs = Dir.chdir(source_dir_name) do
+      # FIXME: we don't need to do this here, it'd be enough just to load
+      # the given gemspec 
+      definition = create_bundler_definition_for_gemspec(gem_name)
+      definition.resolve_remotely!
+    end
+
+    spec = get_spec_for_gem(resolved_specs, gem_name)
+    validate_spec(spec, source_dir_name, expected_version)
+
+    morph = generate_chunk_morph_for_gem(spec)
+    write_morph(STDOUT, morph)
+  end
+end
+
+RubyGemChunkMorphologyGenerator.new.run
diff --git a/baserockimport/exts/rubygems.to_lorry b/baserockimport/exts/rubygems.to_lorry
new file mode 100755
index 0000000..6807b21
--- /dev/null
+++ b/baserockimport/exts/rubygems.to_lorry
@@ -0,0 +1,164 @@
+#!/usr/bin/python
+#
+# Create a Baserock .lorry file for a given RubyGem
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import requests
+import requests_cache
+import yaml
+
+import logging
+import json
+import os
+import sys
+import urlparse
+
+from importer_base import ImportException, ImportExtension
+
+
+class GenerateLorryException(ImportException):
+    pass
+
+
+class RubyGemsWebServiceClient(object):
+    def __init__(self):
+        # Save hammering the rubygems.org API: 'requests' API calls are
+        # transparently cached in an SQLite database, instead.
+        requests_cache.install_cache('rubygems_api_cache')
+
+    def _request(self, url):
+        r = requests.get(url)
+        if r.ok:
+            return json.loads(r.text)
+        else:
+            raise GenerateLorryException(
+                'Request to %s failed: %s' % (r.url, r.reason))
+
+    def get_gem_info(self, gem_name):
+        info = self._request(
+            'http://rubygems.org/api/v1/gems/%s.json' % gem_name)
+
+        if info['name'] != gem_name:
+            # Sanity check
+            raise GenerateLorryException(
+                 'Received info for Gem "%s", requested "%s"' % info['name'],
+                  gem_name)
+
+        return info
+
+
+class RubyGemLorryGenerator(ImportExtension):
+    def __init__(self):
+        super(RubyGemLorryGenerator, self).__init__()
+
+        with open(self.local_data_path('rubygems.yaml'), 'r') as f:
+            local_data = yaml.load(f.read())
+
+        self.lorry_prefix = local_data['lorry-prefix']
+        self.known_source_uris = local_data['known-source-uris']
+
+        logging.debug(
+            "Loaded %i known source URIs from local metadata.", len(self.known_source_uris))
+
+    def process_args(self, args):
+        if len(args) != 1:
+            raise ImportException(
+                'Please call me with the name of a RubyGem as an argument.')
+
+        gem_name = args[0]
+
+        lorry = self.generate_lorry_for_gem(gem_name)
+        self.write_lorry(sys.stdout, lorry)
+
+    def find_upstream_repo_for_gem(self, gem_name, gem_info):
+        source_code_uri = gem_info['source_code_uri']
+
+        if gem_name in self.known_source_uris:
+            logging.debug('Found %s in known-source-uris', gem_name)
+            known_uri = self.known_source_uris[gem_name]
+            if source_code_uri is not None and known_uri != source_code_uri:
+                sys.stderr.write(
+                    '%s: Hardcoded source URI %s doesn\'t match spec URI %s\n' %
+                    (gem_name, known_uri, source_code_uri))
+            return known_uri
+
+        if source_code_uri is not None and len(source_code_uri) > 0:
+            logging.debug('Got source_code_uri %s', source_code_uri)
+            if source_code_uri.endswith('/tree'):
+                source_code_uri = source_code_uri[:-len('/tree')]
+
+            return source_code_uri
+
+        homepage_uri = gem_info['homepage_uri']
+        if homepage_uri is not None and len(homepage_uri) > 0:
+            logging.debug('Got homepage_uri %s', source_code_uri)
+            netloc = urlparse.urlsplit(homepage_uri)[1]
+            if netloc == 'github.com':
+                return homepage_uri
+
+        # Further possible leads on locating source code.
+        # http://ruby-toolbox.com/projects/$gemname -> sometimes contains an
+        #   upstream link, even if the gem info does not.
+        # https://github.com/search?q=$gemname -> often the first result is
+        #   the correct one, but you can never know.
+
+        raise GenerateLorryException(
+            "Gem metadata for '%s' does not point to its source code "
+            "repository." % gem_name)
+
+    def project_name_from_repo(self, repo_url):
+        if repo_url.endswith('/tree/master'):
+            repo_url = repo_url[:-len('/tree/master')]
+        if repo_url.endswith('/'):
+            repo_url = repo_url[:-1]
+        if repo_url.endswith('.git'):
+            repo_url = repo_url[:-len('.git')]
+        return os.path.basename(repo_url)
+
+    def generate_lorry_for_gem(self, gem_name):
+        rubygems_client = RubyGemsWebServiceClient()
+
+        gem_info = rubygems_client.get_gem_info(gem_name)
+
+        gem_source_url = self.find_upstream_repo_for_gem(gem_name, gem_info)
+        logging.info('Got URL <%s> for %s', gem_source_url, gem_name)
+
+        project_name = self.project_name_from_repo(gem_source_url)
+        lorry_name = self.lorry_prefix + project_name
+
+        # One repo may produce multiple Gems. It's up to the caller to merge
+        # multiple .lorry files that get generated for the same repo.
+
+        lorry = {
+            lorry_name: {
+                'type': 'git',
+                'url': gem_source_url,
+                'x-products-rubygems': [gem_name]
+            }
+        }
+
+        return lorry
+
+    def write_lorry(self, stream, lorry):
+        json.dump(lorry, stream, indent=4)
+        # Needed so the morphlib.extensions code will pick up the last line.
+        stream.write('\n')
+
+
+if __name__ == '__main__':
+    RubyGemLorryGenerator().run()