#!/usr/bin/env ruby # # Create a chunk morphology to integrate a RubyGem in Baserock # # Copyright (C) 2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. require 'bundler' require 'logger' require 'optparse' require 'yaml' # Log information was passed in from the main import process, probably. # This global constant approach seems a little ugly, but it seems to be # recommended here: # # log_fd = ENV['MORPH_LOG_FD'] if log_fd log_stream = IO.new(Integer(log_fd), 'w') Log = Logger.new(log_stream) Log.level = Logger::DEBUG Log.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" } else Log = Logger.new('/dev/null') end class << Bundler def default_gemfile # This is a hack to make things not crash when there's no Gemfile Pathname.new('.') end end def spec_is_from_current_source_tree(spec) spec.source.instance_of? Bundler::Source::Path and spec.source.path.fnmatch?('.') end class RubyGemChunkMorphologyGenerator def initialize local_data = YAML.load_file("rubygems.yaml") @build_dependency_whitelist = local_data['build-dependency-whitelist'] end def parse_options(arguments) # No options so far .. opts = OptionParser.new opts.banner = "Usage: rubygems.to_chunk SOURCE_DIR GEM_NAME [VERSION]" opts.separator "" opts.separator "This tool reads the Gemfile and optionally the " + "Gemfile.lock from a Ruby project " opts.separator "source tree in SOURCE_DIR. It outputs a chunk " + "morphology for GEM_NAME on stdout." opts.separator "If VERSION is supplied, it is used to check that " + "the build instructions will" opts.separator "produce the expected version of the Gem." opts.separator "" opts.separator "It is intended for use with the `baserock-import` tool." parsed_arguments = opts.parse!(arguments) if parsed_arguments.length != 2 && parsed_arguments.length != 3 STDERR.puts opts.help exit 1 end source_dir, gem_name, expected_version = parsed_arguments if expected_version != nil expected_version = Gem::Version.new(expected_version) end [source_dir, gem_name, expected_version] end def error(message) Log.error(message) STDERR.puts(message) end def load_local_gemspecs() # Look for .gemspec files in the source repo. # # If there is no .gemspec, but you set 'name' and 'version' then # inside Bundler::Source::Path.load_spec_files this call will create a # fake gemspec matching that name and version. That's probably not useful. dir = '.' source = Bundler::Source::Path.new({ 'path' => dir, }) Log.info "Loaded #{source.specs.count} specs from source dir." source.specs.each do |spec| Log.debug " * #{spec.inspect} #{spec.dependencies.inspect}" end source end def get_spec_for_gem(specs, gem_name) found = specs[gem_name].select {|s| Gem::Platform.match(s.platform)} if found.empty? raise Exception, "No Gemspecs found matching '#{gem_name}'" elsif found.length != 1 raise Exception, "Unsure which Gem to use for #{gem_name}, got #{found}" end found[0] end def chunk_name_for_gemspec(spec) # Chunk names are the Gem's "full name" (name + version number), so # that we don't break in the rare but possible case that two different # versions of the same Gem are required for something to work. It'd be # nicer to only use the full_name if we detect such a conflict. spec.full_name end def is_signed_gem(spec) spec.signing_key != nil end def generate_chunk_morph_for_gem(spec) description = 'Automatically generated by rubygems.to_chunk' bin_dir = "\"$DESTDIR/$PREFIX/bin\"" gem_dir = "\"$DESTDIR/$(gem environment home)\"" # There's more splitting to be done, but putting the docs in the # correct artifact is the single biggest win for enabling smaller # system images. # # Adding this to Morph's default ruleset is painful, because: # - Changing the default split rules triggers a rebuild of everything. # - The whole split rule code needs reworking to prevent overlaps and to # make it possible to extend rules without creating overlaps. It's # otherwise impossible to reason about. split_rules = [ { 'artifact' => "#{spec.full_name}-doc", 'include' => [ 'usr/lib/ruby/gems/\d[\w.]*/doc/.*' ] } ] # It'd be rather tricky to include these build instructions as a # BuildSystem implementation in Morph. The problem is that there's no # way for the default commands to know what .gemspec file they should # be building. It doesn't help that the .gemspec may be in a subdirectory # (as in Rails, for example). # # Note that `gem help build` says the following: # # The best way to build a gem is to use a Rakefile and the # Gem::PackageTask which ships with RubyGems. # # It's often possible to run `rake gem`, but this may require Hoe, # rake-compiler, Jeweler or other assistance tools to be present at Gem # construction time. It seems that many Ruby projects that use these tools # also maintain an up-to-date generated .gemspec file, which means that we # can get away with using `gem build` just fine in many cases. # # Were we to use `setup.rb install` or `rake install`, programs that loaded # with the 'rubygems' library would complain that required Gems were not # installed. We must have the Gem metadata available, and `gem build; gem # install` seems the easiest way to achieve that. configure_commands = [] if is_signed_gem(spec) # This is a best-guess hack for allowing unsigned builds of Gems that are # normally built signed. There's no value in building signed Gems when we # control the build and deployment environment, and we obviously can't # provide the private key of the Gem's maintainer. configure_commands << "sed -e '/cert_chain\\s*=/d' -e '/signing_key\\s*=/d' -i " + "#{spec.name}.gemspec" end build_commands = [ "gem build #{spec.name}.gemspec", ] install_commands = [ "mkdir -p #{gem_dir}", "gem install --install-dir #{gem_dir} --bindir #{bin_dir} " + "--ignore-dependencies --local ./#{spec.full_name}.gem" ] { 'name' => chunk_name_for_gemspec(spec), 'kind' => 'chunk', 'description' => description, 'build-system' => 'manual', 'products' => split_rules, 'configure-commands' => configure_commands, 'build-commands' => build_commands, 'install-commands' => install_commands, } end def build_deps_for_gem(spec) deps = spec.dependencies.select do |d| d.type == :development && @build_dependency_whitelist.member?(d.name) end end def runtime_deps_for_gem(spec) spec.dependencies.select {|d| d.type == :runtime} end def write_morph(file, morph) file.write(YAML.dump(morph)) end def run source_dir_name, gem_name, expected_version = parse_options(ARGV) Log.info("Creating chunk morph for #{gem_name} based on " + "source code in #{source_dir_name}") Dir.chdir(source_dir_name) # Instead of reading the real Gemfile, invent one that simply includes the # chosen .gemspec. If present, the Gemfile.lock will be honoured. fake_gemfile = Bundler::Dsl.new fake_gemfile.source('https://rubygems.org') fake_gemfile.gemspec({:name => gem_name}) definition = fake_gemfile.to_definition('Gemfile.lock', true) resolved_specs = definition.resolve_remotely! spec = get_spec_for_gem(resolved_specs, gem_name) if not spec_is_from_current_source_tree(spec) error "Specified gem '#{spec.name}' doesn't live in the source in " + "'#{source_dir_name}'" Log.debug "SPEC: #{spec.inspect} #{spec.source}" exit 1 end if expected_version != nil && spec.version != expected_version # This check is brought to you by Coderay, which changes its version # number based on an environment variable. Other Gems may do this too. error "Source in #{source_dir_name} produces #{spec.full_name}, but " + "the expected version was #{expected_version}." exit 1 end morph = generate_chunk_morph_for_gem(spec) # One might think that you could use the Bundler::Dependency.groups # field to filter but it doesn't seem to be useful. Instead we go back to # the Gem::Specification of the target Gem and use the dependencies fild # there. We look up each dependency in the resolved_specset to find out # what version Bundler has chosen of it. def format_deps_for_morphology(specset, dep_list) info = dep_list.collect do |dep| spec = specset[dep][0] [spec.name, spec.version.to_s] end Hash[info] end build_deps = format_deps_for_morphology( resolved_specs, build_deps_for_gem(spec)) runtime_deps = format_deps_for_morphology( resolved_specs, runtime_deps_for_gem(spec)) morph['x-build-dependencies-rubygems'] = build_deps morph['x-runtime-dependencies-rubygems'] = runtime_deps write_morph(STDOUT, morph) end end RubyGemChunkMorphologyGenerator.new.run