Merge branch 'sam/final-fixes'

7 files changed, 157 insertions, 52 deletions

diff --git a/README.omnibus b/README.omnibus
index 840bbab..54d0577 100644
--- a/README.omnibus
+++ b/README.omnibus
@@ -10,8 +10,42 @@ First, clone the Git repository corresponding to the Omnibus project you want
 to import. For example, if you want to import the Chef Server, clone:
 <https://github.com/opscode/omnibus-chef-server>
 
-As per Omnibus' instructions, you should then run `bundle install --binstubs`
-in the checkout to make available the various dependent repos and Gems of the
-project definitions.
+You should then run `bundle install --binstubs --path=~/.gem/ruby/2.0.0` in the
+checkout to make available the various dependent repos and Gems of the project
+definitions.
 
+The extra '--path' flag is needed because currently in Baserock you'll be
+running as the 'root' user, which Bundler and Gem will take as a sign to
+install the Gems it installs into /usr. It's not recommended to manually
+install stuff into /usr in Baserock systems: it makes it harder to reason
+about their provenance and behaviour, the changes will not persist across
+system upgrades, and it will probably not be possible in future (even as
+'root').  By installing the Gems into your home directory (as would happen
+if you ran Bundler as non-root) you avoid these issues.
 
+Omnibus packaging often includes components which already exist in other
+Baserock strata. You will need to resolve this manually after running the
+import tool. If you find that the import is failing to generate a stratum
+because of errors in some components that you're not going to use anyway,
+you can use the `--force-stratum-generation` flag to ignore those errors
+and get on with the integration.
+
+Errors you might see
+--------------------
+
+If you see errors from Bundler itself about missing Gems, such as this one:
+
+    rubygems.to_chunk failed with code 1: /usr/lib/ruby/gems/2.0.0/gems/bundler-1.6.2/lib/bundler/spec_set.rb:92:in `block in materialize': Could not find addressable-2.3.6 in any of the sources (Bundler::GemNotFound)
+        from /usr/lib/ruby/gems/2.0.0/gems/bundler-1.6.2/lib/bundler/spec_set.rb:85:in `map!'
+        from /usr/lib/ruby/gems/2.0.0/gems/bundler-1.6.2/lib/bundler/spec_set.rb:85:in `materialize'
+        from /usr/lib/ruby/gems/2.0.0/gems/bundler-1.6.2/lib/bundler/definition.rb:133:in `specs'
+        from /usr/lib/ruby/gems/2.0.0/gems/bundler-1.6.2/lib/bundler/definition.rb:178:in `specs_for'
+        from /usr/lib/ruby/gems/2.0.0/gems/bundler-1.6.2/lib/bundler/definition.rb:167:in `requested_specs'
+        from /usr/lib/ruby/gems/2.0.0/gems/bundler-1.6.2/lib/bundler/environment.rb:18:in `requested_specs'
+        from /usr/lib/ruby/gems/2.0.0/gems/bundler-1.6.2/lib/bundler/runtime.rb:13:in `setup'
+        from /usr/lib/ruby/gems/2.0.0/gems/bundler-1.6.2/lib/bundler.rb:120:in `setup'
+        from /usr/lib/ruby/gems/2.0.0/gems/bundler-1.6.2/lib/bundler/setup.rb:17:in `<top (required)>'
+        from /usr/lib/ruby/2.0.0/rubygems/core_ext/kernel_require.rb:55:in `require'
+        from /usr/lib/ruby/2.0.0/rubygems/core_ext/kernel_require.rb:55:in `require'
+
+Try rerunning the Bundler `install` command as described above.
diff --git a/baserockimport/app.py b/baserockimport/app.py
index 6ae3424..6f4d7c3 100644
--- a/baserockimport/app.py
+++ b/baserockimport/app.py
@@ -44,6 +44,11 @@ class BaserockImportApplication(cliapp.Application):
                              metavar="PATH",
                              default=os.path.abspath('./lorry-working-dir'))
 
+        self.settings.boolean(['force-stratum-generation', 'force-stratum'],
+                              "always create a stratum, overwriting any "
+                              "existing stratum morphology, and ignoring any "
+                              "components where errors occurred during import",
+                              default=False)
         self.settings.boolean(['update-existing'],
                               "update all the checked-out Git trees and "
                               "generated definitions",
@@ -76,7 +81,7 @@ class BaserockImportApplication(cliapp.Application):
         self.add_subcommand('omnibus', self.import_omnibus,
                             arg_synopsis='REPO PROJECT_NAME SOFTWARE_NAME')
         self.add_subcommand('rubygems', self.import_rubygems,
-                            arg_synopsis='GEM_NAME')
+                            arg_synopsis='GEM_NAME [GEM_VERSION]')
 
         self.stdout_has_colours = self._stream_has_colours(sys.stdout)
 
@@ -165,12 +170,16 @@ class BaserockImportApplication(cliapp.Application):
 
     def import_rubygems(self, args):
         '''Import one or more RubyGems.'''
-        if len(args) != 1:
+        if len(args) not in [1, 2]:
             raise cliapp.AppException(
-                'Please pass the name of a RubyGem on the commandline.')
+                'Please pass the name and version of a RubyGem on the '
+                'commandline.')
+
+        goal_name = args[0]
+        goal_version = args[1] if len(args) == 2 else 'master'
 
         loop = baserockimport.mainloop.ImportLoop(
             app=self,
-            goal_kind='rubygems', goal_name=args[0], goal_version='master')
+            goal_kind='rubygems', goal_name=goal_name, goal_version=goal_version)
         loop.enable_importer('rubygems')
         loop.run()
diff --git a/baserockimport/data/rubygems.yaml b/baserockimport/data/rubygems.yaml
index e1e6fcc..8c13689 100644
--- a/baserockimport/data/rubygems.yaml
+++ b/baserockimport/data/rubygems.yaml
@@ -8,9 +8,24 @@ lorry-prefix: ruby-gems/
 # tools because of the number of circular dependencies. Instead, only those
 # tools which are known to be required at Gem build time are listed as
 # build-dependencies, and any other :development dependencies are ignored.
-build-dependency-whitelist:
+#
+# This list is currently empty, because everything that was in it has been
+# added to the 'ruby' stratum and so is present implicitly.
+build-dependency-whitelist: []
+
+# List of Gems which are built into Ruby or included in the 'ruby' stratum
+# in http://git.baserock.org/cgi-bin/cgit.cgi/baserock/baserock/definitions.git
+#
+# This doesn't take the versions that are provided into account (and it'd be
+# quite a maintenance burden if it did). Thus it's not an ideal solution, as
+# something may for example depend on Rake 4.9 when Rake 2.0 is actually the
+# only thing available and this won't be noticed until the user tries to build
+# the generated definitions.
+ignore-list:
+  - bundler
   - hoe
-  # rake is bundled with Ruby, so it is not included in the whitelist.
+  - rake
+  - rake-compiler
 
 # The following Gems don't provide a source_code_uri in their Gem metadata.
 # Ideally ... they would do.
diff --git a/baserockimport/exts/importer_bundler_extensions.rb b/baserockimport/exts/importer_bundler_extensions.rb
index 034b3c2..88c82e2 100644
--- a/baserockimport/exts/importer_bundler_extensions.rb
+++ b/baserockimport/exts/importer_bundler_extensions.rb
@@ -28,7 +28,25 @@ end
 
 module Importer
   module BundlerExtensions
-    def create_bundler_definition_for_gemspec(gem_name)
+    def locate_gemspec(gem_name, path)
+      target = "#{gem_name}.gemspec"
+      matches = Dir["#{path}/#{Bundler::Source::Path::DEFAULT_GLOB}"].select do |filename|
+        File.basename(filename) == target
+      end
+      if matches.empty?
+        error "Did not find any files matching #{target} within #{path}."
+        exit 1
+      elsif matches.length > 1
+        error "Multiple files matching #{target} found within #{path}. It's " \
+              "not clear which one to use!"
+        exit 1
+      end
+      matches[0]
+    end
+
+    def create_bundler_definition_for_gemspec(gem_name, path)
+      gemspec_file = locate_gemspec(gem_name, path)
+
       # Using the real Gemfile doesn't get great results, because people can put
       # lots of stuff in there that is handy for developers to have but
       # irrelevant if you just want to produce a .gem. Also, there is only one
@@ -40,13 +58,8 @@ module Importer
       # chosen .gemspec. If present, the Gemfile.lock will be honoured.
       fake_gemfile = Bundler::Dsl.new
       fake_gemfile.source('https://rubygems.org')
-      begin
-        fake_gemfile.gemspec({:name => gem_name})
-      rescue Bundler::InvalidOption
-        error "Did not find #{gem_name}.gemspec in current directory."
-        exit 1
-      end
-
+      fake_gemfile.gemspec({:name => gem_name,
+                            :path => File.dirname(gemspec_file)})
       fake_gemfile.to_definition('Gemfile.lock', true)
     end
 
@@ -60,11 +73,13 @@ module Importer
       found[0]
     end
 
+    def directory_is_within(path, expected_subpath)
+      File.realpath(expected_subpath).start_with?(File.realpath(path))
+    end
+
     def spec_is_from_current_source_tree(spec, source_dir)
-      Dir.chdir(source_dir) do
-        spec.source.instance_of? Bundler::Source::Path and
-          File.identical?(spec.source.path, '.')
-      end
+      spec.source.instance_of? Bundler::Source::Path and
+        directory_is_within(source_dir, spec.source.path)
     end
 
     def validate_spec(spec, source_dir_name, expected_version)
diff --git a/baserockimport/exts/rubygems.find_deps b/baserockimport/exts/rubygems.find_deps
index 228c88b..ae08b65 100755
--- a/baserockimport/exts/rubygems.find_deps
+++ b/baserockimport/exts/rubygems.find_deps
@@ -36,6 +36,7 @@ class RubyGemDependencyFinder < Importer::Base
   def initialize
     local_data = YAML.load_file(local_data_path("rubygems.yaml"))
     @build_dependency_whitelist = local_data['build-dependency-whitelist']
+    @ignore_list = local_data['ignore-list']
   end
 
   def parse_options(arguments)
@@ -64,7 +65,9 @@ class RubyGemDependencyFinder < Importer::Base
   end
 
   def runtime_deps_for_gem(spec)
-    spec.dependencies.select {|d| d.type == :runtime}
+    spec.dependencies.select do |d|
+      d.type == :runtime && ! @ignore_list.member?(d.name)
+    end
   end
 
   def run
@@ -74,7 +77,7 @@ class RubyGemDependencyFinder < Importer::Base
              "#{source_dir_name}")
 
     resolved_specs = Dir.chdir(source_dir_name) do
-      definition = create_bundler_definition_for_gemspec(gem_name)
+      definition = create_bundler_definition_for_gemspec(gem_name, source_dir_name)
       definition.resolve_remotely!
     end
 
diff --git a/baserockimport/exts/rubygems.to_chunk b/baserockimport/exts/rubygems.to_chunk
index c1a3e7c..1573d8b 100755
--- a/baserockimport/exts/rubygems.to_chunk
+++ b/baserockimport/exts/rubygems.to_chunk
@@ -154,9 +154,11 @@ class RubyGemChunkMorphologyGenerator < Importer::Base
              "source code in #{source_dir_name}")
 
     resolved_specs = Dir.chdir(source_dir_name) do
-      # FIXME: we don't need to do this here, it'd be enough just to load
-      # the given gemspec 
-      definition = create_bundler_definition_for_gemspec(gem_name)
+      # FIXME: resolving the specs for all the dependencies of the target gem
+      # isn't necessary here. In fact, just reading/executing the .gemspec
+      # would be enough, and would speed this program up and remove a lot of
+      # pointless network access to rubygems.org.
+      definition = create_bundler_definition_for_gemspec(gem_name, source_dir_name)
       definition.resolve_remotely!
     end
 
diff --git a/baserockimport/mainloop.py b/baserockimport/mainloop.py
index ccb695d..4ca677d 100644
--- a/baserockimport/mainloop.py
+++ b/baserockimport/mainloop.py
@@ -182,14 +182,7 @@ class ImportLoop(object):
                     current_item, current_item.dependencies, to_process,
                     processed)
 
-        if len(errors) > 0:
-            self.app.status(
-                '\nErrors encountered, not generating a stratum morphology.')
-            self.app.status(
-                'See the README files for guidance.')
-        else:
-            self._generate_stratum_morph_if_none_exists(
-                processed, self.goal_name)
+        self._maybe_generate_stratum(processed, errors, self.goal_name)
 
         duration = time.time() - start_time
         end_displaytime = time.strftime('%x %X %Z', time.localtime())
@@ -210,10 +203,19 @@ class ImportLoop(object):
         lorry = self._find_or_create_lorry_file(kind, name)
         source_repo, url = self._fetch_or_update_source(lorry)
 
-        checked_out_version, ref = self._checkout_source_version(
-            source_repo, name, version)
+        checked_out_version, ref = self._checkout_source_version_for_package(
+            source_repo, package)
         package.set_version_in_use(checked_out_version)
 
+        repo_path = os.path.relpath(source_repo.dirname)
+        if morphlib.git.is_valid_sha1(ref):
+            self.app.status(
+                "%s %s: using %s commit %s", name, version, repo_path, ref)
+        else:
+            self.app.status(
+                "%s %s: using %s ref %s (commit %s)", name, version, repo_path,
+                ref, source_repo.resolve_ref_to_commit(ref))
+
         # 2. Create a chunk morphology with build instructions.
 
         chunk_morph = self._find_or_create_chunk_morph(
@@ -317,7 +319,8 @@ class ImportLoop(object):
         if kind not in self.importers:
             raise Exception('Importer for %s was not enabled.' % kind)
         extra_args = self.importers[kind]['extra_args']
-        self.app.status('Calling %s to generate lorry for %s', tool, name)
+        self.app.status(
+            '%s: calling %s to generate lorry', name, tool)
         lorry_text = run_extension(tool, extra_args + [name])
         try:
             lorry = json.loads(lorry_text)
@@ -373,9 +376,11 @@ class ImportLoop(object):
 
         return repo, url
 
-    def _checkout_source_version(self, source_repo, name, version):
+    def _checkout_source_version_for_package(self, source_repo, package):
         # FIXME: we need to be a bit smarter than this. Right now we assume
         # that 'version' is a valid Git ref.
+        name = package.name
+        version = package.version
 
         possible_names = [
             version,
@@ -397,7 +402,7 @@ class ImportLoop(object):
                 ref = version = 'master'
             else:
                 raise BaserockImportException(
-                    'Could not find ref for %s version %s.' % (name, version))
+                    'Could not find ref for %s.' % package)
 
         return version, ref
 
@@ -449,8 +454,7 @@ class ImportLoop(object):
         extra_args = self.importers[kind]['extra_args']
 
         self.app.status(
-            'Calling %s to generate chunk morph for %s %s', tool, name,
-            version)
+            '%s %s: calling %s to generate chunk morph', name, version, tool)
 
         args = extra_args + [source_repo.dirname, name]
         if version != 'master':
@@ -493,8 +497,7 @@ class ImportLoop(object):
         extra_args = self.importers[kind]['extra_args']
 
         self.app.status(
-            'Calling %s to calculate dependencies for %s %s', tool, name,
-            version)
+            '%s %s: calling %s to calculate dependencies', name, version, tool)
 
         args = extra_args + [source_repo.dirname, name]
         if version != 'master':
@@ -520,34 +523,58 @@ class ImportLoop(object):
                 'One or more cycles detected in build graph: %s' %
                 (', '.join(all_loops_str)))
 
-    def _generate_stratum_morph_if_none_exists(self, graph, goal_name):
+    def _maybe_generate_stratum(self, graph, errors, goal_name):
         filename = os.path.join(
             self.app.settings['definitions-dir'], 'strata', '%s.morph' %
             goal_name)
+        update_existing = self.app.settings['update-existing']
 
-        if os.path.exists(filename):
-            if not self.app.settings['update-existing']:
-                self.app.status(
-                    msg='Found stratum morph for %s at %s, not overwriting' %
-                    (goal_name, filename))
-                return
+        if self.app.settings['force-stratum-generation']:
+            self._generate_stratum(
+                graph, goal_name, filename, ignore_errors=True)
+        elif len(errors) > 0:
+            self.app.status(
+                '\nErrors encountered, not generating a stratum morphology.')
+            self.app.status(
+                'See the README files for guidance.')
+        elif os.path.exists(filename) and not update_existing:
+            self.app.status(
+                msg='Found stratum morph for %s at %s, not overwriting' %
+                (goal_name, filename))
+        else:
+            self._generate_stratum(graph, goal_name, filename)
 
+    def _generate_stratum(self, graph, goal_name, filename,
+                          ignore_errors=False):
         self.app.status(msg='Generating stratum morph for %s' % goal_name)
 
         chunk_entries = []
 
         for package in self._sort_chunks_by_build_order(graph):
             m = package.morphology
+
             if m is None:
-                raise cliapp.AppException('No morphology for %s' % package)
+                if ignore_errors:
+                    logging.warn(
+                        'Ignoring %s because there is no chunk morphology.')
+                    continue
+                else:
+                    raise cliapp.AppException('No morphology for %s' % package)
 
             def format_build_dep(name, version):
                 dep_package = find(graph, lambda p: p.match(name, version))
                 return '%s-%s' % (name, dep_package.version_in_use)
 
+            def get_build_deps(morphology):
+                deps = dict()
+                for kind in self.importers:
+                    field = 'x-build-dependencies-%s' % kind
+                    deps.update(morphology.get(field, []))
+                return deps
+
             build_depends = [
                 format_build_dep(name, version) for name, version in
-                m['x-build-dependencies-rubygems'].iteritems()
+                get_build_deps(m).iteritems()
             ]
 
             entry = {