Auto merge of #5908 - bundler:seg-globally-cache-built-extensions, r=indirect

[2.0] Globally cache built extensions for RG & Git sources

### What was the end-user problem that led to this PR?

The problem was with the switch to defaulting the install path to `./.bundle`, you'd need to rebuild extensions for each app you use a gem in.

Closes #5905

### What was your diagnosis of the problem?

My diagnosis was we could globally cache built extensions in a per-ruby, per-source directory, similar to how 2.0 global `.gem` caching and the git cache works.

### What is your fix for the problem, implemented in this PR?

My fix implements that cache.

### Remaining questions:

- [x] Tests
- [x] Right now, the directory structure is `cache/gems/remote_name/extensions/ruby_scope/gem_full_name`. I'm wondering whether we should make it `cache/extensions/ruby_scope/source_slug/gem_full_name` instead to make removing the cache for an old ruby version easier?
- [x] Should the cache path include something about the local OS? ie. do we want to force a re-compile when the user updates OS versions?

9 files changed, 108 insertions, 8 deletions

diff --git a/lib/bundler/rubygems_gem_installer.rb b/lib/bundler/rubygems_gem_installer.rb
index 1d9da0e329..e7e1a7e940 100644
--- a/lib/bundler/rubygems_gem_installer.rb
+++ b/lib/bundler/rubygems_gem_installer.rb
@@ -18,6 +18,28 @@ module Bundler
       super && validate_bundler_checksum(options[:bundler_expected_checksum])
     end
 
+    def build_extensions
+      extension_cache_path = options[:bundler_extension_cache_path]
+      return super unless extension_cache_path && extension_dir = Bundler.rubygems.spec_extension_dir(spec)
+
+      extension_dir = Pathname.new(extension_dir)
+      build_complete = SharedHelpers.filesystem_access(extension_cache_path.join("gem.build_complete"), :read, &:file?)
+      if build_complete && !options[:force]
+        SharedHelpers.filesystem_access(extension_dir.parent, &:mkpath)
+        SharedHelpers.filesystem_access(extension_cache_path) do
+          FileUtils.cp_r extension_cache_path, spec.extension_dir
+        end
+      else
+        super
+        if extension_dir.directory? # not made for gems without extensions
+          SharedHelpers.filesystem_access(extension_cache_path.parent, &:mkpath)
+          SharedHelpers.filesystem_access(extension_cache_path) do
+            FileUtils.cp_r extension_dir, extension_cache_path
+          end
+        end
+      end
+    end
+
   private
 
     def validate_bundler_checksum(checksum)
diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb
index dc1c35b990..202903d981 100644
--- a/lib/bundler/rubygems_integration.rb
+++ b/lib/bundler/rubygems_integration.rb
@@ -93,6 +93,11 @@ module Bundler
       end.flatten(1)
     end
 
+    def spec_extension_dir(spec)
+      return unless spec.respond_to?(:extension_dir)
+      spec.extension_dir
+    end
+
     def stub_set_spec(stub, spec)
       stub.instance_variable_set(:@spec, spec)
     end
diff --git a/lib/bundler/source.rb b/lib/bundler/source.rb
index 38e001535c..956cf39d56 100644
--- a/lib/bundler/source.rb
+++ b/lib/bundler/source.rb
@@ -73,5 +73,18 @@ module Bundler
         Bundler.ui.info message
       end
     end
+
+    def extension_cache_path(spec)
+      return unless Bundler.feature_flag.global_gem_cache?
+      return unless source_slug = extension_cache_slug(spec)
+      Bundler.user_cache.join(
+        "extensions", Gem::Platform.local.to_s, Bundler.ruby_scope,
+        source_slug, spec.full_name
+      )
+    end
+
+    def extension_cache_slug(_)
+      nil
+    end
   end
 end
diff --git a/lib/bundler/source/git.rb b/lib/bundler/source/git.rb
index fd8f6debea..c66cbb17fb 100644
--- a/lib/bundler/source/git.rb
+++ b/lib/bundler/source/git.rb
@@ -209,8 +209,6 @@ module Bundler
       # When using local git repos, this is set to the local repo.
       def cache_path
         @cache_path ||= begin
-          git_scope = "#{base_name}-#{uri_hash}"
-
           if Bundler.requires_sudo? || Bundler.feature_flag.global_gem_cache?
             Bundler.user_cache
           else
@@ -318,6 +316,14 @@ module Bundler
           StubSpecification.from_stub(stub)
         end
       end
+
+      def git_scope
+        "#{base_name}-#{uri_hash}"
+      end
+
+      def extension_cache_slug(_)
+        extension_dir_name
+      end
     end
   end
 end
diff --git a/lib/bundler/source/path.rb b/lib/bundler/source/path.rb
index 806ba81935..afd8723168 100644
--- a/lib/bundler/source/path.rb
+++ b/lib/bundler/source/path.rb
@@ -227,7 +227,8 @@ module Bundler
           spec,
           :env_shebang => false,
           :disable_extensions => options[:disable_extensions],
-          :build_args => options[:build_args]
+          :build_args => options[:build_args],
+          :bundler_extension_cache_path => extension_cache_path(spec)
         )
         installer.post_install
       rescue Gem::InvalidSpecificationException => e
diff --git a/lib/bundler/source/path/installer.rb b/lib/bundler/source/path/installer.rb
index 64e63766dd..a0357ffa39 100644
--- a/lib/bundler/source/path/installer.rb
+++ b/lib/bundler/source/path/installer.rb
@@ -7,6 +7,7 @@ module Bundler
         attr_reader :spec
 
         def initialize(spec, options = {})
+          @options            = options
           @spec               = spec
           @gem_dir            = Bundler.rubygems.path(spec.full_gem_path)
           @wrappers           = true
diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb
index 70dc5ac038..fa60bb0c84 100644
--- a/lib/bundler/source/rubygems.rb
+++ b/lib/bundler/source/rubygems.rb
@@ -148,7 +148,8 @@ module Bundler
               :wrappers            => true,
               :env_shebang         => true,
               :build_args          => opts[:build_args],
-              :bundler_expected_checksum => spec.respond_to?(:checksum) && spec.checksum
+              :bundler_expected_checksum => spec.respond_to?(:checksum) && spec.checksum,
+              :bundler_extension_cache_path => extension_cache_path(spec)
             ).install
           end
           spec.full_gem_path = installed_spec.full_gem_path
@@ -498,6 +499,11 @@ module Bundler
 
         Bundler.user_cache.join("gems", cache_slug, spec.file_name)
       end
+
+      def extension_cache_slug(spec)
+        return unless remote = spec.remote
+        remote.cache_slug
+      end
     end
   end
 end
diff --git a/spec/install/global_cache_spec.rb b/spec/install/global_cache_spec.rb
index c48bb1bd89..3664d3963a 100644
--- a/spec/install/global_cache_spec.rb
+++ b/spec/install/global_cache_spec.rb
@@ -186,4 +186,50 @@ RSpec.describe "global gem caching" do
       end
     end
   end
+
+  describe "extension caching", :rubygems => "2.2" do
+    it "works" do
+      build_git "very_simple_git_binary", &:add_c_extension
+      build_lib "very_simple_path_binary", &:add_c_extension
+      revision = revision_for(lib_path("very_simple_git_binary-1.0"))[0, 12]
+
+      install_gemfile! <<-G
+        source "file:#{gem_repo1}"
+
+        gem "very_simple_binary"
+        gem "very_simple_git_binary", :git => "#{lib_path("very_simple_git_binary-1.0")}"
+        gem "very_simple_path_binary", :path => "#{lib_path("very_simple_path_binary-1.0")}"
+      G
+
+      gem_binary_cache = home(".bundle", "cache", "extensions", specific_local_platform.to_s, Bundler.ruby_scope,
+        Digest(:MD5).hexdigest("#{gem_repo1}/"), "very_simple_binary-1.0")
+      git_binary_cache = home(".bundle", "cache", "extensions", specific_local_platform.to_s, Bundler.ruby_scope,
+        "very_simple_git_binary-1.0-#{revision}", "very_simple_git_binary-1.0")
+
+      cached_extensions = Pathname.glob(home(".bundle", "cache", "extensions", "*", "*", "*", "*", "*")).sort
+      expect(cached_extensions).to eq [gem_binary_cache, git_binary_cache].sort
+
+      run! <<-R
+        require 'very_simple_binary_c'; puts ::VERY_SIMPLE_BINARY_IN_C
+        require 'very_simple_git_binary_c'; puts ::VERY_SIMPLE_GIT_BINARY_IN_C
+      R
+      expect(out).to eq "VERY_SIMPLE_BINARY_IN_C\nVERY_SIMPLE_GIT_BINARY_IN_C"
+
+      FileUtils.rm Dir[home(".bundle", "cache", "extensions", "**", "*binary_c*")]
+
+      gem_binary_cache.join("very_simple_binary_c.rb").open("w") {|f| f << "puts File.basename(__FILE__)" }
+      git_binary_cache.join("very_simple_git_binary_c.rb").open("w") {|f| f << "puts File.basename(__FILE__)" }
+
+      bundle! "config --local path different_path"
+      bundle! :install
+
+      expect(Dir[home(".bundle", "cache", "extensions", "**", "*binary_c*")]).to all(end_with(".rb"))
+
+      run! <<-R
+        require 'very_simple_binary_c'
+        require 'very_simple_git_binary_c'
+      R
+      expect(out).to eq "very_simple_binary_c.rb\nvery_simple_git_binary_c.rb"
+    end
+  end
 end
diff --git a/spec/support/builders.rb b/spec/support/builders.rb
index 93b173a5d0..e8208eacd9 100644
--- a/spec/support/builders.rb
+++ b/spec/support/builders.rb
@@ -566,7 +566,7 @@ module Spec
 
           # exit 1 unless with_config("simple")
 
-          extension_name = "very_simple_binary_c"
+          extension_name = "#{name}_c"
           if extra_lib_dir = with_config("ext-lib")
             # add extra libpath if --with-ext-lib is
             # passed in as a build_arg
@@ -576,11 +576,11 @@ module Spec
           end
           create_makefile extension_name
         RUBY
-        write "ext/very_simple_binary.c", <<-C
+        write "ext/#{name}.c", <<-C
           #include "ruby.h"
 
-          void Init_very_simple_binary_c() {
-            rb_define_module("VerySimpleBinaryInC");
+          void Init_#{name}_c() {
+            rb_define_module("#{Builders.constantize(name)}_IN_C");
           }
         C
       end