Merge branch 'lazy-blobs' into 'master'

Lazy Git blobs

To prevent madness like loading a 100MB file into Ruby's memory just
to decide whether it is binary or text.

See merge request !2663

8 files changed, 18 insertions, 6 deletions

diff --git a/Gemfile b/Gemfile
index c9d428a1798..3f83a47a3c2 100644
--- a/Gemfile
+++ b/Gemfile
@@ -49,7 +49,7 @@ gem "browser", '~> 1.0.0'
 
 # Extracting information from a git repository
 # Provide access to Gitlab::Git library
-gem "gitlab_git", '~> 7.2.23'
+gem "gitlab_git", '~> 8.0.0'
 
 # LDAP Auth
 # GitLab fork with several improvements to original library. For full list of changes
diff --git a/Gemfile.lock b/Gemfile.lock
index b4f7587c419..8c6e01ede84 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -356,7 +356,7 @@ GEM
       posix-spawn (~> 0.3)
     gitlab_emoji (0.2.0)
       gemojione (~> 2.1)
-    gitlab_git (7.2.24)
+    gitlab_git (8.0.0)
       activesupport (~> 4.0)
       charlock_holmes (~> 0.7.3)
       github-linguist (~> 4.7.0)
@@ -934,7 +934,7 @@ DEPENDENCIES
   github-markup (~> 1.3.1)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab_emoji (~> 0.2.0)
-  gitlab_git (~> 7.2.23)
+  gitlab_git (~> 8.0.0)
   gitlab_meta (= 7.0)
   gitlab_omniauth-ldap (~> 1.2.1)
   gollum-lib (~> 4.1.0)
diff --git a/app/controllers/projects/blob_controller.rb b/app/controllers/projects/blob_controller.rb
index bb72232edd7..495a432347e 100644
--- a/app/controllers/projects/blob_controller.rb
+++ b/app/controllers/projects/blob_controller.rb
@@ -33,6 +33,7 @@ class Projects::BlobController < Projects::ApplicationController
 
   def edit
     @last_commit = Gitlab::Git::Commit.last_for_path(@repository, @ref, @path).sha
+    blob.load_all_data!(@repository)
   end
 
   def update
@@ -51,6 +52,7 @@ class Projects::BlobController < Projects::ApplicationController
 
   def preview
     @content = params[:content]
+    @blob.load_all_data!(@repository)
     diffy = Diffy::Diff.new(@blob.data, @content, diff: '-U 3', include_diff_info: true)
     diff_lines = diffy.diff.scan(/.*\n/)[2..-1]
     diff_lines = Gitlab::Diff::Parser.new.parse(diff_lines)
diff --git a/app/models/ci/commit.rb b/app/models/ci/commit.rb
index d2a29236942..ecbd2078b1d 100644
--- a/app/models/ci/commit.rb
+++ b/app/models/ci/commit.rb
@@ -205,7 +205,11 @@ module Ci
     end
 
     def ci_yaml_file
-      @ci_yaml_file ||= project.repository.blob_at(sha, '.gitlab-ci.yml').data
+      @ci_yaml_file ||= begin
+        blob = project.repository.blob_at(sha, '.gitlab-ci.yml')
+        blob.load_all_data!(project.repository)
+        blob.data
+      end
     rescue
       nil
     end
diff --git a/app/models/tree.rb b/app/models/tree.rb
index b28f31cdd6e..7c4ed6e393b 100644
--- a/app/models/tree.rb
+++ b/app/models/tree.rb
@@ -39,6 +39,8 @@ class Tree
 
     git_repo = repository.raw_repository
     @readme = Gitlab::Git::Blob.find(git_repo, sha, readme_path)
+    @readme.load_all_data!(git_repo)
+    @readme
   end
 
   def trees
diff --git a/app/views/projects/blob/_text.html.haml b/app/views/projects/blob/_text.html.haml
index 906e5ccb360..d09cd73558c 100644
--- a/app/views/projects/blob/_text.html.haml
+++ b/app/views/projects/blob/_text.html.haml
@@ -1,3 +1,4 @@
+- blob.load_all_data!(@repository)
 - if markup?(blob.name)
   .file-content.wiki
     = render_markup(blob.name, blob.data)
diff --git a/lib/api/files.rb b/lib/api/files.rb
index 8ad2c1883c7..c1d86f313b0 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -58,9 +58,11 @@ module API
         commit = user_project.commit(ref)
         not_found! 'Commit' unless commit
 
-        blob = user_project.repository.blob_at(commit.sha, file_path)
+        repo = user_project.repository
+        blob = repo.blob_at(commit.sha, file_path)
 
         if blob
+          blob.load_all_data!(repo)
           status(200)
 
           {
@@ -72,7 +74,7 @@ module API
             ref: ref,
             blob_id: blob.id,
             commit_id: commit.id,
-            last_commit_id: user_project.repository.last_commit_for_path(commit.sha, file_path).id
+            last_commit_id: repo.last_commit_for_path(commit.sha, file_path).id
           }
         else
           not_found! 'File'
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index 4ddb4fea977..cac76442321 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -8,6 +8,7 @@ module Gitlab
       blob = repository.blob_at(ref, file_name)
       return [] unless blob
 
+      blob.load_all_data!(repository)
       highlight(file_name, blob.data).lines.map!(&:html_safe)
     end