Search blobs without limitjprovazn-blob-count

4 files changed, 97 insertions, 56 deletions

diff --git a/app/views/search/_category.html.haml b/app/views/search/_category.html.haml
index 73ae5c6201a..cc0afb08baa 100644
--- a/app/views/search/_category.html.haml
+++ b/app/views/search/_category.html.haml
@@ -83,7 +83,7 @@
           = link_to search_filter_path(scope: 'blobs') do
             Code
             %span.badge.badge-pill
-              = limited_count(@search_results.blobs_count)
+              = @search_results.blobs_count
         %li{ class: active_when(@scope == 'commits') }
           = link_to search_filter_path(scope: 'commits') do
             Commits
@@ -93,4 +93,4 @@
           = link_to search_filter_path(scope: 'wiki_blobs') do
             Wiki
             %span.badge.badge-pill
-              = limited_count(@search_results.wiki_blobs_count)
+              = @search_results.wiki_blobs_count
diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb
index b4db3f93c9c..f59ca45e656 100644
--- a/lib/gitlab/file_finder.rb
+++ b/lib/gitlab/file_finder.rb
@@ -4,64 +4,115 @@
 # the result is joined and sorted by file name
 module Gitlab
   class FileFinder
-    BATCH_SIZE = 100
+    class BaseMatch
+      attr_reader :match
 
-    attr_reader :project, :ref
+      def initialize(match)
+        @match = match
+      end
+    end
+
+    class ContentMatch < BaseMatch
+      FILENAME_REGEXP = /\A(?<ref>[^:]*):(?<filename>[^\x00]*)\x00/.freeze
+
+      def filename
+        # FIXME - use const, match may not be found
+        # FIXME - check if utf8 is needed
+        @filename ||= match.match(FILENAME_REGEXP)[:filename]
+      end
+
+      def found_blob(project, ref)
+        Gitlab::ProjectSearchResults.parse_search_result(match, project)
+      end
+    end
+
+    class FilenameMatch < BaseMatch
+      attr_accessor :blob, :ref
+
+      def filename
+        match
+      end
+
+      def found_blob(project, ref)
+        Gitlab::SearchResults::FoundBlob.new(
+          id: blob.id,
+          filename: blob.path,
+          basename: File.basename(blob.path, File.extname(blob.path)),
+          ref: ref,
+          startline: 1,
+          data: blob.data,
+          project: project
+        )
+      end
+    end
+
+    attr_reader :project, :ref, :page, :per_page, :query
 
     delegate :repository, to: :project
 
-    def initialize(project, ref)
+    def initialize(project, ref, page: 1, per_page: 20)
       @project = project
       @ref = ref
+      @page = page
+      @per_page = [per_page.to_i, 100].min
     end
 
     def find(query)
-      query = Gitlab::Search::Query.new(query) do
+      @query = Gitlab::Search::Query.new(query) do
         filter :filename, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}$/i }
         filter :path, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}/i }
         filter :extension, matcher: ->(filter, blob) { blob.filename =~ /\.#{filter[:regex_value]}$/i }
       end
 
-      by_content = find_by_content(query.term)
+      results = Kaminari.paginate_array(find_by_filename + find_by_content).page(page).per(per_page)
 
-      already_found = Set.new(by_content.map(&:filename))
-      by_filename = find_by_filename(query.term, except: already_found)
+      # convert to blob only items on the selected page of array
+      replace_filename_blobs!(results)
+      replace_content_blobs!(results)
 
-      files = (by_content + by_filename)
-              .sort_by(&:filename)
+      results.each_with_index { |blob, idx| results[idx] = [blob.filename, blob] }
 
-      query.filter_results(files).map { |blob| [blob.filename, blob] }
+      results
     end
 
     private
 
-    def find_by_content(query)
-      results = repository.search_files_by_content(query, ref).first(BATCH_SIZE)
-      results.map { |result| Gitlab::ProjectSearchResults.parse_search_result(result, project) }
+    def replace_content_blobs!(array)
+      array.each_with_index do |item, idx|
+        next unless item.is_a?(ContentMatch)
+        array[idx] = item.found_blob(project, ref)
+      end
     end
 
-    def find_by_filename(query, except: [])
-      filenames = search_filenames(query, except)
+    def replace_filename_blobs!(array)
+      filenames = array.select { |a| a.is_a?(FilenameMatch) }.map(&:match)
+      blobs = blobs(filenames)
 
-      blobs(filenames).map do |blob|
-        Gitlab::SearchResults::FoundBlob.new(
-          id: blob.id,
-          filename: blob.path,
-          basename: File.basename(blob.path, File.extname(blob.path)),
-          ref: ref,
-          startline: 1,
-          data: blob.data,
-          project: project
-        )
+      array.each_with_index do |item, idx|
+        next unless item.is_a?(FilenameMatch)
+        item.blob = blobs.find { |b| b.path == item.match }
+        array[idx] = item.found_blob(project, ref)
       end
     end
 
-    def search_filenames(query, except)
-      filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE)
+    def find_by_content
+      files = repository.search_files_by_content(query.term, ref)
+
+      filter_matches(ContentMatch, files)
+    end
+
+    def find_by_filename
+      filenames = repository.search_files_by_name(query.term, ref)
+
+      filter_matches(FilenameMatch, filenames)
+    end
+
+    def filter_matches(match_class, matches)
+      matches = matches.map { |match| match_class.new(match) }
 
-      filenames.delete_if { |filename| except.include?(filename) } unless except.empty?
+      matches = query.filter_results(matches) if query.filters.any?
 
-      filenames
+      matches
     end
 
     def blob_refs(filenames)
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 04df881bf03..fce6ff1fe8b 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -17,9 +17,9 @@ module Gitlab
       when 'notes'
         notes.page(page).per(per_page)
       when 'blobs'
-        Kaminari.paginate_array(blobs).page(page).per(per_page)
+        blobs(page: page)
       when 'wiki_blobs'
-        Kaminari.paginate_array(wiki_blobs).page(page).per(per_page)
+        wiki_blobs(page: page)
       when 'commits'
         Kaminari.paginate_array(commits).page(page).per(per_page)
       else
@@ -28,7 +28,7 @@ module Gitlab
     end
 
     def blobs_count
-      @blobs_count ||= blobs.count
+      @blobs_count ||= blobs.total_count
     end
 
     # rubocop: disable CodeReuse/ActiveRecord
@@ -48,7 +48,7 @@ module Gitlab
     # rubocop: enable CodeReuse/ActiveRecord
 
     def wiki_blobs_count
-      @wiki_blobs_count ||= wiki_blobs.count
+      @wiki_blobs_count ||= wiki_blobs.total_count
     end
 
     def commits_count
@@ -97,24 +97,22 @@ module Gitlab
 
     private
 
-    def blobs
-      return [] unless Ability.allowed?(@current_user, :download_code, @project)
+    def blobs(page: nil)
+      return Kaminari.paginate_array([]) unless Ability.allowed?(@current_user, :download_code, @project)
 
-      @blobs ||= Gitlab::FileFinder.new(project, repository_project_ref).find(query)
+      @blobs ||= Gitlab::FileFinder.new(project, repository_project_ref, page: page, per_page: per_page).find(query)
     end
 
-    def wiki_blobs
-      return [] unless Ability.allowed?(@current_user, :read_wiki, @project)
+    def wiki_blobs(page: nil)
+      unless Ability.allowed?(@current_user, :read_wiki, @project)
+        return Kaminari.paginate_array([])
+      end
 
       @wiki_blobs ||= begin
-        if project.wiki_enabled? && query.present?
-          unless project.wiki.empty?
-            Gitlab::WikiFileFinder.new(project, repository_wiki_ref).find(query)
-          else
-            []
-          end
+        if project.wiki_enabled? && query.present? && !project.wiki.empty?
+          Gitlab::WikiFileFinder.new(project, repository_wiki_ref, page: page, per_page: per_page, limit: count_limit).find(query)
         else
-          []
+          Kaminari.paginate_array([])
         end
       end
     end
diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb
index a00cd65594c..224781a002a 100644
--- a/lib/gitlab/wiki_file_finder.rb
+++ b/lib/gitlab/wiki_file_finder.rb
@@ -2,20 +2,12 @@
 
 module Gitlab
   class WikiFileFinder < FileFinder
-    attr_reader :repository
-
-    def initialize(project, ref)
-      @project = project
-      @ref = ref
-      @repository = project.wiki.repository
-    end
-
     private
 
     def search_filenames(query, except)
       safe_query = Regexp.escape(query.tr(' ', '-'))
       safe_query = Regexp.new(safe_query, Regexp::IGNORECASE)
-      filenames = repository.ls_files(ref)
+      filenames = project.wiki.repository.ls_files(ref)
 
       filenames.delete_if { |filename| except.include?(filename) } unless except.empty?