15 files changed, 234 insertions, 161 deletions

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 1cc8fcb8408..bb48a86fe9e 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -349,6 +349,10 @@ module API
       expose :developers_can_merge do |repo_branch, options|
         options[:project].protected_branches.developers_can?(:merge, repo_branch.name)
       end
+
+      expose :can_push do |repo_branch, options|
+        Gitlab::UserAccess.new(options[:current_user], project: options[:project]).can_push_to_branch?(repo_branch.name)
+      end
     end
 
     class TreeObject < Grape::Entity
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 0119c5d6851..221ba52b490 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -4,7 +4,6 @@ require_relative 'helper'
 module Backup
   class Repository
     include Backup::Helper
-    # rubocop:disable Metrics/AbcSize
 
     attr_reader :progress
 
@@ -18,61 +17,26 @@ module Backup
       Project.find_each(batch_size: 1000) do |project|
         progress.print " * #{display_repo_path(project)} ... "
 
-        path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-          path_to_repo(project)
-        end
-        path_to_project_bundle = path_to_bundle(project)
-
-        # Create namespace dir or hashed path if missing
         if project.hashed_storage?(:repository)
           FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
         else
           FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
         end
 
-        if empty_repo?(project)
-          progress.puts "[SKIPPED]".color(:cyan)
+        if !empty_repo?(project)
+          backup_project(project)
+          progress.puts "[DONE]".color(:green)
         else
-          in_path(path_to_project_repo) do |dir|
-            FileUtils.mkdir_p(path_to_tars(project))
-            cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
-            output, status = Gitlab::Popen.popen(cmd)
-
-            unless status.zero?
-              progress_warn(project, cmd.join(' '), output)
-            end
-          end
-
-          cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
-          output, status = Gitlab::Popen.popen(cmd)
-
-          if status.zero?
-            progress.puts "[DONE]".color(:green)
-          else
-            progress_warn(project, cmd.join(' '), output)
-          end
+          progress.puts "[SKIPPED]".color(:cyan)
         end
 
         wiki = ProjectWiki.new(project)
-        path_to_wiki_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-          path_to_repo(wiki)
-        end
-        path_to_wiki_bundle = path_to_bundle(wiki)
 
-        if File.exist?(path_to_wiki_repo)
-          progress.print " * #{display_repo_path(wiki)} ... "
-
-          if empty_repo?(wiki)
-            progress.puts " [SKIPPED]".color(:cyan)
-          else
-            cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all)
-            output, status = Gitlab::Popen.popen(cmd)
-            if status.zero?
-              progress.puts " [DONE]".color(:green)
-            else
-              progress_warn(wiki, cmd.join(' '), output)
-            end
-          end
+        if !empty_repo?(wiki)
+          backup_project(wiki)
+          progress.puts "[DONE] Wiki".color(:green)
+        else
+          progress.puts "[SKIPPED] Wiki".color(:cyan)
         end
       end
     end
@@ -83,6 +47,38 @@ module Backup
       end
     end
 
+    def backup_project(project)
+      gitaly_migrate(:repository_backup) do |is_enabled|
+        if is_enabled
+          backup_project_gitaly(project)
+        else
+          backup_project_local(project)
+        end
+      end
+
+      backup_custom_hooks(project)
+    rescue => e
+      progress_warn(project, e, 'Failed to backup repo')
+    end
+
+    def backup_project_gitaly(project)
+      path_to_project_bundle = path_to_bundle(project)
+      Gitlab::GitalyClient::RepositoryService.new(project.repository)
+        .create_bundle(path_to_project_bundle)
+    end
+
+    def backup_project_local(project)
+      path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+        path_to_repo(project)
+      end
+
+      path_to_project_bundle = path_to_bundle(project)
+
+      cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
+      output, status = Gitlab::Popen.popen(cmd)
+      progress_warn(project, cmd.join(' '), output) unless status.zero?
+    end
+
     def delete_all_repositories(name, repository_storage)
       gitaly_migrate(:delete_all_repositories) do |is_enabled|
         if is_enabled
@@ -97,8 +93,6 @@ module Backup
       path = repository_storage.legacy_disk_path
       return unless File.exist?(path)
 
-      # Move all files in the existing repos directory except . and .. to
-      # repositories.old.<timestamp> directory
       bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
       FileUtils.mkdir_p(bk_repos_path, mode: 0700)
       files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
@@ -129,13 +123,47 @@ module Backup
         .restore_custom_hooks(custom_hooks_path)
     end
 
+    def local_backup_custom_hooks(project)
+      in_path(path_to_tars(project)) do |dir|
+        path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          path_to_repo(project)
+        end
+        break unless File.exist?(File.join(path_to_project_repo, dir))
+
+        FileUtils.mkdir_p(path_to_tars(project))
+        cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir})
+        output, status = Gitlab::Popen.popen(cmd)
+
+        unless status.zero?
+          progress_warn(project, cmd.join(' '), output)
+        end
+      end
+    end
+
+    def gitaly_backup_custom_hooks(project)
+      FileUtils.mkdir_p(path_to_tars(project))
+      custom_hooks_path = path_to_tars(project, 'custom_hooks')
+      Gitlab::GitalyClient::RepositoryService.new(project.repository)
+        .backup_custom_hooks(custom_hooks_path)
+    end
+
+    def backup_custom_hooks(project)
+      gitaly_migrate(:backup_custom_hooks) do |is_enabled|
+        if is_enabled
+          gitaly_backup_custom_hooks(project)
+        else
+          local_backup_custom_hooks(project)
+        end
+      end
+    end
+
     def restore_custom_hooks(project)
       in_path(path_to_tars(project)) do |dir|
         gitaly_migrate(:restore_custom_hooks) do |is_enabled|
           if is_enabled
-            local_restore_custom_hooks(project, dir)
-          else
             gitaly_restore_custom_hooks(project, dir)
+          else
+            local_restore_custom_hooks(project, dir)
           end
         end
       end
@@ -186,7 +214,6 @@ module Backup
         end
       end
     end
-    # rubocop:enable Metrics/AbcSize
 
     protected
 
@@ -224,9 +251,7 @@ module Backup
 
     def prepare
       FileUtils.rm_rf(backup_repos_path)
-      # Ensure the parent dir of backup_repos_path exists
       FileUtils.mkdir_p(Gitlab.config.backup.path)
-      # Fail if somebody raced to create backup_repos_path before us
       FileUtils.mkdir(backup_repos_path, mode: 0700)
     end
 
@@ -242,7 +267,6 @@ module Backup
     end
 
     def empty_repo?(project_or_wiki)
-      # Protect against stale caches
       project_or_wiki.repository.expire_emptiness_caches
       project_or_wiki.repository.empty?
     end
diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb
index c9c3050cfc2..87af4a90572 100644
--- a/lib/gitlab/checks/force_push.rb
+++ b/lib/gitlab/checks/force_push.rb
@@ -7,18 +7,10 @@ module Gitlab
         # Created or deleted branch
         return false if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev)
 
-        GitalyClient.migrate(:force_push) do |is_enabled|
-          if is_enabled
-            !project
-              .repository
-              .gitaly_commit_client
-              .ancestor?(oldrev, newrev)
-          else
-            Gitlab::Git::RevList.new(
-              project.repository.raw, oldrev: oldrev, newrev: newrev
-            ).missed_ref.present?
-          end
-        end
+        !project
+          .repository
+          .gitaly_commit_client
+          .ancestor?(oldrev, newrev)
       end
     end
   end
diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb
index faf7016d73a..d512fc58e46 100644
--- a/lib/gitlab/favicon.rb
+++ b/lib/gitlab/favicon.rb
@@ -2,10 +2,10 @@ module Gitlab
   class Favicon
     class << self
       def main
-        return appearance_favicon.url if appearance_favicon.exists?
-
         image_name =
-          if Gitlab::Utils.to_boolean(ENV['CANARY'])
+          if appearance_favicon.exists?
+            appearance_favicon.url
+          elsif Gitlab::Utils.to_boolean(ENV['CANARY'])
             'favicon-yellow.png'
           elsif Rails.env.development?
             'favicon-blue.png'
@@ -13,7 +13,7 @@ module Gitlab
             'favicon.png'
           end
 
-        ActionController::Base.helpers.image_path(image_name)
+        ActionController::Base.helpers.image_path(image_name, host: host)
       end
 
       def status_overlay(status_name)
@@ -22,7 +22,7 @@ module Gitlab
           "#{status_name}.png"
         )
 
-        ActionController::Base.helpers.image_path(path)
+        ActionController::Base.helpers.image_path(path, host: host)
       end
 
       def available_status_names
@@ -35,6 +35,16 @@ module Gitlab
 
       private
 
+      # we only want to create full urls when there's a different asset_host
+      # configured.
+      def host
+        if Gitlab::Application.config.asset_host.nil? || Gitlab::Application.config.asset_host == Gitlab.config.gitlab.base_url
+          nil
+        else
+          Gitlab.config.gitlab.base_url
+        end
+      end
+
       def appearance
         RequestStore.store[:appearance] ||= (Appearance.current || Appearance.new)
       end
diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb
index f42088f980e..af8270c8db8 100644
--- a/lib/gitlab/file_finder.rb
+++ b/lib/gitlab/file_finder.rb
@@ -14,14 +14,21 @@ module Gitlab
     end
 
     def find(query)
-      by_content = find_by_content(query)
+      query = Gitlab::Search::Query.new(query) do
+        filter :filename, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}$/i }
+        filter :path, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}/i }
+        filter :extension, matcher: ->(filter, blob) { blob.filename =~ /\.#{filter[:regex_value]}$/i }
+      end
+
+      by_content = find_by_content(query.term)
 
       already_found = Set.new(by_content.map(&:filename))
-      by_filename = find_by_filename(query, except: already_found)
+      by_filename = find_by_filename(query.term, except: already_found)
+
+      files = (by_content + by_filename)
+              .sort_by(&:filename)
 
-      (by_content + by_filename)
-        .sort_by(&:filename)
-        .map { |blob| [blob.filename, blob] }
+      query.filter_results(files).map { |blob| [blob.filename, blob] }
     end
 
     private
diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb
index f3cc388ea41..f0fab1e76a3 100644
--- a/lib/gitlab/git/lfs_changes.rb
+++ b/lib/gitlab/git/lfs_changes.rb
@@ -7,67 +7,11 @@ module Gitlab
       end
 
       def new_pointers(object_limit: nil, not_in: nil)
-        @repository.gitaly_migrate(:blob_get_new_lfs_pointers) do |is_enabled|
-          if is_enabled
-            @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in)
-          else
-            git_new_pointers(object_limit, not_in)
-          end
-        end
+        @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in)
       end
 
       def all_pointers
-        @repository.gitaly_migrate(:blob_get_all_lfs_pointers) do |is_enabled|
-          if is_enabled
-            @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev)
-          else
-            git_all_pointers
-          end
-        end
-      end
-
-      private
-
-      def git_new_pointers(object_limit, not_in)
-        @new_pointers ||= begin
-          rev_list.new_objects(rev_list_params(not_in: not_in)) do |object_ids|
-            object_ids = object_ids.take(object_limit) if object_limit
-
-            Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids)
-          end
-        end
-      end
-
-      def git_all_pointers
-        params = {}
-        if rev_list_supports_new_options?
-          params[:options] = ["--filter=blob:limit=#{Gitlab::Git::Blob::LFS_POINTER_MAX_SIZE}"]
-        end
-
-        rev_list.all_objects(rev_list_params(params)) do |object_ids|
-          Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids)
-        end
-      end
-
-      def rev_list
-        Gitlab::Git::RevList.new(@repository, newrev: @newrev)
-      end
-
-      # We're passing the `--in-commit-order` arg to ensure we don't wait
-      # for git to traverse all commits before returning pointers.
-      # This is required in order to improve the performance of LFS integrity check
-      def rev_list_params(params = {})
-        params[:options] ||= []
-        params[:options] << "--in-commit-order" if rev_list_supports_new_options?
-        params[:require_path] = true
-
-        params
-      end
-
-      def rev_list_supports_new_options?
-        return @option_supported if defined?(@option_supported)
-
-        @option_supported = Gitlab::Git.version >= Gitlab::VersionInfo.parse('2.16.0')
+        @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev)
       end
     end
   end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 7af40e62395..77543206e13 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -1479,10 +1479,6 @@ module Gitlab
         run_git!(args, lazy_block: block)
       end
 
-      def missed_ref(oldrev, newrev)
-        run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"])
-      end
-
       def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:)
         base_args = %w(worktree add --detach)
 
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
index 4e661eceffb..5fdad077eea 100644
--- a/lib/gitlab/git/rev_list.rb
+++ b/lib/gitlab/git/rev_list.rb
@@ -1,5 +1,3 @@
-# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites.
-
 module Gitlab
   module Git
     class RevList
@@ -45,13 +43,6 @@ module Gitlab
                     &lazy_block)
       end
 
-      # This methods returns an array of missed references
-      #
-      # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348.
-      def missed_ref
-        repository.missed_ref(oldrev, newrev).split("\n")
-      end
-
       private
 
       def execute(args)
diff --git a/lib/gitlab/git/version.rb b/lib/gitlab/git/version.rb
index 11184ca3457..1e14e8b652a 100644
--- a/lib/gitlab/git/version.rb
+++ b/lib/gitlab/git/version.rb
@@ -4,7 +4,7 @@ module Gitlab
       extend Gitlab::Git::Popen
 
       def self.git_version
-        Gitlab::VersionInfo.parse(popen(%W(#{Gitlab.config.git.bin_path} --version), nil).first)
+        Gitlab::VersionInfo.parse(Gitaly::Server.all.first.git_binary_version)
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 4340f779e53..ca986434221 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -196,20 +196,21 @@ module Gitlab
       end
 
       def create_bundle(save_path)
-        request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(
-          @storage,
-          :repository_service,
+        gitaly_fetch_stream_to_file(
+          save_path,
           :create_bundle,
-          request,
-          timeout: GitalyClient.default_timeout
+          Gitaly::CreateBundleRequest,
+          GitalyClient.default_timeout
         )
+      end
 
-        File.open(save_path, 'wb') do |f|
-          response.each do |message|
-            f.write(message.data)
-          end
-        end
+      def backup_custom_hooks(save_path)
+        gitaly_fetch_stream_to_file(
+          save_path,
+          :backup_custom_hooks,
+          Gitaly::BackupCustomHooksRequest,
+          GitalyClient.default_timeout
+        )
       end
 
       def create_from_bundle(bundle_path)
@@ -309,6 +310,25 @@ module Gitlab
 
       private
 
+      def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout)
+        request = request_class.new(repository: @gitaly_repo)
+        response = GitalyClient.call(
+          @storage,
+          :repository_service,
+          rpc_name,
+          request,
+          timeout: timeout
+        )
+
+        File.open(save_path, 'wb') do |f|
+          response.each do |message|
+            f.write(message.data)
+          end
+        end
+        # If the file is empty means that we recieved an empty stream, we delete the file
+        FileUtils.rm(save_path) if File.zero?(save_path)
+      end
+
       def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout)
         request = request_class.new(repository: @gitaly_repo)
         enum = Enumerator.new do |y|
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
index fcbf266b80b..050fe7a5173 100644
--- a/lib/gitlab/health_checks/fs_shards_check.rb
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -1,5 +1,6 @@
 module Gitlab
   module HealthChecks
+    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1218
     class FsShardsCheck
       extend BaseAbstractCheck
       RANDOM_STRING = SecureRandom.hex(1000).freeze
diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb
index 30af3e97b4a..d2133a6d65b 100644
--- a/lib/gitlab/kubernetes/helm/install_command.rb
+++ b/lib/gitlab/kubernetes/helm/install_command.rb
@@ -2,11 +2,12 @@ module Gitlab
   module Kubernetes
     module Helm
       class InstallCommand < BaseCommand
-        attr_reader :name, :chart, :repository, :values
+        attr_reader :name, :chart, :version, :repository, :values
 
-        def initialize(name, chart:, values:, repository: nil)
+        def initialize(name, chart:, values:, version: nil, repository: nil)
           @name = name
           @chart = chart
+          @version = version
           @values = values
           @repository = repository
         end
@@ -39,9 +40,13 @@ module Gitlab
 
         def script_command
           <<~HEREDOC
-          helm install #{chart} --name #{name} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null
+          helm install #{chart} --name #{name}#{optional_version_flag} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null
           HEREDOC
         end
+
+        def optional_version_flag
+          " --version #{version}" if version
+        end
       end
     end
   end
diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb
new file mode 100644
index 00000000000..23595f23f01
--- /dev/null
+++ b/lib/gitlab/search/parsed_query.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Search
+    class ParsedQuery
+      attr_reader :term, :filters
+
+      def initialize(term, filters)
+        @term = term
+        @filters = filters
+      end
+
+      def filter_results(results)
+        filters = @filters.reject { |filter| filter[:matcher].nil? }
+        return unless filters
+
+        results.select do |result|
+          filters.all? do |filter|
+            filter[:matcher].call(filter, result)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb
new file mode 100644
index 00000000000..8583bce7792
--- /dev/null
+++ b/lib/gitlab/search/query.rb
@@ -0,0 +1,55 @@
+module Gitlab
+  module Search
+    class Query < SimpleDelegator
+      def initialize(query, filter_opts = {}, &block)
+        @raw_query = query.dup
+        @filters = []
+        @filter_options = { default_parser: :downcase.to_proc }.merge(filter_opts)
+
+        self.instance_eval(&block) if block_given?
+
+        @query = Gitlab::Search::ParsedQuery.new(*extract_filters)
+        # set the ParsedQuery as our default delegator thanks to SimpleDelegator
+        super(@query)
+      end
+
+      private
+
+      def filter(name, **attributes)
+        filter = { parser: @filter_options[:default_parser], name: name }.merge(attributes)
+
+        @filters << filter
+      end
+
+      def filter_options(**options)
+        @filter_options.merge!(options)
+      end
+
+      def extract_filters
+        fragments = []
+
+        filters = @filters.each_with_object([]) do |filter, parsed_filters|
+          match = @raw_query.split.find { |part| part =~ /\A#{filter[:name]}:/ }
+          next unless match
+
+          input = match.split(':')[1..-1].join
+          next if input.empty?
+
+          filter[:value] = parse_filter(filter, input)
+          filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?')
+          fragments << match
+
+          parsed_filters << filter
+        end
+
+        query = (@raw_query.split - fragments).join(' ')
+
+        [query, filters]
+      end
+
+      def parse_filter(filter, input)
+        filter[:parser].call(input)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index 4a87f43597e..b2d75aac1d0 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -24,6 +24,7 @@ module Gitlab
             address = val['gitaly_address']
           end
 
+          # https://gitlab.com/gitlab-org/gitaly/issues/1238
           Gitlab::GitalyClient::StorageSettings.allow_disk_access do
             storages << { name: key, path: val.legacy_disk_path }
           end