Merge branch 'master' into 'catch-redis-address-error'

# Conflicts:
#   lib/gitlab/current_settings.rb

109 files changed, 2521 insertions, 476 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index d767af36e8e..efcf0976a81 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -2,6 +2,8 @@ module API
   class API < Grape::API
     include APIGuard
 
+    allow_access_with_scope :api
+
     version %w(v3 v4), using: :path
 
     version 'v3', using: :path do
@@ -44,7 +46,6 @@ module API
       mount ::API::V3::Variables
     end
 
-    before { allow_access_with_scope :api }
     before { header['X-Frame-Options'] = 'SAMEORIGIN' }
     before { Gitlab::I18n.locale = current_user&.preferred_language }
 
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index 9fcf04efa38..0d2d71e336a 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -23,6 +23,23 @@ module API
       install_error_responders(base)
     end
 
+    class_methods do
+      # Set the authorization scope(s) allowed for an API endpoint.
+      #
+      # A call to this method maps the given scope(s) to the current API
+      # endpoint class. If this method is called multiple times on the same class,
+      # the scopes are all aggregated.
+      def allow_access_with_scope(scopes, options = {})
+        Array(scopes).each do |scope|
+          allowed_scopes << Scope.new(scope, options)
+        end
+      end
+
+      def allowed_scopes
+        @scopes ||= []
+      end
+    end
+
     # Helper Methods for Grape Endpoint
     module HelperMethods
       # Invokes the doorkeeper guard.
@@ -47,7 +64,7 @@ module API
         access_token = find_access_token
         return nil unless access_token
 
-        case AccessTokenValidationService.new(access_token).validate(scopes: scopes)
+        case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes)
         when AccessTokenValidationService::INSUFFICIENT_SCOPE
           raise InsufficientScopeError.new(scopes)
 
@@ -74,18 +91,6 @@ module API
         @current_user
       end
 
-      # Set the authorization scope(s) allowed for the current request.
-      #
-      # Note: A call to this method adds to any previous scopes in place. This is done because
-      # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then
-      # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the
-      # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they
-      # need to be stored.
-      def allow_access_with_scope(*scopes)
-        @scopes ||= []
-        @scopes.concat(scopes.map(&:to_s))
-      end
-
       private
 
       def find_user_by_authentication_token(token_string)
@@ -96,7 +101,7 @@ module API
         access_token = PersonalAccessToken.active.find_by_token(token_string)
         return unless access_token
 
-        if AccessTokenValidationService.new(access_token).include_any_scope?(scopes)
+        if AccessTokenValidationService.new(access_token, request: request).include_any_scope?(scopes)
           User.find(access_token.user_id)
         end
       end
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index c6fc17cc391..bcb842b9211 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -67,7 +67,7 @@ module API
         result = ::Files::MultiService.new(user_project, current_user, attrs).execute
 
         if result[:status] == :success
-          commit_detail = user_project.repository.commits(result[:result], limit: 1).first
+          commit_detail = user_project.repository.commit(result[:result])
           present commit_detail, with: Entities::RepoCommitDetail
         else
           render_api_error!(result[:message], 400)
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index aa91451c9f4..945f2821d72 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -112,6 +112,7 @@ module API
       expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
       expose :public_builds, as: :public_jobs
+      expose :ci_config_path
       expose :shared_with_groups do |project, options|
         SharedGroup.represent(project.project_group_links.all, options)
       end
@@ -254,7 +255,7 @@ module API
 
     class ProjectEntity < Grape::Entity
       expose :id, :iid
-      expose(:project_id) { |entity| entity.project.id }
+      expose(:project_id) { |entity| entity&.project.try(:id) }
       expose :title, :description
       expose :state, :created_at, :updated_at
     end
@@ -266,7 +267,12 @@ module API
       expose :deleted_file?, as: :deleted_file
     end
 
-    class Milestone < ProjectEntity
+    class Milestone < Grape::Entity
+      expose :id, :iid
+      expose(:project_id) { |entity| entity&.project_id }
+      expose(:group_id) { |entity| entity&.group_id }
+      expose :title, :description
+      expose :state, :created_at, :updated_at
       expose :due_date
       expose :start_date
     end
@@ -310,10 +316,26 @@ module API
 
     class MergeRequestBasic < ProjectEntity
       expose :target_branch, :source_branch
-      expose :upvotes, :downvotes
+      expose :upvotes do |merge_request, options|
+        if options[:issuable_metadata]
+          options[:issuable_metadata][merge_request.id].upvotes
+        else
+          merge_request.upvotes
+        end
+      end
+      expose :downvotes do |merge_request, options|
+        if options[:issuable_metadata]
+          options[:issuable_metadata][merge_request.id].downvotes
+        else
+          merge_request.downvotes
+        end
+      end
       expose :author, :assignee, using: Entities::UserBasic
       expose :source_project_id, :target_project_id
-      expose :label_names, as: :labels
+      expose :labels do |merge_request, options|
+        # Avoids an N+1 query since labels are preloaded
+        merge_request.labels.map(&:title).sort
+      end
       expose :work_in_progress?, as: :work_in_progress
       expose :milestone, using: Entities::Milestone
       expose :merge_when_pipeline_succeeds
@@ -434,7 +456,7 @@ module API
         target_url    = "namespace_project_#{target_type}_url"
         target_anchor = "note_#{todo.note_id}" if todo.note_id?
 
-        Gitlab::Application.routes.url_helpers.public_send(target_url,
+        Gitlab::Routing.url_helpers.public_send(target_url,
           todo.project.namespace, todo.project, todo.target, anchor: target_anchor)
       end
 
@@ -444,7 +466,15 @@ module API
     end
 
     class Namespace < Grape::Entity
-      expose :id, :name, :path, :kind, :full_path
+      expose :id, :name, :path, :kind, :full_path, :parent_id
+
+      expose :members_count_with_descendants, if: -> (namespace, opts) { expose_members_count_with_descendants?(namespace, opts) } do |namespace, _|
+        namespace.users_with_descendants.count
+      end
+
+      def expose_members_count_with_descendants?(namespace, opts)
+        namespace.kind == 'group' && Ability.allowed?(opts[:current_user], :admin_group, namespace)
+      end
     end
 
     class MemberAccess < Grape::Entity
@@ -494,12 +524,20 @@ module API
     class ProjectWithAccess < Project
       expose :permissions do
         expose :project_access, using: Entities::ProjectAccess do |project, options|
-          project.project_members.find_by(user_id: options[:current_user].id)
+          if options.key?(:project_members)
+            (options[:project_members] || []).find { |member| member.source_id == project.id }
+          else
+            project.project_members.find_by(user_id: options[:current_user].id)
+          end
         end
 
         expose :group_access, using: Entities::GroupAccess do |project, options|
           if project.group
-            project.group.group_members.find_by(user_id: options[:current_user].id)
+            if options.key?(:group_members)
+              (options[:group_members] || []).find { |member| member.source_id == project.namespace_id }
+            else
+              project.group.group_members.find_by(user_id: options[:current_user].id)
+            end
           end
         end
       end
@@ -823,7 +861,7 @@ module API
       end
 
       class Cache < Grape::Entity
-        expose :key, :untracked, :paths
+        expose :key, :untracked, :paths, :policy
       end
 
       class Credentials < Grape::Entity
@@ -866,5 +904,11 @@ module API
         expose :dependencies, using: Dependency
       end
     end
+
+    class UserAgentDetail < Grape::Entity
+      expose :user_agent
+      expose :ip_address
+      expose :submitted, as: :akismet_submitted
+    end
   end
 end
diff --git a/lib/api/features.rb b/lib/api/features.rb
index cff0ba2ddff..9385c6ca174 100644
--- a/lib/api/features.rb
+++ b/lib/api/features.rb
@@ -2,6 +2,27 @@ module API
   class Features < Grape::API
     before { authenticated_as_admin! }
 
+    helpers do
+      def gate_value(params)
+        case params[:value]
+        when 'true'
+          true
+        when '0', 'false'
+          false
+        else
+          params[:value].to_i
+        end
+      end
+
+      def gate_targets(params)
+        targets = []
+        targets << Feature.group(params[:feature_group]) if params[:feature_group]
+        targets << User.find_by_username(params[:user]) if params[:user]
+
+        targets
+      end
+    end
+
     resource :features do
       desc 'Get a list of all features' do
         success Entities::Feature
@@ -17,16 +38,29 @@ module API
       end
       params do
         requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time'
+        optional :feature_group, type: String, desc: 'A Feature group name'
+        optional :user, type: String, desc: 'A GitLab username'
       end
       post ':name' do
         feature = Feature.get(params[:name])
+        targets = gate_targets(params)
+        value = gate_value(params)
 
-        if %w(0 false).include?(params[:value])
-          feature.disable
-        elsif params[:value] == 'true'
-          feature.enable
+        case value
+        when true
+          if targets.present?
+            targets.each { |target| feature.enable(target) }
+          else
+            feature.enable
+          end
+        when false
+          if targets.present?
+            targets.each { |target| feature.disable(target) }
+          else
+            feature.disable
+          end
         else
-          feature.enable_percentage_of_time(params[:value].to_i)
+          feature.enable_percentage_of_time(value)
         end
 
         present feature, with: Entities::Feature, current_user: current_user
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 2c73a6fdc4e..0f4791841d2 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -268,6 +268,7 @@ module API
       finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility]
       finder_params[:archived] = params[:archived]
       finder_params[:search] = params[:search] if params[:search]
+      finder_params[:user] = params.delete(:user) if params[:user]
       finder_params
     end
 
@@ -313,7 +314,7 @@ module API
 
     def present_artifacts!(artifacts_file)
       return not_found! unless artifacts_file.exists?
-  
+
       if artifacts_file.file_storage?
         present_file!(artifacts_file.path, artifacts_file.filename)
       else
@@ -342,8 +343,8 @@ module API
     def initial_current_user
       return @initial_current_user if defined?(@initial_current_user)
       Gitlab::Auth::UniqueIpsLimiter.limit_user! do
-        @initial_current_user ||= find_user_by_private_token(scopes: @scopes)
-        @initial_current_user ||= doorkeeper_guard(scopes: @scopes)
+        @initial_current_user ||= find_user_by_private_token(scopes: scopes_registered_for_endpoint)
+        @initial_current_user ||= doorkeeper_guard(scopes: scopes_registered_for_endpoint)
         @initial_current_user ||= find_user_from_warden
 
         unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed?
@@ -407,5 +408,22 @@ module API
 
       exception.status == 500
     end
+
+    # An array of scopes that were registered (using `allow_access_with_scope`)
+    # for the current endpoint class. It also returns scopes registered on
+    # `API::API`, since these are meant to apply to all API routes.
+    def scopes_registered_for_endpoint
+      @scopes_registered_for_endpoint ||=
+        begin
+          endpoint_classes = [options[:for].presence, ::API::API].compact
+          endpoint_classes.reduce([]) do |memo, endpoint|
+            if endpoint.respond_to?(:allowed_scopes)
+              memo.concat(endpoint.allowed_scopes)
+            else
+              memo
+            end
+          end
+        end
+    end
   end
 end
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index 5e9cf5e68b1..ecb79317093 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -1,6 +1,11 @@
 module API
   module Helpers
     module InternalHelpers
+      SSH_GITALY_FEATURES = {
+        'git-receive-pack' => :ssh_receive_pack,
+        'git-upload-pack' => :ssh_upload_pack
+      }.freeze
+
       def wiki?
         set_project unless defined?(@wiki)
         @wiki
@@ -10,7 +15,7 @@ module API
         set_project unless defined?(@project)
         @project
       end
-      
+
       def redirected_path
         @redirected_path
       end
@@ -54,15 +59,33 @@ module API
         Gitlab::GlRepository.gl_repository(project, wiki?)
       end
 
-      # Return the repository full path so that gitlab-shell has it when
-      # handling ssh commands
-      def repository_path
+      # Return the repository depending on whether we want the wiki or the
+      # regular repository
+      def repository
         if wiki?
-          project.wiki.repository.path_to_repo
+          project.wiki.repository
         else
-          project.repository.path_to_repo
+          project.repository
         end
       end
+
+      # Return the repository full path so that gitlab-shell has it when
+      # handling ssh commands
+      def repository_path
+        repository.path_to_repo
+      end
+
+      # Return the Gitaly Address if it is enabled
+      def gitaly_payload(action)
+        feature = SSH_GITALY_FEATURES[action]
+        return unless feature && Gitlab::GitalyClient.feature_enabled?(feature)
+
+        {
+          repository: repository.gitaly_repository,
+          address: Gitlab::GitalyClient.address(project.repository_storage),
+          token: Gitlab::GitalyClient.token(project.repository_storage)
+        }
+      end
     end
   end
 end
diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
index 1369b021ea4..f8645e364ce 100644
--- a/lib/api/helpers/runner.rb
+++ b/lib/api/helpers/runner.rb
@@ -46,7 +46,8 @@ module API
 
         yield if block_given?
 
-        forbidden!('Project has been deleted!') unless job.project
+        project = job.project
+        forbidden!('Project has been deleted!') if project.nil? || project.pending_delete?
         forbidden!('Job has been erased!') if job.erased?
       end
 
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index f1c79970ba4..ef2c08e902c 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -47,7 +47,8 @@ module API
         {
           status: true,
           gl_repository: gl_repository,
-          repository_path: repository_path
+          repository_path: repository_path,
+          gitaly: gitaly_payload(params[:action])
         }
       end
 
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 09dca0dff8b..64be08094ed 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -241,6 +241,22 @@ module API
 
         present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project
       end
+
+      desc 'Get the user agent details for an issue' do
+        success Entities::UserAgentDetail
+      end
+      params do
+        requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
+      end
+      get ":id/issues/:issue_iid/user_agent_detail" do
+        authenticated_as_admin!
+
+        issue = find_project_issue(params[:issue_iid])
+
+        return not_found!('UserAgentDetail') unless issue.user_agent_detail
+
+        present issue.user_agent_detail, with: Entities::UserAgentDetail
+      end
     end
   end
 end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 1118fc7465b..4ad1eef4ff1 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -10,6 +10,8 @@ module API
     resource :projects, requirements: { id: %r{[^/]+} } do
       include TimeTrackingEndpoints
 
+      helpers ::Gitlab::IssuableMetadata
+
       helpers do
         def handle_merge_request_errors!(errors)
           if errors[:project_access].any?
@@ -41,7 +43,8 @@ module API
           args[:milestone_title] = args.delete(:milestone)
           args[:label_name] = args.delete(:labels)
 
-          merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations
+          merge_requests = MergeRequestsFinder.new(current_user, args).execute
+                             .preload(:notes, :target_project, :author, :assignee, :milestone, :merge_request_diff, :labels)
 
           merge_requests.reorder(args[:order_by] => args[:sort])
         end
@@ -80,8 +83,9 @@ module API
         authorize! :read_merge_request, user_project
 
         merge_requests = find_merge_requests(project_id: user_project.id)
+        issuable_metadata = issuable_meta_data(merge_requests, 'MergeRequest')
 
-        present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project
+        present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project, issuable_metadata: issuable_metadata
       end
 
       desc 'Create a merge request' do
diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb
index 30761cb9b55..f1eaff6b0eb 100644
--- a/lib/api/namespaces.rb
+++ b/lib/api/namespaces.rb
@@ -17,7 +17,7 @@ module API
 
         namespaces = namespaces.search(params[:search]) if params[:search].present?
 
-        present paginate(namespaces), with: Entities::Namespace
+        present paginate(namespaces), with: Entities::Namespace, current_user: current_user
       end
     end
   end
diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb
index 93d89209934..dbeaf9e17ef 100644
--- a/lib/api/pipeline_schedules.rb
+++ b/lib/api/pipeline_schedules.rb
@@ -74,9 +74,10 @@ module API
         optional :active, type: Boolean, desc: 'The activation of pipeline schedule'
       end
       put ':id/pipeline_schedules/:pipeline_schedule_id' do
-        authorize! :update_pipeline_schedule, user_project
+        authorize! :read_pipeline_schedule, user_project
 
         not_found!('PipelineSchedule') unless pipeline_schedule
+        authorize! :update_pipeline_schedule, pipeline_schedule
 
         if pipeline_schedule.update(declared_params(include_missing: false))
           present pipeline_schedule, with: Entities::PipelineScheduleDetails
@@ -92,9 +93,10 @@ module API
         requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
       end
       post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do
-        authorize! :update_pipeline_schedule, user_project
+        authorize! :read_pipeline_schedule, user_project
 
         not_found!('PipelineSchedule') unless pipeline_schedule
+        authorize! :update_pipeline_schedule, pipeline_schedule
 
         if pipeline_schedule.own!(current_user)
           present pipeline_schedule, with: Entities::PipelineScheduleDetails
@@ -110,9 +112,10 @@ module API
         requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
       end
       delete ':id/pipeline_schedules/:pipeline_schedule_id' do
-        authorize! :admin_pipeline_schedule, user_project
+        authorize! :read_pipeline_schedule, user_project
 
         not_found!('PipelineSchedule') unless pipeline_schedule
+        authorize! :admin_pipeline_schedule, pipeline_schedule
 
         status :accepted
         present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails
diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb
index 64efe82a937..3320eadff0d 100644
--- a/lib/api/project_snippets.rb
+++ b/lib/api/project_snippets.rb
@@ -131,6 +131,22 @@ module API
         content_type 'text/plain'
         present snippet.content
       end
+
+      desc 'Get the user agent details for a project snippet' do
+        success Entities::UserAgentDetail
+      end
+      params do
+        requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
+      end
+      get ":id/snippets/:snippet_id/user_agent_detail" do
+        authenticated_as_admin!
+
+        snippet = Snippet.find_by!(id: params[:id])
+
+        return not_found!('UserAgentDetail') unless snippet.user_agent_detail
+
+        present snippet.user_agent_detail, with: Entities::UserAgentDetail
+      end
     end
   end
 end
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index c5df45b7902..c459257158d 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -1,3 +1,5 @@
+require_dependency 'declarative_policy'
+
 module API
   # Projects API
   class Projects < Grape::API
@@ -8,6 +10,7 @@ module API
     helpers do
       params :optional_params_ce do
         optional :description, type: String, desc: 'The description of the project'
+        optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`'
         optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled'
         optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled'
         optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled'
@@ -33,61 +36,86 @@ module API
       params :statistics_params do
         optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
       end
-    end
 
-    resource :projects do
-      helpers do
-        params :collection_params do
-          use :sort_params
-          use :filter_params
-          use :pagination
-
-          optional :simple, type: Boolean, default: false,
-                            desc: 'Return only the ID, URL, name, and path of each project'
-        end
+      params :collection_params do
+        use :sort_params
+        use :filter_params
+        use :pagination
 
-        params :sort_params do
-          optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
-                              default: 'created_at', desc: 'Return projects ordered by field'
-          optional :sort, type: String, values: %w[asc desc], default: 'desc',
-                          desc: 'Return projects sorted in ascending and descending order'
-        end
+        optional :simple, type: Boolean, default: false,
+                          desc: 'Return only the ID, URL, name, and path of each project'
+      end
 
-        params :filter_params do
-          optional :archived, type: Boolean, default: false, desc: 'Limit by archived status'
-          optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values,
-                                desc: 'Limit by visibility'
-          optional :search, type: String, desc: 'Return list of projects matching the search criteria'
-          optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
-          optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
-          optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of'
-          optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
-          optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
-        end
+      params :sort_params do
+        optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
+                            default: 'created_at', desc: 'Return projects ordered by field'
+        optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                        desc: 'Return projects sorted in ascending and descending order'
+      end
 
-        params :create_params do
-          optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
-          optional :import_url, type: String, desc: 'URL from which the project is imported'
-        end
+      params :filter_params do
+        optional :archived, type: Boolean, default: false, desc: 'Limit by archived status'
+        optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values,
+                              desc: 'Limit by visibility'
+        optional :search, type: String, desc: 'Return list of projects matching the search criteria'
+        optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
+        optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
+        optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of'
+        optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
+        optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
+      end
+
+      params :create_params do
+        optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
+        optional :import_url, type: String, desc: 'URL from which the project is imported'
+      end
+
+      def present_projects(options = {})
+        projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
+        projects = reorder_projects(projects)
+        projects = projects.with_statistics if params[:statistics]
+        projects = projects.with_issues_enabled if params[:with_issues_enabled]
+        projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
 
-        def present_projects(options = {})
-          projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
-          projects = reorder_projects(projects)
-          projects = projects.with_statistics if params[:statistics]
-          projects = projects.with_issues_enabled if params[:with_issues_enabled]
-          projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
-
-          options = options.reverse_merge(
-            with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
-            statistics: params[:statistics],
-            current_user: current_user
-          )
-          options[:with] = Entities::BasicProjectDetails if params[:simple]
-
-          present paginate(projects), options
+        if current_user
+          projects = projects.includes(:route, :taggings, namespace: :route)
+          project_members = current_user.project_members
+          group_members = current_user.group_members
         end
+
+        options = options.reverse_merge(
+          with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
+          statistics: params[:statistics],
+          project_members: project_members,
+          group_members: group_members,
+          current_user: current_user
+        )
+        options[:with] = Entities::BasicProjectDetails if params[:simple]
+
+        present paginate(projects), options
       end
+    end
 
+    resource :users, requirements: { user_id: %r{[^/]+} } do
+      desc 'Get a user projects' do
+        success Entities::BasicProjectDetails
+      end
+      params do
+        requires :user_id, type: String, desc: 'The ID or username of the user'
+        use :collection_params
+        use :statistics_params
+      end
+      get ":user_id/projects" do
+        user = find_user(params[:user_id])
+        not_found!('User') unless user
+
+        params[:user] = user
+
+        present_projects
+      end
+    end
+
+    resource :projects do
       desc 'Get a list of visible projects for authenticated user' do
         success Entities::BasicProjectDetails
       end
@@ -396,7 +424,7 @@ module API
         use :pagination
       end
       get ':id/users' do
-        users = user_project.team.users
+        users = DeclarativePolicy.subject_scope { user_project.team.users }
         users = users.search(params[:search]) if params[:search].present?
 
         present paginate(users), with: Entities::UserBasic
diff --git a/lib/api/scope.rb b/lib/api/scope.rb
new file mode 100644
index 00000000000..d5165b2e482
--- /dev/null
+++ b/lib/api/scope.rb
@@ -0,0 +1,23 @@
+# Encapsulate a scope used for authorization, such as `api`, or `read_user`
+module API
+  class Scope
+    attr_reader :name, :if
+
+    def initialize(name, options = {})
+      @name = name.to_sym
+      @if = options[:if]
+    end
+
+    # Are the `scopes` passed in sufficient to adequately authorize the passed
+    # request for the scope represented by the current instance of this class?
+    def sufficient?(scopes, request)
+      scopes.include?(self.name) && verify_if_condition(request)
+    end
+
+    private
+
+    def verify_if_condition(request)
+      self.if.nil? || self.if.call(request)
+    end
+  end
+end
diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb
index c630c24c339..fd634037a77 100644
--- a/lib/api/snippets.rb
+++ b/lib/api/snippets.rb
@@ -140,6 +140,22 @@ module API
         content_type 'text/plain'
         present snippet.content
       end
+
+      desc 'Get the user agent details for a snippet' do
+        success Entities::UserAgentDetail
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of a snippet'
+      end
+      get ":id/user_agent_detail" do
+        authenticated_as_admin!
+
+        snippet = Snippet.find_by!(id: params[:id])
+
+        return not_found!('UserAgentDetail') unless snippet.user_agent_detail
+
+        present snippet.user_agent_detail, with: Entities::UserAgentDetail
+      end
     end
   end
 end
diff --git a/lib/api/users.rb b/lib/api/users.rb
index f9555842daf..c469751c31c 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -1,13 +1,15 @@
 module API
   class Users < Grape::API
     include PaginationParams
+    include APIGuard
 
-    before do
-      allow_access_with_scope :read_user if request.get?
-      authenticate!
-    end
+    allow_access_with_scope :read_user, if: -> (request) { request.get? }
 
     resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do
+      before do
+        authenticate_non_get!
+      end
+
       helpers do
         def find_user(params)
           id = params[:user_id] || params[:id]
@@ -46,20 +48,33 @@ module API
         optional :active, type: Boolean, default: false, desc: 'Filters only active users'
         optional :external, type: Boolean, default: false, desc: 'Filters only external users'
         optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users'
+        optional :created_after, type: DateTime, desc: 'Return users created after the specified time'
+        optional :created_before, type: DateTime, desc: 'Return users created before the specified time'
         all_or_none_of :extern_uid, :provider
 
         use :pagination
       end
       get do
-        unless can?(current_user, :read_users_list)
-          render_api_error!("Not authorized.", 403)
-        end
-
         authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?)
 
+        unless current_user&.admin?
+          params.except!(:created_after, :created_before)
+        end
+
         users = UsersFinder.new(current_user, params).execute
 
-        entity = current_user.admin? ? Entities::UserWithAdmin : Entities::UserBasic
+        authorized = can?(current_user, :read_users_list)
+
+        # When `current_user` is not present, require that the `username`
+        # parameter is passed, to prevent an unauthenticated user from accessing
+        # a list of all the users on the GitLab instance. `UsersFinder` performs
+        # an exact match on the `username` parameter, so we are guaranteed to
+        # get either 0 or 1 `users` here.
+        authorized &&= params[:username].present? if current_user.blank?
+
+        forbidden!("Not authorized to access /api/v4/users") unless authorized
+
+        entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic
         present paginate(users), with: entity
       end
 
@@ -398,6 +413,10 @@ module API
     end
 
     resource :user do
+      before do
+        authenticate!
+      end
+
       desc 'Get the currently authenticated user' do
         success Entities::UserPublic
       end
diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb
index 37020019e07..cf106f2552d 100644
--- a/lib/api/v3/users.rb
+++ b/lib/api/v3/users.rb
@@ -2,9 +2,11 @@ module API
   module V3
     class Users < Grape::API
       include PaginationParams
+      include APIGuard
+
+      allow_access_with_scope :read_user, if: -> (request) { request.get? }
 
       before do
-        allow_access_with_scope :read_user if request.get?
         authenticate!
       end
 
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
index 10374995497..7fa528fb2d3 100644
--- a/lib/api/variables.rb
+++ b/lib/api/variables.rb
@@ -45,7 +45,9 @@ module API
         optional :protected, type: String, desc: 'Whether the variable is protected'
       end
       post ':id/variables' do
-        variable = user_project.variables.create(declared_params(include_missing: false))
+        variable_params = declared_params(include_missing: false)
+
+        variable = user_project.variables.create(variable_params)
 
         if variable.valid?
           present variable, with: Entities::Variable
@@ -67,7 +69,9 @@ module API
 
         return not_found!('Variable') unless variable
 
-        if variable.update(declared_params(include_missing: false).except(:key))
+        variable_params = declared_params(include_missing: false).except(:key)
+
+        if variable.update(variable_params)
           present variable, with: Entities::Variable
         else
           render_validation_error!(variable)
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index 8bc2dd18bda..7a262dd025c 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -216,12 +216,7 @@ module Banzai
         @references_per_project ||= begin
           refs = Hash.new { |hash, key| hash[key] = Set.new }
 
-          regex =
-            if uses_reference_pattern?
-              Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern)
-            else
-              object_class.link_reference_pattern
-            end
+          regex = Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern)
 
           nodes.each do |node|
             node.to_html.scan(regex) do
@@ -323,14 +318,6 @@ module Banzai
           value
         end
       end
-
-      # There might be special cases like filters
-      # that should ignore reference pattern
-      # eg: IssueReferenceFilter when using a external issues tracker
-      # In those cases this method should be overridden on the filter subclass
-      def uses_reference_pattern?
-        true
-      end
     end
   end
 end
diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb
index eaacb9591b1..21bcb1c5ca8 100644
--- a/lib/banzai/filter/commit_range_reference_filter.rb
+++ b/lib/banzai/filter/commit_range_reference_filter.rb
@@ -30,7 +30,7 @@ module Banzai
 
       def url_for_object(range, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_compare_url(project.namespace, project,
+        h.project_compare_url(project,
                                         range.to_param.merge(only_path: context[:only_path]))
       end
 
diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb
index 69c06117eda..714e0319025 100644
--- a/lib/banzai/filter/commit_reference_filter.rb
+++ b/lib/banzai/filter/commit_reference_filter.rb
@@ -24,7 +24,7 @@ module Banzai
 
       def url_for_object(commit, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_commit_url(project.namespace, project, commit,
+        h.project_commit_url(project, commit,
                                         only_path: context[:only_path])
       end
 
diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb
index dce4de3ceaf..53a229256a5 100644
--- a/lib/banzai/filter/external_issue_reference_filter.rb
+++ b/lib/banzai/filter/external_issue_reference_filter.rb
@@ -3,6 +3,8 @@ module Banzai
     # HTML filter that replaces external issue tracker references with links.
     # References are ignored if the project doesn't use an external issue
     # tracker.
+    #
+    # This filter does not support cross-project references.
     class ExternalIssueReferenceFilter < ReferenceFilter
       self.reference_type = :external_issue
 
@@ -87,7 +89,7 @@ module Banzai
       end
 
       def issue_reference_pattern
-        external_issues_cached(:issue_reference_pattern)
+        external_issues_cached(:external_issue_reference_pattern)
       end
 
       private
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
index 044d18ff824..ba1a5ac84b3 100644
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ b/lib/banzai/filter/issue_reference_filter.rb
@@ -15,10 +15,6 @@ module Banzai
         Issue
       end
 
-      def uses_reference_pattern?
-        context[:project].default_issues_tracker?
-      end
-
       def find_object(project, iid)
         issues_per_project[project][iid]
       end
@@ -38,13 +34,7 @@ module Banzai
 
           projects_per_reference.each do |path, project|
             issue_ids = references_per_project[path]
-
-            issues =
-              if project.default_issues_tracker?
-                project.issues.where(iid: issue_ids.to_a)
-              else
-                issue_ids.map { |id| ExternalIssue.new(id, project) }
-              end
+            issues = project.issues.where(iid: issue_ids.to_a)
 
             issues.each do |issue|
               hash[project][issue.iid.to_i] = issue
@@ -55,26 +45,6 @@ module Banzai
         end
       end
 
-      def object_link_title(object)
-        if object.is_a?(ExternalIssue)
-          "Issue in #{object.project.external_issue_tracker.title}"
-        else
-          super
-        end
-      end
-
-      def data_attributes_for(text, project, object, link: false)
-        if object.is_a?(ExternalIssue)
-          data_attribute(
-            project: project.id,
-            external_issue: object.id,
-            reference_type: ExternalIssueReferenceFilter.reference_type
-          )
-        else
-          super
-        end
-      end
-
       def projects_relation_for_paths(paths)
         super(paths).includes(:gitlab_issue_tracker_service)
       end
diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb
index a605dea149e..5364984c9d3 100644
--- a/lib/banzai/filter/label_reference_filter.rb
+++ b/lib/banzai/filter/label_reference_filter.rb
@@ -61,8 +61,7 @@ module Banzai
 
       def url_for_object(label, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_issues_url(project.namespace, project, label_name: label.name,
-                                                                   only_path:  context[:only_path])
+        h.project_issues_url(project, label_name: label.name, only_path: context[:only_path])
       end
 
       def object_link_text(object, matches)
diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb
index 3888acf935e..0eab865ac04 100644
--- a/lib/banzai/filter/merge_request_reference_filter.rb
+++ b/lib/banzai/filter/merge_request_reference_filter.rb
@@ -17,7 +17,7 @@ module Banzai
 
       def url_for_object(mr, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_merge_request_url(project.namespace, project, mr,
+        h.project_merge_request_url(project, mr,
                                             only_path: context[:only_path])
       end
 
diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb
index f12014e191f..45c033d32a8 100644
--- a/lib/banzai/filter/milestone_reference_filter.rb
+++ b/lib/banzai/filter/milestone_reference_filter.rb
@@ -49,7 +49,7 @@ module Banzai
 
       def url_for_object(milestone, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_milestone_url(project.namespace, project, milestone,
+        h.project_milestone_url(project, milestone,
                                         only_path: context[:only_path])
       end
 
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 6640168bfa2..a6f8650ed3d 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -30,6 +30,8 @@ module Banzai
         attributes = attributes.reject { |_, v| v.nil? }
 
         attributes[:reference_type] ||= self.class.reference_type
+        attributes[:container] ||= 'body'
+        attributes[:placement] ||= 'bottom'
         attributes.delete(:original) if context[:no_original_data]
         attributes.map do |key, value|
           %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}")
diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb
index 212a0bbf2a0..134a192c22b 100644
--- a/lib/banzai/filter/snippet_reference_filter.rb
+++ b/lib/banzai/filter/snippet_reference_filter.rb
@@ -17,7 +17,7 @@ module Banzai
 
       def url_for_object(snippet, project)
         h = Gitlab::Routing.url_helpers
-        h.namespace_project_snippet_url(project.namespace, project, snippet,
+        h.project_snippet_url(project, snippet,
                                         only_path: context[:only_path])
       end
     end
diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb
index a798927823f..f3356d6c51e 100644
--- a/lib/banzai/filter/user_reference_filter.rb
+++ b/lib/banzai/filter/user_reference_filter.rb
@@ -107,7 +107,7 @@ module Banzai
         if author && !project.team.member?(author)
           link_content
         else
-          url = urls.namespace_project_url(project.namespace, project,
+          url = urls.project_url(project,
                                            only_path: context[:only_path])
 
           data = data_attribute(project: project.id, author: author.try(:id))
diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb
index 9fd4bd68d43..a65bbe23958 100644
--- a/lib/banzai/reference_parser/issue_parser.rb
+++ b/lib/banzai/reference_parser/issue_parser.rb
@@ -4,9 +4,6 @@ module Banzai
       self.reference_type = :issue
 
       def nodes_visible_to_user(user, nodes)
-        # It is not possible to check access rights for external issue trackers
-        return nodes if project && project.external_issue_tracker
-
         issues = issues_for_nodes(nodes)
 
         readable_issues = Ability
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index 5109dc9670f..a40b6ab6c9f 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -28,7 +28,8 @@ module Ci
 
         yield if block_given?
 
-        forbidden!('Project has been deleted!') unless build.project
+        project = build.project
+        forbidden!('Project has been deleted!') if project.nil? || project.pending_delete?
         forbidden!('Build has been erased!') if build.erased?
       end
 
diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb
new file mode 100644
index 00000000000..d9959bc1aff
--- /dev/null
+++ b/lib/declarative_policy.rb
@@ -0,0 +1,58 @@
+require_dependency 'declarative_policy/cache'
+require_dependency 'declarative_policy/condition'
+require_dependency 'declarative_policy/dsl'
+require_dependency 'declarative_policy/preferred_scope'
+require_dependency 'declarative_policy/rule'
+require_dependency 'declarative_policy/runner'
+require_dependency 'declarative_policy/step'
+
+require_dependency 'declarative_policy/base'
+
+module DeclarativePolicy
+  class << self
+    def policy_for(user, subject, opts = {})
+      cache = opts[:cache] || {}
+      key = Cache.policy_key(user, subject)
+
+      cache[key] ||= class_for(subject).new(user, subject, opts)
+    end
+
+    def class_for(subject)
+      return GlobalPolicy if subject == :global
+      return NilPolicy if subject.nil?
+
+      subject = find_delegate(subject)
+
+      subject.class.ancestors.each do |klass|
+        next unless klass.name
+
+        begin
+          policy_class = "#{klass.name}Policy".constantize
+
+          # NOTE: the < operator here tests whether policy_class
+          # inherits from Base. We can't use #is_a? because that
+          # tests for *instances*, not *subclasses*.
+          return policy_class if policy_class < Base
+        rescue NameError
+          nil
+        end
+      end
+
+      raise "no policy for #{subject.class.name}"
+    end
+
+    private
+
+    def find_delegate(subject)
+      seen = Set.new
+
+      while subject.respond_to?(:declarative_policy_delegate)
+        raise ArgumentError, "circular delegations" if seen.include?(subject.object_id)
+        seen << subject.object_id
+        subject = subject.declarative_policy_delegate
+      end
+
+      subject
+    end
+  end
+end
diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb
new file mode 100644
index 00000000000..df94cafb6a1
--- /dev/null
+++ b/lib/declarative_policy/base.rb
@@ -0,0 +1,329 @@
+module DeclarativePolicy
+  class Base
+    # A map of ability => list of rules together with :enable
+    # or :prevent actions. Used to look up which rules apply to
+    # a given ability. See Base.ability_map
+    class AbilityMap
+      attr_reader :map
+      def initialize(map = {})
+        @map = map
+      end
+
+      # This merge behavior is different than regular hashes - if both
+      # share a key, the values at that key are concatenated, rather than
+      # overridden.
+      def merge(other)
+        conflict_proc = proc { |key, my_val, other_val| my_val + other_val }
+        AbilityMap.new(@map.merge(other.map, &conflict_proc))
+      end
+
+      def actions(key)
+        @map[key] ||= []
+      end
+
+      def enable(key, rule)
+        actions(key) << [:enable, rule]
+      end
+
+      def prevent(key, rule)
+        actions(key) << [:prevent, rule]
+      end
+    end
+
+    class << self
+      # The `own_ability_map` vs `ability_map` distinction is used so that
+      # the data structure is properly inherited - with subclasses recursively
+      # merging their parent class.
+      #
+      # This pattern is also used for conditions, global_actions, and delegations.
+      def ability_map
+        if self == Base
+          own_ability_map
+        else
+          superclass.ability_map.merge(own_ability_map)
+        end
+      end
+
+      def own_ability_map
+        @own_ability_map ||= AbilityMap.new
+      end
+
+      # an inheritable map of conditions, by name
+      def conditions
+        if self == Base
+          own_conditions
+        else
+          superclass.conditions.merge(own_conditions)
+        end
+      end
+
+      def own_conditions
+        @own_conditions ||= {}
+      end
+
+      # a list of global actions, generated by `prevent_all`. these aren't
+      # stored in `ability_map` because they aren't indexed by a particular
+      # ability.
+      def global_actions
+        if self == Base
+          own_global_actions
+        else
+          superclass.global_actions + own_global_actions
+        end
+      end
+
+      def own_global_actions
+        @own_global_actions ||= []
+      end
+
+      # an inheritable map of delegations, indexed by name (which may be
+      # autogenerated)
+      def delegations
+        if self == Base
+          own_delegations
+        else
+          superclass.delegations.merge(own_delegations)
+        end
+      end
+
+      def own_delegations
+        @own_delegations ||= {}
+      end
+
+      # all the [rule, action] pairs that apply to a particular ability.
+      # we combine the specific ones looked up in ability_map with the global
+      # ones.
+      def configuration_for(ability)
+        ability_map.actions(ability) + global_actions
+      end
+
+      ### declaration methods ###
+
+      def delegate(name = nil, &delegation_block)
+        if name.nil?
+          @delegate_name_counter ||= 0
+          @delegate_name_counter += 1
+          name = :"anonymous_#{@delegate_name_counter}"
+        end
+
+        name = name.to_sym
+
+        if delegation_block.nil?
+          delegation_block = proc { @subject.__send__(name) }
+        end
+
+        own_delegations[name] = delegation_block
+      end
+
+      # Declares a rule, constructed using RuleDsl, and returns
+      # a PolicyDsl which is used for registering the rule with
+      # this class. PolicyDsl will call back into Base.enable_when,
+      # Base.prevent_when, and Base.prevent_all_when.
+      def rule(&b)
+        rule = RuleDsl.new(self).instance_eval(&b)
+        PolicyDsl.new(self, rule)
+      end
+
+      # A hash in which to store calls to `desc` and `with_scope`, etc.
+      def last_options
+        @last_options ||= {}.with_indifferent_access
+      end
+
+      # retrieve and zero out the previously set options (used in .condition)
+      def last_options!
+        last_options.tap { @last_options = nil }
+      end
+
+      # Declare a description for the following condition. Currently unused,
+      # but opens the potential for explaining to users why they were or were
+      # not able to do something.
+      def desc(description)
+        last_options[:description] = description
+      end
+
+      def with_options(opts = {})
+        last_options.merge!(opts)
+      end
+
+      def with_scope(scope)
+        with_options scope: scope
+      end
+
+      def with_score(score)
+        with_options score: score
+      end
+
+      # Declares a condition. It gets stored in `own_conditions`, and generates
+      # a query method based on the condition's name.
+      def condition(name, opts = {}, &value)
+        name = name.to_sym
+
+        opts = last_options!.merge(opts)
+        opts[:context_key] ||= self.name
+
+        condition = Condition.new(name, opts, &value)
+
+        self.own_conditions[name] = condition
+
+        define_method(:"#{name}?") { condition(name).pass? }
+      end
+
+      # These next three methods are mainly called from PolicyDsl,
+      # and are responsible for "inverting" the relationship between
+      # an ability and a rule. We store in `ability_map` a map of
+      # abilities to rules that affect them, together with a
+      # symbol indicating :prevent or :enable.
+      def enable_when(abilities, rule)
+        abilities.each { |a| own_ability_map.enable(a, rule) }
+      end
+
+      def prevent_when(abilities, rule)
+        abilities.each { |a| own_ability_map.prevent(a, rule) }
+      end
+
+      # we store global prevents (from `prevent_all`) separately,
+      # so that they can be combined into every decision made.
+      def prevent_all_when(rule)
+        own_global_actions << [:prevent, rule]
+      end
+    end
+
+    # A policy object contains a specific user and subject on which
+    # to compute abilities. For this reason it's sometimes called
+    # "context" within the framework.
+    #
+    # It also stores a reference to the cache, so it can be used
+    # to cache computations by e.g. ManifestCondition.
+    attr_reader :user, :subject, :cache
+    def initialize(user, subject, opts = {})
+      @user = user
+      @subject = subject
+      @cache = opts[:cache] || {}
+    end
+
+    # helper for checking abilities on this and other subjects
+    # for the current user.
+    def can?(ability, new_subject = :_self)
+      return allowed?(ability) if new_subject == :_self
+
+      policy_for(new_subject).allowed?(ability)
+    end
+
+    # This is the main entry point for permission checks. It constructs
+    # or looks up a Runner for the given ability and asks it if it passes.
+    def allowed?(*abilities)
+      abilities.all? { |a| runner(a).pass? }
+    end
+
+    # The inverse of #allowed?, used mainly in specs.
+    def disallowed?(*abilities)
+      abilities.all? { |a| !runner(a).pass? }
+    end
+
+    # computes the given ability and prints a helpful debugging output
+    # showing which 
+    def debug(ability, *a)
+      runner(ability).debug(*a)
+    end
+
+    desc "Unknown user"
+    condition(:anonymous, scope: :user, score: 0) { @user.nil? }
+
+    desc "By default"
+    condition(:default, scope: :global, score: 0) { true }
+
+    def repr
+      subject_repr =
+        if @subject.respond_to?(:id)
+          "#{@subject.class.name}/#{@subject.id}"
+        else
+          @subject.inspect
+        end
+
+      user_repr =
+        if @user
+          @user.to_reference
+        else
+          "<anonymous>"
+        end
+
+      "(#{user_repr} : #{subject_repr})"
+    end
+
+    def inspect
+      "#<#{self.class.name} #{repr}>"
+    end
+
+    # returns a Runner for the given ability, capable of computing whether
+    # the ability is allowed. Runners are cached on the policy (which itself
+    # is cached on @cache), and caches its result. This is how we perform caching
+    # at the ability level.
+    def runner(ability)
+      ability = ability.to_sym
+      @runners ||= {}
+      @runners[ability] ||=
+        begin
+          delegated_runners = delegated_policies.values.compact.map { |p| p.runner(ability) }
+          own_runner = Runner.new(own_steps(ability))
+          delegated_runners.inject(own_runner, &:merge_runner)
+        end
+    end
+
+    # Helpers for caching. Used by ManifestCondition in performing condition
+    # computation.
+    #
+    # NOTE we can't use ||= here because the value might be the
+    # boolean `false`
+    def cache(key, &b)
+      return @cache[key] if cached?(key)
+      @cache[key] = yield
+    end
+
+    def cached?(key)
+      !@cache[key].nil?
+    end
+
+    # returns a ManifestCondition capable of computing itself. The computation
+    # will use our own @cache.
+    def condition(name)
+      name = name.to_sym
+      @_conditions ||= {}
+      @_conditions[name] ||=
+        begin
+          raise "invalid condition #{name}" unless self.class.conditions.key?(name)
+          ManifestCondition.new(self.class.conditions[name], self)
+        end
+    end
+
+    # used in specs - returns true if there is no possible way for any action
+    # to be allowed, determined only by the global :prevent_all rules.
+    def banned?
+      global_steps = self.class.global_actions.map { |(action, rule)| Step.new(self, rule, action) }
+      !Runner.new(global_steps).pass?
+    end
+
+    # A list of other policies that we've delegated to (see `Base.delegate`)
+    def delegated_policies
+      @delegated_policies ||= self.class.delegations.transform_values do |block|
+        new_subject = instance_eval(&block)
+
+        # never delegate to nil, as that would immediately prevent_all
+        next if new_subject.nil?
+
+        policy_for(new_subject)
+      end
+    end
+
+    def policy_for(other_subject)
+      DeclarativePolicy.policy_for(@user, other_subject, cache: @cache)
+    end
+
+    protected
+
+    # constructs steps that come from this policy and not from any delegations
+    def own_steps(ability)
+      rules = self.class.configuration_for(ability)
+      rules.map { |(action, rule)| Step.new(self, rule, action) }
+    end
+  end
+end
diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb
new file mode 100644
index 00000000000..b8cc60074c7
--- /dev/null
+++ b/lib/declarative_policy/cache.rb
@@ -0,0 +1,32 @@
+module DeclarativePolicy
+  module Cache
+    class << self
+      def user_key(user)
+        return '<anonymous>' if user.nil?
+        id_for(user)
+      end
+
+      def policy_key(user, subject)
+        u = user_key(user)
+        s = subject_key(subject)
+        "/dp/policy/#{u}/#{s}"
+      end
+
+      def subject_key(subject)
+        return '<nil>' if subject.nil?
+        return subject.inspect if subject.is_a?(Symbol)
+        "#{subject.class.name}:#{id_for(subject)}"
+      end
+
+      private
+
+      def id_for(obj)
+        if obj.respond_to?(:id) && obj.id
+          obj.id.to_s
+        else
+          "##{obj.object_id}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/declarative_policy/condition.rb b/lib/declarative_policy/condition.rb
new file mode 100644
index 00000000000..9d7cf6b9726
--- /dev/null
+++ b/lib/declarative_policy/condition.rb
@@ -0,0 +1,102 @@
+module DeclarativePolicy
+  # A Condition is the data structure that is created by the
+  # `condition` declaration on DeclarativePolicy::Base. It is
+  # more or less just a struct of the data passed to that
+  # declaration. It holds on to the block to be instance_eval'd
+  # on a context (instance of Base) later, via #compute.
+  class Condition
+    attr_reader :name, :description, :scope
+    attr_reader :manual_score
+    attr_reader :context_key
+    def initialize(name, opts = {}, &compute)
+      @name = name
+      @compute = compute
+      @scope = opts.fetch(:scope, :normal)
+      @description = opts.delete(:description)
+      @context_key = opts[:context_key]
+      @manual_score = opts.fetch(:score, nil)
+    end
+
+    def compute(context)
+      !!context.instance_eval(&@compute)
+    end
+
+    def key
+      "#{@context_key}/#{@name}"
+    end
+  end
+
+  # In contrast to a Condition, a ManifestCondition contains
+  # a Condition and a context object, and is capable of calculating
+  # a result itself. This is the return value of Base#condition.
+  class ManifestCondition
+    def initialize(condition, context)
+      @condition = condition
+      @context = context
+    end
+
+    # The main entry point - does this condition pass? We reach into
+    # the context's cache here so that we can share in the global
+    # cache (often RequestStore or similar).
+    def pass?
+      @context.cache(cache_key) { @condition.compute(@context) }
+    end
+
+    # Whether we've already computed this condition.
+    def cached?
+      @context.cached?(cache_key)
+    end
+
+    # This is used to score Rule::Condition. See Rule::Condition#score
+    # and Runner#steps_by_score for how scores are used.
+    #
+    # The number here is intended to represent, abstractly, how
+    # expensive it would be to calculate this condition.
+    #
+    # See #cache_key for info about @condition.scope.
+    def score
+      # If we've been cached, no computation is necessary.
+      return 0 if cached?
+
+      # Use the override from condition(score: ...) if present
+      return @condition.manual_score if @condition.manual_score
+
+      # Global scope rules are cheap due to max cache sharing
+      return 2 if  @condition.scope == :global
+
+      # "Normal" rules can't share caches with any other policies
+      return 16 if @condition.scope == :normal
+
+      # otherwise, we're :user or :subject scope, so it's 4 if
+      # the caller has declared a preference
+      return 4 if @condition.scope == DeclarativePolicy.preferred_scope
+
+      # and 8 for all other :user or :subject scope conditions.
+      8
+    end
+
+    private
+
+    # This method controls the caching for the condition. This is where
+    # the condition(scope: ...) option comes into play. Notice that
+    # depending on the scope, we may cache only by the user or only by
+    # the subject, resulting in sharing across different policy objects.
+    def cache_key
+      case @condition.scope
+      when :normal  then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}"
+      when :user    then "/dp/condition/#{@condition.key}/#{user_key}"
+      when :subject then "/dp/condition/#{@condition.key}/#{subject_key}"
+      when :global  then "/dp/condition/#{@condition.key}"
+      else raise 'invalid scope'
+      end
+    end
+
+    def user_key
+      Cache.user_key(@context.user)
+    end
+
+    def subject_key
+      Cache.subject_key(@context.subject)
+    end
+  end
+end
diff --git a/lib/declarative_policy/dsl.rb b/lib/declarative_policy/dsl.rb
new file mode 100644
index 00000000000..b26807a7622
--- /dev/null
+++ b/lib/declarative_policy/dsl.rb
@@ -0,0 +1,103 @@
+module DeclarativePolicy
+  # The DSL evaluation context inside rule { ... } blocks.
+  # Responsible for creating and combining Rule objects.
+  #
+  # See Base.rule
+  class RuleDsl
+    def initialize(context_class)
+      @context_class = context_class
+    end
+
+    def can?(ability)
+      Rule::Ability.new(ability)
+    end
+
+    def all?(*rules)
+      Rule::And.make(rules)
+    end
+
+    def any?(*rules)
+      Rule::Or.make(rules)
+    end
+
+    def none?(*rules)
+      ~Rule::Or.new(rules)
+    end
+
+    def cond(condition)
+      Rule::Condition.new(condition)
+    end
+
+    def delegate(delegate_name, condition)
+      Rule::DelegatedCondition.new(delegate_name, condition)
+    end
+
+    def method_missing(m, *a, &b)
+      return super unless a.size == 0 && !block_given?
+
+      if @context_class.delegations.key?(m)
+        DelegateDsl.new(self, m)
+      else
+        cond(m.to_sym)
+      end
+    end
+  end
+
+  # Used when the name of a delegate is mentioned in
+  # the rule DSL.
+  class DelegateDsl
+    def initialize(rule_dsl, delegate_name)
+      @rule_dsl = rule_dsl
+      @delegate_name = delegate_name
+    end
+
+    def method_missing(m, *a, &b)
+      return super unless a.size == 0 && !block_given?
+
+      @rule_dsl.delegate(@delegate_name, m)
+    end
+  end
+
+  # The return value of a rule { ... } declaration.
+  # Can call back to register rules with the containing
+  # Policy class (context_class here). See Base.rule
+  #
+  # Note that the #policy method just performs an #instance_eval,
+  # which is useful for multiple #enable or #prevent callse.
+  #
+  # Also provides a #method_missing proxy to the context
+  # class's class methods, so that helper methods can be
+  # defined and used in a #policy { ... } block.
+  class PolicyDsl
+    def initialize(context_class, rule)
+      @context_class = context_class
+      @rule = rule
+    end
+
+    def policy(&b)
+      instance_eval(&b)
+    end
+
+    def enable(*abilities)
+      @context_class.enable_when(abilities, @rule)
+    end
+
+    def prevent(*abilities)
+      @context_class.prevent_when(abilities, @rule)
+    end
+
+    def prevent_all
+      @context_class.prevent_all_when(@rule)
+    end
+
+    def method_missing(m, *a, &b)
+      return super unless @context_class.respond_to?(m)
+
+      @context_class.__send__(m, *a, &b)
+    end
+
+    def respond_to_missing?(m)
+      @context_class.respond_to?(m) || super
+    end
+  end
+end
diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb
new file mode 100644
index 00000000000..b0754098149
--- /dev/null
+++ b/lib/declarative_policy/preferred_scope.rb
@@ -0,0 +1,28 @@
+module DeclarativePolicy
+  PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope"
+
+  class << self
+    def with_preferred_scope(scope, &b)
+      Thread.current[PREFERRED_SCOPE_KEY], old_scope = scope, Thread.current[PREFERRED_SCOPE_KEY]
+      yield
+    ensure
+      Thread.current[PREFERRED_SCOPE_KEY] = old_scope
+    end
+
+    def preferred_scope
+      Thread.current[PREFERRED_SCOPE_KEY]
+    end
+
+    def user_scope(&b)
+      with_preferred_scope(:user, &b)
+    end
+
+    def subject_scope(&b)
+      with_preferred_scope(:subject, &b)
+    end
+
+    def preferred_scope=(scope)
+      Thread.current[PREFERRED_SCOPE_KEY] = scope
+    end
+  end
+end
diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb
new file mode 100644
index 00000000000..bfcec241489
--- /dev/null
+++ b/lib/declarative_policy/rule.rb
@@ -0,0 +1,301 @@
+module DeclarativePolicy
+  module Rule
+    # A Rule is the object that results from the `rule` declaration,
+    # usually built using the DSL in `RuleDsl`. It is a basic logical
+    # combination of building blocks, and is capable of deciding,
+    # given a context (instance of DeclarativePolicy::Base) whether it
+    # passes or not. Note that this decision doesn't by itself know
+    # how that affects the actual ability decision - for that, a
+    # `Step` is used.
+    class Base
+      def self.make(*a)
+        new(*a).simplify
+      end
+
+      # true or false whether this rule passes.
+      # `context` is a policy - an instance of
+      # DeclarativePolicy::Base.
+      def pass?(context)
+        raise 'abstract'
+      end
+
+      # same as #pass? except refuses to do any I/O,
+      # returning nil if the result is not yet cached.
+      # used for accurately scoring And/Or
+      def cached_pass?(context)
+        raise 'abstract'
+      end
+
+      # abstractly, how long would it take to compute
+      # this rule? lower-scored rules are tried first.
+      def score(context)
+        raise 'abstract'
+      end
+
+      # unwrap double negatives and nested and/or
+      def simplify
+        self
+      end
+
+      # convenience combination methods
+      def or(other)
+        Or.make([self, other])
+      end
+
+      def and(other)
+        And.make([self, other])
+      end
+
+      def negate
+        Not.make(self)
+      end
+
+      alias_method :|, :or
+      alias_method :&, :and
+      alias_method :~@, :negate
+
+      def inspect
+        "#<Rule #{repr}>"
+      end
+    end
+
+    # A rule that checks a condition. This is the
+    # type of rule that results from a basic bareword
+    # in the rule dsl (see RuleDsl#method_missing).
+    class Condition < Base
+      def initialize(name)
+        @name = name
+      end
+
+      # we delegate scoring to the condition. See
+      # ManifestCondition#score.
+      def score(context)
+        context.condition(@name).score
+      end
+
+      # Let the ManifestCondition from the context
+      # decide whether we pass.
+      def pass?(context)
+        context.condition(@name).pass?
+      end
+
+      # returns nil unless it's already cached
+      def cached_pass?(context)
+        condition = context.condition(@name)
+        return nil unless condition.cached?
+        condition.pass?
+      end
+
+      def description(context)
+        context.class.conditions[@name].description
+      end
+
+      def repr
+        @name.to_s
+      end
+    end
+
+    # A rule constructed from DelegateDsl - using a condition from a
+    # delegated policy.
+    class DelegatedCondition < Base
+      # Internal use only - this is rescued each time it's raised.
+      MissingDelegate = Class.new(StandardError)
+
+      def initialize(delegate_name, name)
+        @delegate_name = delegate_name
+        @name = name
+      end
+
+      def delegated_context(context)
+        policy = context.delegated_policies[@delegate_name]
+        raise MissingDelegate if policy.nil?
+        policy
+      end
+
+      def score(context)
+        delegated_context(context).condition(@name).score
+      rescue MissingDelegate
+        0
+      end
+
+      def cached_pass?(context)
+        condition = delegated_context(context).condition(@name)
+        return nil unless condition.cached?
+        condition.pass?
+      rescue MissingDelegate
+        false
+      end
+
+      def pass?(context)
+        delegated_context(context).condition(@name).pass?
+      rescue MissingDelegate
+        false
+      end
+
+      def repr
+        "#{@delegate_name}.#{@name}"
+      end
+    end
+
+    # A rule constructed from RuleDsl#can?. Computes a different ability
+    # on the same subject.
+    class Ability < Base
+      attr_reader :ability
+      def initialize(ability)
+        @ability = ability
+      end
+
+      # We ask the ability's runner for a score
+      def score(context)
+        context.runner(@ability).score
+      end
+
+      def pass?(context)
+        context.allowed?(@ability)
+      end
+
+      def cached_pass?(context)
+        runner = context.runner(@ability)
+        return nil unless runner.cached?
+        runner.pass?
+      end
+
+      def description(context)
+        "User can #{@ability.inspect}"
+      end
+
+      def repr
+        "can?(#{@ability.inspect})"
+      end
+    end
+
+    # Logical `and`, containing a list of rules. Only passes
+    # if all of them do.
+    class And < Base
+      attr_reader :rules
+      def initialize(rules)
+        @rules = rules
+      end
+
+      def simplify
+        simplified_rules = @rules.flat_map do |rule|
+          simplified = rule.simplify
+          case simplified
+          when And then simplified.rules
+          else [simplified]
+          end
+        end
+
+        And.new(simplified_rules)
+      end
+
+      def score(context)
+        return 0 unless cached_pass?(context).nil?
+
+        # note that cached rules will have score 0 anyways.
+        @rules.map { |r| r.score(context) }.inject(0, :+)
+      end
+
+      def pass?(context)
+        # try to find a cached answer before
+        # checking in order
+        cached = cached_pass?(context)
+        return cached unless cached.nil?
+
+        @rules.all? { |r| r.pass?(context) }
+      end
+
+      def cached_pass?(context)
+        passes = @rules.map { |r| r.cached_pass?(context) }
+        return false if passes.any? { |p| p == false }
+        return true if passes.all? { |p| p == true }
+
+        nil
+      end
+
+      def repr
+        "all?(#{rules.map(&:repr).join(', ')})"
+      end
+    end
+
+    # Logical `or`. Mirrors And.
+    class Or < Base
+      attr_reader :rules
+      def initialize(rules)
+        @rules = rules
+      end
+
+      def pass?(context)
+        cached = cached_pass?(context)
+        return cached unless cached.nil?
+
+        @rules.any? { |r| r.pass?(context) }
+      end
+
+      def simplify
+        simplified_rules = @rules.flat_map do |rule|
+          simplified = rule.simplify
+          case simplified
+          when Or then simplified.rules
+          else [simplified]
+          end
+        end
+
+        Or.new(simplified_rules)
+      end
+
+      def cached_pass?(context)
+        passes = @rules.map { |r| r.cached_pass?(context) }
+        return true if passes.any? { |p| p == true }
+        return false if passes.all? { |p| p == false }
+
+        nil
+      end
+
+      def score(context)
+        return 0 unless cached_pass?(context).nil?
+        @rules.map { |r| r.score(context) }.inject(0, :+)
+      end
+
+      def repr
+        "any?(#{@rules.map(&:repr).join(', ')})"
+      end
+    end
+
+    class Not < Base
+      attr_reader :rule
+      def initialize(rule)
+        @rule = rule
+      end
+
+      def simplify
+        case @rule
+        when And then Or.new(@rule.rules.map(&:negate)).simplify
+        when Or then And.new(@rule.rules.map(&:negate)).simplify
+        when Not then @rule.rule.simplify
+        else Not.new(@rule.simplify)
+        end
+      end
+
+      def pass?(context)
+        !@rule.pass?(context)
+      end
+
+      def cached_pass?(context)
+        case @rule.cached_pass?(context)
+        when nil then nil
+        when true then false
+        when false then true
+        end
+      end
+
+      def score(context)
+        @rule.score(context)
+      end
+
+      def repr
+        "~#{@rule.repr}"
+      end
+    end
+  end
+end
diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb
new file mode 100644
index 00000000000..b5c615da4e3
--- /dev/null
+++ b/lib/declarative_policy/runner.rb
@@ -0,0 +1,181 @@
+module DeclarativePolicy
+  class Runner
+    class State
+      def initialize
+        @enabled = false
+        @prevented = false
+      end
+
+      def enable!
+        @enabled = true
+      end
+
+      def enabled?
+        @enabled
+      end
+
+      def prevent!
+        @prevented = true
+      end
+
+      def prevented?
+        @prevented
+      end
+
+      def pass?
+        !prevented? && enabled?
+      end
+    end
+
+    # a Runner contains a list of Steps to be run.
+    attr_reader :steps
+    def initialize(steps)
+      @steps = steps
+    end
+
+    # We make sure only to run any given Runner once,
+    # and just continue to use the resulting @state
+    # that's left behind.
+    def cached?
+      !!@state
+    end
+
+    # used by Rule::Ability. See #steps_by_score
+    def score
+      return 0 if cached?
+      steps.map(&:score).inject(0, :+)
+    end
+
+    def merge_runner(other)
+      Runner.new(@steps + other.steps)
+    end
+
+    # The main entry point, called for making an ability decision.
+    # See #run and DeclarativePolicy::Base#can?
+    def pass?
+      run unless cached?
+
+      @state.pass?
+    end
+
+    # see DeclarativePolicy::Base#debug
+    def debug(out = $stderr)
+      run(out)
+    end
+
+    private
+
+    def flatten_steps!
+      @steps = @steps.flat_map { |s| s.flattened(@steps) }
+    end
+
+    # This method implements the semantic of "one enable and no prevents".
+    # It relies on #steps_by_score for the main loop, and updates @state
+    # with the result of the step.
+    def run(debug = nil)
+      @state = State.new
+
+      steps_by_score do |step, score|
+        passed = nil
+        case step.action
+        when :enable then
+          # we only check :enable actions if they have a chance of
+          # changing the outcome - if no other rule has enabled or
+          # prevented.
+          unless @state.enabled? || @state.prevented?
+            passed = step.pass?
+            @state.enable! if passed
+          end
+
+          debug << inspect_step(step, score, passed) if debug
+        when :prevent then
+          # we only check :prevent actions if the state hasn't already
+          # been prevented.
+          unless @state.prevented?
+            passed = step.pass?
+            if passed
+              @state.prevent!
+              return unless debug
+            end
+          end
+
+          debug << inspect_step(step, score, passed) if debug
+        else raise "invalid action #{step.action.inspect}"
+        end
+      end
+
+      @state
+    end
+
+    # This is the core spot where all those `#score` methods matter.
+    # It is critcal for performance to run steps in the correct order,
+    # so that we don't compute expensive conditions (potentially n times
+    # if we're called on, say, a large list of users).
+    #
+    # In order to determine the cheapest step to run next, we rely on
+    # Step#score, which returns a numerical rating of how expensive
+    # it would be to calculate - the lower the better. It would be
+    # easy enough to statically sort by these scores, but we can do
+    # a little better - the scores are cache-aware (conditions that
+    # are already in the cache have score 0), which means that running
+    # a step can actually change the scores of other steps.
+    #
+    # So! The way we sort here involves re-scoring at every step. This
+    # is by necessity quadratic, but most of the time the number of steps
+    # will be low. But just in case, if the number of steps exceeds 50,
+    # we print a warning and fall back to a static sort.
+    #
+    # For each step, we yield the step object along with the computed score
+    # for debugging purposes.
+    def steps_by_score(&b)
+      flatten_steps!
+
+      if @steps.size > 50
+        warn "DeclarativePolicy: large number of steps (#{steps.size}), falling back to static sort"
+
+        @steps.map { |s| [s.score, s] }.sort_by { |(score, _)| score }.each do |(score, step)|
+          yield step, score
+        end
+
+        return
+      end
+
+      steps = Set.new(@steps)
+
+      loop do
+        return if steps.empty?
+
+        # if the permission hasn't yet been enabled and we only have
+        # prevent steps left, we short-circuit the state here
+        @state.prevent! if !@state.enabled? && steps.all?(&:prevent?)
+
+        lowest_score = Float::INFINITY
+        next_step = nil
+
+        steps.each do |step|
+          score = step.score
+          if score < lowest_score
+            next_step = step
+            lowest_score = score
+          end
+        end
+
+        steps.delete(next_step)
+
+        yield next_step, lowest_score
+      end
+    end
+
+    # Formatter for debugging output.
+    def inspect_step(step, original_score, passed)
+      symbol =
+        case passed
+        when true then '+'
+        when false then '-'
+        when nil then ' '
+        end
+
+      "#{symbol} [#{original_score.to_i}] #{step.repr}\n"
+    end
+  end
+end
diff --git a/lib/declarative_policy/step.rb b/lib/declarative_policy/step.rb
new file mode 100644
index 00000000000..3469fe9f991
--- /dev/null
+++ b/lib/declarative_policy/step.rb
@@ -0,0 +1,86 @@
+module DeclarativePolicy
+  # This object represents one step in the runtime decision of whether
+  # an ability is allowed. It contains a Rule and a context (instance
+  # of DeclarativePolicy::Base), which contains the user, the subject,
+  # and the cache. It also contains an "action", which is the symbol
+  # :prevent or :enable.
+  class Step
+    attr_reader :context, :rule, :action
+    def initialize(context, rule, action)
+      @context = context
+      @rule = rule
+      @action = action
+    end
+
+    # In the flattening process, duplicate steps may be generated in the
+    # same rule. This allows us to eliminate those (see Runner#steps_by_score
+    # and note its use of a Set)
+    def ==(other)
+      @context == other.context && @rule == other.rule && @action == other.action
+    end
+
+    # In the runner, steps are sorted dynamically by score, so that
+    # we are sure to compute them in close to the optimal order.
+    #
+    # See also Rule#score, ManifestCondition#score, and Runner#steps_by_score.
+    def score
+      # we slightly prefer the preventative actions
+      # since they are more likely to short-circuit
+      case @action
+      when :prevent
+        @rule.score(@context) * (7.0 / 8)
+      when :enable
+        @rule.score(@context)
+      end
+    end
+
+    def with_action(action)
+      Step.new(@context, @rule, action)
+    end
+
+    def enable?
+      @action == :enable
+    end
+
+    def prevent?
+      @action == :prevent
+    end
+
+    # This rather complex method allows us to split rules into parts so that
+    # they can be sorted independently for better optimization
+    def flattened(roots)
+      case @rule
+      when Rule::Or
+        # A single `Or` step is the same as each of its elements as separate steps
+        @rule.rules.flat_map { |r| Step.new(@context, r, @action).flattened(roots) }
+      when Rule::Ability
+        # This looks like a weird micro-optimization but it buys us quite a lot
+        # in some cases. If we depend on an Ability (i.e. a `can?(...)` rule),
+        # and that ability *only* has :enable actions (modulo some actions that
+        # we already have taken care of), then its rules can be safely inlined.
+        steps = @context.runner(@rule.ability).steps.reject { |s| roots.include?(s) }
+
+        if steps.all?(&:enable?)
+          # in the case that we are a :prevent step, each inlined step becomes
+          # an independent :prevent, even though it was an :enable in its initial
+          # context.
+          steps.map! { |s| s.with_action(:prevent) } if prevent?
+
+          steps.flat_map { |s| s.flattened(roots) }
+        else
+          [self]
+        end
+      else
+        [self]
+      end
+    end
+
+    def pass?
+      @rule.pass?(@context)
+    end
+
+    def repr
+      "#{@action} when #{@rule.repr} (#{@context.repr})"
+    end
+  end
+end
diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb
index dd864eea3fa..721ed97bb6b 100644
--- a/lib/extracts_path.rb
+++ b/lib/extracts_path.rb
@@ -126,8 +126,7 @@ module ExtractsPath
     raise InvalidPathError unless @commit
 
     @hex_path = Digest::SHA1.hexdigest(@path)
-    @logs_path = logs_file_namespace_project_ref_path(@project.namespace,
-                                                      @project, @ref, @path)
+    @logs_path = logs_file_project_ref_path(@project, @ref, @path)
 
   rescue RuntimeError, NoMethodError, InvalidPathError
     render_404
diff --git a/lib/feature.rb b/lib/feature.rb
index d3d972564af..4bd29aed687 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -12,6 +12,8 @@ class Feature
   end
 
   class << self
+    delegate :group, to: :flipper
+
     def all
       flipper.features.to_a
     end
@@ -27,16 +29,24 @@ class Feature
       all.map(&:name).include?(feature.name)
     end
 
-    def enabled?(key)
-      get(key).enabled?
+    def enabled?(key, thing = nil)
+      get(key).enabled?(thing)
+    end
+
+    def enable(key, thing = true)
+      get(key).enable(thing)
     end
 
-    def enable(key)
-      get(key).enable
+    def disable(key, thing = false)
+      get(key).disable(thing)
     end
 
-    def disable(key)
-      get(key).disable
+    def enable_group(key, group)
+      get(key).enable_group(group)
+    end
+
+    def disable_group(key, group)
+      get(key).disable_group(group)
     end
 
     def flipper
@@ -47,5 +57,11 @@ class Feature
         Flipper.new(adapter)
       end
     end
+
+    # This method is called from config/initializers/flipper.rb and can be used
+    # to register Flipper groups.
+    # See https://docs.gitlab.com/ee/development/feature_flags.html#feature-groups
+    def register_feature_groups
+    end
   end
 end
diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb
index e4f7cad2b79..45c2b01dd8f 100644
--- a/lib/gitlab/allowable.rb
+++ b/lib/gitlab/allowable.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Allowable
-    def can?(user, action, subject = :global)
-      Ability.allowed?(user, action, subject)
+    def can?(*args)
+      Ability.allowed?(*args)
     end
   end
 end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 3933c3b04dd..ccb5d886bab 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -130,13 +130,13 @@ module Gitlab
 
         token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password)
 
-        if token && valid_scoped_token?(token, AVAILABLE_SCOPES.map(&:to_s))
+        if token && valid_scoped_token?(token, AVAILABLE_SCOPES)
           Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes))
         end
       end
 
       def valid_oauth_token?(token)
-        token && token.accessible? && valid_scoped_token?(token, ["api"])
+        token && token.accessible? && valid_scoped_token?(token, [:api])
       end
 
       def valid_scoped_token?(token, scopes)
diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb
new file mode 100644
index 00000000000..91540127ea9
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb
@@ -0,0 +1,19 @@
+module Gitlab
+  module BackgroundMigration
+    class MigrateBuildStageIdReference
+      def perform(start_id, stop_id)
+        sql = <<-SQL.strip_heredoc
+          UPDATE ci_builds
+            SET stage_id =
+              (SELECT id FROM ci_stages
+                WHERE ci_stages.pipeline_id = ci_builds.commit_id
+                AND ci_stages.name = ci_builds.stage)
+          WHERE ci_builds.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
+            AND ci_builds.stage_id IS NULL
+        SQL
+
+        ActiveRecord::Base.connection.execute(sql)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/badge/build/metadata.rb b/lib/gitlab/badge/build/metadata.rb
index f87a7b7942e..2ee35a0d4c1 100644
--- a/lib/gitlab/badge/build/metadata.rb
+++ b/lib/gitlab/badge/build/metadata.rb
@@ -15,12 +15,11 @@ module Gitlab
         end
 
         def image_url
-          build_namespace_project_badges_url(@project.namespace,
-                                             @project, @ref, format: :svg)
+          build_project_badges_url(@project, @ref, format: :svg)
         end
 
         def link_url
-          namespace_project_commits_url(@project.namespace, @project, id: @ref)
+          project_commits_url(@project, id: @ref)
         end
       end
     end
diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb
index 53588185622..e898f5d790e 100644
--- a/lib/gitlab/badge/coverage/metadata.rb
+++ b/lib/gitlab/badge/coverage/metadata.rb
@@ -16,13 +16,11 @@ module Gitlab
         end
 
         def image_url
-          coverage_namespace_project_badges_url(@project.namespace,
-                                                @project, @ref,
-                                                format: :svg)
+          coverage_project_badges_url(@project, @ref, format: :svg)
         end
 
         def link_url
-          namespace_project_commits_url(@project.namespace, @project, id: @ref)
+          project_commits_url(@project, @ref)
         end
       end
     end
diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb
index 4a049ef758d..8ad6f3cb986 100644
--- a/lib/gitlab/badge/metadata.rb
+++ b/lib/gitlab/badge/metadata.rb
@@ -4,7 +4,7 @@ module Gitlab
     # Abstract class for badge metadata
     #
     class Metadata
-      include Gitlab::Application.routes.url_helpers
+      include Gitlab::Routing
       include ActionView::Helpers::AssetTagHelper
       include ActionView::Helpers::UrlHelper
 
diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb
index f074df9c7a1..d7e09acbbf3 100644
--- a/lib/gitlab/ci/config/entry/cache.rb
+++ b/lib/gitlab/ci/config/entry/cache.rb
@@ -7,11 +7,14 @@ module Gitlab
         #
         class Cache < Node
           include Configurable
+          include Attributable
 
-          ALLOWED_KEYS = %i[key untracked paths].freeze
+          ALLOWED_KEYS = %i[key untracked paths policy].freeze
+          DEFAULT_POLICY = 'pull-push'.freeze
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
+            validates :policy, inclusion: { in: %w[pull-push push pull], message: 'should be pull-push, push, or pull' }, allow_blank: true
           end
 
           entry :key, Entry::Key,
@@ -25,8 +28,15 @@ module Gitlab
 
           helpers :key
 
+          attributes :policy
+
           def value
-            super.merge(key: key_value)
+            result = super
+
+            result[:key] = key_value
+            result[:policy] = policy || DEFAULT_POLICY
+
+            result
           end
         end
       end
diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb
index 897dcff8012..6555c589173 100644
--- a/lib/gitlab/ci/config/entry/image.rb
+++ b/lib/gitlab/ci/config/entry/image.rb
@@ -15,7 +15,7 @@ module Gitlab
             validates :config, allowed_keys: ALLOWED_KEYS
 
             validates :name, type: String, presence: true
-            validates :entrypoint, type: String, allow_nil: true
+            validates :entrypoint, array_of_strings: true, allow_nil: true
           end
 
           def hash?
diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb
index b52faf48b58..3e2ebcff31a 100644
--- a/lib/gitlab/ci/config/entry/service.rb
+++ b/lib/gitlab/ci/config/entry/service.rb
@@ -15,8 +15,8 @@ module Gitlab
             validates :config, allowed_keys: ALLOWED_KEYS
 
             validates :name, type: String, presence: true
-            validates :entrypoint, type: String, allow_nil: true
-            validates :command, type: String, allow_nil: true
+            validates :entrypoint, array_of_strings: true, allow_nil: true
+            validates :command, array_of_strings: true, allow_nil: true
             validates :alias, type: String, allow_nil: true
           end
 
diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb
index 439ef0ce015..8ad3e57e59d 100644
--- a/lib/gitlab/ci/status/build/cancelable.rb
+++ b/lib/gitlab/ci/status/build/cancelable.rb
@@ -12,9 +12,7 @@ module Gitlab
           end
 
           def action_path
-            cancel_namespace_project_job_path(subject.project.namespace,
-                                                subject.project,
-                                                subject)
+            cancel_project_job_path(subject.project, subject)
           end
 
           def action_method
diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb
index b173c23fba4..c0c7c7f5b5d 100644
--- a/lib/gitlab/ci/status/build/common.rb
+++ b/lib/gitlab/ci/status/build/common.rb
@@ -8,9 +8,7 @@ module Gitlab
           end
 
           def details_path
-            namespace_project_job_path(subject.project.namespace,
-                                         subject.project,
-                                         subject)
+            project_job_path(subject.project, subject)
           end
         end
       end
diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb
index e80f3263794..c7726543599 100644
--- a/lib/gitlab/ci/status/build/play.rb
+++ b/lib/gitlab/ci/status/build/play.rb
@@ -20,9 +20,7 @@ module Gitlab
           end
 
           def action_path
-            play_namespace_project_job_path(subject.project.namespace,
-                                              subject.project,
-                                              subject)
+            play_project_job_path(subject.project, subject)
           end
 
           def action_method
diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb
index 56303e4cb17..8c8fdc56d75 100644
--- a/lib/gitlab/ci/status/build/retryable.rb
+++ b/lib/gitlab/ci/status/build/retryable.rb
@@ -16,9 +16,7 @@ module Gitlab
           end
 
           def action_path
-            retry_namespace_project_job_path(subject.project.namespace,
-                                               subject.project,
-                                               subject)
+            retry_project_job_path(subject.project, subject)
           end
 
           def action_method
diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb
index 2778d6f3b52..d464738deaf 100644
--- a/lib/gitlab/ci/status/build/stop.rb
+++ b/lib/gitlab/ci/status/build/stop.rb
@@ -20,9 +20,7 @@ module Gitlab
           end
 
           def action_path
-            play_namespace_project_job_path(subject.project.namespace,
-                                              subject.project,
-                                              subject)
+            play_project_job_path(subject.project, subject)
           end
 
           def action_method
diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb
index 76bfd18bf40..61bb07beb0f 100644
--- a/lib/gitlab/ci/status/pipeline/common.rb
+++ b/lib/gitlab/ci/status/pipeline/common.rb
@@ -8,9 +8,7 @@ module Gitlab
           end
 
           def details_path
-            namespace_project_pipeline_path(subject.project.namespace,
-                                            subject.project,
-                                            subject)
+            project_pipeline_path(subject.project, subject)
           end
 
           def has_action?
diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb
index 7852f492e1d..bc99d925347 100644
--- a/lib/gitlab/ci/status/stage/common.rb
+++ b/lib/gitlab/ci/status/stage/common.rb
@@ -8,10 +8,7 @@ module Gitlab
           end
 
           def details_path
-            namespace_project_pipeline_path(subject.project.namespace,
-                                            subject.project,
-                                            subject.pipeline,
-                                            anchor: subject.name)
+            project_pipeline_path(subject.project, subject.pipeline, anchor: subject.name)
           end
 
           def has_action?
diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb
index 75a213ef752..98dfe900044 100644
--- a/lib/gitlab/conflict/file.rb
+++ b/lib/gitlab/conflict/file.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Conflict
     class File
-      include Gitlab::Routing.url_helpers
+      include Gitlab::Routing
       include IconsHelper
 
       MissingResolution = Class.new(ResolutionError)
@@ -205,9 +205,7 @@ module Gitlab
           old_path: their_path,
           new_path: our_path,
           blob_icon: file_type_icon_class('file', our_mode, our_path),
-          blob_path: namespace_project_blob_path(merge_request.project.namespace,
-                                                 merge_request.project,
-                                                 ::File.join(merge_request.diff_refs.head_sha, our_path))
+          blob_path: project_blob_path(merge_request.project, ::File.join(merge_request.diff_refs.head_sha, our_path))
         }
 
         json_hash.tap do |json_hash|
@@ -223,11 +221,10 @@ module Gitlab
       end
 
       def content_path
-        conflict_for_path_namespace_project_merge_request_path(merge_request.project.namespace,
-                                                               merge_request.project,
-                                                               merge_request,
-                                                               old_path: their_path,
-                                                               new_path: our_path)
+        conflict_for_path_project_merge_request_path(merge_request.project,
+                                                     merge_request,
+                                                     old_path: their_path,
+                                                     new_path: our_path)
       end
 
       # Don't try to print merge_request or repository.
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 7f1932da3b7..7fa02f3d7b3 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -33,12 +33,7 @@ module Gitlab
     def uncached_application_settings
       return fake_application_settings unless connect_to_db?
 
-      # This loads from the database into the cache, so handle Redis errors
-      begin
-        db_settings = ::ApplicationSetting.current
-      rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL
-        # In case Redis isn't running or the Redis UNIX socket file is not available
-      end
+      db_settings = ::ApplicationSetting.current
 
       # If there are pending migrations, it's possible there are columns that
       # need to be added to the application settings. To prevent Rake tasks
diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb
index 9d25ef078e8..f5d08c0b658 100644
--- a/lib/gitlab/cycle_analytics/metrics_tables.rb
+++ b/lib/gitlab/cycle_analytics/metrics_tables.rb
@@ -13,6 +13,10 @@ module Gitlab
         MergeRequestDiff.arel_table
       end
 
+      def mr_diff_commits_table
+        MergeRequestDiffCommit.arel_table
+      end
+
       def mr_closing_issues_table
         MergeRequestsClosingIssues.arel_table
       end
diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
index 7d342a2d2cb..b260822788d 100644
--- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
@@ -2,40 +2,59 @@ module Gitlab
   module CycleAnalytics
     class PlanEventFetcher < BaseEventFetcher
       def initialize(*args)
-        @projections = [mr_diff_table[:st_commits].as('commits'),
+        @projections = [mr_diff_table[:id],
+                        mr_diff_table[:st_commits],
                         issue_metrics_table[:first_mentioned_in_commit_at]]
 
         super(*args)
       end
 
       def events_query
-        base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id]))
+        base_query
+          .join(mr_diff_table)
+          .on(mr_diff_table[:merge_request_id].eq(mr_table[:id]))
 
         super
       end
 
       private
 
+      def merge_request_diff_commits
+        @merge_request_diff_commits ||=
+          MergeRequestDiffCommit
+            .where(merge_request_diff_id: event_result.map { |event| event['id'] })
+            .group_by(&:merge_request_diff_id)
+      end
+
       def serialize(event)
-        st_commit = first_time_reference_commit(event.delete('commits'), event)
+        commit = first_time_reference_commit(event)
 
-        return unless st_commit
+        return unless commit
 
-        serialize_commit(event, st_commit, query)
+        serialize_commit(event, commit, query)
       end
 
-      def first_time_reference_commit(commits, event)
+      def first_time_reference_commit(event)
+        return nil unless event && merge_request_diff_commits
+
+        commits =
+          if event['st_commits'].present?
+            YAML.load(event['st_commits'])
+          else
+            merge_request_diff_commits[event['id'].to_i]
+          end
+
         return nil if commits.blank?
 
-        YAML.load(commits).find do |commit|
+        commits.find do |commit|
           next unless commit[:committed_date] && event['first_mentioned_in_commit_at']
 
           commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i
         end
       end
 
-      def serialize_commit(event, st_commit, query)
-        commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project)
+      def serialize_commit(event, commit, query)
+        commit = Commit.new(Gitlab::Git::Commit.new(commit.to_hash), @project)
 
         AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit)
       end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
index 89530082cd2..f333ff22300 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
@@ -29,6 +29,11 @@ module Gitlab
           paths = Array(paths)
           RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
         end
+
+        def revert_renames
+          RenameProjects.new([], self).revert_renames
+          RenameNamespaces.new([], self).revert_renames
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
index d8163d7da11..33f8939bc61 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
@@ -6,7 +6,10 @@ module Gitlab
           attr_reader :paths, :migration
 
           delegate :update_column_in_batches,
+                   :execute,
                    :replace_sql,
+                   :quote_string,
+                   :say,
                    to: :migration
 
           def initialize(paths, migration)
@@ -26,24 +29,45 @@ module Gitlab
             new_path = rename_path(namespace_path, old_path)
             new_full_path = join_routable_path(namespace_path, new_path)
 
+            perform_rename(routable, old_full_path, new_full_path)
+
+            [old_full_path, new_full_path]
+          end
+
+          def perform_rename(routable, old_full_path, new_full_path)
             # skips callbacks & validations
+            new_path = new_full_path.split('/').last
             routable.class.where(id: routable)
               .update_all(path: new_path)
 
             rename_routes(old_full_path, new_full_path)
-
-            [old_full_path, new_full_path]
           end
 
           def rename_routes(old_full_path, new_full_path)
+            routes = Route.arel_table
+
+            quoted_old_full_path = quote_string(old_full_path)
+            quoted_old_wildcard_path = quote_string("#{old_full_path}/%")
+
+            filter = if Database.mysql?
+                       "lower(routes.path) = lower('#{quoted_old_full_path}') "\
+                       "OR routes.path LIKE '#{quoted_old_wildcard_path}'"
+                     else
+                       "routes.id IN "\
+                       "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\
+                       "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )"
+                     end
+
             replace_statement = replace_sql(Route.arel_table[:path],
                                             old_full_path,
                                             new_full_path)
 
-            update_column_in_batches(:routes, :path, replace_statement)  do |table, query|
-              path_or_children = table[:path].matches_any([old_full_path, "#{old_full_path}/%"])
-              query.where(path_or_children)
-            end
+            update = Arel::UpdateManager.new(ActiveRecord::Base)
+                       .table(routes)
+                       .set([[routes[:path], replace_statement]])
+                       .where(Arel::Nodes::SqlLiteral.new(filter))
+
+            execute(update.to_sql)
           end
 
           def rename_path(namespace_path, path_was)
@@ -86,32 +110,74 @@ module Gitlab
 
           def move_folders(directory, old_relative_path, new_relative_path)
             old_path = File.join(directory, old_relative_path)
-            return unless File.directory?(old_path)
+            unless File.directory?(old_path)
+              say "#{old_path} doesn't exist, skipping"
+              return
+            end
 
             new_path = File.join(directory, new_relative_path)
             FileUtils.mv(old_path, new_path)
           end
 
           def remove_cached_html_for_projects(project_ids)
-            update_column_in_batches(:projects, :description_html, nil) do |table, query|
-              query.where(table[:id].in(project_ids))
-            end
-
-            update_column_in_batches(:issues, :description_html, nil) do |table, query|
-              query.where(table[:project_id].in(project_ids))
+            project_ids.each do |project_id|
+              update_column_in_batches(:projects, :description_html, nil) do |table, query|
+                query.where(table[:id].eq(project_id))
+              end
+
+              update_column_in_batches(:issues, :description_html, nil) do |table, query|
+                query.where(table[:project_id].eq(project_id))
+              end
+
+              update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
+                query.where(table[:target_project_id].eq(project_id))
+              end
+
+              update_column_in_batches(:notes, :note_html, nil) do |table, query|
+                query.where(table[:project_id].eq(project_id))
+              end
+
+              update_column_in_batches(:milestones, :description_html, nil) do |table, query|
+                query.where(table[:project_id].eq(project_id))
+              end
             end
+          end
 
-            update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
-              query.where(table[:target_project_id].in(project_ids))
+          def track_rename(type, old_path, new_path)
+            key = redis_key_for_type(type)
+            Gitlab::Redis.with do |redis|
+              redis.lpush(key, [old_path, new_path].to_json)
+              redis.expire(key, 2.weeks.to_i)
             end
+            say "tracked rename: #{key}: #{old_path} -> #{new_path}"
+          end
 
-            update_column_in_batches(:notes, :note_html, nil) do |table, query|
-              query.where(table[:project_id].in(project_ids))
+          def reverts_for_type(type)
+            key = redis_key_for_type(type)
+
+            Gitlab::Redis.with do |redis|
+              failed_reverts = []
+
+              while rename_info = redis.lpop(key)
+                path_before_rename, path_after_rename = JSON.parse(rename_info)
+                say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}"
+                begin
+                  yield(path_before_rename, path_after_rename)
+                rescue StandardError => e
+                  failed_reverts << rename_info
+                  say "Renaming #{type} from #{path_after_rename} back to "\
+                      "#{path_before_rename} failed. Review the error and try "\
+                      "again by running the `down` action. \n"\
+                      "#{e.message}: \n #{e.backtrace.join("\n")}"
+                end
+              end
+
+              failed_reverts.each { |rename_info| redis.lpush(key, rename_info) }
             end
+          end
 
-            update_column_in_batches(:milestones, :description_html, nil) do |table, query|
-              query.where(table[:project_id].in(project_ids))
-            end
+          def redis_key_for_type(type)
+            "rename:#{migration.name}:#{type}"
           end
 
           def file_storage?
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
index da7e2cb2e85..05b86f32ce2 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
@@ -26,6 +26,12 @@ module Gitlab
           def rename_namespace(namespace)
             old_full_path, new_full_path = rename_path_for_routable(namespace)
 
+            track_rename('namespace', old_full_path, new_full_path)
+
+            rename_namespace_dependencies(namespace, old_full_path, new_full_path)
+          end
+
+          def rename_namespace_dependencies(namespace, old_full_path, new_full_path)
             move_repositories(namespace, old_full_path, new_full_path)
             move_uploads(old_full_path, new_full_path)
             move_pages(old_full_path, new_full_path)
@@ -33,6 +39,23 @@ module Gitlab
             remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
           end
 
+          def revert_renames
+            reverts_for_type('namespace') do |path_before_rename, current_path|
+              matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
+              namespace = MigrationClasses::Namespace.joins(:route)
+                            .where(matches_path).first&.becomes(MigrationClasses::Namespace)
+
+              if namespace
+                perform_rename(namespace, current_path, path_before_rename)
+
+                rename_namespace_dependencies(namespace, current_path, path_before_rename)
+              else
+                say "Couldn't rename namespace from #{current_path} back to #{path_before_rename}, "\
+                    "namespace was renamed, or no longer exists at the expected path"
+              end
+            end
+          end
+
           def rename_user(old_username, new_username)
             MigrationClasses::User.where(username: old_username)
               .update_all(username: new_username)
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
index 448717eb744..75a75f61953 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
@@ -16,12 +16,37 @@ module Gitlab
           def rename_project(project)
             old_full_path, new_full_path = rename_path_for_routable(project)
 
+            track_rename('project', old_full_path, new_full_path)
+
+            move_project_folders(project, old_full_path, new_full_path)
+          end
+
+          def move_project_folders(project, old_full_path, new_full_path)
             move_repository(project, old_full_path, new_full_path)
             move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
             move_uploads(old_full_path, new_full_path)
             move_pages(old_full_path, new_full_path)
           end
 
+          def revert_renames
+            reverts_for_type('project') do |path_before_rename, current_path|
+              matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
+              project = MigrationClasses::Project.joins(:route)
+                          .where(matches_path).first
+
+              if project
+                perform_rename(project, current_path, path_before_rename)
+
+                move_project_folders(project, current_path, path_before_rename)
+              else
+                say "Couldn't rename project from #{current_path} back to "\
+                    "#{path_before_rename}, project was renamed or no longer "\
+                    "exists at the expected path."
+
+              end
+            end
+          end
+
           def move_repository(project, old_path, new_path)
             unless gitlab_shell.mv_repository(project.repository_storage_path,
                                               old_path,
diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb
new file mode 100644
index 00000000000..d9400e04b83
--- /dev/null
+++ b/lib/gitlab/database/sha_attribute.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Database
+    BINARY_TYPE = if Gitlab::Database.postgresql?
+                    # PostgreSQL defines its own class with slightly different
+                    # behaviour from the default Binary type.
+                    ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
+                  else
+                    ActiveRecord::Type::Binary
+                  end
+
+    # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa).
+    #
+    # Using ShaAttribute allows you to store SHA1 values as binary while still
+    # using them as if they were stored as string values. This gives you the
+    # ease of use of string values, but without the storage overhead.
+    class ShaAttribute < BINARY_TYPE
+      PACK_FORMAT = 'H*'.freeze
+
+      # Casts binary data to a SHA1 in hexadecimal.
+      def type_cast_from_database(value)
+        value = super
+
+        value ? value.unpack(PACK_FORMAT)[0] : nil
+      end
+
+      # Casts a SHA1 in hexadecimal to the proper binary format.
+      def type_cast_for_database(value)
+        arg = value ? [value].pack(PACK_FORMAT) : nil
+
+        super(arg)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb
index 7bbd154eb03..d2360583741 100644
--- a/lib/gitlab/dependency_linker/base_linker.rb
+++ b/lib/gitlab/dependency_linker/base_linker.rb
@@ -52,7 +52,7 @@ module Gitlab
       #   # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"`
       def link_regex(regex, &url_proc)
         highlighted_lines.map!.with_index do |rich_line, i|
-          marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe)
+          marker = StringRegexMarker.new(plain_lines[i].chomp, rich_line.html_safe)
 
           marker.mark(regex, group: :name) do |text, left:, right:|
             url = yield(text)
diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
index 1a5887dab7e..edd7795eef0 100644
--- a/lib/gitlab/ee_compat_check.rb
+++ b/lib/gitlab/ee_compat_check.rb
@@ -337,7 +337,7 @@ module Gitlab
           # In the EE repo
           $ git push origin #{ee_branch_prefix}
 
-        ⚠️ Also, don't forget to create a new merge request on gitlab-ce and
+        ⚠️ Also, don't forget to create a new merge request on gitlab-ee and
         cross-link it with the CE merge request.
 
         Once this is done, you can retry this failed build, and it should pass.
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index ea035e33eff..dd1d9dcd555 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -4,7 +4,7 @@ module Gitlab
       class RepositoryPush
         attr_reader :author_id, :ref, :action
 
-        include Gitlab::Routing.url_helpers
+        include Gitlab::Routing
         include DiffHelper
 
         delegate :namespace, :name_with_namespace, to: :project, prefix: :project
@@ -96,20 +96,13 @@ module Gitlab
         def target_url
           if @action == :push && commits
             if commits.length > 1
-              namespace_project_compare_url(project_namespace,
-                                            project,
-                                            from: compare.start_commit,
-                                            to:   compare.head_commit)
+              project_compare_url(project, from: compare.start_commit, to: compare.head_commit)
             else
-              namespace_project_commit_url(project_namespace,
-                                           project,
-                                           commits.first)
+              project_commit_url(project, commits.first)
             end
           else
             unless @action == :delete
-              namespace_project_tree_url(project_namespace,
-                                         project,
-                                         ref_name)
+              project_tree_url(project, ref_name)
             end
           end
         end
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index 936606152e9..4175746be39 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -7,8 +7,10 @@ module Gitlab
     CommandError = Class.new(StandardError)
 
     class << self
+      include Gitlab::EncodingHelper
+
       def ref_name(ref)
-        ref.sub(/\Arefs\/(tags|heads)\//, '')
+        encode! ref.sub(/\Arefs\/(tags|heads)\//, '')
       end
 
       def branch_name(ref)
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index a7aceab4c14..ea386c2ddcb 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -41,10 +41,6 @@ module Gitlab
               commit_id: sha
             )
           when :BLOB
-            # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
-            # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
-            # which is what we use below to keep a consistent behavior.
-            detect = CharlockHolmes::EncodingDetector.new(8000).detect(entry.data)
             new(
               id: entry.oid,
               name: name,
@@ -53,7 +49,7 @@ module Gitlab
               mode: entry.mode.to_s(8),
               path: path,
               commit_id: sha,
-              binary: detect && detect[:type] == :binary
+              binary: binary?(entry.data)
             )
           end
         end
@@ -87,14 +83,28 @@ module Gitlab
         end
 
         def raw(repository, sha)
-          blob = repository.lookup(sha)
+          Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled|
+            if is_enabled
+              Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
+            else
+              blob = repository.lookup(sha)
+    
+              new(
+                id: blob.oid,
+                size: blob.size,
+                data: blob.content(MAX_DATA_DISPLAY_SIZE),
+                binary: blob.binary?
+              )
+            end
+          end
+        end
 
-          new(
-            id: blob.oid,
-            size: blob.size,
-            data: blob.content(MAX_DATA_DISPLAY_SIZE),
-            binary: blob.binary?
-          )
+        def binary?(data)
+          # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
+          # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
+          # which is what we use below to keep a consistent behavior.
+          detect = CharlockHolmes::EncodingDetector.new(8000).detect(data)
+          detect && detect[:type] == :binary
         end
 
         # Recursive search of blob id by path
@@ -165,8 +175,17 @@ module Gitlab
         return if @data == '' # don't mess with submodule blobs
         return @data if @loaded_all_data
 
+        Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled|
+          @data = begin
+            if is_enabled
+              Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: id, limit: -1).data
+            else
+              repository.lookup(id).content
+            end
+          end
+        end
+
         @loaded_all_data = true
-        @data = repository.lookup(id).content
         @loaded_size = @data.bytesize
         @binary = nil
       end
@@ -175,6 +194,10 @@ module Gitlab
         encode! @name
       end
 
+      def path
+        encode! @path
+      end
+
       def truncated?
         size && (size > loaded_size)
       end
diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb
index bd90d24a2ec..5042916343b 100644
--- a/lib/gitlab/git/hook.rb
+++ b/lib/gitlab/git/hook.rb
@@ -4,9 +4,10 @@ module Gitlab
       GL_PROTOCOL = 'web'.freeze
       attr_reader :name, :repo_path, :path
 
-      def initialize(name, repo_path)
+      def initialize(name, project)
         @name = name
-        @repo_path = repo_path
+        @project = project
+        @repo_path = project.repository.path
         @path = File.join(repo_path.strip, 'hooks', name)
       end
 
@@ -38,7 +39,8 @@ module Gitlab
         vars = {
           'GL_ID' => gl_id,
           'PWD' => repo_path,
-          'GL_PROTOCOL' => GL_PROTOCOL
+          'GL_PROTOCOL' => GL_PROTOCOL,
+          'GL_REPOSITORY' => Gitlab::GlRepository.gl_repository(@project, false)
         }
 
         options = {
diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb
index 1add037fa5f..666743006e5 100644
--- a/lib/gitlab/git/index.rb
+++ b/lib/gitlab/git/index.rb
@@ -110,10 +110,6 @@ module Gitlab
           if segment == '..'
             raise IndexError, 'Path cannot include directory traversal'
           end
-
-          unless segment =~ Gitlab::Regex.file_name_regex
-            raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}"
-          end
         end
 
         pathname.to_s
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 0a0c6f76cd3..e51966313d4 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -113,9 +113,7 @@ module Gitlab
       def local_branches(sort_by: nil)
         gitaly_migrate(:local_branches) do |is_enabled|
           if is_enabled
-            gitaly_ref_client.local_branches(sort_by: sort_by).map do |gitaly_branch|
-              Gitlab::Git::Branch.new(self, gitaly_branch.name, gitaly_branch)
-            end
+            gitaly_ref_client.local_branches(sort_by: sort_by)
           else
             branches(filter: :local, sort_by: sort_by)
           end
@@ -549,41 +547,38 @@ module Gitlab
         rugged.rev_parse(oid_or_ref_name)
       end
 
-      # Return hash with submodules info for this repository
+      # Returns url for submodule
       #
       # Ex.
-      #   {
-      #     "current_path/rack"  => {
-      #       "name" => "original_path/rack",
-      #       "id" => "c67be4624545b4263184c4a0e8f887efd0a66320",
-      #       "url" => "git://github.com/chneukirchen/rack.git"
-      #     },
-      #     "encoding" => {
-      #       "id" => ....
-      #     }
-      #   }
+      #   @repository.submodule_url_for('master', 'rack')
+      #   # => git@localhost:rack.git
       #
-      def submodules(ref)
-        commit = rev_parse_target(ref)
-        return {} unless commit
-
-        begin
-          content = blob_content(commit, ".gitmodules")
-        rescue InvalidBlobName
-          return {}
+      def submodule_url_for(ref, path)
+        Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled|
+          if is_enabled
+            gitaly_submodule_url_for(ref, path)
+          else
+            if submodules(ref).any?
+              submodule = submodules(ref)[path]
+              submodule['url'] if submodule
+            end
+          end
         end
-
-        parser = GitmodulesParser.new(content)
-        fill_submodule_ids(commit, parser.parse)
       end
 
       # Return total commits count accessible from passed ref
       def commit_count(ref)
-        walker = Rugged::Walker.new(rugged)
-        walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
-        oid = rugged.rev_parse_oid(ref)
-        walker.push(oid)
-        walker.count
+        gitaly_migrate(:commit_count) do |is_enabled|
+          if is_enabled
+            gitaly_commit_client.commit_count(ref)
+          else
+            walker = Rugged::Walker.new(rugged)
+            walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
+            oid = rugged.rev_parse_oid(ref)
+            walker.push(oid)
+            walker.count
+          end
+        end
       end
 
       # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or
@@ -912,6 +907,35 @@ module Gitlab
 
       private
 
+      # We are trying to deprecate this method because it does a lot of work
+      # but it seems to be used only to look up submodule URL's.
+      # https://gitlab.com/gitlab-org/gitaly/issues/329
+      def submodules(ref)
+        commit = rev_parse_target(ref)
+        return {} unless commit
+
+        begin
+          content = blob_content(commit, ".gitmodules")
+        rescue InvalidBlobName
+          return {}
+        end
+
+        parser = GitmodulesParser.new(content)
+        fill_submodule_ids(commit, parser.parse)
+      end
+
+      def gitaly_submodule_url_for(ref, path)
+        # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited.
+        commit_object = gitaly_commit_client.tree_entry(ref, path, 1)
+
+        return unless commit_object && commit_object.type == :COMMIT
+
+        gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Blob::MAX_DATA_DISPLAY_SIZE)
+        found_module = GitmodulesParser.new(gitmodules.data).parse[path]
+
+        found_module && found_module['url']
+      end
+
       def alternate_object_directories
         Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact
       end
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index b9afa05c819..b6d4e6cfe46 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -80,6 +80,10 @@ module Gitlab
         encode! @name
       end
 
+      def path
+        encode! @path
+      end
+
       def dir?
         type == :tree
       end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index f605c06dfc3..197a94487eb 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -70,12 +70,8 @@ module Gitlab
       params['gitaly_token'].presence || Gitlab.config.gitaly['token']
     end
 
-    def self.enabled?
-      Gitlab.config.gitaly.enabled
-    end
-
     def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN)
-      return false if !enabled? || status == MigrationStatus::DISABLED
+      return false if status == MigrationStatus::DISABLED
 
       feature = Feature.get("gitaly_#{feature}")
 
diff --git a/lib/gitlab/gitaly_client/blob.rb b/lib/gitlab/gitaly_client/blob.rb
new file mode 100644
index 00000000000..0c398b46a08
--- /dev/null
+++ b/lib/gitlab/gitaly_client/blob.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module GitalyClient
+    class Blob
+      def initialize(repository)
+        @gitaly_repo = repository.gitaly_repository
+      end
+      
+      def get_blob(oid:, limit:)
+        request = Gitaly::GetBlobRequest.new(
+          repository: @gitaly_repo,
+          oid: oid,
+          limit: limit
+        )
+        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request)
+
+        blob = response.first
+        return unless blob.oid.present?
+
+        data = response.reduce(blob.data.dup) { |memo, msg| memo << msg.data.dup }
+
+        Gitlab::Git::Blob.new(
+          id: blob.oid,
+          size: blob.size,
+          data: data,
+          binary: Gitlab::Git::Blob.binary?(data)
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb
index b8877619797..aafc0520664 100644
--- a/lib/gitlab/gitaly_client/commit.rb
+++ b/lib/gitlab/gitaly_client/commit.rb
@@ -56,6 +56,15 @@ module Gitlab
         entry
       end
 
+      def commit_count(ref)
+        request = Gitaly::CountCommitsRequest.new(
+          repository: @gitaly_repo,
+          revision: ref
+        )
+
+        GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count
+      end
+
       private
 
       def commit_diff_request_params(commit, options = {})
diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb
index 6d5f54dd959..6edc69de078 100644
--- a/lib/gitlab/gitaly_client/ref.rb
+++ b/lib/gitlab/gitaly_client/ref.rb
@@ -1,8 +1,11 @@
 module Gitlab
   module GitalyClient
     class Ref
+      include Gitlab::EncodingHelper
+
       # 'repository' is a Gitlab::Git::Repository
       def initialize(repository)
+        @repository = repository
         @gitaly_repo = repository.gitaly_repository
         @storage = repository.storage
       end
@@ -16,13 +19,13 @@ module Gitlab
       def branch_names
         request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo)
         response = GitalyClient.call(@storage, :ref, :find_all_branch_names, request)
-        consume_refs_response(response, prefix: 'refs/heads/')
+        consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) }
       end
 
       def tag_names
         request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo)
         response = GitalyClient.call(@storage, :ref, :find_all_tag_names, request)
-        consume_refs_response(response, prefix: 'refs/tags/')
+        consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) }
       end
 
       def find_ref_name(commit_id, ref_prefix)
@@ -31,7 +34,7 @@ module Gitlab
           commit_id: commit_id,
           prefix: ref_prefix
         )
-        GitalyClient.call(@storage, :ref, :find_ref_name, request).name
+        encode!(GitalyClient.call(@storage, :ref, :find_ref_name, request).name.dup)
       end
 
       def count_tag_names
@@ -51,20 +54,28 @@ module Gitlab
 
       private
 
-      def consume_refs_response(response, prefix:)
-        response.flat_map do |r|
-          r.names.map { |name| name.sub(/\A#{Regexp.escape(prefix)}/, '') }
-        end
+      def consume_refs_response(response)
+        response.flat_map { |message| message.names.map { |name| yield(name) } }
       end
 
       def sort_by_param(sort_by)
+        sort_by = 'name' if sort_by == 'name_asc'
+
         enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym)
         raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value
         enum_value
       end
 
       def consume_branches_response(response)
-        response.flat_map { |r| r.branches }
+        response.flat_map do |message|
+          message.branches.map do |gitaly_branch|
+            Gitlab::Git::Branch.new(
+              @repository,
+              encode!(gitaly_branch.name.dup),
+              gitaly_branch.commit_id
+            )
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 319633656ff..2d1ae6a5925 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -2,11 +2,14 @@
 
 module Gitlab
   module GonHelper
+    include WebpackHelper
+
     def add_gon_variables
       gon.api_version            = 'v4'
       gon.default_avatar_url     = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s
       gon.max_file_size          = current_application_settings.max_attachment_size
       gon.asset_host             = ActionController::Base.asset_host
+      gon.webpack_public_path    = webpack_public_path
       gon.relative_url_root      = Gitlab.config.gitlab.relative_url_root
       gon.shortcuts_path         = help_page_path('shortcuts')
       gon.user_color_scheme      = Gitlab::ColorSchemes.for_user(current_user).css_class
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
index e78b7f22e03..70da4080cae 100644
--- a/lib/gitlab/health_checks/fs_shards_check.rb
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -52,7 +52,7 @@ module Gitlab
             ]
           end
         rescue RuntimeError => ex
-          Rails.logger("unexpected error #{ex} when checking #{ok_metric}")
+          Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}")
           [metric(ok_metric, 0, **labels)]
         end
 
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index db7cdf4b5c7..f3d489aad0d 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -12,7 +12,8 @@ module Gitlab
       'zh_HK' => '繁體中文(香港)',
       'zh_TW' => '繁體中文(臺灣)',
       'bg' => 'български',
-      'eo' => 'Esperanto'
+      'eo' => 'Esperanto',
+      'it' => 'Italiano'
     }.freeze
 
     def available_locales
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 1860352c96d..c8ad3a7a5e0 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -27,6 +27,7 @@ project_tree:
       - :author
       - :events
     - merge_request_diff:
+      - :merge_request_diff_commits
       - :merge_request_diff_files
     - :events
     - :timelogs
diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb
new file mode 100644
index 00000000000..977c05910d3
--- /dev/null
+++ b/lib/gitlab/issuable_metadata.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module IssuableMetadata
+    def issuable_meta_data(issuable_collection, collection_type)
+      # map has to be used here since using pluck or select will
+      # throw an error when ordering issuables by priority which inserts
+      # a new order into the collection.
+      # We cannot use reorder to not mess up the paginated collection.
+      issuable_ids = issuable_collection.map(&:id)
+
+      return {} if issuable_ids.empty?
+
+      issuable_note_count = ::Note.count_for_collection(issuable_ids, collection_type)
+      issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type)
+      issuable_merge_requests_count =
+        if collection_type == 'Issue'
+          ::MergeRequestsClosingIssues.count_for_collection(issuable_ids)
+        else
+          []
+        end
+
+      issuable_ids.each_with_object({}) do |id, issuable_meta|
+        downvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? }
+        upvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? }
+        notes = issuable_note_count.find { |notes| notes.noteable_id == id }
+        merge_requests = issuable_merge_requests_count.find { |mr| mr.first == id }
+
+        issuable_meta[id] = ::Issuable::IssuableMeta.new(
+          upvotes.try(:count).to_i,
+          downvotes.try(:count).to_i,
+          notes.try(:count).to_i,
+          merge_requests.try(:last).to_i
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb
index c56c1a4322f..cdbdfa10d0e 100644
--- a/lib/gitlab/kubernetes.rb
+++ b/lib/gitlab/kubernetes.rb
@@ -76,5 +76,44 @@ module Gitlab
 
       url.to_s
     end
+
+    def to_kubeconfig(url:, namespace:, token:, ca_pem: nil)
+      config = {
+        apiVersion: 'v1',
+        clusters: [
+          name: 'gitlab-deploy',
+          cluster: {
+            server: url
+          }
+        ],
+        contexts: [
+          name: 'gitlab-deploy',
+          context: {
+            cluster: 'gitlab-deploy',
+            namespace: namespace,
+            user: 'gitlab-deploy'
+          }
+        ],
+        'current-context': 'gitlab-deploy',
+        kind: 'Config',
+        users: [
+          {
+            name: 'gitlab-deploy',
+            user: { token: token }
+          }
+        ]
+      }
+
+      kubeconfig_embed_ca_pem(config, ca_pem) if ca_pem
+
+      config.deep_stringify_keys
+    end
+
+    private
+
+    def kubeconfig_embed_ca_pem(config, ca_pem)
+      cluster = config.dig(:clusters, 0, :cluster)
+      cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem)
+    end
   end
 end
diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb
index 8779577258b..fb68627dedf 100644
--- a/lib/gitlab/ldap/access.rb
+++ b/lib/gitlab/ldap/access.rb
@@ -16,7 +16,7 @@ module Gitlab
       def self.allowed?(user)
         self.open(user) do |access|
           if access.allowed?
-            Users::UpdateService.new(user, last_credential_check_a: Time.now).execute
+            Users::UpdateService.new(user, last_credential_check_at: Time.now).execute
 
             true
           else
diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb
new file mode 100644
index 00000000000..219accfc029
--- /dev/null
+++ b/lib/gitlab/metrics/base_sampler.rb
@@ -0,0 +1,94 @@
+require 'logger'
+module Gitlab
+  module Metrics
+    class BaseSampler
+      def self.initialize_instance(*args)
+        raise "#{name} singleton instance already initialized" if @instance
+        @instance = new(*args)
+        at_exit(&@instance.method(:stop))
+        @instance
+      end
+
+      def self.instance
+        @instance
+      end
+
+      attr_reader :running
+
+      # interval - The sampling interval in seconds.
+      def initialize(interval)
+        interval_half = interval.to_f / 2
+
+        @interval = interval
+        @interval_steps = (-interval_half..interval_half).step(0.1).to_a
+
+        @mutex = Mutex.new
+      end
+
+      def enabled?
+        true
+      end
+
+      def start
+        return unless enabled?
+
+        @mutex.synchronize do
+          return if running
+          @running = true
+
+          @thread = Thread.new do
+            sleep(sleep_interval)
+
+            while running
+              safe_sample
+
+              sleep(sleep_interval)
+            end
+          end
+        end
+      end
+
+      def stop
+        @mutex.synchronize do
+          return unless running
+
+          @running = false
+
+          if @thread
+            @thread.wakeup if @thread.alive?
+            @thread.join
+            @thread = nil
+          end
+        end
+      end
+
+      def safe_sample
+        sample
+      rescue => e
+        Rails.logger.warn("#{self.class}: #{e}, stopping")
+        stop
+      end
+
+      def sample
+        raise NotImplementedError
+      end
+
+      # Returns the sleep interval with a random adjustment.
+      #
+      # The random adjustment is put in place to ensure we:
+      #
+      # 1. Don't generate samples at the exact same interval every time (thus
+      #    potentially missing anything that happens in between samples).
+      # 2. Don't sample data at the same interval two times in a row.
+      def sleep_interval
+        while step = @interval_steps.sample
+          if step != @last_step
+            @last_step = step
+
+            return @interval + @last_step
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/connection_rack_middleware.rb b/lib/gitlab/metrics/connection_rack_middleware.rb
new file mode 100644
index 00000000000..b3da360be8f
--- /dev/null
+++ b/lib/gitlab/metrics/connection_rack_middleware.rb
@@ -0,0 +1,45 @@
+module Gitlab
+  module Metrics
+    class ConnectionRackMiddleware
+      def initialize(app)
+        @app = app
+      end
+
+      def self.rack_request_count
+        @rack_request_count ||= Gitlab::Metrics.counter(:rack_request, 'Rack request count')
+      end
+
+      def self.rack_response_count
+        @rack_response_count ||= Gitlab::Metrics.counter(:rack_response, 'Rack response count')
+      end
+
+      def self.rack_uncaught_errors_count
+        @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors, 'Rack connections handling uncaught errors count')
+      end
+
+      def self.rack_execution_time
+        @rack_execution_time ||= Gitlab::Metrics.histogram(:rack_execution_time, 'Rack connection handling execution time',
+                                                           {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 1.5, 2, 2.5, 3, 5, 7, 10])
+      end
+
+      def call(env)
+        method = env['REQUEST_METHOD'].downcase
+        started = Time.now.to_f
+        begin
+          ConnectionRackMiddleware.rack_request_count.increment(method: method)
+
+          status, headers, body = @app.call(env)
+
+          ConnectionRackMiddleware.rack_response_count.increment(method: method, status: status)
+          [status, headers, body]
+        rescue
+          ConnectionRackMiddleware.rack_uncaught_errors_count.increment
+          raise
+        ensure
+          elapsed = Time.now.to_f - started
+          ConnectionRackMiddleware.rack_execution_time.observe({}, elapsed)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/influx_sampler.rb
index 0000450d9bb..6db1dd755b7 100644
--- a/lib/gitlab/metrics/sampler.rb
+++ b/lib/gitlab/metrics/influx_sampler.rb
@@ -5,14 +5,11 @@ module Gitlab
     # This class is used to gather statistics that can't be directly associated
     # with a transaction such as system memory usage, garbage collection
     # statistics, etc.
-    class Sampler
+    class InfluxSampler < BaseSampler
       # interval - The sampling interval in seconds.
       def initialize(interval = Metrics.settings[:sample_interval])
-        interval_half = interval.to_f / 2
-
-        @interval       = interval
-        @interval_steps = (-interval_half..interval_half).step(0.1).to_a
-        @last_step      = nil
+        super(interval)
+        @last_step = nil
 
         @metrics = []
 
@@ -26,18 +23,6 @@ module Gitlab
         end
       end
 
-      def start
-        Thread.new do
-          Thread.current.abort_on_exception = true
-
-          loop do
-            sleep(sleep_interval)
-
-            sample
-          end
-        end
-      end
-
       def sample
         sample_memory_usage
         sample_file_descriptors
@@ -86,7 +71,7 @@ module Gitlab
       end
 
       def sample_gc
-        time  = GC::Profiler.total_time * 1000.0
+        time = GC::Profiler.total_time * 1000.0
         stats = GC.stat.merge(total_time: time)
 
         # We want the difference of GC runs compared to the last sample, not the
@@ -111,23 +96,6 @@ module Gitlab
       def sidekiq?
         Sidekiq.server?
       end
-
-      # Returns the sleep interval with a random adjustment.
-      #
-      # The random adjustment is put in place to ensure we:
-      #
-      # 1. Don't generate samples at the exact same interval every time (thus
-      #    potentially missing anything that happens in between samples).
-      # 2. Don't sample data at the same interval two times in a row.
-      def sleep_interval
-        while step = @interval_steps.sample
-          if step != @last_step
-            @last_step = step
-
-            return @interval + @last_step
-          end
-        end
-      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb
index 9d314a56e58..fb7bbc7cfc7 100644
--- a/lib/gitlab/metrics/prometheus.rb
+++ b/lib/gitlab/metrics/prometheus.rb
@@ -29,8 +29,8 @@ module Gitlab
         provide_metric(name) || registry.summary(name, docstring, base_labels)
       end
 
-      def gauge(name, docstring, base_labels = {})
-        provide_metric(name) || registry.gauge(name, docstring, base_labels)
+      def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all)
+        provide_metric(name) || registry.gauge(name, docstring, base_labels, multiprocess_mode)
       end
 
       def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS)
diff --git a/lib/gitlab/metrics/unicorn_sampler.rb b/lib/gitlab/metrics/unicorn_sampler.rb
new file mode 100644
index 00000000000..f6987252039
--- /dev/null
+++ b/lib/gitlab/metrics/unicorn_sampler.rb
@@ -0,0 +1,48 @@
+module Gitlab
+  module Metrics
+    class UnicornSampler < BaseSampler
+      def initialize(interval)
+        super(interval)
+      end
+
+      def unicorn_active_connections
+        @unicorn_active_connections ||= Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max)
+      end
+
+      def unicorn_queued_connections
+        @unicorn_queued_connections ||= Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max)
+      end
+
+      def enabled?
+        # Raindrops::Linux.tcp_listener_stats is only present on Linux
+        unicorn_with_listeners? && Raindrops::Linux.respond_to?(:tcp_listener_stats)
+      end
+
+      def sample
+        Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats|
+          unicorn_active_connections.set({ type: 'tcp', address: addr }, stats.active)
+          unicorn_queued_connections.set({ type: 'tcp', address: addr }, stats.queued)
+        end
+
+        Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats|
+          unicorn_active_connections.set({ type: 'unix', address: addr }, stats.active)
+          unicorn_queued_connections.set({ type: 'unix', address: addr }, stats.queued)
+        end
+      end
+
+      private
+
+      def tcp_listeners
+        @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z})
+      end
+
+      def unix_listeners
+        @unix_listeners ||= Unicorn.listener_names - tcp_listeners
+      end
+
+      def unicorn_with_listeners?
+        defined?(Unicorn) && Unicorn.listener_names.any?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb
index 0d541935bc6..22332474945 100644
--- a/lib/gitlab/otp_key_rotator.rb
+++ b/lib/gitlab/otp_key_rotator.rb
@@ -34,7 +34,7 @@ module Gitlab
 
       write_csv do |csv|
         ActiveRecord::Base.transaction do
-          User.with_two_factor.in_batches do |relation|
+          User.with_two_factor.in_batches do |relation| # rubocop: disable Cop/InBatches
             rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt)
             rows.each do |row|
               user = %i[id ciphertext iv salt].zip(row).to_h
diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
index 10eb99fb461..d81f825ef96 100644
--- a/lib/gitlab/path_regex.rb
+++ b/lib/gitlab/path_regex.rb
@@ -112,6 +112,7 @@ module Gitlab
     # this group would not be accessible through `/groups/parent/activity` since
     # this would map to the activity-page of its parent.
     GROUP_ROUTES = %w[
+      -
       activity
       analytics
       audit_events
diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb
index 163a40ad306..2da2ce45ebc 100644
--- a/lib/gitlab/performance_bar.rb
+++ b/lib/gitlab/performance_bar.rb
@@ -1,7 +1,33 @@
 module Gitlab
   module PerformanceBar
-    def self.enabled?
-      Feature.enabled?('gitlab_performance_bar')
+    include Gitlab::CurrentSettings
+
+    ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids'.freeze
+
+    def self.enabled?(user = nil)
+      return false unless user && allowed_group_id
+
+      allowed_user_ids.include?(user.id)
+    end
+
+    def self.allowed_group_id
+      current_application_settings.performance_bar_allowed_group_id
+    end
+
+    def self.allowed_user_ids
+      Rails.cache.fetch(ALLOWED_USER_IDS_KEY) do
+        group = Group.find_by_id(allowed_group_id)
+
+        if group
+          GroupMembersFinder.new(group).execute.pluck(:user_id)
+        else
+          []
+        end
+      end
+    end
+
+    def self.expire_allowed_user_ids_cache
+      Rails.cache.delete(ALLOWED_USER_IDS_KEY)
     end
   end
 end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index b706434217d..c1ee20b6977 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -19,14 +19,6 @@ module Gitlab
       "It must start with letter, digit, emoji or '_'."
     end
 
-    def file_name_regex
-      @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze
-    end
-
-    def file_name_regex_message
-      "can contain only letters, digits, '_', '-', '@', '+' and '.'."
-    end
-
     def container_registry_reference_regex
       Gitlab::PathRegex.git_reference_regex
     end
@@ -38,8 +30,12 @@ module Gitlab
       @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z}
     end
 
+    def environment_name_regex_chars
+      'a-zA-Z0-9_/\\$\\{\\}\\. -'
+    end
+
     def environment_name_regex
-      @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze
+      @environment_name_regex ||= /\A[#{environment_name_regex_chars}]+\z/.freeze
     end
 
     def environment_name_regex_message
diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb
deleted file mode 100644
index 36d1a8a6f64..00000000000
--- a/lib/gitlab/routes/legacy_builds.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-module Gitlab
-  module Routes
-    class LegacyBuilds
-      def initialize(map)
-        @map = map
-      end
-
-      def draw
-        @map.instance_eval do
-          resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do
-            collection do
-              resources :artifacts, only: [], controller: 'build_artifacts' do
-                collection do
-                  get :latest_succeeded,
-                    path: '*ref_name_and_path',
-                    format: false
-                end
-              end
-            end
-
-            member do
-              get :raw
-            end
-
-            resource :artifacts, only: [], controller: 'build_artifacts' do
-              get :download
-              get :browse, path: 'browse(/*path)', format: false
-              get :file, path: 'file/*path', format: false
-              get :raw, path: 'raw/*path', format: false
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb
index 632e2d87500..e57890f1143 100644
--- a/lib/gitlab/routing.rb
+++ b/lib/gitlab/routing.rb
@@ -2,10 +2,35 @@ module Gitlab
   module Routing
     extend ActiveSupport::Concern
 
+    mattr_accessor :_includers
+    self._includers = []
+
     included do
+      Gitlab::Routing.includes_helpers(self)
+
       include Gitlab::Routing.url_helpers
     end
 
+    def self.includes_helpers(klass)
+      self._includers << klass
+    end
+
+    def self.add_helpers(mod)
+      url_helpers.include mod
+      url_helpers.extend mod
+
+      GitlabRoutingHelper.include mod
+      GitlabRoutingHelper.extend mod
+
+      app_url_helpers = Gitlab::Application.routes.named_routes.url_helpers_module
+      app_url_helpers.include mod
+      app_url_helpers.extend mod
+
+      _includers.each do |klass|
+        klass.include mod
+      end
+    end
+
     # Returns the URL helpers Module.
     #
     # This method caches the output as Rails' "url_helpers" method creates an
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index 22554236c38..0baea092e6a 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -2,6 +2,8 @@ require 'securerandom'
 
 module Gitlab
   class Shell
+    GITLAB_SHELL_ENV_VARS = %w(GIT_TERMINAL_PROMPT).freeze
+
     Error = Class.new(StandardError)
 
     KeyAdder = Struct.new(:io) do
@@ -67,8 +69,8 @@ module Gitlab
     #   add_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
     def add_repository(storage, name)
-      Gitlab::Utils.system_silent([gitlab_shell_projects_path,
-                                   'add-project', storage, "#{name}.git"])
+      gitlab_shell_fast_execute([gitlab_shell_projects_path,
+                                 'add-project', storage, "#{name}.git"])
     end
 
     # Import repository
@@ -82,10 +84,9 @@ module Gitlab
     def import_repository(storage, name, url)
       # Timeout should be less than 900 ideally, to prevent the memory killer
       # to silently kill the process without knowing we are timing out here.
-      output, status = Popen.popen([gitlab_shell_projects_path, 'import-project',
-                                    storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"])
-      raise Error, output unless status.zero?
-      true
+      cmd = [gitlab_shell_projects_path, 'import-project',
+             storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]
+      gitlab_shell_fast_execute_raise_error(cmd)
     end
 
     # Fetch remote for repository
@@ -103,9 +104,7 @@ module Gitlab
       args << '--force' if forced
       args << '--no-tags' if no_tags
 
-      output, status = Popen.popen(args)
-      raise Error, output unless status.zero?
-      true
+      gitlab_shell_fast_execute_raise_error(args)
     end
 
     # Move repository
@@ -117,8 +116,8 @@ module Gitlab
     #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
     #
     def mv_repository(storage, path, new_path)
-      Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project',
-                                   storage, "#{path}.git", "#{new_path}.git"])
+      gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project',
+                                 storage, "#{path}.git", "#{new_path}.git"])
     end
 
     # Fork repository to new namespace
@@ -131,9 +130,9 @@ module Gitlab
     #  fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx")
     #
     def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace)
-      Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project',
-                                   forked_from_storage, "#{path}.git", forked_to_storage,
-                                   fork_namespace])
+      gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project',
+                                 forked_from_storage, "#{path}.git", forked_to_storage,
+                                 fork_namespace])
     end
 
     # Remove repository from file system
@@ -145,8 +144,8 @@ module Gitlab
     #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
     def remove_repository(storage, name)
-      Gitlab::Utils.system_silent([gitlab_shell_projects_path,
-                                   'rm-project', storage, "#{name}.git"])
+      gitlab_shell_fast_execute([gitlab_shell_projects_path,
+                                 'rm-project', storage, "#{name}.git"])
     end
 
     # Add new key to gitlab-shell
@@ -155,8 +154,8 @@ module Gitlab
     #   add_key("key-42", "sha-rsa ...")
     #
     def add_key(key_id, key_content)
-      Gitlab::Utils.system_silent([gitlab_shell_keys_path,
-                                   'add-key', key_id, self.class.strip_key(key_content)])
+      gitlab_shell_fast_execute([gitlab_shell_keys_path,
+                                 'add-key', key_id, self.class.strip_key(key_content)])
     end
 
     # Batch-add keys to authorized_keys
@@ -175,8 +174,10 @@ module Gitlab
     #   remove_key("key-342", "sha-rsa ...")
     #
     def remove_key(key_id, key_content)
-      Gitlab::Utils.system_silent([gitlab_shell_keys_path,
-                                   'rm-key', key_id, key_content])
+      args = [gitlab_shell_keys_path, 'rm-key', key_id]
+      args << key_content if key_content
+
+      gitlab_shell_fast_execute(args)
     end
 
     # Remove all ssh keys from gitlab shell
@@ -185,7 +186,7 @@ module Gitlab
     #   remove_all_keys
     #
     def remove_all_keys
-      Gitlab::Utils.system_silent([gitlab_shell_keys_path, 'clear'])
+      gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear'])
     end
 
     # Add empty directory for storing repositories
@@ -267,5 +268,31 @@ module Gitlab
     def gitlab_shell_keys_path
       File.join(gitlab_shell_path, 'bin', 'gitlab-keys')
     end
+
+    private
+
+    def gitlab_shell_fast_execute(cmd)
+      output, status = gitlab_shell_fast_execute_helper(cmd)
+
+      return true if status.zero?
+
+      Rails.logger.error("gitlab-shell failed with error #{status}: #{output}")
+      false
+    end
+
+    def gitlab_shell_fast_execute_raise_error(cmd)
+      output, status = gitlab_shell_fast_execute_helper(cmd)
+
+      raise Error, output unless status.zero?
+      true
+    end
+
+    def gitlab_shell_fast_execute_helper(cmd)
+      vars = ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS)
+
+      # Don't pass along the entire parent environment to prevent gitlab-shell
+      # from wasting I/O by searching through GEM_PATH
+      Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
+    end
   end
 end
diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb
index 27696436574..e13808a2720 100644
--- a/lib/gitlab/slash_commands/presenters/base.rb
+++ b/lib/gitlab/slash_commands/presenters/base.rb
@@ -2,7 +2,7 @@ module Gitlab
   module SlashCommands
     module Presenters
       class Base
-        include Gitlab::Routing.url_helpers
+        include Gitlab::Routing
 
         def initialize(resource = nil)
           @resource = resource
diff --git a/lib/gitlab/sql/glob.rb b/lib/gitlab/sql/glob.rb
new file mode 100644
index 00000000000..5e89e12b2b1
--- /dev/null
+++ b/lib/gitlab/sql/glob.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module SQL
+    module Glob
+      extend self
+
+      # Convert a simple glob pattern with wildcard (*) to SQL LIKE pattern
+      # with SQL expression
+      def to_like(pattern)
+        <<~SQL
+          REPLACE(REPLACE(REPLACE(#{pattern},
+                                  #{q('%')}, #{q('\\%')}),
+                          #{q('_')}, #{q('\\_')}),
+                  #{q('*')}, #{q('%')})
+        SQL
+      end
+
+      def q(string)
+        ActiveRecord::Base.connection.quote(string)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index 23af9318d1a..35792d2d67f 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -1,6 +1,6 @@
 module Gitlab
   class UrlBuilder
-    include Gitlab::Routing.url_helpers
+    include Gitlab::Routing
     include GitlabRoutingHelper
     include ActionView::RecordIdentifier
 
@@ -23,9 +23,9 @@ module Gitlab
       when WikiPage
         wiki_page_url
       when ProjectSnippet
-        project_snippet_url(object)
+        project_snippet_url(object.project, object)
       when Snippet
-        personal_snippet_url(object)
+        snippet_url(object)
       else
         raise NotImplementedError.new("No URL builder defined for #{object.class}")
       end
@@ -65,13 +65,13 @@ module Gitlab
         if snippet.is_a?(PersonalSnippet)
           snippet_url(snippet, anchor: dom_id(object))
         else
-          project_snippet_url(snippet, anchor: dom_id(object))
+          project_snippet_url(snippet.project, snippet, anchor: dom_id(object))
         end
       end
     end
 
     def wiki_page_url
-      namespace_project_wiki_url(object.wiki.project.namespace, object.wiki.project, object.slug)
+      project_wiki_url(object.wiki.project, object.slug)
     end
   end
 end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 38dc82493cf..f19b325a126 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -20,7 +20,8 @@ module Gitlab
           counts: {
             boards: Board.count,
             ci_builds: ::Ci::Build.count,
-            ci_pipelines: ::Ci::Pipeline.count,
+            ci_internal_pipelines: ::Ci::Pipeline.internal.count,
+            ci_external_pipelines: ::Ci::Pipeline.external.count,
             ci_runners: ::Ci::Runner.count,
             ci_triggers: ::Ci::Trigger.count,
             ci_pipeline_schedules: ::Ci::PipelineSchedule.count,
diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb
index dbfe0941e4d..841fb681435 100644
--- a/lib/gitlab/view/presenter/base.rb
+++ b/lib/gitlab/view/presenter/base.rb
@@ -15,6 +15,11 @@ module Gitlab
           super(user, action, overriden_subject || subject)
         end
 
+        # delegate all #can? queries to the subject
+        def declarative_policy_delegate
+          subject
+        end
+
         class_methods do
           def presenter?
             true
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index f96ee69096d..4aef23b6aee 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -25,27 +25,25 @@ module Gitlab
           RepoPath: repo_path
         }
 
-        if Gitlab.config.gitaly.enabled
-          server = {
-            address: Gitlab::GitalyClient.address(project.repository_storage),
-            token: Gitlab::GitalyClient.token(project.repository_storage)
-          }
-          params[:Repository] = repository.gitaly_repository.to_h
-
-          feature_enabled = case action.to_s
-                            when 'git_receive_pack'
-                              Gitlab::GitalyClient.feature_enabled?(:post_receive_pack)
-                            when 'git_upload_pack'
-                              Gitlab::GitalyClient.feature_enabled?(:post_upload_pack)
-                            when 'info_refs'
-                              true
-                            else
-                              raise "Unsupported action: #{action}"
-                            end
-          if feature_enabled
-            params[:GitalyAddress] = server[:address] # This field will be deprecated
-            params[:GitalyServer] = server
-          end
+        server = {
+          address: Gitlab::GitalyClient.address(project.repository_storage),
+          token: Gitlab::GitalyClient.token(project.repository_storage)
+        }
+        params[:Repository] = repository.gitaly_repository.to_h
+
+        feature_enabled = case action.to_s
+                          when 'git_receive_pack'
+                            Gitlab::GitalyClient.feature_enabled?(:post_receive_pack)
+                          when 'git_upload_pack'
+                            Gitlab::GitalyClient.feature_enabled?(:post_upload_pack)
+                          when 'info_refs'
+                            true
+                          else
+                            raise "Unsupported action: #{action}"
+                          end
+        if feature_enabled
+          params[:GitalyAddress] = server[:address] # This field will be deprecated
+          params[:GitalyServer] = server
         end
 
         params
diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb
index 99f9c2c9b04..7cfe76b7b71 100644
--- a/lib/peek/rblineprof/custom_controller_helpers.rb
+++ b/lib/peek/rblineprof/custom_controller_helpers.rb
@@ -41,9 +41,14 @@ module Peek
             ]
           end.sort_by{ |a,b,c,d,e,f| -f }
 
-          output = ''
-          per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort|
+          output = "<div class='modal-dialog modal-full'><div class='modal-content'>"
+          output << "<div class='modal-header'>"
+          output << "<button class='close btn btn-link btn-sm' type='button' data-dismiss='modal'>X</button>"
+          output << "<h4>Line profiling: #{human_description(params[:lineprofiler])}</h4>"
+          output << "</div>"
+          output << "<div class='modal-body'>"
 
+          per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort|
             output << "<div class='peek-rblineprof-file'><div class='heading'>"
 
             show_src = file_sort > min
@@ -86,11 +91,32 @@ module Peek
             output << "</div></div>" # .data then .peek-rblineprof-file
           end
 
-          response.body += "<div class='peek-rblineprof-modal' id='line-profile'>#{output}</div>".html_safe
+          output << "</div></div></div>"
+
+          response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output}</div>".html_safe
         end
 
         ret
       end
+
+      private
+
+      def human_description(lineprofiler_param)
+        case lineprofiler_param
+        when 'app'
+          'app/ & lib/'
+        when 'views'
+          'app/view/'
+        when 'gems'
+          'vendor/gems'
+        when 'all'
+          'everything in Rails.root'
+        when 'stdlib'
+          'everything in the Ruby standard library'
+        else
+          'app/, config/, lib/, vendor/ & plugin/'
+        end
+      end
     end
   end
 end
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index e3883278886..e9fb6a008b0 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -42,8 +42,7 @@ namespace :gitlab do
       http_clone_url = project.http_url_to_repo
       ssh_clone_url  = project.ssh_url_to_repo
 
-      omniauth_providers = Gitlab.config.omniauth.providers
-      omniauth_providers.map! { |provider| provider['name'] }
+      omniauth_providers = Gitlab.config.omniauth.providers.map { |provider| provider['name'] }
 
       puts ""
       puts "GitLab information".color(:yellow)