Merge branch 'master' into sh-headless-chrome-support

* master: (97 commits)
  Eliminate N+1 queries in loading discussions.json endpoint
  Clean up read_registry scope changes
  Add missing import statements
  Improve “New project“ page description
  Fix notification message when admin label was modified
  Remove gaps under nav on build page
  Replace the 'project/snippets.feature' spinach test with an rspec analog
  Use correct group members path for members flyout link
  Fix docs for lightweight tag creation via API
  Replace the 'project/commits/revert.feature' spinach test with an rspec analog
  Merge branch 'rs-incoming-email-domain-docs' into 'security-10-0'
  Replace the 'project/archived.feature' spinach test with an rspec analog
  Fix broken link in docs/api/wiki.md
  Fixed the new sidebars width when browser has scrollbars
  Improve 'spec/features/profiles/*' specs
  Replace the 'search.feature' spinach test with an rspec analog
  dedupe yarn packages
  add dependency approvals (all MIT license)
  update build image to latest with node 8.x, yarn 1.0.2, and chrome 61
  Ensure we use `Entities::User` for non-admin `users/:id` API requests
  ...

12 files changed, 266 insertions, 36 deletions

diff --git a/lib/api/users.rb b/lib/api/users.rb
index 1825c90a23b..bdebda58d3f 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -88,7 +88,7 @@ module API
         user = User.find_by(id: params[:id])
         not_found!('User') unless user && can?(current_user, :read_user, user)
 
-        opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : {}
+        opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : { with: Entities::User }
         present user, opts
       end
 
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 11ace83c15c..87aeb76b66a 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Auth
     MissingPersonalTokenError = Class.new(StandardError)
 
-    REGISTRY_SCOPES = Gitlab.config.registry.enabled ? [:read_registry].freeze : [].freeze
+    REGISTRY_SCOPES = [:read_registry].freeze
 
     # Scopes used for GitLab API access
     API_SCOPES = [:api, :read_user].freeze
@@ -13,11 +13,6 @@ module Gitlab
     # Default scopes for OAuth applications that don't define their own
     DEFAULT_SCOPES = [:api].freeze
 
-    AVAILABLE_SCOPES = (API_SCOPES + REGISTRY_SCOPES).freeze
-
-    # Other available scopes
-    OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze
-
     class << self
       include Gitlab::CurrentSettings
 
@@ -132,7 +127,7 @@ module Gitlab
 
         token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password)
 
-        if token && valid_scoped_token?(token, AVAILABLE_SCOPES)
+        if token && valid_scoped_token?(token, available_scopes)
           Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes))
         end
       end
@@ -230,6 +225,21 @@ module Gitlab
       def read_user_scope_authentication_abilities
         []
       end
+
+      def available_scopes
+        API_SCOPES + registry_scopes
+      end
+
+      # Other available scopes
+      def optional_scopes
+        available_scopes + OPENID_SCOPES - DEFAULT_SCOPES
+      end
+
+      def registry_scopes
+        return [] unless Gitlab.config.registry.enabled
+
+        REGISTRY_SCOPES
+      end
     end
   end
 end
diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb
new file mode 100644
index 00000000000..b1411be3016
--- /dev/null
+++ b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb
@@ -0,0 +1,41 @@
+module Gitlab
+  module BackgroundMigration
+    class DeleteConflictingRedirectRoutesRange
+      class Route < ActiveRecord::Base
+        self.table_name = 'routes'
+      end
+
+      class RedirectRoute < ActiveRecord::Base
+        self.table_name = 'redirect_routes'
+      end
+
+      # start_id - The start ID of the range of events to process
+      # end_id - The end ID of the range to process.
+      def perform(start_id, end_id)
+        return unless migrate?
+
+        conflicts = RedirectRoute.where(routes_match_redirects_clause(start_id, end_id))
+        num_rows = conflicts.delete_all
+
+        Rails.logger.info("Gitlab::BackgroundMigration::DeleteConflictingRedirectRoutesRange [#{start_id}, #{end_id}] - Deleted #{num_rows} redirect routes that were conflicting with routes.")
+      end
+
+      def migrate?
+        Route.table_exists? && RedirectRoute.table_exists?
+      end
+
+      def routes_match_redirects_clause(start_id, end_id)
+        <<~ROUTES_MATCH_REDIRECTS
+          EXISTS (
+            SELECT 1 FROM routes
+            WHERE (
+              LOWER(redirect_routes.path) = LOWER(routes.path)
+              OR LOWER(redirect_routes.path) LIKE LOWER(CONCAT(routes.path, '/%'))
+            )
+            AND routes.id BETWEEN #{start_id} AND #{end_id}
+          )
+        ROUTES_MATCH_REDIRECTS
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index fb14798efe6..2c35da8f1aa 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -1,6 +1,9 @@
 module Gitlab
   module Database
     module MigrationHelpers
+      BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job
+      BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time
+
       # Adds `created_at` and `updated_at` columns with timezone information.
       #
       # This method is an improved version of Rails' built-in method `add_timestamps`.
@@ -653,6 +656,91 @@ into similar problems in the future (e.g. when new tables are created).
           EOF
         end
       end
+
+      # Bulk queues background migration jobs for an entire table, batched by ID range.
+      # "Bulk" meaning many jobs will be pushed at a time for efficiency.
+      # If you need a delay interval per job, then use `queue_background_migration_jobs_by_range_at_intervals`.
+      #
+      # model_class - The table being iterated over
+      # job_class_name - The background migration job class as a string
+      # batch_size - The maximum number of rows per job
+      #
+      # Example:
+      #
+      #     class Route < ActiveRecord::Base
+      #       include EachBatch
+      #       self.table_name = 'routes'
+      #     end
+      #
+      #     bulk_queue_background_migration_jobs_by_range(Route, 'ProcessRoutes')
+      #
+      # Where the model_class includes EachBatch, and the background migration exists:
+      #
+      #     class Gitlab::BackgroundMigration::ProcessRoutes
+      #       def perform(start_id, end_id)
+      #         # do something
+      #       end
+      #     end
+      def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE)
+        raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id')
+
+        jobs = []
+
+        model_class.each_batch(of: batch_size) do |relation|
+          start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
+
+          if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE
+            # Note: This code path generally only helps with many millions of rows
+            # We push multiple jobs at a time to reduce the time spent in
+            # Sidekiq/Redis operations. We're using this buffer based approach so we
+            # don't need to run additional queries for every range.
+            BackgroundMigrationWorker.perform_bulk(jobs)
+            jobs.clear
+          end
+
+          jobs << [job_class_name, [start_id, end_id]]
+        end
+
+        BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty?
+      end
+
+      # Queues background migration jobs for an entire table, batched by ID range.
+      # Each job is scheduled with a `delay_interval` in between.
+      # If you use a small interval, then some jobs may run at the same time.
+      #
+      # model_class - The table being iterated over
+      # job_class_name - The background migration job class as a string
+      # delay_interval - The duration between each job's scheduled time (must respond to `to_f`)
+      # batch_size - The maximum number of rows per job
+      #
+      # Example:
+      #
+      #     class Route < ActiveRecord::Base
+      #       include EachBatch
+      #       self.table_name = 'routes'
+      #     end
+      #
+      #     queue_background_migration_jobs_by_range_at_intervals(Route, 'ProcessRoutes', 1.minute)
+      #
+      # Where the model_class includes EachBatch, and the background migration exists:
+      #
+      #     class Gitlab::BackgroundMigration::ProcessRoutes
+      #       def perform(start_id, end_id)
+      #         # do something
+      #       end
+      #     end
+      def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE)
+        raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id')
+
+        model_class.each_batch(of: batch_size) do |relation, index|
+          start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
+
+          # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for
+          # the same time, which is not helpful in most cases where we wish to
+          # spread the work over time.
+          BackgroundMigrationWorker.perform_in(delay_interval * index, job_class_name, [start_id, end_id])
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/database/read_only_relation.rb b/lib/gitlab/database/read_only_relation.rb
new file mode 100644
index 00000000000..4571ad122ce
--- /dev/null
+++ b/lib/gitlab/database/read_only_relation.rb
@@ -0,0 +1,16 @@
+module Gitlab
+  module Database
+    # Module that can be injected into a ActiveRecord::Relation to make it
+    # read-only.
+    module ReadOnlyRelation
+      [:delete, :delete_all, :update, :update_all].each do |method|
+        define_method(method) do |*args|
+          raise(
+            ActiveRecord::ReadOnlyRecord,
+            "This relation is marked as read-only"
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb
index 919965100ae..010b4be7b40 100644
--- a/lib/gitlab/diff/inline_diff_marker.rb
+++ b/lib/gitlab/diff/inline_diff_marker.rb
@@ -2,9 +2,10 @@ module Gitlab
   module Diff
     class InlineDiffMarker < Gitlab::StringRangeMarker
       def mark(line_inline_diffs, mode: nil)
-        super(line_inline_diffs) do |text, left:, right:|
+        mark = super(line_inline_diffs) do |text, left:, right:|
           %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>}
         end
+        mark.html_safe
       end
 
       private
diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb
index 9e6fca8c80c..dcdec818f5e 100644
--- a/lib/gitlab/git/operation_service.rb
+++ b/lib/gitlab/git/operation_service.rb
@@ -1,11 +1,18 @@
 module Gitlab
   module Git
     class OperationService
-      attr_reader :committer, :repository
+      WithBranchResult = Struct.new(:newrev, :repo_created, :branch_created) do
+        alias_method :repo_created?, :repo_created
+        alias_method :branch_created?, :branch_created
+      end
+
+      attr_reader :user, :repository
 
-      def initialize(committer, new_repository)
-        committer = Gitlab::Git::Committer.from_user(committer) if committer.is_a?(User)
-        @committer = committer
+      def initialize(user, new_repository)
+        if user
+          user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id)
+          @user = user
+        end
 
         # Refactoring aid
         unless new_repository.is_a?(Gitlab::Git::Repository)
@@ -105,7 +112,7 @@ module Gitlab
         ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name
         update_ref_in_hooks(ref, newrev, oldrev)
 
-        [newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)]
+        WithBranchResult.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev))
       end
 
       def find_oldrev_from_branch(newrev, branch)
@@ -128,7 +135,7 @@ module Gitlab
 
       def with_hooks(ref, newrev, oldrev)
         Gitlab::Git::HooksService.new.execute(
-          committer,
+          user,
           repository,
           oldrev,
           newrev,
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index efa13590a2c..c499ff101b5 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -610,49 +610,86 @@ module Gitlab
         # TODO: implement this method
       end
 
-      def add_branch(branch_name, committer:, target:)
+      def add_branch(branch_name, user:, target:)
         target_object = Ref.dereference_object(lookup(target))
         raise InvalidRef.new("target not found: #{target}") unless target_object
 
-        OperationService.new(committer, self).add_branch(branch_name, target_object.oid)
+        OperationService.new(user, self).add_branch(branch_name, target_object.oid)
         find_branch(branch_name)
       rescue Rugged::ReferenceError => ex
         raise InvalidRef, ex
       end
 
-      def add_tag(tag_name, committer:, target:, message: nil)
+      def add_tag(tag_name, user:, target:, message: nil)
         target_object = Ref.dereference_object(lookup(target))
         raise InvalidRef.new("target not found: #{target}") unless target_object
 
-        committer = Committer.from_user(committer) if committer.is_a?(User)
+        user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id)
 
         options = nil # Use nil, not the empty hash. Rugged cares about this.
         if message
           options = {
             message: message,
-            tagger: Gitlab::Git.committer_hash(email: committer.email, name: committer.name)
+            tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name)
           }
         end
 
-        OperationService.new(committer, self).add_tag(tag_name, target_object.oid, options)
+        OperationService.new(user, self).add_tag(tag_name, target_object.oid, options)
 
         find_tag(tag_name)
       rescue Rugged::ReferenceError => ex
         raise InvalidRef, ex
       end
 
-      def rm_branch(branch_name, committer:)
-        OperationService.new(committer, self).rm_branch(find_branch(branch_name))
+      def rm_branch(branch_name, user:)
+        OperationService.new(user, self).rm_branch(find_branch(branch_name))
       end
 
-      def rm_tag(tag_name, committer:)
-        OperationService.new(committer, self).rm_tag(find_tag(tag_name))
+      def rm_tag(tag_name, user:)
+        OperationService.new(user, self).rm_tag(find_tag(tag_name))
       end
 
       def find_tag(name)
         tags.find { |tag| tag.name == name }
       end
 
+      def merge(user, source_sha, target_branch, message)
+        committer = Gitlab::Git.committer_hash(email: user.email, name: user.name)
+
+        OperationService.new(user, self).with_branch(target_branch) do |start_commit|
+          our_commit = start_commit.sha
+          their_commit = source_sha
+
+          raise 'Invalid merge target' unless our_commit
+          raise 'Invalid merge source' unless their_commit
+
+          merge_index = rugged.merge_commits(our_commit, their_commit)
+          break if merge_index.conflicts?
+
+          options = {
+            parents: [our_commit, their_commit],
+            tree: merge_index.write_tree(rugged),
+            message: message,
+            author: committer,
+            committer: committer
+          }
+
+          commit_id = create_commit(options)
+
+          yield commit_id
+
+          commit_id
+        end
+      rescue Gitlab::Git::CommitError # when merge_index.conflicts?
+        nil
+      end
+
+      def create_commit(params = {})
+        params[:message].delete!("\r")
+
+        Rugged::Commit.create(rugged, params)
+      end
+
       # Delete the specified branch from the repository
       def delete_branch(branch_name)
         gitaly_migrate(:delete_branch) do |is_enabled|
diff --git a/lib/gitlab/git/committer.rb b/lib/gitlab/git/user.rb
index 1f4bcf7a3a0..ea634d39668 100644
--- a/lib/gitlab/git/committer.rb
+++ b/lib/gitlab/git/user.rb
@@ -1,10 +1,14 @@
 module Gitlab
   module Git
-    class Committer
+    class User
       attr_reader :name, :email, :gl_id
 
-      def self.from_user(user)
-        new(user.name, user.email, Gitlab::GlId.gl_id(user))
+      def self.from_gitlab(gitlab_user)
+        new(gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user))
+      end
+
+      def self.from_gitaly(gitaly_user)
+        new(gitaly_user.name, gitaly_user.email, gitaly_user.gl_id)
       end
 
       def initialize(name, email, gl_id)
diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb
index 5a31e56cb30..635f52131f9 100644
--- a/lib/gitlab/group_hierarchy.rb
+++ b/lib/gitlab/group_hierarchy.rb
@@ -22,7 +22,7 @@ module Gitlab
     def base_and_ancestors
       return ancestors_base unless Group.supports_nested_groups?
 
-      base_and_ancestors_cte.apply_to(model.all)
+      read_only(base_and_ancestors_cte.apply_to(model.all))
     end
 
     # Returns a relation that includes the descendants_base set of groups
@@ -30,7 +30,7 @@ module Gitlab
     def base_and_descendants
       return descendants_base unless Group.supports_nested_groups?
 
-      base_and_descendants_cte.apply_to(model.all)
+      read_only(base_and_descendants_cte.apply_to(model.all))
     end
 
     # Returns a relation that includes the base groups, their ancestors,
@@ -67,11 +67,13 @@ module Gitlab
       union = SQL::Union.new([model.unscoped.from(ancestors_table),
                               model.unscoped.from(descendants_table)])
 
-      model
+      relation = model
         .unscoped
         .with
         .recursive(ancestors.to_arel, descendants.to_arel)
         .from("(#{union.to_sql}) #{model.table_name}")
+
+      read_only(relation)
     end
 
     private
@@ -107,5 +109,12 @@ module Gitlab
     def groups_table
       model.arel_table
     end
+
+    def read_only(relation)
+      # relations using a CTE are not safe to use with update_all as it will
+      # throw away the CTE, hence we mark them as read-only.
+      relation.extend(Gitlab::Database::ReadOnlyRelation)
+      relation
+    end
   end
 end
diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb
index 1f331b1e91d..5b5ed449f94 100644
--- a/lib/gitlab/o_auth/auth_hash.rb
+++ b/lib/gitlab/o_auth/auth_hash.rb
@@ -13,7 +13,7 @@ module Gitlab
       end
 
       def provider
-        @provider ||= Gitlab::Utils.force_utf8(auth_hash.provider.to_s)
+        @provider ||= auth_hash.provider.to_s
       end
 
       def name
diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb
index 703adae12cb..4e1ec1402ea 100644
--- a/lib/gitlab/url_sanitizer.rb
+++ b/lib/gitlab/url_sanitizer.rb
@@ -19,13 +19,12 @@ module Gitlab
     end
 
     def initialize(url, credentials: nil)
-      @url = Addressable::URI.parse(url.to_s.strip)
-
       %i[user password].each do |symbol|
         credentials[symbol] = credentials[symbol].presence if credentials&.key?(symbol)
       end
 
       @credentials = credentials
+      @url = parse_url(url)
     end
 
     def sanitized_url
@@ -49,12 +48,30 @@ module Gitlab
 
     private
 
+    def parse_url(url)
+      url             = url.to_s.strip
+      match           = url.match(%r{\A(?:git|ssh|http(?:s?))\://(?:(.+)(?:@))?(.+)})
+      raw_credentials = match[1] if match
+
+      if raw_credentials.present?
+        url.sub!("#{raw_credentials}@", '')
+
+        user, password = raw_credentials.split(':')
+        @credentials ||= { user: user.presence, password: password.presence }
+      end
+
+      url = Addressable::URI.parse(url)
+      url.password = password if password.present?
+      url.user = user if user.present?
+      url
+    end
+
     def generate_full_url
       return @url unless valid_credentials?
       @full_url = @url.dup
 
-      @full_url.password = credentials[:password]
-      @full_url.user = credentials[:user]
+      @full_url.password = credentials[:password] if credentials[:password].present?
+      @full_url.user = credentials[:user] if credentials[:user].present?
 
       @full_url
     end