Merge branch 'master' into 40016-log-header40016-log-header

* master: (21 commits)
  Changing OAuth lookup to be case insensitive
  Delete orphaned fork networks in a migration
  Delete the fork network when removing the last membership
  Resolve "Performance issues when loading large number of wiki pages"
  Exports a couple of project related code as es6 modules
  Fix go-import meta data when enabled_git_access_protocol is a blank string
  Add dropdowns documentation
  Convert migration to populate latest merge request ID into a background migration
  Set 0.69.0 instead of latest for codeclimate image
  Fix hashed storage with project transfers to another namespace
  De-duplicate background migration matchers defined in spec/support/migrations_helpers.rb
  Update database_debugging.md
  Update database_debugging.md
  Move installation of apps higher
  Change to Google Kubernetes Cluster and add internal links
  Add Ingress description from official docs
  Add info on creating your own k8s cluster from the cluster page
  Add info about the installed apps in the Cluster docs
  Update HA README.md to clarify GitLab support does not troubleshoot DRBD.
  Optimise getting the pipeline status of commits
  ...

5 files changed, 79 insertions, 19 deletions

diff --git a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb
new file mode 100644
index 00000000000..7e109e96e73
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module BackgroundMigration
+    class PopulateMergeRequestsLatestMergeRequestDiffId
+      BATCH_SIZE = 1_000
+
+      class MergeRequest < ActiveRecord::Base
+        self.table_name = 'merge_requests'
+
+        include ::EachBatch
+      end
+
+      def perform(start_id, stop_id)
+        update = '
+          latest_merge_request_diff_id = (
+            SELECT MAX(id)
+            FROM merge_request_diffs
+            WHERE merge_requests.id = merge_request_diffs.merge_request_id
+          )'.squish
+
+        MergeRequest
+          .where(id: start_id..stop_id)
+          .where(latest_merge_request_diff_id: nil)
+          .each_batch(of: BATCH_SIZE) do |relation|
+
+          relation.update_all(update)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb
index 022d1f249a9..d4a53d32c28 100644
--- a/lib/gitlab/git/wiki.rb
+++ b/lib/gitlab/git/wiki.rb
@@ -58,12 +58,12 @@ module Gitlab
         end
       end
 
-      def pages
-        @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled|
+      def pages(limit: nil)
+        @repository.gitaly_migrate(:wiki_get_all_pages, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled|
           if is_enabled
             gitaly_get_all_pages
           else
-            gollum_get_all_pages
+            gollum_get_all_pages(limit: limit)
           end
         end
       end
@@ -88,14 +88,23 @@ module Gitlab
         end
       end
 
-      def page_versions(page_path)
+      # options:
+      #  :page     - The Integer page number.
+      #  :per_page - The number of items per page.
+      #  :limit    - Total number of items to return.
+      def page_versions(page_path, options = {})
         current_page = gollum_page_by_path(page_path)
-        current_page.versions.map do |gollum_git_commit|
-          gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id)
-          new_version(gollum_page, gollum_git_commit.id)
+
+        commits_from_page(current_page, options).map do |gitlab_git_commit|
+          gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id)
+          Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format)
         end
       end
 
+      def count_page_versions(page_path)
+        @repository.count_commits(ref: 'HEAD', path: page_path)
+      end
+
       def preview_slug(title, format)
         # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid
         # using Rugged through a Gollum::Wiki instance
@@ -110,6 +119,22 @@ module Gitlab
 
       private
 
+      # options:
+      #  :page     - The Integer page number.
+      #  :per_page - The number of items per page.
+      #  :limit    - Total number of items to return.
+      def commits_from_page(gollum_page, options = {})
+        unless options[:limit]
+          options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page
+          options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i
+        end
+
+        @repository.log(ref: gollum_page.last_version.id,
+                        path: gollum_page.path,
+                        limit: options[:limit],
+                        offset: options[:offset])
+      end
+
       def gollum_wiki
         @gollum_wiki ||= Gollum::Wiki.new(@repository.path)
       end
@@ -126,8 +151,17 @@ module Gitlab
       end
 
       def new_version(gollum_page, commit_id)
-        commit = Gitlab::Git::Commit.find(@repository, commit_id)
-        Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format)
+        Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format)
+      end
+
+      def version(commit_id)
+        commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) }
+
+        if RequestStore.active?
+          RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call }
+        else
+          commit_find_proc.call
+        end
       end
 
       def assert_type!(object, klass)
@@ -185,8 +219,8 @@ module Gitlab
         Gitlab::Git::WikiFile.new(gollum_file)
       end
 
-      def gollum_get_all_pages
-        gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) }
+      def gollum_get_all_pages(limit: nil)
+        gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) }
       end
 
       def gitaly_write_page(name, format, content, commit_details)
diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb
index 4d5c67ed892..3945df27eed 100644
--- a/lib/gitlab/ldap/user.rb
+++ b/lib/gitlab/ldap/user.rb
@@ -9,11 +9,8 @@ module Gitlab
     class User < Gitlab::OAuth::User
       class << self
         def find_by_uid_and_provider(uid, provider)
-          uid = Gitlab::LDAP::Person.normalize_dn(uid)
+          identity = ::Identity.with_extern_uid(provider, uid).take
 
-          identity = ::Identity
-            .where(provider: provider)
-            .where(extern_uid: uid).last
           identity && identity.user
         end
       end
diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index cfc6b2a2029..1edda50e4b0 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -42,12 +42,11 @@ module Gitlab
         project_url = URI.join(config.gitlab.url, path)
         import_prefix = strip_url(project_url.to_s)
 
-        repository_url = case current_application_settings.enabled_git_access_protocol
-                         when 'ssh'
+        repository_url = if current_application_settings.enabled_git_access_protocol == 'ssh'
                            shell = config.gitlab_shell
                            port = ":#{shell.ssh_port}" unless shell.ssh_port == 22
                            "ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git"
-                         when 'http', nil
+                         else
                            "#{project_url}.git"
                          end
 
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index b4b3b00c84d..552133234a3 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -157,7 +157,7 @@ module Gitlab
       end
 
       def find_by_uid_and_provider
-        identity = Identity.find_by(provider: auth_hash.provider, extern_uid: auth_hash.uid)
+        identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take
         identity && identity.user
       end