Merge remote-tracking branch 'upstream/master' into 19737-read-only-auditor19737-read-only-auditor

* upstream/master: (1277 commits)
  Grapify the labels API
  Fix typo in project settings that prevents users from enabling container registry.
  Fix old monitoring links to point to the new location
  Added path parameter to Commits API
  fixes build with cache:clear issue
  Merge branch 'security-fix-leaking-namespace-name' into 'security'
  Fix authored vote from notes
  Grapify builds API
  Add changelog item for groups 404 on relative url
  Add relative url support to routing contrainers
  Update project member controller to match recent master logic
  Add parentheses around return redirect_to method
  Trigger change even in select2 test helper to produce production-like behaviour
  Refactor js that disable form submit if no members selected
  Improve create project member test at project_members_controller_spec
  Move changelog item to 8.14
  Refactor create member tests from group_members_controller_spec
  Refactor groups/projects members controller
  Gracefully handle adding of no users to projects and groups
  Revert "Change "Group#web_url" to return "/groups/twitter" rather than "/twitter"."
  ...

10 files changed, 155 insertions, 45 deletions

diff --git a/lib/tasks/.gitkeep b/lib/tasks/.gitkeep
deleted file mode 100644
index e69de29bb2d..00000000000
--- a/lib/tasks/.gitkeep
+++ /dev/null
diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index 2214f855200..78ae187817a 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -1,22 +1,33 @@
 namespace :cache do
-  CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000
-  REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan
+  namespace :clear do
+    REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000
+    REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan
 
-  desc "GitLab | Clear redis cache"
-  task :clear => :environment do
-    Gitlab::Redis.with do |redis|
-      cursor = REDIS_SCAN_START_STOP
-      loop do
-        cursor, keys = redis.scan(
-          cursor,
-          match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", 
-          count: CLEAR_BATCH_SIZE
-        )
-  
-        redis.del(*keys) if keys.any?
-  
-        break if cursor == REDIS_SCAN_START_STOP
+    desc "GitLab | Clear redis cache"
+    task redis: :environment do
+      Gitlab::Redis.with do |redis|
+        cursor = REDIS_SCAN_START_STOP
+        loop do
+          cursor, keys = redis.scan(
+            cursor,
+            match: "#{Gitlab::Redis::CACHE_NAMESPACE}*",
+            count: REDIS_CLEAR_BATCH_SIZE
+          )
+
+          redis.del(*keys) if keys.any?
+
+          break if cursor == REDIS_SCAN_START_STOP
+        end
       end
     end
+
+    desc "GitLab | Clear database cache (in the background)"
+    task db: :environment do
+      ClearDatabaseCacheWorker.perform_async
+    end
+
+    task all: [:db, :redis]
   end
+
+  task clear: 'cache:clear:redis'
 end
diff --git a/lib/tasks/ce_to_ee_merge_check.rake b/lib/tasks/ce_to_ee_merge_check.rake
new file mode 100644
index 00000000000..424e7883060
--- /dev/null
+++ b/lib/tasks/ce_to_ee_merge_check.rake
@@ -0,0 +1,4 @@
+desc 'Checks if the branch would apply cleanly to EE'
+task ce_to_ee_merge_check: :environment do
+  Rake::Task['gitlab:dev:ce_to_ee_merge_check'].invoke
+end
diff --git a/lib/tasks/ci/.gitkeep b/lib/tasks/ci/.gitkeep
deleted file mode 100644
index e69de29bb2d..00000000000
--- a/lib/tasks/ci/.gitkeep
+++ /dev/null
diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake
deleted file mode 100644
index 3bfe999ae74..00000000000
--- a/lib/tasks/flog.rake
+++ /dev/null
@@ -1,25 +0,0 @@
-desc 'Code complexity analyze via flog'
-task :flog do
-  output = %x(bundle exec flog -m app/ lib/gitlab)
-  exit_code = 0
-  minimum_score = 70
-  output = output.lines
-
-  # Skip total complexity score
-  output.shift
-
-  # Skip some trash info
-  output.shift
-
-  output.each do |line|
-    score, method = line.split(" ")
-    score = score.to_i
-
-    if score > minimum_score
-      exit_code = 1
-      puts "High complexity in #{method}. Score: #{score}"
-    end
-  end
-
-  exit exit_code
-end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index b43ee5b3383..a9f1255e8cf 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -51,6 +51,7 @@ namespace :gitlab do
         $progress.puts 'done'.color(:green)
         Rake::Task['gitlab:backup:db:restore'].invoke
       end
+
       Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories')
       Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads')
       Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds')
@@ -58,6 +59,7 @@ namespace :gitlab do
       Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs')
       Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry')
       Rake::Task['gitlab:shell:setup'].invoke
+      Rake::Task['cache:clear'].invoke
 
       backup.cleanup
     end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 5f4a6bbfa35..2ae48a970ce 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -671,7 +671,7 @@ namespace :gitlab do
           "Enable mail_room in the init.d configuration."
         )
         for_more_information(
-          "doc/incoming_email/README.md"
+          "doc/administration/reply_by_email.md"
         )
         fix_and_rerun
       end
@@ -690,7 +690,7 @@ namespace :gitlab do
           "Enable mail_room in your Procfile."
         )
         for_more_information(
-          "doc/incoming_email/README.md"
+          "doc/administration/reply_by_email.md"
         )
         fix_and_rerun
       end
@@ -747,7 +747,7 @@ namespace :gitlab do
           "Check that the information in config/gitlab.yml is correct"
         )
         for_more_information(
-          "doc/incoming_email/README.md"
+          "doc/administration/reply_by_email.md"
         )
         fix_and_rerun
       end
diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake
new file mode 100644
index 00000000000..47bdb2d32d2
--- /dev/null
+++ b/lib/tasks/gitlab/dev.rake
@@ -0,0 +1,107 @@
+namespace :gitlab do
+  namespace :dev do
+    desc 'Checks if the branch would apply cleanly to EE'
+    task ce_to_ee_merge_check: :environment do
+      return if defined?(Gitlab::License)
+      return unless ENV['CI']
+
+      ce_repo = ENV['CI_BUILD_REPO']
+      ce_branch = ENV['CI_BUILD_REF_NAME']
+
+      ee_repo = 'https://gitlab.com/gitlab-org/gitlab-ee.git'
+      ee_branch = "#{ce_branch}-ee"
+      ee_dir = 'gitlab-ee-merge-check'
+
+      puts "\n=> Cloning #{ee_repo} into #{ee_dir}\n"
+      `git clone #{ee_repo} #{ee_dir} --depth 1`
+      Dir.chdir(ee_dir) do
+        puts "\n => Fetching #{ce_repo}/#{ce_branch}\n"
+        `git fetch #{ce_repo} #{ce_branch} --depth 1`
+
+        # Try to merge the current tested branch to EE/master...
+        puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n"
+        `git merge FETCH_HEAD`
+
+        exit 0 if $?.success?
+
+        # Check if the <branch>-ee branch exists...
+        puts "\n => Check if #{ee_repo}/#{ee_branch} exists\n"
+        `git rev-parse --verify #{ee_branch}`
+
+        # The <branch>-ee doesn't exist
+        unless $?.success?
+          puts
+          puts <<-MSG.strip_heredoc
+            =================================================================
+            The #{ce_branch} branch cannot be merged without conflicts to the
+            current EE/master, and no #{ee_branch} branch was detected in
+            the EE repository.
+
+            Please create a #{ee_branch} branch that includes changes from
+            #{ce_branch} but also specific changes than can be applied cleanly
+            to EE/master.
+
+            You can create this branch as follows:
+
+            1. In the EE repo:
+              $ git fetch origin
+              $ git fetch #{ce_repo} #{ce_branch}
+              $ git checkout -b #{ee_branch} FETCH_HEAD
+              $ git rebase origin/master
+            2. At this point you will likely have conflicts, solve them, and
+              continue/finish the rebase. Note: You can squash the CE commits
+              before rebasing.
+            3. You can squash all the original #{ce_branch} commits into a
+              single "Port of #{ce_branch} to EE".
+            4. Push your branch to #{ee_repo}:
+              $ git push origin #{ee_branch}
+            =================================================================\n
+          MSG
+
+          exit 1
+        end
+
+        # Try to merge the <branch>-ee branch to EE/master...
+        puts "\n => Merging #{ee_repo}/#{ee_branch} into #{ee_repo}/master\n"
+        `git merge #{ee_branch} master`
+
+        # The <branch>-ee cannot be merged cleanly to EE/master...
+        unless $?.success?
+          puts
+          puts <<-MSG.strip_heredoc
+            =================================================================
+            The #{ce_branch} branch cannot be merged without conflicts to
+            EE/master, and even though the #{ee_branch} branch exists in the EE
+            repository, it cannot be merged without conflicts to EE/master.
+
+            Please update the #{ee_branch}, push it again to #{ee_repo}, and
+            retry this job.
+            =================================================================\n
+          MSG
+
+          exit 2
+        end
+
+        puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n"
+        `git merge FETCH_HEAD`
+        exit 0 if $?.success?
+
+        # The <branch>-ee can be merged cleanly to EE/master, but <branch> still
+        # cannot be merged cleanly to EE/master...
+        puts
+        puts <<-MSG.strip_heredoc
+          =================================================================
+          The #{ce_branch} branch cannot be merged without conflicts to EE, and
+          even though the #{ee_branch} branch exists in the EE repository and
+          applies cleanly to EE/master, it doesn't prevent conflicts when
+          merging #{ce_branch} into EE.
+
+          We may be in a complex situation here.
+          =================================================================\n
+        MSG
+
+        exit 3
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index bb7eb852f1b..210899882b4 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -78,7 +78,7 @@ namespace :gitlab do
         f.puts "PATH=#{ENV['PATH']}"
       end
 
-      Gitlab::Shell.new.generate_and_link_secret_token
+      Gitlab::Shell.ensure_secret_token!
     end
 
     desc "GitLab | Setup gitlab-shell"
diff --git a/lib/tasks/gitlab/users.rake b/lib/tasks/gitlab/users.rake
new file mode 100644
index 00000000000..3a16ace60bd
--- /dev/null
+++ b/lib/tasks/gitlab/users.rake
@@ -0,0 +1,11 @@
+namespace :gitlab do
+  namespace :users do
+    desc "GitLab | Clear the authentication token for all users"
+    task clear_all_authentication_tokens: :environment  do |t, args|
+      # Do small batched updates because these updates will be slow and locking
+      User.select(:id).find_in_batches(batch_size: 100) do |batch|
+        User.where(id: batch.map(&:id)).update_all(authentication_token: nil)
+      end
+    end
+  end
+end