Merge branch 'master' into 13931-custom-emoji-implementation13931-custom-emoji-implementation

 Conflicts:
	app/assets/javascripts/dispatcher.js
	app/assets/javascripts/pages/projects/shared/project_avatar.js

16 files changed, 120 insertions, 106 deletions

diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake
index e65609d7001..4beb94eeb8e 100644
--- a/lib/tasks/dev.rake
+++ b/lib/tasks/dev.rake
@@ -7,4 +7,9 @@ namespace :dev do
     Rake::Task["gitlab:setup"].invoke
     Rake::Task["gitlab:shell:setup"].invoke
   end
+
+  desc "GitLab | Eager load application"
+  task load: :environment do
+    Rails.application.eager_load!
+  end
 end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 9dcf44fdc3e..2383bcf954b 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -46,6 +46,7 @@ namespace :gitlab do
             puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow)
             sleep(5)
           end
+
           # Drop all tables Load the schema to ensure we don't have any newer tables
           # hanging out from a failed upgrade
           $progress.puts 'Cleaning the database ... '.color(:blue)
@@ -222,6 +223,7 @@ namespace :gitlab do
 
       task restore: :environment do
         $progress.puts "Restoring container registry images ... ".color(:blue)
+
         if Gitlab.config.registry.enabled
           Backup::Registry.new.restore
           $progress.puts "done".color(:green)
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index dfade1f3885..a584eb97cf5 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -180,6 +180,7 @@ namespace :gitlab do
         puts "can't check, you have no projects".color(:magenta)
         return
       end
+
       puts ""
 
       Project.find_each(batch_size: 100) do |project|
@@ -210,6 +211,7 @@ namespace :gitlab do
       gitlab_shell_repo_base = gitlab_shell_path
       check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base)
       puts "Running #{check_cmd}"
+
       if system(check_cmd, chdir: gitlab_shell_repo_base)
         puts 'gitlab-shell self-check successful'.color(:green)
       else
@@ -285,6 +287,7 @@ namespace :gitlab do
       return if process_count.zero?
 
       print 'Number of Sidekiq processes ... '
+
       if process_count == 1
         puts '1'.color(:green)
       else
@@ -387,14 +390,8 @@ namespace :gitlab do
   namespace :repo do
     desc "GitLab | Check the integrity of the repositories managed by GitLab"
     task check: :environment do
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*'))
-
-        namespace_dirs.each do |namespace_dir|
-          repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
-          repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
-        end
-      end
+      puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red)
+      Rake::Task["gitlab:git:fsck"].execute
     end
   end
 
@@ -430,7 +427,7 @@ namespace :gitlab do
   namespace :user do
     desc "GitLab | Check the integrity of a specific user's repositories"
     task :check_repos, [:username] => :environment do |t, args|
-      username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue))
+      username = args[:username] || prompt("Check repository integrity for username? ".color(:blue))
       user = User.find_by(username: username)
       if user
         repo_dirs = user.authorized_projects.map do |p|
@@ -461,35 +458,4 @@ namespace :gitlab do
       puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red)
     end
   end
-
-  def check_repo_integrity(repo_dir)
-    puts "\nChecking repo at #{repo_dir.color(:yellow)}"
-
-    git_fsck(repo_dir)
-    check_config_lock(repo_dir)
-    check_ref_locks(repo_dir)
-  end
-
-  def git_fsck(repo_dir)
-    puts "Running `git fsck`".color(:yellow)
-    system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir)
-  end
-
-  def check_config_lock(repo_dir)
-    config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
-    config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
-    puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
-  end
-
-  def check_ref_locks(repo_dir)
-    lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
-    if lock_files.present?
-      puts "Ref lock files exist:".color(:red)
-      lock_files.each do |lock_file|
-        puts "  #{lock_file}"
-      end
-    else
-      puts "No ref lock files exist".color(:green)
-    end
-  end
 end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index eb0f757aea7..04d56509ac6 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -84,6 +84,7 @@ namespace :gitlab do
         next unless user.ldap_user?
 
         print "#{user.name} (#{user.ldap_identity.extern_uid}) ..."
+
         if Gitlab::LDAP::Access.allowed?(user)
           puts " [OK]".color(:green)
         else
diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake
index ba221e44e5d..77c28615856 100644
--- a/lib/tasks/gitlab/dev.rake
+++ b/lib/tasks/gitlab/dev.rake
@@ -14,6 +14,7 @@ namespace :gitlab do
             puts "Must specify a branch as an argument".color(:red)
             exit 1
           end
+
           args
         end
 
diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake
index cf82134d97e..3f5dd2ae3b3 100644
--- a/lib/tasks/gitlab/git.rake
+++ b/lib/tasks/gitlab/git.rake
@@ -30,6 +30,20 @@ namespace :gitlab do
       end
     end
 
+    desc 'GitLab | Git | Check all repos integrity'
+    task fsck: :environment do
+      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo|
+        check_config_lock(repo)
+        check_ref_locks(repo)
+      end
+
+      if failures.empty?
+        puts "Done".color(:green)
+      else
+        output_failures(failures)
+      end
+    end
+
     def perform_git_cmd(cmd, message)
       puts "Starting #{message} on all repositories"
 
@@ -40,6 +54,8 @@ namespace :gitlab do
         else
           failures << repo
         end
+
+        yield(repo) if block_given?
       end
 
       failures
@@ -49,5 +65,24 @@ namespace :gitlab do
       puts "The following repositories reported errors:".color(:red)
       failures.each { |f| puts "- #{f}" }
     end
+
+    def check_config_lock(repo_dir)
+      config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
+      config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
+
+      puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
+    end
+
+    def check_ref_locks(repo_dir)
+      lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
+
+      if lock_files.present?
+        puts "Ref lock files exist:".color(:red)
+
+        lock_files.each { |lock_file| puts "  #{lock_file}" }
+      else
+        puts "No ref lock files exist".color(:green)
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index 4d880c05f99..a2e68c0471b 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -5,9 +5,11 @@ namespace :gitlab do
       require 'toml'
 
       warn_user_is_not_gitlab
+
       unless args.dir.present?
         abort %(Please specify the directory where you want to install gitaly:\n  rake "gitlab:gitaly:install[/home/git/gitaly]")
       end
+
       args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git')
 
       version = Gitlab::GitalyClient.expected_server_version
@@ -21,8 +23,8 @@ namespace :gitlab do
 
       command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test?
 
+      Gitlab::SetupHelper.create_gitaly_configuration(args.dir)
       Dir.chdir(args.dir) do
-        create_gitaly_configuration
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
           Bundler.with_original_env { run_command!(command) }
@@ -39,60 +41,7 @@ namespace :gitlab do
 
       # Exclude gitaly-ruby configuration because that depends on the gitaly
       # installation directory.
-      puts gitaly_configuration_toml(gitaly_ruby: false)
-    end
-
-    private
-
-    # We cannot create config.toml files for all possible Gitaly configuations.
-    # For instance, if Gitaly is running on another machine then it makes no
-    # sense to write a config.toml file on the current machine. This method will
-    # only generate a configuration for the most common and simplest case: when
-    # we have exactly one Gitaly process and we are sure it is running locally
-    # because it uses a Unix socket.
-    # For development and testing purposes, an extra storage is added to gitaly,
-    # which is not known to Rails, but must be explicitly stubbed.
-    def gitaly_configuration_toml(gitaly_ruby: true)
-      storages = []
-      address = nil
-
-      Gitlab.config.repositories.storages.each do |key, val|
-        if address
-          if address != val['gitaly_address']
-            raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address."
-          end
-        elsif URI(val['gitaly_address']).scheme != 'unix'
-          raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses."
-        else
-          address = val['gitaly_address']
-        end
-
-        storages << { name: key, path: val['path'] }
-      end
-
-      if Rails.env.test?
-        storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
-      end
-
-      config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
-      config[:auth] = { token: 'secret' } if Rails.env.test?
-      config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
-      config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
-      config[:bin_dir] = Gitlab.config.gitaly.client_path
-
-      TOML.dump(config)
-    end
-
-    def create_gitaly_configuration
-      File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f|
-        f.puts gitaly_configuration_toml
-      end
-    rescue Errno::EEXIST
-      puts "Skipping config.toml generation:"
-      puts "A configuration file already exists."
-    rescue ArgumentError => e
-      puts "Skipping config.toml generation:"
-      puts e.message
+      puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false)
     end
   end
 end
diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake
index b732db9db6e..d7f28691098 100644
--- a/lib/tasks/gitlab/list_repos.rake
+++ b/lib/tasks/gitlab/list_repos.rake
@@ -8,6 +8,7 @@ namespace :gitlab do
       namespace_ids = Namespace.where(['updated_at > ?', date]).pluck(:id).sort
       scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids)
     end
+
     scope.find_each do |project|
       base = File.join(project.repository_storage_path, project.disk_path)
       puts base + '.git'
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 0e6aed32c52..12ae4199b69 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -54,16 +54,6 @@ namespace :gitlab do
       # (Re)create hooks
       Rake::Task['gitlab:shell:create_hooks'].invoke
 
-      # Required for debian packaging with PKGR: Setup .ssh/environment with
-      # the current PATH, so that the correct ruby version gets loaded
-      # Requires to set "PermitUserEnvironment yes" in sshd config (should not
-      # be an issue since it is more than likely that there are no "normal"
-      # user accounts on a gitlab server). The alternative is for the admin to
-      # install a ruby (1.9.3+) in the global path.
-      File.open(File.join(user_home, ".ssh", "environment"), "w+") do |f|
-        f.puts "PATH=#{ENV['PATH']}"
-      end
-
       Gitlab::Shell.ensure_secret_token!
     end
 
diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb
index 6723662703c..c1182af1014 100644
--- a/lib/tasks/gitlab/task_helpers.rb
+++ b/lib/tasks/gitlab/task_helpers.rb
@@ -130,7 +130,7 @@ module Gitlab
 
     def all_repos
       Gitlab.config.repositories.storages.each_value do |repository_storage|
-        IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
+        IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find|
           find.each_line do |path|
             yield path.chomp
           end
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index f44abc2b81b..a25f7ce59c7 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -10,6 +10,7 @@ namespace :gitlab do
       puts "This rake task is not meant fo production instances".red
       exit(1)
     end
+
     admin = User.find_by(admin: true)
 
     unless admin
diff --git a/lib/tasks/gitlab/uploads.rake b/lib/tasks/gitlab/uploads.rake
new file mode 100644
index 00000000000..df31567ce64
--- /dev/null
+++ b/lib/tasks/gitlab/uploads.rake
@@ -0,0 +1,44 @@
+namespace :gitlab do
+  namespace :uploads do
+    desc 'GitLab | Uploads | Check integrity of uploaded files'
+    task check: :environment do
+      puts 'Checking integrity of uploaded files'
+
+      uploads_batches do |batch|
+        batch.each do |upload|
+          puts "- Checking file (#{upload.id}): #{upload.absolute_path}".color(:green)
+
+          if upload.exist?
+            check_checksum(upload)
+          else
+            puts "  * File does not exist on the file system".color(:red)
+          end
+        end
+      end
+
+      puts 'Done!'
+    end
+
+    def batch_size
+      ENV.fetch('BATCH', 200).to_i
+    end
+
+    def calculate_checksum(absolute_path)
+      Digest::SHA256.file(absolute_path).hexdigest
+    end
+
+    def check_checksum(upload)
+      checksum = calculate_checksum(upload.absolute_path)
+
+      if checksum != upload.checksum
+        puts "  * File checksum (#{checksum}) does not match the one in the database (#{upload.checksum})".color(:red)
+      end
+    end
+
+    def uploads_batches(&block)
+      Upload.all.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
+        yield relation
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake
index e7ac0b5859f..308ffb0e284 100644
--- a/lib/tasks/gitlab/workhorse.rake
+++ b/lib/tasks/gitlab/workhorse.rake
@@ -3,9 +3,11 @@ namespace :gitlab do
     desc "GitLab | Install or upgrade gitlab-workhorse"
     task :install, [:dir, :repo] => :environment do |t, args|
       warn_user_is_not_gitlab
+
       unless args.dir.present?
         abort %(Please specify the directory where you want to install gitlab-workhorse:\n  rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]")
       end
+
       args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git')
 
       version = Gitlab::Workhorse.version
diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake
index 7b63e93db0e..3ab406eff2c 100644
--- a/lib/tasks/lint.rake
+++ b/lib/tasks/lint.rake
@@ -1,5 +1,17 @@
 unless Rails.env.production?
   namespace :lint do
+    task :static_verification_env do
+      ENV['STATIC_VERIFICATION'] = 'true'
+    end
+
+    desc "GitLab | lint | Static verification"
+    task static_verification: %w[
+      lint:static_verification_env
+      dev:load
+    ] do
+      Gitlab::Utils::Override.verify!
+    end
+
     desc "GitLab | lint | Lint JavaScript files using ESLint"
     task :javascript do
       Rake::Task['eslint'].invoke
diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake
index fc2cea8c016..aa2d01730d7 100644
--- a/lib/tasks/migrate/migrate_iids.rake
+++ b/lib/tasks/migrate/migrate_iids.rake
@@ -4,6 +4,7 @@ task migrate_iids: :environment do
   Issue.where(iid: nil).find_each(batch_size: 100) do |issue|
     begin
       issue.set_iid
+
       if issue.update_attribute(:iid, issue.iid)
         print '.'
       else
@@ -19,6 +20,7 @@ task migrate_iids: :environment do
   MergeRequest.where(iid: nil).find_each(batch_size: 100) do |mr|
     begin
       mr.set_iid
+
       if mr.update_attribute(:iid, mr.iid)
         print '.'
       else
@@ -34,6 +36,7 @@ task migrate_iids: :environment do
   Milestone.where(iid: nil).find_each(batch_size: 100) do |m|
     begin
       m.set_iid
+
       if m.update_attribute(:iid, m.iid)
         print '.'
       else
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index c9e3eed82f2..c996537cfbe 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -7,6 +7,7 @@ require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like')
 require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes')
 require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like')
 require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb')
+require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb')
 
 desc 'GitLab | Sets up PostgreSQL'
 task setup_postgresql: :environment do
@@ -17,4 +18,5 @@ task setup_postgresql: :environment do
   AddLowerPathIndexToRedirectRoutes.new.up
   IndexRedirectRoutesPathForLike.new.up
   AddIndexOnNamespacesLowerName.new.up
+  ReworkRedirectRoutesIndexes.new.up
 end