Merge branch 'master' into fix/visibility-level-setting-in-forked-projects

* master: (723 commits)
  Bump Rack Attack to v4.3.1 for security fix
  Remove duplicate entry in the changelog
  Remove extra spaces after branchname
  Fix merge-request-reopen button title
  Add branch and tag operation to tree dropdown
  Use gitlab-shell 2.6.9
  Clarify Windows shell executor artifact upload support
  Fix feature specs: we always show the build status if ci_commit is present
  Do not display project group/name when issue and MR are in same project
  Don't create CI status for refs that doesn't have .gitlab-ci.yml, even if the builds are enabled
  Use gitlab-workhorse 0.5.1
  Fix ci_projects migration by using the value only from latest row [ci skip]
  Revert sidebar position for issue and merge request
  Add info on using private Docker registries in CI [ci skip]
  Upgrade Poltergeist to 1.8.1. #4131
  Fix ux issue with "This issue will be closed automatically" message
  Move MR Builds tab next to Commits
  Api support for requesting starred projects for user
  Fix Rubocop complain.
  Fix merge widget JS for buttons
  ...

Conflicts:
	app/models/project.rb

9 files changed, 208 insertions, 26 deletions

diff --git a/lib/tasks/ci/schedule_builds.rake b/lib/tasks/ci/schedule_builds.rake
deleted file mode 100644
index 49435504c67..00000000000
--- a/lib/tasks/ci/schedule_builds.rake
+++ /dev/null
@@ -1,6 +0,0 @@
-namespace :ci do
-  desc "GitLab CI | Clean running builds"
-  task schedule_builds: :environment do
-    Ci::Scheduler.new.perform
-  end
-end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 3c46bcea40e..cb4abe13799 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -13,6 +13,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:uploads:create"].invoke
       Rake::Task["gitlab:backup:builds:create"].invoke
       Rake::Task["gitlab:backup:artifacts:create"].invoke
+      Rake::Task["gitlab:backup:lfs:create"].invoke
 
       backup = Backup::Manager.new
       backup.pack
@@ -34,6 +35,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
       Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
       Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
+      Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs")
       Rake::Task["gitlab:shell:setup"].invoke
 
       backup.cleanup
@@ -134,6 +136,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :lfs do
+      task create: :environment do
+        $progress.puts "Dumping lfs objects ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("lfs")
+          $progress.puts "[SKIPPED]".cyan
+        else
+          Backup::Lfs.new.dump
+          $progress.puts "done".green
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring lfs objects ... ".blue
+        Backup::Lfs.new.restore
+        $progress.puts "done".green
+      end
+    end
+
     def configure_cron_mode
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index a25fac62cfc..0469c5a61c3 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -331,7 +331,7 @@ namespace :gitlab do
     end
 
     def check_redis_version
-      min_redis_version = "2.4.0"
+      min_redis_version = "2.8.0"
       print "Redis version >= #{min_redis_version}? ... "
 
       redis_version = run(%W(redis-cli --version))
@@ -822,10 +822,27 @@ namespace :gitlab do
 
       namespace_dirs.each do |namespace_dir|
         repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
-        repo_dirs.each do |dir|
-          puts "\nChecking repo at #{dir}"
-          system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: dir)
-        end
+        repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
+      end
+    end
+  end
+
+  namespace :user do
+    desc "GitLab | Check the integrity of a specific user's repositories"
+    task :check_repos, [:username] => :environment do |t, args|
+      username = args[:username] || prompt("Check repository integrity for which username? ".blue)
+      user = User.find_by(username: username)
+      if user
+        repo_dirs = user.authorized_projects.map do |p|
+                      File.join(
+                        Gitlab.config.gitlab_shell.repos_path,
+                        "#{p.path_with_namespace}.git"
+                      )
+                    end
+
+        repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
+      else
+        puts "\nUser '#{username}' not found".red
       end
     end
   end
@@ -952,4 +969,35 @@ namespace :gitlab do
       false
     end
   end
+
+  def check_repo_integrity(repo_dir)
+    puts "\nChecking repo at #{repo_dir.yellow}"
+
+    git_fsck(repo_dir)
+    check_config_lock(repo_dir)
+    check_ref_locks(repo_dir)
+  end
+
+  def git_fsck(repo_dir)
+    puts "Running `git fsck`".yellow
+    system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir)
+  end
+
+  def check_config_lock(repo_dir)
+    config_exists = File.exist?(File.join(repo_dir,'config.lock'))
+    config_output = config_exists ? 'yes'.red : 'no'.green
+    puts "'config.lock' file exists?".yellow + " ... #{config_output}"
+  end
+
+  def check_ref_locks(repo_dir)
+    lock_files = Dir.glob(File.join(repo_dir,'refs/heads/*.lock'))
+    if lock_files.present?
+      puts "Ref lock files exist:".red
+      lock_files.each do |lock_file|
+        puts "  #{lock_file}"
+      end
+    else
+      puts "No ref lock files exist".green
+    end
+  end
 end
diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake
new file mode 100644
index 00000000000..65ee430d550
--- /dev/null
+++ b/lib/tasks/gitlab/git.rake
@@ -0,0 +1,55 @@
+namespace :gitlab do
+  namespace :git do
+
+    desc "GitLab | Git | Repack"
+    task repack: :environment do
+      failures = perform_git_cmd(%W(git repack -a --quiet), "Repacking repo")
+      if failures.empty?
+        puts "Done".green
+      else
+        output_failures(failures)
+      end
+    end
+
+    desc "GitLab | Git | Run garbage collection on all repos"
+    task gc: :environment do
+      failures = perform_git_cmd(%W(git gc --auto --quiet), "Garbage Collecting")
+      if failures.empty?
+        puts "Done".green
+      else
+        output_failures(failures)
+      end
+    end
+    
+    desc "GitLab | Git | Prune all repos"
+    task prune: :environment do
+      failures = perform_git_cmd(%W(git prune), "Git Prune")
+      if failures.empty?
+        puts "Done".green
+      else
+        output_failures(failures)
+      end
+    end
+
+    def perform_git_cmd(cmd, message)
+      puts "Starting #{message} on all repositories"
+
+      failures = []
+      all_repos do |repo|
+        if system(*cmd, chdir: repo)
+          puts "Performed #{message} at #{repo}"
+        else
+          failures << repo
+        end
+      end
+
+      failures
+    end
+
+    def output_failures(failures)
+      puts "The following repositories reported errors:".red
+      failures.each { |f| puts "- #{f}" }
+    end
+
+  end
+end
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index c1ee271ae2b..1c04f47f08f 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -64,6 +64,8 @@ namespace :gitlab do
 
           if project.persisted?
             puts " * Created #{project.name} (#{repo_path})".green
+            project.update_repository_size
+            project.update_commit_count
           else
             puts " * Failed trying to create #{project.name} (#{repo_path})".red
             puts "   Errors: #{project.errors.messages}".red
diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake
new file mode 100644
index 00000000000..c7596e7abcb
--- /dev/null
+++ b/lib/tasks/gitlab/list_repos.rake
@@ -0,0 +1,17 @@
+namespace :gitlab do
+  task list_repos: :environment do
+    scope = Project
+    if ENV['SINCE']
+      date = Time.parse(ENV['SINCE'])
+      warn "Listing repositories with activity or changes since #{date}"
+      project_ids = Project.where('last_activity_at > ? OR updated_at > ?', date, date).pluck(:id).sort
+      namespace_ids = Namespace.where(['updated_at > ?', date]).pluck(:id).sort
+      scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids)
+    end
+    scope.find_each do |project|
+      base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace)
+      puts base + '.git'
+      puts base + '.wiki.git'
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake
index c95b6540ebc..ebe516ec879 100644
--- a/lib/tasks/gitlab/task_helpers.rake
+++ b/lib/tasks/gitlab/task_helpers.rake
@@ -2,16 +2,6 @@ module Gitlab
   class TaskAbortedByUserError < StandardError; end
 end
 
-unless STDOUT.isatty
-  module Colored
-    extend self
-
-    def colorize(string, options={})
-      string
-    end
-  end
-end
-
 namespace :gitlab do
 
   # Ask if the user wants to continue
@@ -103,7 +93,7 @@ namespace :gitlab do
       gitlab_user = Gitlab.config.gitlab.user
       current_user = run(%W(whoami)).chomp
       unless current_user == gitlab_user
-        puts "#{Colored.color(:black)+Colored.color(:on_yellow)} Warning #{Colored.extra(:clear)}"
+        puts " Warning ".colorize(:black).on_yellow
         puts "  You are running as user #{current_user.magenta}, we hope you know what you are doing."
         puts "  Things may work\/fail for the wrong reasons."
         puts "  For correct results you should run this as user #{gitlab_user.magenta}."
@@ -128,4 +118,12 @@ namespace :gitlab do
       false
     end
   end
+
+  def all_repos
+    IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
+      find.each_line do |path|
+        yield path.chomp
+      end
+    end
+  end
 end
diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake
index 365ff2defd4..0985ef3a669 100644
--- a/lib/tasks/spec.rake
+++ b/lib/tasks/spec.rake
@@ -19,6 +19,33 @@ namespace :spec do
     run_commands(cmds)
   end
 
+  desc 'GitLab | Rspec | Run model specs'
+  task :models do
+    cmds = [
+      %W(rake gitlab:setup),
+      %W(rspec spec --tag @models)
+    ]
+    run_commands(cmds)
+  end
+
+  desc 'GitLab | Rspec | Run service specs'
+  task :services do
+    cmds = [
+      %W(rake gitlab:setup),
+      %W(rspec spec --tag @services)
+    ]
+    run_commands(cmds)
+  end
+
+  desc 'GitLab | Rspec | Run lib specs'
+  task :lib do
+    cmds = [
+      %W(rake gitlab:setup),
+      %W(rspec spec --tag @lib)
+    ]
+    run_commands(cmds)
+  end
+
   desc 'GitLab | Rspec | Run benchmark specs'
   task :benchmark do
     cmds = [
@@ -32,7 +59,7 @@ namespace :spec do
   task :other do
     cmds = [
       %W(rake gitlab:setup),
-      %W(rspec spec --tag ~@api --tag ~@feature --tag ~@benchmark)
+      %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services --tag ~@benchmark)
     ]
     run_commands(cmds)
   end
diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake
index d5a96fd38f4..3acfc6e2075 100644
--- a/lib/tasks/spinach.rake
+++ b/lib/tasks/spinach.rake
@@ -1,11 +1,31 @@
 Rake::Task["spinach"].clear if Rake::Task.task_defined?('spinach')
 
 namespace :spinach do
+  namespace :project do
+    desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features"
+    task :half do
+      cmds = [
+        %W(rake gitlab:setup),
+        %W(spinach --tags @project_commits,@project_issues,@project_merge_requests),
+      ]
+      run_commands(cmds)
+    end
+
+    desc "GitLab | Spinach | Run remaining project spinach features"
+    task :rest do
+      cmds = [
+        %W(rake gitlab:setup),
+        %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests),
+      ]
+      run_commands(cmds)
+    end
+  end
+
   desc "GitLab | Spinach | Run project spinach features"
   task :project do
     cmds = [
       %W(rake gitlab:setup),
-      %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@commits),
+      %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets),
     ]
     run_commands(cmds)
   end
@@ -14,7 +34,7 @@ namespace :spinach do
   task :other do
     cmds = [
       %W(rake gitlab:setup),
-      %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets,@commits),
+      %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets),
     ]
     run_commands(cmds)
   end
@@ -33,4 +53,4 @@ def run_commands(cmds)
   cmds.each do |cmd|
     system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!")
   end
-end
\ No newline at end of file
+end