Merge branch 'master' into sync-all-repos

14 files changed, 158 insertions, 176 deletions

diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake
deleted file mode 100644
index 1de664c85e1..00000000000
--- a/lib/tasks/ci/migrate.rake
+++ /dev/null
@@ -1,87 +0,0 @@
-namespace :ci do
-  desc 'GitLab | Import and migrate CI database'
-  task migrate: :environment do
-    warn_user_is_not_gitlab
-    configure_cron_mode
-
-    unless ENV['force'] == 'yes'
-      puts 'This will remove all CI related data and restore it from the provided backup.'
-      ask_to_continue
-      puts ''
-    end
-
-    # disable CI for time of migration
-    enable_ci(false)
-
-    # unpack archives
-    migrate = Ci::Migrate::Manager.new
-    migrate.unpack
-
-    Rake::Task['ci:migrate:db'].invoke
-    Rake::Task['ci:migrate:builds'].invoke
-    Rake::Task['ci:migrate:tags'].invoke
-    Rake::Task['ci:migrate:services'].invoke
-
-    # enable CI for time of migration
-    enable_ci(true)
-
-    migrate.cleanup
-  end
-
-  namespace :migrate do
-    desc 'GitLab | Import CI database'
-    task db: :environment do
-      configure_cron_mode
-      $progress.puts 'Restoring database ... '.blue
-      Ci::Migrate::Database.new.restore
-      $progress.puts 'done'.green
-    end
-
-    desc 'GitLab | Import CI builds'
-    task builds: :environment do
-      configure_cron_mode
-      $progress.puts 'Restoring builds ... '.blue
-      Ci::Migrate::Builds.new.restore
-      $progress.puts 'done'.green
-    end
-
-    desc 'GitLab | Migrate CI tags'
-    task tags: :environment do
-      configure_cron_mode
-      $progress.puts 'Migrating tags ... '.blue
-      ::Ci::Migrate::Tags.new.restore
-      $progress.puts 'done'.green
-    end
-
-    desc 'GitLab | Migrate CI auto-increments'
-    task autoincrements: :environment do
-      c = ActiveRecord::Base.connection
-      c.tables.select { |t| t.start_with?('ci_') }.each do |table|
-        result = c.select_one("SELECT id FROM #{table} ORDER BY id DESC LIMIT 1")
-        if result
-          ai_val = result['id'].to_i + 1
-          puts "Resetting auto increment ID for #{table} to #{ai_val}"
-          if c.adapter_name == 'PostgreSQL'
-            c.execute("ALTER SEQUENCE #{table}_id_seq RESTART WITH #{ai_val}")
-          else
-            c.execute("ALTER TABLE #{table} AUTO_INCREMENT = #{ai_val}")
-          end
-        end
-      end
-    end
-
-    desc 'GitLab | Migrate CI services'
-    task services: :environment do
-      $progress.puts 'Migrating services ... '.blue
-      c = ActiveRecord::Base.connection
-      c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
-      $progress.puts 'done'.green
-    end
-  end
-
-  def enable_ci(enabled)
-    settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults
-    settings.ci_enabled = enabled
-    settings.save!
-  end
-end
diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake
new file mode 100644
index 00000000000..e9587595fef
--- /dev/null
+++ b/lib/tasks/flay.rake
@@ -0,0 +1,9 @@
+desc 'Code duplication analyze via flay'
+task :flay do
+  output = %x(bundle exec flay --mass 35 app/ lib/gitlab/)
+
+  if output.include? "Similar code found"
+    puts output
+    exit 1
+  end
+end
diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake
new file mode 100644
index 00000000000..3bfe999ae74
--- /dev/null
+++ b/lib/tasks/flog.rake
@@ -0,0 +1,25 @@
+desc 'Code complexity analyze via flog'
+task :flog do
+  output = %x(bundle exec flog -m app/ lib/gitlab)
+  exit_code = 0
+  minimum_score = 70
+  output = output.lines
+
+  # Skip total complexity score
+  output.shift
+
+  # Skip some trash info
+  output.shift
+
+  output.each do |line|
+    score, method = line.split(" ")
+    score = score.to_i
+
+    if score > minimum_score
+      exit_code = 1
+      puts "High complexity in #{method}. Score: #{score}"
+    end
+  end
+
+  exit exit_code
+end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index f20c7f71ba5..cb4abe13799 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -12,6 +12,8 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:repo:create"].invoke
       Rake::Task["gitlab:backup:uploads:create"].invoke
       Rake::Task["gitlab:backup:builds:create"].invoke
+      Rake::Task["gitlab:backup:artifacts:create"].invoke
+      Rake::Task["gitlab:backup:lfs:create"].invoke
 
       backup = Backup::Manager.new
       backup.pack
@@ -32,6 +34,8 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories")
       Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads")
       Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds")
+      Rake::Task["gitlab:backup:artifacts:restore"].invoke unless backup.skipped?("artifacts")
+      Rake::Task["gitlab:backup:lfs:restore"].invoke unless backup.skipped?("lfs")
       Rake::Task["gitlab:shell:setup"].invoke
 
       backup.cleanup
@@ -113,6 +117,44 @@ namespace :gitlab do
       end
     end
 
+    namespace :artifacts do
+      task create: :environment do
+        $progress.puts "Dumping artifacts ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
+          $progress.puts "[SKIPPED]".cyan
+        else
+          Backup::Artifacts.new.dump
+          $progress.puts "done".green
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring artifacts ... ".blue
+        Backup::Artifacts.new.restore
+        $progress.puts "done".green
+      end
+    end
+
+    namespace :lfs do
+      task create: :environment do
+        $progress.puts "Dumping lfs objects ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("lfs")
+          $progress.puts "[SKIPPED]".cyan
+        else
+          Backup::Lfs.new.dump
+          $progress.puts "done".green
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring lfs objects ... ".blue
+        Backup::Lfs.new.restore
+        $progress.puts "done".green
+      end
+    end
+
     def configure_cron_mode
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 66f1ecf385f..a25fac62cfc 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -335,7 +335,7 @@ namespace :gitlab do
       print "Redis version >= #{min_redis_version}? ... "
 
       redis_version = run(%W(redis-cli --version))
-      redis_version = redis_version.try(:match, /redis-cli (.*)/)
+      redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/)
       if redis_version &&
           (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version))
         puts "yes".green
@@ -642,7 +642,6 @@ namespace :gitlab do
 
       if Gitlab.config.incoming_email.enabled
         check_address_formatted_correctly
-        check_mail_room_config_exists
         check_imap_authentication
 
         if Rails.env.production?
@@ -744,42 +743,16 @@ namespace :gitlab do
       end
     end
 
-    def check_mail_room_config_exists
-      print "MailRoom config exists? ... "
-
-      mail_room_config_file = Rails.root.join("config", "mail_room.yml")
-
-      if File.exists?(mail_room_config_file)
-        puts "yes".green
-      else
-        puts "no".red
-        try_fixing_it(
-          "Copy config/mail_room.yml.example to config/mail_room.yml",
-          "Check that the information in config/mail_room.yml is correct"
-        )
-        for_more_information(
-          "doc/incoming_email/README.md"
-        )
-        fix_and_rerun
-      end
-    end
-
     def check_imap_authentication
       print "IMAP server credentials are correct? ... "
 
-      mail_room_config_file = Rails.root.join("config", "mail_room.yml")
-
-      unless File.exists?(mail_room_config_file)
-        puts "can't check because of previous errors".magenta
-        return
-      end
-
-      config = YAML.load_file(mail_room_config_file)[:mailboxes].first rescue nil
+      config = Gitlab.config.incoming_email
 
       if config
         begin
-          imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl])
-          imap.login(config[:email], config[:password])
+          imap = Net::IMAP.new(config.host, port: config.port, ssl: config.ssl)
+          imap.starttls if config.start_tls
+          imap.login(config.user, config.password)
           connected = true
         rescue
           connected = false
@@ -791,7 +764,7 @@ namespace :gitlab do
       else
         puts "no".red
         try_fixing_it(
-          "Check that the information in config/mail_room.yml is correct"
+          "Check that the information in config/gitlab.yml is correct"
         )
         for_more_information(
           "doc/incoming_email/README.md"
@@ -851,7 +824,7 @@ namespace :gitlab do
         repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
         repo_dirs.each do |dir|
           puts "\nChecking repo at #{dir}"
-          system(*%w(git fsck), chdir: dir)
+          system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: dir)
         end
       end
     end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 6b1e3716147..9f5852ac613 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -46,43 +46,24 @@ namespace :gitlab do
     desc "GitLab | Cleanup | Clean repositories"
     task repos: :environment  do
       warn_user_is_not_gitlab
-      remove_flag = ENV['REMOVE']
-
-      git_base_path = Gitlab.config.gitlab_shell.repos_path
-      all_dirs = Dir.glob(git_base_path + '/*')
-
-      global_projects = Project.in_namespace(nil).pluck(:path)
-
-      puts git_base_path.yellow
-      puts "Looking for global repos to remove... "
-
-      # skip non git repo
-      all_dirs.select! do |dir|
-        dir =~ /.git$/
-      end
-
-      # skip existing repos
-      all_dirs.reject! do |dir|
-        repo_name = File.basename dir
-        path = repo_name.gsub(/\.git$/, "")
-        global_projects.include?(path)
-      end
 
-      all_dirs.each do |dir_path|
-        if remove_flag
-          if FileUtils.rm_rf dir_path
-            puts "Removed...#{dir_path}".red
-          else
-            puts "Cannot remove #{dir_path}".red
-          end
-        else
-          puts "Can be removed: #{dir_path}".red
+      move_suffix = "+orphaned+#{Time.now.to_i}"
+      repo_root = Gitlab.config.gitlab_shell.repos_path
+      # Look for global repos (legacy, depth 1) and normal repos (depth 2)
+      IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
+        find.each_line do |path|
+          path.chomp!
+          repo_with_namespace = path.
+            sub(repo_root, '').
+            sub(%r{^/*}, '').
+            chomp('.git').
+            chomp('.wiki')
+          next if Project.find_with_namespace(repo_with_namespace)
+          new_path = path + move_suffix
+          puts path.inspect + ' -> ' + new_path.inspect
+          File.rename(path, new_path)
         end
       end
-
-      unless remove_flag
-        puts "To cleanup this directories run this command with REMOVE=true".yellow
-      end
     end
 
     desc "GitLab | Cleanup | Block users that have been removed in LDAP"
diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake
index 0ac4b0fa8a3..4cbccf2ca89 100644
--- a/lib/tasks/gitlab/setup.rake
+++ b/lib/tasks/gitlab/setup.rake
@@ -16,6 +16,7 @@ namespace :gitlab do
 
     Rake::Task["db:setup"].invoke
     Rake::Task["add_limits_mysql"].invoke
+    Rake::Task["setup_postgresql"].invoke
     Rake::Task["db:seed_fu"].invoke
   rescue Gitlab::TaskAbortedByUserError
     puts "Quitting...".red
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 3c0cc763d17..dd61632e557 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -17,7 +17,7 @@ namespace :gitlab do
 
       # Clone if needed
       unless File.directory?(target_dir)
-        system(*%W(git clone -- #{args.repo} #{target_dir}))
+        system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir}))
       end
 
       # Make sure we're on the right tag
@@ -27,7 +27,7 @@ namespace :gitlab do
         reseted = reset_to_commit(args)
 
         unless reseted
-          system(*%W(git fetch origin))
+          system(*%W(#{Gitlab.config.git.bin_path} fetch origin))
           reset_to_commit(args)
         end
 
@@ -128,14 +128,14 @@ namespace :gitlab do
   end
 
   def reset_to_commit(args)
-    tag, status = Gitlab::Popen.popen(%W(git describe -- #{args.tag}))
+    tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag}))
 
     unless status.zero?
-      tag, status = Gitlab::Popen.popen(%W(git describe -- origin/#{args.tag}))
+      tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag}))
     end
 
     tag = tag.strip
-    system(*%W(git reset --hard #{tag}))
+    system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag}))
   end
 end
 
diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake
index c95b6540ebc..efb863a8764 100644
--- a/lib/tasks/gitlab/task_helpers.rake
+++ b/lib/tasks/gitlab/task_helpers.rake
@@ -2,16 +2,6 @@ module Gitlab
   class TaskAbortedByUserError < StandardError; end
 end
 
-unless STDOUT.isatty
-  module Colored
-    extend self
-
-    def colorize(string, options={})
-      string
-    end
-  end
-end
-
 namespace :gitlab do
 
   # Ask if the user wants to continue
@@ -103,7 +93,7 @@ namespace :gitlab do
       gitlab_user = Gitlab.config.gitlab.user
       current_user = run(%W(whoami)).chomp
       unless current_user == gitlab_user
-        puts "#{Colored.color(:black)+Colored.color(:on_yellow)} Warning #{Colored.extra(:clear)}"
+        puts " Warning ".colorize(:black).on_yellow
         puts "  You are running as user #{current_user.magenta}, we hope you know what you are doing."
         puts "  Things may work\/fail for the wrong reasons."
         puts "  For correct results you should run this as user #{gitlab_user.magenta}."
diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake
new file mode 100644
index 00000000000..9196677a017
--- /dev/null
+++ b/lib/tasks/gitlab/two_factor.rake
@@ -0,0 +1,23 @@
+namespace :gitlab do
+  namespace :two_factor do
+    desc "GitLab | Disable Two-factor authentication (2FA) for all users"
+    task disable_for_all_users: :environment do
+      scope = User.with_two_factor
+      count = scope.count
+
+      if count > 0
+        puts "This will disable 2FA for #{count.to_s.red} users..."
+
+        begin
+          ask_to_continue
+          scope.find_each(&:disable_two_factor!)
+          puts "Successfully disabled 2FA for #{count} users.".green
+        rescue Gitlab::TaskAbortedByUserError
+          puts "Quitting...".red
+        end
+      else
+        puts "There are currently no users with 2FA enabled.".yellow
+      end
+    end
+  end
+end
diff --git a/lib/tasks/grape.rake b/lib/tasks/grape.rake
new file mode 100644
index 00000000000..9980e0b7984
--- /dev/null
+++ b/lib/tasks/grape.rake
@@ -0,0 +1,8 @@
+namespace :grape do
+  desc 'Print compiled grape routes'
+  task routes: :environment do
+    API::API.routes.each do |route|
+      puts route
+    end
+  end
+end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
new file mode 100644
index 00000000000..141a0b74ec0
--- /dev/null
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -0,0 +1,8 @@
+require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes')
+require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes')
+
+desc 'GitLab | Sets up PostgreSQL'
+task setup_postgresql: :environment do
+  NamespacesProjectsPathLowerIndexes.new.up
+  AddUsersLowerUsernameEmailIndexes.new.up
+end
diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake
index 831746815d7..365ff2defd4 100644
--- a/lib/tasks/spec.rake
+++ b/lib/tasks/spec.rake
@@ -19,11 +19,20 @@ namespace :spec do
     run_commands(cmds)
   end
 
+  desc 'GitLab | Rspec | Run benchmark specs'
+  task :benchmark do
+    cmds = [
+      %W(rake gitlab:setup),
+      %W(rspec spec --tag @benchmark)
+    ]
+    run_commands(cmds)
+  end
+
   desc 'GitLab | Rspec | Run other specs'
   task :other do
     cmds = [
       %W(rake gitlab:setup),
-      %W(rspec spec --tag ~@api --tag ~@feature)
+      %W(rspec spec --tag ~@api --tag ~@feature --tag ~@benchmark)
     ]
     run_commands(cmds)
   end
@@ -33,7 +42,7 @@ desc "GitLab | Run specs"
 task :spec do
   cmds = [
     %W(rake gitlab:setup),
-    %W(rspec spec),
+    %W(rspec spec --tag ~@benchmark),
   ]
   run_commands(cmds)
 end
diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake
index c8881be0954..d5a96fd38f4 100644
--- a/lib/tasks/spinach.rake
+++ b/lib/tasks/spinach.rake
@@ -5,7 +5,7 @@ namespace :spinach do
   task :project do
     cmds = [
       %W(rake gitlab:setup),
-      %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets),
+      %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@commits),
     ]
     run_commands(cmds)
   end
@@ -14,7 +14,7 @@ namespace :spinach do
   task :other do
     cmds = [
       %W(rake gitlab:setup),
-      %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets),
+      %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets,@commits),
     ]
     run_commands(cmds)
   end
@@ -33,4 +33,4 @@ def run_commands(cmds)
   cmds.each do |cmd|
     system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!")
   end
-end
+end
\ No newline at end of file