Merge commit 'b39d0c318921bae2e3a11df9ee6828291dad9864' into object-storage-ee-to-ce-backport

10 files changed, 88 insertions, 215 deletions

diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
index b48e4dce445..35ba729c156 100644
--- a/lib/tasks/gettext.rake
+++ b/lib/tasks/gettext.rake
@@ -19,4 +19,46 @@ namespace :gettext do
     Rake::Task['gettext:pack'].invoke
     Rake::Task['gettext:po_to_json'].invoke
   end
+
+  desc 'Lint all po files in `locale/'
+  task lint: :environment do
+    require 'simple_po_parser'
+
+    FastGettext.silence_errors
+    files = Dir.glob(Rails.root.join('locale/*/gitlab.po'))
+
+    linters = files.map do |file|
+      locale = File.basename(File.dirname(file))
+
+      Gitlab::I18n::PoLinter.new(file, locale)
+    end
+
+    pot_file = Rails.root.join('locale/gitlab.pot')
+    linters.unshift(Gitlab::I18n::PoLinter.new(pot_file))
+
+    failed_linters = linters.select { |linter| linter.errors.any? }
+
+    if failed_linters.empty?
+      puts 'All PO files are valid.'
+    else
+      failed_linters.each do |linter|
+        report_errors_for_file(linter.po_path, linter.errors)
+      end
+
+      raise "Not all PO-files are valid: #{failed_linters.map(&:po_path).to_sentence}"
+    end
+  end
+
+  def report_errors_for_file(file, errors_for_file)
+    puts "Errors in `#{file}`:"
+
+    errors_for_file.each do |message_id, errors|
+      puts "  #{message_id}"
+      errors.each do |error|
+        spaces = ' ' * 4
+        error = error.lines.join("#{spaces}")
+        puts "#{spaces}#{error}"
+      end
+    end
+  end
 end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index dbb3b827b9a..654f638c454 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -33,6 +33,7 @@ namespace :gitlab do
         SystemCheck::App::RedisVersionCheck,
         SystemCheck::App::RubyVersionCheck,
         SystemCheck::App::GitVersionCheck,
+        SystemCheck::App::GitUserDefaultSSHConfigCheck,
         SystemCheck::App::ActiveUsersCheck
       ]
 
@@ -41,8 +42,6 @@ namespace :gitlab do
   end
 
   namespace :gitlab_shell do
-    include SystemCheck::Helpers
-
     desc "GitLab | Check the configuration of GitLab Shell"
     task check: :environment  do
       warn_user_is_not_gitlab
@@ -249,8 +248,6 @@ namespace :gitlab do
   end
 
   namespace :sidekiq do
-    include SystemCheck::Helpers
-
     desc "GitLab | Check the configuration of Sidekiq"
     task check: :environment  do
       warn_user_is_not_gitlab
@@ -309,143 +306,30 @@ namespace :gitlab do
   end
 
   namespace :incoming_email do
-    include SystemCheck::Helpers
-
     desc "GitLab | Check the configuration of Reply by email"
     task check: :environment  do
       warn_user_is_not_gitlab
-      start_checking "Reply by email"
 
       if Gitlab.config.incoming_email.enabled
-        check_imap_authentication
+        checks = [
+          SystemCheck::IncomingEmail::ImapAuthenticationCheck
+        ]
 
         if Rails.env.production?
-          check_initd_configured_correctly
-          check_mail_room_running
+          checks << SystemCheck::IncomingEmail::InitdConfiguredCheck
+          checks << SystemCheck::IncomingEmail::MailRoomRunningCheck
         else
-          check_foreman_configured_correctly
+          checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck
         end
-      else
-        puts 'Reply by email is disabled in config/gitlab.yml'
-      end
-
-      finished_checking "Reply by email"
-    end
-
-    # Checks
-    ########################
-
-    def check_initd_configured_correctly
-      return if omnibus_gitlab?
-
-      print "Init.d configured correctly? ... "
 
-      path = "/etc/default/gitlab"
-
-      if File.exist?(path) && File.read(path).include?("mail_room_enabled=true")
-        puts "yes".color(:green)
+        SystemCheck.run('Reply by email', checks)
       else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Enable mail_room in the init.d configuration."
-        )
-        for_more_information(
-          "doc/administration/reply_by_email.md"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_foreman_configured_correctly
-      print "Foreman configured correctly? ... "
-
-      path = Rails.root.join("Procfile")
-
-      if File.exist?(path) && File.read(path) =~ /^mail_room:/
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Enable mail_room in your Procfile."
-        )
-        for_more_information(
-          "doc/administration/reply_by_email.md"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_mail_room_running
-      return if omnibus_gitlab?
-
-      print "MailRoom running? ... "
-
-      path = "/etc/default/gitlab"
-
-      unless File.exist?(path) && File.read(path).include?("mail_room_enabled=true")
-        puts "can't check because of previous errors".color(:magenta)
-        return
-      end
-
-      if mail_room_running?
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          sudo_gitlab("RAILS_ENV=production bin/mail_room start")
-        )
-        for_more_information(
-          see_installation_guide_section("Install Init Script"),
-          "see log/mail_room.log for possible errors"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_imap_authentication
-      print "IMAP server credentials are correct? ... "
-
-      config_path = Rails.root.join('config', 'mail_room.yml').to_s
-      erb = ERB.new(File.read(config_path))
-      erb.filename = config_path
-      config_file = YAML.load(erb.result)
-
-      config = config_file[:mailboxes].first
-
-      if config
-        begin
-          imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl])
-          imap.starttls if config[:start_tls]
-          imap.login(config[:email], config[:password])
-          connected = true
-        rescue
-          connected = false
-        end
-      end
-
-      if connected
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Check that the information in config/gitlab.yml is correct"
-        )
-        for_more_information(
-          "doc/administration/reply_by_email.md"
-        )
-        fix_and_rerun
+        puts 'Reply by email is disabled in config/gitlab.yml'
       end
     end
-
-    def mail_room_running?
-      ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww))
-      ps_ux.include?("mail_room")
-    end
   end
 
   namespace :ldap do
-    include SystemCheck::Helpers
-
     task :check, [:limit] => :environment do |_, args|
       # Only show up to 100 results because LDAP directories can be very big.
       # This setting only affects the `rake gitlab:check` script.
@@ -501,8 +385,6 @@ namespace :gitlab do
   end
 
   namespace :repo do
-    include SystemCheck::Helpers
-
     desc "GitLab | Check the integrity of the repositories managed by GitLab"
     task check: :environment do
       Gitlab.config.repositories.storages.each do |name, repository_storage|
@@ -517,8 +399,6 @@ namespace :gitlab do
   end
 
   namespace :user do
-    include SystemCheck::Helpers
-
     desc "GitLab | Check the integrity of a specific user's repositories"
     task :check_repos, [:username] => :environment do |t, args|
       username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue))
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index f76bef5f4bf..8ae1b6a626a 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -111,7 +111,7 @@ namespace :gitlab do
           next unless id > max_iid
 
           project.deployments.find(id).create_ref
-          rugged.references.delete(ref)
+          project.repository.delete_refs(ref)
         end
       end
     end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index aaf00bd703a..08677a98fc1 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -15,13 +15,17 @@ namespace :gitlab do
       checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
 
       _, status = Gitlab::Popen.popen(%w[which gmake])
-      command = status.zero? ? 'gmake' : 'make'
+      command = status.zero? ? ['gmake'] : ['make']
+
+      if Rails.env.test?
+        command += %W[BUNDLE_PATH=#{Bundler.bundle_path}]
+      end
 
       Dir.chdir(args.dir) do
         create_gitaly_configuration
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
-          Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT] + [command]) }
+          Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + command) }
         end
       end
     end
diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake
index dd2d5861481..b0a24790c4a 100644
--- a/lib/tasks/gitlab/helpers.rake
+++ b/lib/tasks/gitlab/helpers.rake
@@ -4,5 +4,5 @@ require 'tasks/gitlab/task_helpers'
 StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON']
 
 namespace :gitlab do
-  include Gitlab::TaskHelpers
+  extend SystemCheck::Helpers
 end
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index 48bd9139ce8..d227a0c8bdb 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -9,71 +9,16 @@ namespace :gitlab do
     #  * The project owner will set to the first administator of the system
     #  * Existing projects will be skipped
     #
+    #
     desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
     task repos: :environment do
-      Gitlab.config.repositories.storages.each_value do |repository_storage|
-        git_base_path = repository_storage['path']
-        repos_to_import = Dir.glob(git_base_path + '/**/*.git')
-
-        repos_to_import.each do |repo_path|
-          # strip repo base path
-          repo_path[0..git_base_path.length] = ''
-
-          path = repo_path.sub(/\.git$/, '')
-          group_name, name = File.split(path)
-          group_name = nil if group_name == '.'
-
-          puts "Processing #{repo_path}".color(:yellow)
-
-          if path.end_with?('.wiki')
-            puts " * Skipping wiki repo"
-            next
-          end
-
-          project = Project.find_by_full_path(path)
-
-          if project
-            puts " * #{project.name} (#{repo_path}) exists"
-          else
-            user = User.admins.reorder("id").first
-
-            project_params = {
-              name: name,
-              path: name
-            }
-
-            # find group namespace
-            if group_name
-              group = Namespace.find_by(path: group_name)
-              # create group namespace
-              unless group
-                group = Group.new(name: group_name)
-                group.path = group_name
-                group.owner = user
-                if group.save
-                  puts " * Created Group #{group.name} (#{group.id})".color(:green)
-                else
-                  puts " * Failed trying to create group #{group.name}".color(:red)
-                end
-              end
-              # set project group
-              project_params[:namespace_id] = group.id
-            end
-
-            project = Projects::CreateService.new(user, project_params).execute
+      if Project.current_application_settings.hashed_storage_enabled
+        puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red)
 
-            if project.persisted?
-              puts " * Created #{project.name} (#{repo_path})".color(:green)
-              ProjectCacheWorker.perform_async(project.id)
-            else
-              puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
-              puts "   Errors: #{project.errors.messages}".color(:red)
-            end
-          end
-        end
+        exit 1
       end
 
-      puts "Done!".color(:green)
+      Gitlab::BareRepositoryImporter.execute
     end
   end
 end
diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake
index dd1825c8a9e..44074397c05 100644
--- a/lib/tasks/gitlab/import_export.rake
+++ b/lib/tasks/gitlab/import_export.rake
@@ -9,5 +9,16 @@ namespace :gitlab do
     task data: :environment do
       puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true)
     end
+
+    desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz'
+    task bump_test_version: :environment do
+      Dir.mktmpdir do |tmp_dir|
+        system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null")
+        File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
+        system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null")
+      end
+
+      puts "Updated to #{Gitlab::ImportExport.version}"
+    end
   end
 end
diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb
index 28b2d86eed2..8a63f486fa3 100644
--- a/lib/tasks/gitlab/task_helpers.rb
+++ b/lib/tasks/gitlab/task_helpers.rb
@@ -5,6 +5,8 @@ module Gitlab
   TaskAbortedByUserError = Class.new(StandardError)
 
   module TaskHelpers
+    extend self
+
     # Ask if the user wants to continue
     #
     # Returns "yes" the user chose to continue
@@ -102,7 +104,7 @@ module Gitlab
       Gitlab.config.gitlab.user
     end
 
-    def is_gitlab_user?
+    def gitlab_user?
       return @is_gitlab_user unless @is_gitlab_user.nil?
 
       current_user = run_command(%w(whoami)).chomp
@@ -112,7 +114,7 @@ module Gitlab
     def warn_user_is_not_gitlab
       return if @warned_user_not_gitlab
 
-      unless is_gitlab_user?
+      unless gitlab_user?
         current_user = run_command(%w(whoami)).chomp
 
         puts " Warning ".color(:black).background(:yellow)
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index a7e30423c7a..f44abc2b81b 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -21,13 +21,18 @@ namespace :gitlab do
       params = {
         import_url: template.clone_url,
         namespace_id: admin.namespace.id,
-        path: template.title,
+        path: template.name,
         skip_wiki: true
       }
 
-      puts "Creating project for #{template.name}"
+      puts "Creating project for #{template.title}"
       project = Projects::CreateService.new(admin, params).execute
 
+      unless project.persisted?
+        puts project.errors.messages
+        exit(1)
+      end
+
       loop do
         if project.finished?
           puts "Import finished for #{template.name}"
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index 96b8f59242c..4d485108cf6 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -72,23 +72,7 @@ class GithubImport
     return @current_user.namespace if names == @current_user.namespace_path
     return @current_user.namespace unless @current_user.can_create_group?
 
-    full_path_namespace = Namespace.find_by_full_path(names)
-
-    return full_path_namespace if full_path_namespace
-
-    names.split('/').inject(nil) do |parent, name|
-      begin
-        namespace = Group.create!(name: name,
-                                  path: name,
-                                  owner: @current_user,
-                                  parent: parent)
-        namespace.add_owner(@current_user)
-
-        namespace
-      rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
-        Namespace.where(parent: parent).find_by_path_or_name(name)
-      end
-    end
+    Groups::NestedCreateService.new(@current_user, group_path: names).execute
   end
 
   def full_path_namespace(names)
@@ -96,7 +80,7 @@ class GithubImport
   end
 
   def visibility_level
-    @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
+    @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility
   end
 end