Merge branch 'master' into flevour/gitlab-ce-fix-merge-request-textareas

21 files changed, 284 insertions, 238 deletions

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 33b6224a810..9620d36ac41 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -45,7 +45,7 @@ module API
 
     class ProjectHook < Hook
       expose :project_id, :push_events
-      expose :issues_events, :merge_requests_events, :tag_push_events
+      expose :issues_events, :merge_requests_events, :tag_push_events, :note_events, :enable_ssl_verification
     end
 
     class ForkedFromProject < Grape::Entity
diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb
index ad4d2e65dfd..882d1a083ad 100644
--- a/lib/api/project_hooks.rb
+++ b/lib/api/project_hooks.rb
@@ -44,7 +44,8 @@ module API
           :issues_events,
           :merge_requests_events,
           :tag_push_events,
-          :note_events
+          :note_events,
+          :enable_ssl_verification
         ]
         @hook = user_project.hooks.new(attrs)
 
@@ -75,7 +76,8 @@ module API
           :issues_events,
           :merge_requests_events,
           :tag_push_events,
-          :note_events
+          :note_events,
+          :enable_ssl_verification
         ]
 
         if @hook.update_attributes attrs
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 813cc379e43..a98d668e02d 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -121,6 +121,17 @@ module API
             User.where(username: attrs[:username]).
                 where.not(id: user.id).count > 0
 
+        identity_attrs = attributes_for_keys [:provider, :extern_uid]
+        if identity_attrs.any?
+          identity = user.identities.find_by(provider: identity_attrs[:provider])
+          if identity
+            identity.update_attributes(identity_attrs)
+          else
+            identity = user.identities.build(identity_attrs)
+            identity.save
+          end
+        end
+
         if user.update_attributes(attrs)
           present user, with: Entities::UserFull
         else
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index ac63f89c6ec..5c42f25f4a2 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -45,7 +45,8 @@ module Backup
       directory = connection.directories.get(remote_directory)
 
       if directory.files.create(key: tar_file, body: File.open(tar_file), public: false,
-          multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size)
+          multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
+          encryption: Gitlab.config.backup.upload.encryption)
         $progress.puts "done".green
       else
         puts "uploading backup to #{remote_directory} failed".red
@@ -55,7 +56,7 @@ module Backup
 
     def cleanup
       $progress.print "Deleting tmp directories ... "
-      
+
       backup_contents.each do |dir|
         next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
 
@@ -75,7 +76,7 @@ module Backup
 
       if keep_time > 0
         removed = 0
-        
+
         Dir.chdir(Gitlab.config.backup.path) do
           file_list = Dir.glob('*_gitlab_backup.tar')
           file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ }
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 7bb8869d61a..5109c84e0ea 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -36,7 +36,6 @@ module Ci
       mount Commits
       mount Runners
       mount Projects
-      mount Forks
       mount Triggers
     end
   end
diff --git a/lib/ci/api/forks.rb b/lib/ci/api/forks.rb
deleted file mode 100644
index 152883a599f..00000000000
--- a/lib/ci/api/forks.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-module Ci
-  module API
-    class Forks < Grape::API
-      resource :forks do
-        # Create a fork
-        #
-        # Parameters:
-        #   project_id (required) - The ID of a project
-        #   project_token (requires) - Project token
-        #   private_token(required) - User private token
-        #   data (required) - GitLab project data (name_with_namespace, web_url, default_branch, ssh_url_to_repo)
-        #
-        #
-        # Example Request:
-        #   POST /forks
-        post do
-          required_attributes! [:project_id, :data, :project_token, :private_token]
-          project = Ci::Project.find_by!(gitlab_id: params[:project_id])
-          authenticate_project_token!(project)
-
-          fork = Ci::CreateProjectService.new.execute(
-            current_user,
-            params[:data],
-            Ci::RoutesHelper.ci_project_url(":project_id"),
-            project
-          )
-
-          if fork
-            present fork, with: Entities::Project
-          else
-            not_found!
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb
index 66bcf65e8c4..d719ad9e8d5 100644
--- a/lib/ci/api/projects.rb
+++ b/lib/ci/api/projects.rb
@@ -75,23 +75,17 @@ module Ci
         # Create Gitlab CI project using Gitlab project info
         #
         # Parameters:
-        #   name (required)            - The name of the project
         #   gitlab_id (required)       - The gitlab id of the project
-        #   path (required)            - The gitlab project path, ex. randx/six
-        #   ssh_url_to_repo (required) - The gitlab ssh url to the repo
         #   default_ref                - The branch to run against (defaults to `master`)
         # Example Request:
         #   POST /projects
         post do
-          required_attributes! [:name, :gitlab_id, :ssh_url_to_repo]
+          required_attributes! [:gitlab_id]
 
           filtered_params = {
-            name:            params[:name],
             gitlab_id:       params[:gitlab_id],
             # we accept gitlab_url for backward compatibility for a while (added to 7.11)
-            path:            params[:path] || params[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1'),
-            default_ref:     params[:default_ref] || 'master',
-            ssh_url_to_repo: params[:ssh_url_to_repo]
+            default_ref:     params[:default_ref] || 'master'
           }
 
           project = Ci::Project.new(filtered_params)
@@ -109,11 +103,7 @@ module Ci
         #
         # Parameters:
         #   id (required)   - The ID of a project
-        #   name            - The name of the project
-        #   gitlab_id       - The gitlab id of the project
-        #   path            - The gitlab project path, ex. randx/six
-        #   ssh_url_to_repo - The gitlab ssh url to the repo
-        #   default_ref     - The branch to run against (defaults to `master`)
+        #   default_ref      - The branch to run against (defaults to `master`)
         # Example Request:
         #   PUT /projects/:id
         put ":id" do
@@ -121,12 +111,7 @@ module Ci
 
           unauthorized! unless can?(current_user, :admin_project, project.gl_project)
 
-          attrs = attributes_for_keys [:name, :gitlab_id, :path, :gitlab_url, :default_ref, :ssh_url_to_repo]
-
-          # we accept gitlab_url for backward compatibility for a while (added to 7.11)
-          if attrs[:gitlab_url] && !attrs[:path]
-            attrs[:path] = attrs[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1')
-          end
+          attrs = attributes_for_keys [:default_ref]
 
           if project.update_attributes(attrs)
             present project, with: Entities::Project
diff --git a/lib/ci/migrate/builds.rb b/lib/ci/migrate/builds.rb
new file mode 100644
index 00000000000..c4f62e55295
--- /dev/null
+++ b/lib/ci/migrate/builds.rb
@@ -0,0 +1,29 @@
+module Ci
+  module Migrate
+    class Builds
+      attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
+
+      def initialize
+        @app_builds_dir = Settings.gitlab_ci.builds_path
+        @backup_dir = Gitlab.config.backup.path
+        @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz')
+      end
+
+      def restore
+        backup_existing_builds_dir
+
+        FileUtils.mkdir_p(app_builds_dir, mode: 0700)
+        unless system('tar', '-C', app_builds_dir, '-zxf', backup_builds_tarball)
+          abort 'Restore failed'.red
+        end
+      end
+
+      def backup_existing_builds_dir
+        timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
+        if File.exists?(app_builds_dir)
+          FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb
index 74f592dcaea..bf9b80f1f62 100644
--- a/lib/ci/migrate/database.rb
+++ b/lib/ci/migrate/database.rb
@@ -9,32 +9,32 @@ module Ci
         @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
       end
 
-      def restore(ci_dump)
-        puts 'Deleting all CI related data ... '
-        truncate_ci_tables
+      def restore
+        decompress_rd, decompress_wr = IO.pipe
+        decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
+        decompress_wr.close
 
-        puts 'Restoring CI data ... '
-        case config["adapter"]
-          when /^mysql/ then
-            print "Restoring MySQL database #{config['database']} ... "
-            # Workaround warnings from MySQL 5.6 about passwords on cmd line
-            ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-            system('mysql', *mysql_args, config['database'], in: ci_dump)
-          when "postgresql" then
-            puts "Restoring PostgreSQL database #{config['database']} ... "
-            pg_env
-            system('psql', config['database'], '-f', ci_dump)
-        end
+        restore_pid = case config["adapter"]
+                        when /^mysql/ then
+                          $progress.print "Restoring MySQL database #{config['database']} ... "
+                          # Workaround warnings from MySQL 5.6 about passwords on cmd line
+                          ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
+                          spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
+                        when "postgresql" then
+                          $progress.print "Restoring PostgreSQL database #{config['database']} ... "
+                          pg_env
+                          spawn('psql', config['database'], in: decompress_rd)
+                      end
+        decompress_rd.close
+
+        success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+        abort 'Restore failed' unless success
       end
 
       protected
 
-      def truncate_ci_tables
-        c = ActiveRecord::Base.connection
-        c.tables.select { |t| t.start_with?('ci_') }.each do |table|
-          puts "Deleting data from #{table}..."
-          c.execute("DELETE FROM #{table}")
-        end
+      def db_file_name
+        File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
       end
 
       def mysql_args
diff --git a/lib/ci/migrate/manager.rb b/lib/ci/migrate/manager.rb
new file mode 100644
index 00000000000..e5e4fb784eb
--- /dev/null
+++ b/lib/ci/migrate/manager.rb
@@ -0,0 +1,72 @@
+module Ci
+  module Migrate
+    class Manager
+      CI_IMPORT_PREFIX = '8.0' # Only allow imports from CI 8.0.x
+
+      def cleanup
+        $progress.print "Deleting tmp directories ... "
+
+        backup_contents.each do |dir|
+          next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
+
+          if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
+            $progress.puts "done".green
+          else
+            puts "deleting tmp directory '#{dir}' failed".red
+            abort 'Backup failed'
+          end
+        end
+      end
+
+      def unpack
+        Dir.chdir(Gitlab.config.backup.path)
+
+        # check for existing backups in the backup dir
+        file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
+        puts "no backups found" if file_list.count == 0
+
+        if file_list.count > 1 && ENV["BACKUP"].nil?
+          puts "Found more than one backup, please specify which one you want to restore:"
+          puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
+          exit 1
+        end
+
+        tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
+
+        unless File.exists?(tar_file)
+          puts "The specified CI backup doesn't exist!"
+          exit 1
+        end
+
+        $progress.print "Unpacking backup ... "
+
+        unless Kernel.system(*%W(tar -xf #{tar_file}))
+          puts "unpacking backup failed".red
+          exit 1
+        else
+          $progress.puts "done".green
+        end
+
+        ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
+
+        # restoring mismatching backups can lead to unexpected problems
+        if !settings[:gitlab_version].start_with?(CI_IMPORT_PREFIX)
+          puts "GitLab CI version mismatch:".red
+          puts "  Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red
+          exit 1
+        end
+      end
+
+      private
+
+      def backup_contents
+        ["db", "builds", "backup_information.yml"]
+      end
+
+      def settings
+        @settings ||= YAML.load_file("backup_information.yml")
+      end
+    end
+  end
+end
+
diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb
index 125a535e9a9..97e043ece27 100644
--- a/lib/ci/migrate/tags.rb
+++ b/lib/ci/migrate/tags.rb
@@ -4,45 +4,38 @@ module Ci
   module Migrate
     class Tags
       def restore
-        puts 'Migrating tags for Runners... '
-        list_objects('Runner').each do |id|
-          putc '.'
-          runner = Ci::Runner.find_by_id(id)
-          if runner
-            tags = list_tags('Runner', id)
-            runner.update_attributes(tag_list: tags)
+        puts 'Inserting tags...'
+        connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag|
+          begin
+            connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})")
+          rescue ActiveRecord::RecordNotUnique
           end
         end
-        puts ''
 
-        puts 'Migrating tags for Builds... '
-        list_objects('Build').each do |id|
-          putc '.'
-          build = Ci::Build.find_by_id(id)
-          if build
-            tags = list_tags('Build', id)
-            build.update_attributes(tag_list: tags)
-          end
+        ActiveRecord::Base.transaction do
+          puts 'Deleting old taggings...'
+          connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'"
+
+          puts 'Inserting taggings...'
+          connection.execute(
+            'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' +
+              "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " +
+              'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' +
+              'JOIN tags ON tags.name = ci_tags.name '
+          )
+
+          puts 'Resetting counters... '
+          connection.execute(
+            'UPDATE tags SET ' +
+              'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)'
+          )
         end
-        puts ''
       end
 
       protected
 
-      def list_objects(type)
-        ids = ActiveRecord::Base.connection.select_all(
-          "select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}"
-        )
-        ids.map { |id| id['taggable_id'] }
-      end
-
-      def list_tags(type, id)
-        tags = ActiveRecord::Base.connection.select_all(
-          'select ci_tags.name from ci_tags ' +
-            'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' +
-            "where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = 'tags'"
-        )
-        tags.map { |tag| tag['name'] }
+      def connection
+        ActiveRecord::Base.connection
       end
     end
   end
diff --git a/lib/ci/project_list_builder.rb b/lib/ci/project_list_builder.rb
deleted file mode 100644
index da26f9a9f47..00000000000
--- a/lib/ci/project_list_builder.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-module Ci
-  class ProjectListBuilder
-    def execute(current_user, search = nil)
-      projects = current_user.authorized_projects
-      projects = projects.search(search) if search
-      
-      projects.
-        joins("LEFT JOIN ci_projects ON projects.id = ci_projects.gitlab_id
-          LEFT JOIN #{last_commit_subquery} AS last_commit ON #{Ci::Project.table_name}.id = last_commit.project_id").
-        reorder("ci_projects.id is NULL ASC,
-          CASE WHEN last_commit.committed_at IS NULL THEN 1 ELSE 0 END,
-          last_commit.committed_at DESC")
-    end
-
-    private
-
-    def last_commit_subquery
-      "(SELECT project_id, MAX(committed_at) committed_at FROM #{Ci::Commit.table_name} GROUP BY project_id)"
-    end
-  end
-end
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 355fbd27898..2b252b32887 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -65,7 +65,7 @@ module Gitlab
       def reply_key
         reply_key = nil
         message.to.each do |address|
-          reply_key = Gitlab::ReplyByEmail.reply_key_from_address(address)
+          reply_key = Gitlab::IncomingEmail.key_from_address(address)
           break if reply_key
         end
 
@@ -98,7 +98,8 @@ module Gitlab
           note:           reply,
           noteable_type:  sent_notification.noteable_type,
           noteable_id:    sent_notification.noteable_id,
-          commit_id:      sent_notification.commit_id
+          commit_id:      sent_notification.commit_id,
+          line_code:      sent_notification.line_code
         ).execute
       end
     end
diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb
index 61e08b23543..496256700b8 100644
--- a/lib/gitlab/fogbugz_import/importer.rb
+++ b/lib/gitlab/fogbugz_import/importer.rb
@@ -154,7 +154,7 @@ module Gitlab
           while comment = comments.shift
             verb = comment['sVerb']
 
-            next if verb == 'Opened' || verb === 'Closed'
+            next if verb == 'Opened'
 
             content = format_content(comment['s'])
             attachments = format_attachments(comment['rgAttachments'])
diff --git a/lib/gitlab/reply_by_email.rb b/lib/gitlab/incoming_email.rb
index c3fe6778f06..856ccc71084 100644
--- a/lib/gitlab/reply_by_email.rb
+++ b/lib/gitlab/incoming_email.rb
@@ -1,5 +1,5 @@
 module Gitlab
-  module ReplyByEmail
+  module IncomingEmail
     class << self
       def enabled?
         config.enabled && address_formatted_correctly?
@@ -7,20 +7,14 @@ module Gitlab
 
       def address_formatted_correctly?
         config.address &&
-          config.address.include?("%{reply_key}")
+          config.address.include?("%{key}")
       end
 
-      def reply_key
-        return nil unless enabled?
-
-        SecureRandom.hex(16)
-      end
-
-      def reply_address(reply_key)
-        config.address.gsub('%{reply_key}', reply_key)
+      def reply_address(key)
+        config.address.gsub('%{key}', key)
       end
 
-      def reply_key_from_address(address)
+      def key_from_address(address)
         regex = address_regex
         return unless regex
 
@@ -33,7 +27,7 @@ module Gitlab
       private
 
       def config
-        Gitlab.config.reply_by_email
+        Gitlab.config.incoming_email
       end
 
       def address_regex
@@ -41,7 +35,7 @@ module Gitlab
         return nil unless wildcard_address
 
         regex = Regexp.escape(wildcard_address)
-        regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.+)")
+        regex = regex.gsub(Regexp.escape('%{key}'), "(.+)")
         Regexp.new(regex).freeze
       end
     end
diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb
index 55deeeacd90..bf4dd9542d5 100644
--- a/lib/gitlab/ldap/auth_hash.rb
+++ b/lib/gitlab/ldap/auth_hash.rb
@@ -6,7 +6,7 @@ module Gitlab
       private
 
       def get_info(key)
-        attributes = ldap_config.attributes[key]
+        attributes = ldap_config.attributes[key.to_s]
         return super unless attributes
 
         attributes = Array(attributes)
@@ -14,6 +14,7 @@ module Gitlab
         value = nil
         attributes.each do |attribute|
           value = get_raw(attribute)
+          value = value.first if value
           break if value.present?
         end
         
diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/gitlab/markdown/relative_link_filter.rb
index 8c5cf51bfe1..6ee3d1ce039 100644
--- a/lib/gitlab/markdown/relative_link_filter.rb
+++ b/lib/gitlab/markdown/relative_link_filter.rb
@@ -59,25 +59,43 @@ module Gitlab
       end
 
       def relative_file_path(path)
-        nested_path = build_nested_path(path, context[:requested_path])
+        nested_path = build_relative_path(path, context[:requested_path])
         file_exists?(nested_path) ? nested_path : path
       end
 
-      # Covering a special case, when the link is referencing file in the same
-      # directory.
-      # If we are at doc/api/README.md and the README.md contains relative
-      # links like [Users](users.md), this takes the request
-      # path(doc/api/README.md) and replaces the README.md with users.md so the
-      # path looks like doc/api/users.md.
-      # If we are at doc/api and the README.md shown in below the tree view
-      # this takes the request path(doc/api) and adds users.md so the path
-      # looks like doc/api/users.md
-      def build_nested_path(path, request_path)
+      # Convert a relative path into its correct location based on the currently
+      # requested path
+      #
+      # path         - Relative path String
+      # request_path - Currently-requested path String
+      #
+      # Examples:
+      #
+      #   # File in the same directory as the current path
+      #   build_relative_path("users.md", "doc/api/README.md")
+      #   # => "doc/api/users.md"
+      #
+      #   # File in the same directory, which is also the current path
+      #   build_relative_path("users.md", "doc/api")
+      #   # => "doc/api/users.md"
+      #
+      #   # Going up one level to a different directory
+      #   build_relative_path("../update/7.14-to-8.0.md", "doc/api/README.md")
+      #   # => "doc/update/7.14-to-8.0.md"
+      #
+      # Returns a String
+      def build_relative_path(path, request_path)
         return request_path if path.empty?
         return path unless request_path
 
         parts = request_path.split('/')
         parts.pop if path_type(request_path) != 'tree'
+
+        while parts.length > 1 && path.start_with?('../')
+          parts.pop
+          path.sub!('../', '')
+        end
+
         parts.push(path).join('/')
       end
 
diff --git a/lib/support/nginx/gitlab_ci b/lib/support/nginx/gitlab_ci
index ce179d6f599..bf05edfd780 100644
--- a/lib/support/nginx/gitlab_ci
+++ b/lib/support/nginx/gitlab_ci
@@ -18,18 +18,6 @@ server {
     proxy_pass $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri;
   }
 
-  # expose build endpoint to allow trigger builds
-  location ~ ^/projects/\d+/build$ {
-    proxy_read_timeout    300;
-    proxy_connect_timeout 300;
-    proxy_redirect        off;
-    proxy_set_header      X-Real-IP $remote_addr;
-
-    # You need to specify your DNS servers that are able to resolve YOUR_GITLAB_SERVER_FQDN
-    resolver 8.8.8.8 8.8.4.4;
-    proxy_pass $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri;
-  }
-
   # redirect all other CI requests
   location / {
     return 301 $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri;
diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake
index e7d41874a11..1de664c85e1 100644
--- a/lib/tasks/ci/migrate.rake
+++ b/lib/tasks/ci/migrate.rake
@@ -1,40 +1,56 @@
 namespace :ci do
   desc 'GitLab | Import and migrate CI database'
   task migrate: :environment do
+    warn_user_is_not_gitlab
+    configure_cron_mode
+
     unless ENV['force'] == 'yes'
-      puts "This will truncate all CI tables and restore it from provided backup."
-      puts "You will lose any previous CI data stored in the database."
+      puts 'This will remove all CI related data and restore it from the provided backup.'
       ask_to_continue
-      puts ""
+      puts ''
     end
 
-    Rake::Task["ci:migrate:db"].invoke
-    Rake::Task["ci:migrate:autoincrements"].invoke
-    Rake::Task["ci:migrate:tags"].invoke
-    Rake::Task["ci:migrate:services"].invoke
+    # disable CI for time of migration
+    enable_ci(false)
+
+    # unpack archives
+    migrate = Ci::Migrate::Manager.new
+    migrate.unpack
+
+    Rake::Task['ci:migrate:db'].invoke
+    Rake::Task['ci:migrate:builds'].invoke
+    Rake::Task['ci:migrate:tags'].invoke
+    Rake::Task['ci:migrate:services'].invoke
+
+    # enable CI for time of migration
+    enable_ci(true)
+
+    migrate.cleanup
   end
 
   namespace :migrate do
     desc 'GitLab | Import CI database'
     task db: :environment do
-      if ENV["CI_DUMP"].nil?
-        puts "No CI SQL dump specified:"
-        puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql"
-        exit 1
-      end
-
-      ci_dump = ENV["CI_DUMP"]
-      unless File.exists?(ci_dump)
-        puts "The specified sql dump doesn't exist!"
-        exit 1
-      end
+      configure_cron_mode
+      $progress.puts 'Restoring database ... '.blue
+      Ci::Migrate::Database.new.restore
+      $progress.puts 'done'.green
+    end
 
-      ::Ci::Migrate::Database.new.restore(ci_dump)
+    desc 'GitLab | Import CI builds'
+    task builds: :environment do
+      configure_cron_mode
+      $progress.puts 'Restoring builds ... '.blue
+      Ci::Migrate::Builds.new.restore
+      $progress.puts 'done'.green
     end
 
     desc 'GitLab | Migrate CI tags'
     task tags: :environment do
+      configure_cron_mode
+      $progress.puts 'Migrating tags ... '.blue
       ::Ci::Migrate::Tags.new.restore
+      $progress.puts 'done'.green
     end
 
     desc 'GitLab | Migrate CI auto-increments'
@@ -56,8 +72,16 @@ namespace :ci do
 
     desc 'GitLab | Migrate CI services'
     task services: :environment do
+      $progress.puts 'Migrating services ... '.blue
       c = ActiveRecord::Base.connection
       c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
+      $progress.puts 'done'.green
     end
   end
+
+  def enable_ci(enabled)
+    settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults
+    settings.ci_enabled = enabled
+    settings.save!
+  end
 end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index b8eb13a4fea..66f1ecf385f 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -2,7 +2,7 @@ namespace :gitlab do
   desc "GitLab | Check the configuration of GitLab and its environment"
   task check: %w{gitlab:gitlab_shell:check
                  gitlab:sidekiq:check
-                 gitlab:reply_by_email:check
+                 gitlab:incoming_email:check
                  gitlab:ldap:check
                  gitlab:app:check}
 
@@ -634,13 +634,13 @@ namespace :gitlab do
   end
 
 
-  namespace :reply_by_email do
+  namespace :incoming_email do
     desc "GitLab | Check the configuration of Reply by email"
     task check: :environment  do
       warn_user_is_not_gitlab
       start_checking "Reply by email"
 
-      if Gitlab.config.reply_by_email.enabled
+      if Gitlab.config.incoming_email.enabled
         check_address_formatted_correctly
         check_mail_room_config_exists
         check_imap_authentication
@@ -665,12 +665,12 @@ namespace :gitlab do
     def check_address_formatted_correctly
       print "Address formatted correctly? ... "
 
-      if Gitlab::ReplyByEmail.address_formatted_correctly?
+      if Gitlab::IncomingEmail.address_formatted_correctly?
         puts "yes".green
       else
         puts "no".red
         try_fixing_it(
-          "Make sure that the address in config/gitlab.yml includes the '%{reply_key}' placeholder."
+          "Make sure that the address in config/gitlab.yml includes the '%{key}' placeholder."
         )
         fix_and_rerun
       end
@@ -679,6 +679,11 @@ namespace :gitlab do
     def check_initd_configured_correctly
       print "Init.d configured correctly? ... "
 
+      if omnibus_gitlab?
+        puts 'skipped (omnibus-gitlab has no init script)'.magenta
+        return
+      end
+
       path = "/etc/default/gitlab"
 
       if File.exist?(path) && File.read(path).include?("mail_room_enabled=true")
@@ -689,7 +694,7 @@ namespace :gitlab do
           "Enable mail_room in the init.d configuration."
         )
         for_more_information(
-          "doc/reply_by_email/README.md"
+          "doc/incoming_email/README.md"
         )
         fix_and_rerun
       end
@@ -708,7 +713,7 @@ namespace :gitlab do
           "Enable mail_room in your Procfile."
         )
         for_more_information(
-          "doc/reply_by_email/README.md"
+          "doc/incoming_email/README.md"
         )
         fix_and_rerun
       end
@@ -753,7 +758,7 @@ namespace :gitlab do
           "Check that the information in config/mail_room.yml is correct"
         )
         for_more_information(
-          "doc/reply_by_email/README.md"
+          "doc/incoming_email/README.md"
         )
         fix_and_rerun
       end
@@ -789,7 +794,7 @@ namespace :gitlab do
           "Check that the information in config/mail_room.yml is correct"
         )
         for_more_information(
-          "doc/reply_by_email/README.md"
+          "doc/incoming_email/README.md"
         )
         fix_and_rerun
       end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 6b1e3716147..9f5852ac613 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -46,43 +46,24 @@ namespace :gitlab do
     desc "GitLab | Cleanup | Clean repositories"
     task repos: :environment  do
       warn_user_is_not_gitlab
-      remove_flag = ENV['REMOVE']
-
-      git_base_path = Gitlab.config.gitlab_shell.repos_path
-      all_dirs = Dir.glob(git_base_path + '/*')
-
-      global_projects = Project.in_namespace(nil).pluck(:path)
-
-      puts git_base_path.yellow
-      puts "Looking for global repos to remove... "
-
-      # skip non git repo
-      all_dirs.select! do |dir|
-        dir =~ /.git$/
-      end
-
-      # skip existing repos
-      all_dirs.reject! do |dir|
-        repo_name = File.basename dir
-        path = repo_name.gsub(/\.git$/, "")
-        global_projects.include?(path)
-      end
 
-      all_dirs.each do |dir_path|
-        if remove_flag
-          if FileUtils.rm_rf dir_path
-            puts "Removed...#{dir_path}".red
-          else
-            puts "Cannot remove #{dir_path}".red
-          end
-        else
-          puts "Can be removed: #{dir_path}".red
+      move_suffix = "+orphaned+#{Time.now.to_i}"
+      repo_root = Gitlab.config.gitlab_shell.repos_path
+      # Look for global repos (legacy, depth 1) and normal repos (depth 2)
+      IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
+        find.each_line do |path|
+          path.chomp!
+          repo_with_namespace = path.
+            sub(repo_root, '').
+            sub(%r{^/*}, '').
+            chomp('.git').
+            chomp('.wiki')
+          next if Project.find_with_namespace(repo_with_namespace)
+          new_path = path + move_suffix
+          puts path.inspect + ' -> ' + new_path.inspect
+          File.rename(path, new_path)
         end
       end
-
-      unless remove_flag
-        puts "To cleanup this directories run this command with REMOVE=true".yellow
-      end
     end
 
     desc "GitLab | Cleanup | Block users that have been removed in LDAP"