Merge remote-tracking branch 'upstream/master' into 15041-Add-Custom-CI-Config-Path15041-Add-Custom-CI-Config-Path

* upstream/master: (12506 commits)
  Update CHANGELOG.md for 9.3.2
  Update architecture.md
  Fix changelog entry file extension
  Fix head pipeline stored in merge request for external pipelines
  updated gitlab-ci.yml to compile locale
  Ignore JSON files generated from PO files
  Update mmap2 gem tha disables mmap_obj.gsub! as current implementation uses method that is no longer part of Ruby API
  Disable rainbow during SimpleExecutor specs to have consistence
  Slightly refactor pipeline schedules form in preparation for additions
  Resolve "Submitting reply to existing diff discussion using Cmd/Ctrl+Enter submits twice and refreshes page"
  Make the SimpleExecutor rescue exceptions in the executing Checks
  Resolve "Unable to access edit comment from dropdown menu in certain screen sizes"
  Update changelog item
  revert removal of requestAnimationFrame and move to a separate MR/discussion
  rename getEmojiCategoryMap and remove unnecessary parameter
  Action Buttons on Prio Labels working again by setting pointer events to none on…
  Remove 'contains' option from Commit.find_all
  Remove Gitlab::Git::Repository#find_all
  Use latest chrome and chrome driver in GitLab QA
  Polish sidebar toggle
  ...

725 files changed, 32320 insertions, 7486 deletions

diff --git a/lib/additional_email_headers_interceptor.rb b/lib/additional_email_headers_interceptor.rb
new file mode 100644
index 00000000000..2358fa6bbfd
--- /dev/null
+++ b/lib/additional_email_headers_interceptor.rb
@@ -0,0 +1,8 @@
+class AdditionalEmailHeadersInterceptor
+  def self.delivering_email(message)
+    message.headers(
+      'Auto-Submitted' => 'auto-generated',
+      'X-Auto-Response-Suppress' => 'All'
+    )
+  end
+end
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb
index 87915b19480..c9b5f58c557 100644
--- a/lib/api/access_requests.rb
+++ b/lib/api/access_requests.rb
@@ -1,5 +1,7 @@
 module API
   class AccessRequests < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
     helpers ::API::Helpers::MembersHelpers
@@ -8,11 +10,14 @@ module API
       params do
         requires :id, type: String, desc: "The #{source_type} ID"
       end
-      resource source_type.pluralize do
+      resource source_type.pluralize, requirements: { id: %r{[^/]+} } do
         desc "Gets a list of access requests for a #{source_type}." do
           detail 'This feature was introduced in GitLab 8.11.'
           success Entities::AccessRequester
         end
+        params do
+          use :pagination
+        end
         get ":id/access_requests" do
           source = find_source(source_type, params[:id])
 
@@ -48,7 +53,7 @@ module API
         put ':id/access_requests/:user_id/approve' do
           source = find_source(source_type, params[:id])
 
-          member = ::Members::ApproveAccessRequestService.new(source, current_user, declared(params)).execute
+          member = ::Members::ApproveAccessRequestService.new(source, current_user, declared_params).execute
 
           status :created
           present member.user, with: Entities::Member, member: member
@@ -63,8 +68,8 @@ module API
         delete ":id/access_requests/:user_id" do
           source = find_source(source_type, params[:id])
 
-          ::Members::DestroyService.new(source, current_user, params).
-            execute(:requesters)
+          ::Members::DestroyService.new(source, current_user, params)
+            .execute(:requesters)
         end
       end
     end
diff --git a/lib/api/api.rb b/lib/api/api.rb
index 0bbf73a1b63..d767af36e8e 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -1,7 +1,54 @@
 module API
   class API < Grape::API
     include APIGuard
-    version 'v3', using: :path
+
+    version %w(v3 v4), using: :path
+
+    version 'v3', using: :path do
+      helpers ::API::V3::Helpers
+      helpers ::API::Helpers::CommonHelpers
+
+      mount ::API::V3::AwardEmoji
+      mount ::API::V3::Boards
+      mount ::API::V3::Branches
+      mount ::API::V3::BroadcastMessages
+      mount ::API::V3::Builds
+      mount ::API::V3::Commits
+      mount ::API::V3::DeployKeys
+      mount ::API::V3::Environments
+      mount ::API::V3::Files
+      mount ::API::V3::Groups
+      mount ::API::V3::Issues
+      mount ::API::V3::Labels
+      mount ::API::V3::Members
+      mount ::API::V3::MergeRequestDiffs
+      mount ::API::V3::MergeRequests
+      mount ::API::V3::Notes
+      mount ::API::V3::Pipelines
+      mount ::API::V3::ProjectHooks
+      mount ::API::V3::Milestones
+      mount ::API::V3::Projects
+      mount ::API::V3::ProjectSnippets
+      mount ::API::V3::Repositories
+      mount ::API::V3::Runners
+      mount ::API::V3::Services
+      mount ::API::V3::Settings
+      mount ::API::V3::Snippets
+      mount ::API::V3::Subscriptions
+      mount ::API::V3::SystemHooks
+      mount ::API::V3::Tags
+      mount ::API::V3::Templates
+      mount ::API::V3::Todos
+      mount ::API::V3::Triggers
+      mount ::API::V3::Users
+      mount ::API::V3::Variables
+    end
+
+    before { allow_access_with_scope :api }
+    before { header['X-Frame-Options'] = 'SAMEORIGIN' }
+    before { Gitlab::I18n.locale = current_user&.preferred_language }
+
+    after { Gitlab::I18n.use_default_locale }
 
     rescue_from Gitlab::Access::AccessDeniedError do
       rack_response({ 'message' => '403 Forbidden' }.to_json, 403)
@@ -12,11 +59,19 @@ module API
     end
 
     # Retain 405 error rather than a 500 error for Grape 0.15.0+.
-    # See: https://github.com/ruby-grape/grape/commit/252bfd27c320466ec3c0751812cf44245e97e5de
+    # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes
+    rescue_from Grape::Exceptions::MethodNotAllowed do |e|
+      error! e.message, e.status, e.headers
+    end
+
     rescue_from Grape::Exceptions::Base do |e|
       error! e.message, e.status, e.headers
     end
 
+    rescue_from Gitlab::Auth::TooManyIps do |e|
+      rack_response({ 'message' => '403 Forbidden' }.to_json, 403)
+    end
+
     rescue_from :all do |exception|
       handle_api_exception(exception)
     end
@@ -27,44 +82,49 @@ module API
     # Ensure the namespace is right, otherwise we might load Grape::API::Helpers
     helpers ::SentryHelper
     helpers ::API::Helpers
+    helpers ::API::Helpers::CommonHelpers
 
     # Keep in alphabetical order
     mount ::API::AccessRequests
     mount ::API::AwardEmoji
+    mount ::API::Boards
     mount ::API::Branches
     mount ::API::BroadcastMessages
-    mount ::API::Builds
-    mount ::API::CommitStatuses
     mount ::API::Commits
+    mount ::API::CommitStatuses
     mount ::API::DeployKeys
     mount ::API::Deployments
     mount ::API::Environments
+    mount ::API::Events
+    mount ::API::Features
     mount ::API::Files
     mount ::API::Groups
     mount ::API::Internal
     mount ::API::Issues
-    mount ::API::Boards
+    mount ::API::Jobs
     mount ::API::Keys
     mount ::API::Labels
-    mount ::API::LicenseTemplates
     mount ::API::Lint
     mount ::API::Members
-    mount ::API::MergeRequests
     mount ::API::MergeRequestDiffs
+    mount ::API::MergeRequests
     mount ::API::Milestones
     mount ::API::Namespaces
     mount ::API::Notes
     mount ::API::NotificationSettings
     mount ::API::Pipelines
+    mount ::API::PipelineSchedules
     mount ::API::ProjectHooks
-    mount ::API::ProjectSnippets
     mount ::API::Projects
+    mount ::API::ProjectSnippets
     mount ::API::Repositories
+    mount ::API::Runner
     mount ::API::Runners
     mount ::API::Services
     mount ::API::Session
     mount ::API::Settings
     mount ::API::SidekiqMetrics
+    mount ::API::Snippets
     mount ::API::Subscriptions
     mount ::API::SystemHooks
     mount ::API::Tags
@@ -73,5 +133,10 @@ module API
     mount ::API::Triggers
     mount ::API::Users
     mount ::API::Variables
+    mount ::API::Version
+
+    route :any, '*path' do
+      error!('404 Not Found', 404)
+    end
   end
 end
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index 8cc7a26f1fa..9fcf04efa38 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -6,6 +6,9 @@ module API
   module APIGuard
     extend ActiveSupport::Concern
 
+    PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN".freeze
+    PRIVATE_TOKEN_PARAM = :private_token
+
     included do |base|
       # OAuth2 Resource Server Authentication
       use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request|
@@ -44,27 +47,60 @@ module API
         access_token = find_access_token
         return nil unless access_token
 
-        case validate_access_token(access_token, scopes)
-        when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
+        case AccessTokenValidationService.new(access_token).validate(scopes: scopes)
+        when AccessTokenValidationService::INSUFFICIENT_SCOPE
           raise InsufficientScopeError.new(scopes)
 
-        when Oauth2::AccessTokenValidationService::EXPIRED
+        when AccessTokenValidationService::EXPIRED
           raise ExpiredError
 
-        when Oauth2::AccessTokenValidationService::REVOKED
+        when AccessTokenValidationService::REVOKED
           raise RevokedError
 
-        when Oauth2::AccessTokenValidationService::VALID
+        when AccessTokenValidationService::VALID
           @current_user = User.find(access_token.resource_owner_id)
         end
       end
 
+      def find_user_by_private_token(scopes: [])
+        token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s
+
+        return nil unless token_string.present?
+
+        find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes)
+      end
+
       def current_user
         @current_user
       end
 
+      # Set the authorization scope(s) allowed for the current request.
+      #
+      # Note: A call to this method adds to any previous scopes in place. This is done because
+      # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then
+      # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the
+      # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they
+      # need to be stored.
+      def allow_access_with_scope(*scopes)
+        @scopes ||= []
+        @scopes.concat(scopes.map(&:to_s))
+      end
+
       private
 
+      def find_user_by_authentication_token(token_string)
+        User.find_by_authentication_token(token_string)
+      end
+
+      def find_user_by_personal_access_token(token_string, scopes)
+        access_token = PersonalAccessToken.active.find_by_token(token_string)
+        return unless access_token
+
+        if AccessTokenValidationService.new(access_token).include_any_scope?(scopes)
+          User.find(access_token.user_id)
+        end
+      end
+
       def find_access_token
         @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods)
       end
@@ -72,24 +108,20 @@ module API
       def doorkeeper_request
         @doorkeeper_request ||= ActionDispatch::Request.new(env)
       end
-
-      def validate_access_token(access_token, scopes)
-        Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes)
-      end
     end
 
     module ClassMethods
       private
 
       def install_error_responders(base)
-        error_classes = [ MissingTokenError, TokenNotFoundError,
-                          ExpiredError, RevokedError, InsufficientScopeError]
+        error_classes = [MissingTokenError, TokenNotFoundError,
+                         ExpiredError, RevokedError, InsufficientScopeError]
 
         base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler
       end
 
       def oauth2_bearer_token_error_handler
-        Proc.new do |e|
+        proc do |e|
           response =
             case e
             when MissingTokenError
@@ -128,13 +160,10 @@ module API
     # Exceptions
     #
 
-    class MissingTokenError < StandardError; end
-
-    class TokenNotFoundError < StandardError; end
-
-    class ExpiredError < StandardError; end
-
-    class RevokedError < StandardError; end
+    MissingTokenError = Class.new(StandardError)
+    TokenNotFoundError = Class.new(StandardError)
+    ExpiredError = Class.new(StandardError)
+    RevokedError = Class.new(StandardError)
 
     class InsufficientScopeError < StandardError
       attr_reader :scopes
diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb
index e9ccba3b465..56f19f89642 100644
--- a/lib/api/award_emoji.rb
+++ b/lib/api/award_emoji.rb
@@ -1,19 +1,28 @@
 module API
   class AwardEmoji < Grape::API
-    before { authenticate! }
-    AWARDABLES = %w[issue merge_request snippet]
+    include PaginationParams
 
-    resource :projects do
-      AWARDABLES.each do |awardable_type|
-        awardable_string = awardable_type.pluralize
-        awardable_id_string = "#{awardable_type}_id"
+    before { authenticate! }
+    AWARDABLES = [
+      { type: 'issue', find_by: :iid },
+      { type: 'merge_request', find_by: :iid },
+      { type: 'snippet', find_by: :id }
+    ].freeze
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      AWARDABLES.each do |awardable_params|
+        awardable_string = awardable_params[:type].pluralize
+        awardable_id_string = "#{awardable_params[:type]}_#{awardable_params[:find_by]}"
 
         params do
-          requires :id, type: String, desc: 'The ID of a project'
           requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet"
         end
 
-        [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji",
+        [
+          ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji",
           ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji"
         ].each do |endpoint|
 
@@ -21,10 +30,13 @@ module API
             detail 'This feature was introduced in 8.9'
             success Entities::AwardEmoji
           end
+          params do
+            use :pagination
+          end
           get endpoint do
             if can_read_awardable?
-              awards = paginate(awardable.award_emoji)
-              present awards, with: Entities::AwardEmoji
+              awards = awardable.award_emoji
+              present paginate(awards), with: Entities::AwardEmoji
             else
               not_found!("Award Emoji")
             end
@@ -77,7 +89,6 @@ module API
             unauthorized! unless award.user == current_user || current_user.admin?
 
             award.destroy
-            present award, with: Entities::AwardEmoji
           end
         end
       end
@@ -99,10 +110,10 @@ module API
               note_id = params.delete(:note_id)
 
               awardable.notes.find(note_id)
-            elsif params.include?(:issue_id)
-              user_project.issues.find(params[:issue_id])
-            elsif params.include?(:merge_request_id)
-              user_project.merge_requests.find(params[:merge_request_id])
+            elsif params.include?(:issue_iid)
+              user_project.issues.find_by!(iid: params[:issue_iid])
+            elsif params.include?(:merge_request_iid)
+              user_project.merge_requests.find_by!(iid: params[:merge_request_iid])
             else
               user_project.snippets.find(params[:snippet_id])
             end
diff --git a/lib/api/boards.rb b/lib/api/boards.rb
index 4d5d144a02e..5a2d7a681e3 100644
--- a/lib/api/boards.rb
+++ b/lib/api/boards.rb
@@ -1,20 +1,34 @@
 module API
-  # Boards API
   class Boards < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
-    resource :projects do
-      # Get the project board
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get all project boards' do
+        detail 'This feature was introduced in 8.13'
+        success Entities::Board
+      end
+      params do
+        use :pagination
+      end
       get ':id/boards' do
         authorize!(:read_board, user_project)
-        present [user_project.board], with: Entities::Board
+        present paginate(user_project.boards), with: Entities::Board
       end
 
+      params do
+        requires :board_id, type: Integer, desc: 'The ID of a board'
+      end
       segment ':id/boards/:board_id' do
         helpers do
           def project_board
-            board = user_project.board
-            if params[:board_id].to_i == board.id
+            board = user_project.boards.first
+
+            if params[:board_id] == board.id
               board
             else
               not_found!('Board')
@@ -26,37 +40,48 @@ module API
           end
         end
 
-        # Get the lists of a project board
-        # Does not include `backlog` and `done` lists
+        desc 'Get the lists of a project board' do
+          detail 'Does not include `done` list. This feature was introduced in 8.13'
+          success Entities::List
+        end
+        params do
+          use :pagination
+        end
         get '/lists' do
           authorize!(:read_board, user_project)
-          present board_lists, with: Entities::List
+          present paginate(board_lists), with: Entities::List
         end
 
-        # Get a list of a project board
+        desc 'Get a list of a project board' do
+          detail 'This feature was introduced in 8.13'
+          success Entities::List
+        end
+        params do
+          requires :list_id, type: Integer, desc: 'The ID of a list'
+        end
         get '/lists/:list_id' do
           authorize!(:read_board, user_project)
           present board_lists.find(params[:list_id]), with: Entities::List
         end
 
-        # Create a new board list
-        #
-        # Parameters:
-        #   id (required)           - The ID of a project
-        #   label_id (required)     - The ID of an existing label
-        # Example Request:
-        #   POST /projects/:id/boards/:board_id/lists
+        desc 'Create a new board list' do
+          detail 'This feature was introduced in 8.13'
+          success Entities::List
+        end
+        params do
+          requires :label_id, type: Integer, desc: 'The ID of an existing label'
+        end
         post '/lists' do
-          required_attributes! [:label_id]
-
-          unless user_project.labels.exists?(params[:label_id])
-            render_api_error!({ error: "Label not found!" }, 400)
+          unless available_labels.exists?(params[:label_id])
+            render_api_error!({ error: 'Label not found!' }, 400)
           end
 
           authorize!(:admin_list, user_project)
 
-          list = ::Boards::Lists::CreateService.new(user_project, current_user,
-              { label_id: params[:label_id] }).execute
+          service = ::Boards::Lists::CreateService.new(user_project, current_user,
+            { label_id: params[:label_id] })
+
+          list = service.execute(project_board)
 
           if list.valid?
             present list, with: Entities::List
@@ -65,48 +90,45 @@ module API
           end
         end
 
-        # Moves a board list to a new position
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   board_id (required) - The ID of a board
-        #   position (required) - The position of the list
-        # Example Request:
-        #   PUT /projects/:id/boards/:board_id/lists/:list_id
+        desc 'Moves a board list to a new position' do
+          detail 'This feature was introduced in 8.13'
+          success Entities::List
+        end
+        params do
+          requires :list_id,  type: Integer, desc: 'The ID of a list'
+          requires :position, type: Integer, desc: 'The position of the list'
+        end
         put '/lists/:list_id' do
           list = project_board.lists.movable.find(params[:list_id])
 
           authorize!(:admin_list, user_project)
 
-          moved = ::Boards::Lists::MoveService.new(user_project, current_user,
-              { position: params[:position].to_i }).execute(list)
+          service = ::Boards::Lists::MoveService.new(user_project, current_user,
+              { position: params[:position] })
 
-          if moved
+          if service.execute(list)
             present list, with: Entities::List
           else
             render_api_error!({ error: "List could not be moved!" }, 400)
           end
         end
 
-        # Delete a board list
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   board_id (required) - The ID of a board
-        #   list_id (required) - The ID of a board list
-        # Example Request:
-        #   DELETE /projects/:id/boards/:board_id/lists/:list_id
+        desc 'Delete a board list' do
+          detail 'This feature was introduced in 8.13'
+          success Entities::List
+        end
+        params do
+          requires :list_id, type: Integer, desc: 'The ID of a board list'
+        end
         delete "/lists/:list_id" do
-          list = board_lists.find_by(id: params[:list_id])
-
           authorize!(:admin_list, user_project)
 
-          if list
-            destroyed_list = ::Boards::Lists::DestroyService.new(
-              user_project, current_user).execute(list)
-            present destroyed_list, with: Entities::List
-          else
-            not_found!('List')
+          list = board_lists.find(params[:list_id])
+
+          service = ::Boards::Lists::DestroyService.new(user_project, current_user)
+
+          unless service.execute(list)
+            render_api_error!({ error: 'List could not be deleted!' }, 400)
           end
         end
       end
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index b615703df93..3d816f8771d 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -1,129 +1,109 @@
 require 'mime/types'
 
 module API
-  # Projects API
   class Branches < Grape::API
-    before { authenticate! }
+    include PaginationParams
+
     before { authorize! :download_code, user_project }
 
-    resource :projects do
-      # Get a project repository branches
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   GET /projects/:id/repository/branches
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get a project repository branches' do
+        success Entities::RepoBranch
+      end
+      params do
+        use :pagination
+      end
       get ":id/repository/branches" do
-        branches = user_project.repository.branches.sort_by(&:name)
+        branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name))
 
-        present branches, with: Entities::RepoBranch, project: user_project
+        present paginate(branches), with: Entities::RepoBranch, project: user_project
       end
 
-      # Get a single branch
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   branch (required) - The name of the branch
-      # Example Request:
-      #   GET /projects/:id/repository/branches/:branch
+      desc 'Get a single branch' do
+        success Entities::RepoBranch
+      end
+      params do
+        requires :branch, type: String, desc: 'The name of the branch'
+      end
       get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do
-        @branch = user_project.repository.branches.find { |item| item.name == params[:branch] }
-        not_found!("Branch") unless @branch
+        branch = user_project.repository.find_branch(params[:branch])
+        not_found!("Branch") unless branch
 
-        present @branch, with: Entities::RepoBranch, project: user_project
+        present branch, with: Entities::RepoBranch, project: user_project
       end
 
-      # Protect a single branch
-      #
       # Note: The internal data model moved from `developers_can_{merge,push}` to `allowed_to_{merge,push}`
       # in `gitlab-org/gitlab-ce!5081`. The API interface has not been changed (to maintain compatibility),
       # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`.
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   branch (required) - The name of the branch
-      #   developers_can_push (optional) - Flag if developers can push to that branch
-      #   developers_can_merge (optional) - Flag if developers can merge to that branch
-      # Example Request:
-      #   PUT /projects/:id/repository/branches/:branch/protect
-      put ':id/repository/branches/:branch/protect',
-          requirements: { branch: /.+/ } do
+      desc 'Protect a single branch' do
+        success Entities::RepoBranch
+      end
+      params do
+        requires :branch, type: String, desc: 'The name of the branch'
+        optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch'
+        optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch'
+      end
+      put ':id/repository/branches/:branch/protect', requirements: { branch: /.+/ } do
         authorize_admin_project
 
-        @branch = user_project.repository.find_branch(params[:branch])
-        not_found!('Branch') unless @branch
-        protected_branch = user_project.protected_branches.find_by(name: @branch.name)
+        branch = user_project.repository.find_branch(params[:branch])
+        not_found!('Branch') unless branch
 
-        developers_can_merge = to_boolean(params[:developers_can_merge])
-        developers_can_push = to_boolean(params[:developers_can_push])
+        protected_branch = user_project.protected_branches.find_by(name: branch.name)
 
         protected_branch_params = {
-          name: @branch.name
+          name: branch.name,
+          developers_can_push: params[:developers_can_push],
+          developers_can_merge: params[:developers_can_merge]
         }
 
-        # If `developers_can_merge` is switched off, _all_ `DEVELOPER`
-        # merge_access_levels need to be deleted.
-        if developers_can_merge == false
-          protected_branch.merge_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all
-        end
+        service_args = [user_project, current_user, protected_branch_params]
 
-        # If `developers_can_push` is switched off, _all_ `DEVELOPER`
-        # push_access_levels need to be deleted.
-        if developers_can_push == false
-          protected_branch.push_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all
-        end
+        protected_branch = if protected_branch
+                             ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch)
+                           else
+                             ProtectedBranches::ApiCreateService.new(*service_args).execute
+                           end
 
-        protected_branch_params.merge!(
-          merge_access_levels_attributes: [{
-            access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER
-          }],
-          push_access_levels_attributes: [{
-            access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER
-          }]
-        )
-
-        if protected_branch
-          service = ProtectedBranches::UpdateService.new(user_project, current_user, protected_branch_params)
-          service.execute(protected_branch)
+        if protected_branch.valid?
+          present branch, with: Entities::RepoBranch, project: user_project
         else
-          service = ProtectedBranches::CreateService.new(user_project, current_user, protected_branch_params)
-          service.execute
+          render_api_error!(protected_branch.errors.full_messages, 422)
         end
-
-        present @branch, with: Entities::RepoBranch, project: user_project
       end
 
-      # Unprotect a single branch
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   branch (required) - The name of the branch
-      # Example Request:
-      #   PUT /projects/:id/repository/branches/:branch/unprotect
-      put ':id/repository/branches/:branch/unprotect',
-          requirements: { branch: /.+/ } do
+      desc 'Unprotect a single branch' do
+        success Entities::RepoBranch
+      end
+      params do
+        requires :branch, type: String, desc: 'The name of the branch'
+      end
+      put ':id/repository/branches/:branch/unprotect', requirements: { branch: /.+/ } do
         authorize_admin_project
 
-        @branch = user_project.repository.find_branch(params[:branch])
-        not_found!("Branch") unless @branch
-        protected_branch = user_project.protected_branches.find_by(name: @branch.name)
-        protected_branch.destroy if protected_branch
+        branch = user_project.repository.find_branch(params[:branch])
+        not_found!("Branch") unless branch
+        protected_branch = user_project.protected_branches.find_by(name: branch.name)
+        protected_branch&.destroy
 
-        present @branch, with: Entities::RepoBranch, project: user_project
+        present branch, with: Entities::RepoBranch, project: user_project
       end
 
-      # Create branch
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   branch_name (required) - The name of the branch
-      #   ref (required) - Create branch from commit sha or existing branch
-      # Example Request:
-      #   POST /projects/:id/repository/branches
+      desc 'Create branch' do
+        success Entities::RepoBranch
+      end
+      params do
+        requires :branch, type: String, desc: 'The name of the branch'
+        requires :ref, type: String, desc: 'Create branch from commit sha or existing branch'
+      end
       post ":id/repository/branches" do
         authorize_push_project
-        result = CreateBranchService.new(user_project, current_user).
-          execute(params[:branch_name], params[:ref])
+
+        result = CreateBranchService.new(user_project, current_user)
+                 .execute(params[:branch], params[:ref])
 
         if result[:status] == :success
           present result[:branch],
@@ -134,27 +114,27 @@ module API
         end
       end
 
-      # Delete branch
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   branch (required) - The name of the branch
-      # Example Request:
-      #   DELETE /projects/:id/repository/branches/:branch
-      delete ":id/repository/branches/:branch",
-          requirements: { branch: /.+/ } do
+      desc 'Delete a branch'
+      params do
+        requires :branch, type: String, desc: 'The name of the branch'
+      end
+      delete ":id/repository/branches/:branch", requirements: { branch: /.+/ } do
         authorize_push_project
-        result = DeleteBranchService.new(user_project, current_user).
-          execute(params[:branch])
 
-        if result[:status] == :success
-          {
-            branch_name: params[:branch]
-          }
-        else
+        result = DeleteBranchService.new(user_project, current_user)
+                 .execute(params[:branch])
+
+        if result[:status] != :success
           render_api_error!(result[:message], result[:return_code])
         end
       end
+
+      desc 'Delete all merged branches'
+      delete ":id/repository/merged_branches" do
+        DeleteMergedBranchesService.new(user_project, current_user).async_execute
+
+        accepted!
+      end
     end
   end
 end
diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb
index fb2a4148011..395c401203c 100644
--- a/lib/api/broadcast_messages.rb
+++ b/lib/api/broadcast_messages.rb
@@ -1,5 +1,7 @@
 module API
   class BroadcastMessages < Grape::API
+    include PaginationParams
+
     before { authenticate! }
     before { authenticated_as_admin! }
 
@@ -15,8 +17,7 @@ module API
         success Entities::BroadcastMessage
       end
       params do
-        optional :page,     type: Integer, desc: 'Current page number'
-        optional :per_page, type: Integer, desc: 'Number of messages per page'
+        use :pagination
       end
       get do
         messages = BroadcastMessage.all
@@ -36,8 +37,7 @@ module API
         optional :font,      type: String,   desc: 'Foreground color'
       end
       post do
-        create_params = declared(params, include_missing: false).to_h
-        message = BroadcastMessage.create(create_params)
+        message = BroadcastMessage.create(declared_params(include_missing: false))
 
         if message.persisted?
           present message, with: Entities::BroadcastMessage
@@ -73,9 +73,8 @@ module API
       end
       put ':id' do
         message = find_message
-        update_params = declared(params, include_missing: false).to_h
 
-        if message.update(update_params)
+        if message.update(declared_params(include_missing: false))
           present message, with: Entities::BroadcastMessage
         else
           render_validation_error!(message)
@@ -92,7 +91,7 @@ module API
       delete ':id' do
         message = find_message
 
-        present message.destroy, with: Entities::BroadcastMessage
+        message.destroy
       end
     end
   end
diff --git a/lib/api/builds.rb b/lib/api/builds.rb
deleted file mode 100644
index 52bdbcae5a8..00000000000
--- a/lib/api/builds.rb
+++ /dev/null
@@ -1,262 +0,0 @@
-module API
-  # Projects builds API
-  class Builds < Grape::API
-    before { authenticate! }
-
-    resource :projects do
-      # Get a project builds
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled;
-      #                      if none provided showing all builds)
-      # Example Request:
-      #   GET /projects/:id/builds
-      get ':id/builds' do
-        builds = user_project.builds.order('id DESC')
-        builds = filter_builds(builds, params[:scope])
-
-        present paginate(builds), with: Entities::Build,
-                                  user_can_download_artifacts: can?(current_user, :read_build, user_project)
-      end
-
-      # Get builds for a specific commit of a project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   sha (required) - The SHA id of a commit
-      #   scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled;
-      #                      if none provided showing all builds)
-      # Example Request:
-      #   GET /projects/:id/repository/commits/:sha/builds
-      get ':id/repository/commits/:sha/builds' do
-        authorize_read_builds!
-
-        return not_found! unless user_project.commit(params[:sha])
-
-        pipelines = user_project.pipelines.where(sha: params[:sha])
-        builds = user_project.builds.where(pipeline: pipelines).order('id DESC')
-        builds = filter_builds(builds, params[:scope])
-
-        present paginate(builds), with: Entities::Build,
-                                  user_can_download_artifacts: can?(current_user, :read_build, user_project)
-      end
-
-      # Get a specific build of a project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   build_id (required) - The ID of a build
-      # Example Request:
-      #   GET /projects/:id/builds/:build_id
-      get ':id/builds/:build_id' do
-        authorize_read_builds!
-
-        build = get_build!(params[:build_id])
-
-        present build, with: Entities::Build,
-                       user_can_download_artifacts: can?(current_user, :read_build, user_project)
-      end
-
-      # Download the artifacts file from build
-      #
-      # Parameters:
-      #   id (required) - The ID of a build
-      #   token (required) - The build authorization token
-      # Example Request:
-      #   GET /projects/:id/builds/:build_id/artifacts
-      get ':id/builds/:build_id/artifacts' do
-        authorize_read_builds!
-
-        build = get_build!(params[:build_id])
-
-        present_artifacts!(build.artifacts_file)
-      end
-
-      # Download the artifacts file from ref_name and job
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   ref_name (required) - The ref from repository
-      #   job (required) - The name for the build
-      # Example Request:
-      #   GET /projects/:id/builds/artifacts/:ref_name/download?job=name
-      get ':id/builds/artifacts/:ref_name/download',
-        requirements: { ref_name: /.+/ } do
-        authorize_read_builds!
-
-        builds = user_project.latest_successful_builds_for(params[:ref_name])
-        latest_build = builds.find_by!(name: params[:job])
-
-        present_artifacts!(latest_build.artifacts_file)
-      end
-
-      # Get a trace of a specific build of a project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   build_id (required) - The ID of a build
-      # Example Request:
-      #   GET /projects/:id/build/:build_id/trace
-      #
-      # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace
-      #       is saved in the DB instead of file). But before that, we need to consider how to replace the value of
-      #       `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse.
-      get ':id/builds/:build_id/trace' do
-        authorize_read_builds!
-
-        build = get_build!(params[:build_id])
-
-        header 'Content-Disposition', "infile; filename=\"#{build.id}.log\""
-        content_type 'text/plain'
-        env['api.format'] = :binary
-
-        trace = build.trace
-        body trace
-      end
-
-      # Cancel a specific build of a project
-      #
-      # parameters:
-      #   id (required) - the id of a project
-      #   build_id (required) - the id of a build
-      # example request:
-      #   post /projects/:id/build/:build_id/cancel
-      post ':id/builds/:build_id/cancel' do
-        authorize_update_builds!
-
-        build = get_build!(params[:build_id])
-
-        build.cancel
-
-        present build, with: Entities::Build,
-                       user_can_download_artifacts: can?(current_user, :read_build, user_project)
-      end
-
-      # Retry a specific build of a project
-      #
-      # parameters:
-      #   id (required) - the id of a project
-      #   build_id (required) - the id of a build
-      # example request:
-      #   post /projects/:id/build/:build_id/retry
-      post ':id/builds/:build_id/retry' do
-        authorize_update_builds!
-
-        build = get_build!(params[:build_id])
-        return forbidden!('Build is not retryable') unless build.retryable?
-
-        build = Ci::Build.retry(build, current_user)
-
-        present build, with: Entities::Build,
-                       user_can_download_artifacts: can?(current_user, :read_build, user_project)
-      end
-
-      # Erase build (remove artifacts and build trace)
-      #
-      # Parameters:
-      #   id (required) - the id of a project
-      #   build_id (required) - the id of a build
-      # example Request:
-      #  post  /projects/:id/build/:build_id/erase
-      post ':id/builds/:build_id/erase' do
-        authorize_update_builds!
-
-        build = get_build!(params[:build_id])
-        return forbidden!('Build is not erasable!') unless build.erasable?
-
-        build.erase(erased_by: current_user)
-        present build, with: Entities::Build,
-                       user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project)
-      end
-
-      # Keep the artifacts to prevent them from being deleted
-      #
-      # Parameters:
-      #   id (required) - the id of a project
-      #   build_id (required) - The ID of a build
-      # Example Request:
-      #   POST /projects/:id/builds/:build_id/artifacts/keep
-      post ':id/builds/:build_id/artifacts/keep' do
-        authorize_update_builds!
-
-        build = get_build!(params[:build_id])
-        return not_found!(build) unless build.artifacts?
-
-        build.keep_artifacts!
-
-        status 200
-        present build, with: Entities::Build,
-                       user_can_download_artifacts: can?(current_user, :read_build, user_project)
-      end
-
-      desc 'Trigger a manual build' do
-        success Entities::Build
-        detail 'This feature was added in GitLab 8.11'
-      end
-      params do
-        requires :build_id, type: Integer, desc: 'The ID of a Build'
-      end
-      post ":id/builds/:build_id/play" do
-        authorize_read_builds!
-
-        build = get_build!(params[:build_id])
-
-        bad_request!("Unplayable Build") unless build.playable?
-
-        build.play(current_user)
-
-        status 200
-        present build, with: Entities::Build,
-                       user_can_download_artifacts: can?(current_user, :read_build, user_project)
-      end
-    end
-
-    helpers do
-      def get_build(id)
-        user_project.builds.find_by(id: id.to_i)
-      end
-
-      def get_build!(id)
-        get_build(id) || not_found!
-      end
-
-      def present_artifacts!(artifacts_file)
-        if !artifacts_file.file_storage?
-          redirect_to(build.artifacts_file.url)
-        elsif artifacts_file.exists?
-          present_file!(artifacts_file.path, artifacts_file.filename)
-        else
-          not_found!
-        end
-      end
-
-      def filter_builds(builds, scope)
-        return builds if scope.nil? || scope.empty?
-
-        available_statuses = ::CommitStatus::AVAILABLE_STATUSES
-        scope =
-          if scope.is_a?(String)
-            [scope]
-          elsif scope.is_a?(Hashie::Mash)
-            scope.values
-          else
-            ['unknown']
-          end
-
-        unknown = scope - available_statuses
-        render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty?
-
-        builds.where(status: available_statuses && scope)
-      end
-
-      def authorize_read_builds!
-        authorize! :read_build, user_project
-      end
-
-      def authorize_update_builds!
-        authorize! :update_build, user_project
-      end
-    end
-  end
-end
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index dfbdd597d29..485b680cd5f 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -1,22 +1,26 @@
 require 'mime/types'
 
 module API
-  # Project commit statuses API
   class CommitStatuses < Grape::API
-    resource :projects do
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      include PaginationParams
+
       before { authenticate! }
 
-      # Get a commit's statuses
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   sha (required) - The commit hash
-      #   ref (optional) - The ref
-      #   stage (optional) - The stage
-      #   name (optional) - The name
-      #   all (optional) - Show all statuses, default: false
-      # Examples:
-      #   GET /projects/:id/repository/commits/:sha/statuses
+      desc "Get a commit's statuses" do
+        success Entities::CommitStatus
+      end
+      params do
+        requires :sha,   type: String, desc: 'The commit hash'
+        optional :ref,   type: String, desc: 'The ref'
+        optional :stage, type: String, desc: 'The stage'
+        optional :name,  type: String, desc: 'The name'
+        optional :all,   type: String, desc: 'Show all statuses, default: false'
+        use :pagination
+      end
       get ':id/repository/commits/:sha/statuses' do
         authorize!(:read_commit_status, user_project)
 
@@ -31,22 +35,23 @@ module API
         present paginate(statuses), with: Entities::CommitStatus
       end
 
-      # Post status to commit
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   sha (required) - The commit hash
-      #   ref (optional) - The ref
-      #   state (required) - The state of the status. Can be: pending, running, success, failed or canceled
-      #   target_url (optional) - The target URL to associate with this status
-      #   description (optional) - A short description of the status
-      #   name or context (optional) - A string label to differentiate this status from the status of other systems. Default: "default"
-      # Examples:
-      #   POST /projects/:id/statuses/:sha
+      desc 'Post status to a commit' do
+        success Entities::CommitStatus
+      end
+      params do
+        requires :sha,         type: String,  desc: 'The commit hash'
+        requires :state,       type: String,  desc: 'The state of the status',
+                               values: %w(pending running success failed canceled)
+        optional :ref,         type: String,  desc: 'The ref'
+        optional :target_url,  type: String,  desc: 'The target URL to associate with this status'
+        optional :description, type: String,  desc: 'A short description of the status'
+        optional :name,        type: String,  desc: 'A string label to differentiate this status from the status of other systems. Default: "default"'
+        optional :context,     type: String,  desc: 'A string label to differentiate this status from the status of other systems. Default: "default"'
+        optional :coverage,    type: Float,   desc: 'The total code coverage'
+      end
       post ':id/statuses/:sha' do
         authorize! :create_commit_status, user_project
-        required_attributes! [:state]
-        attrs = attributes_for_keys [:target_url, :description]
+
         commit = @project.commit(params[:sha])
         not_found! 'Commit' unless commit
 
@@ -63,15 +68,31 @@ module API
 
         name = params[:name] || params[:context] || 'default'
 
-        pipeline = @project.ensure_pipeline(ref, commit.sha, current_user)
+        pipeline = @project.pipeline_for(ref, commit.sha)
+        unless pipeline
+          pipeline = @project.pipelines.create!(
+            source: :external,
+            sha: commit.sha,
+            ref: ref,
+            user: current_user)
+        end
 
         status = GenericCommitStatus.running_or_pending.find_or_initialize_by(
-          project: @project, pipeline: pipeline,
-          user: current_user, name: name, ref: ref)
-        status.attributes = attrs
+          project: @project,
+          pipeline: pipeline,
+          name: name,
+          ref: ref,
+          user: current_user
+        )
+
+        optional_attributes =
+          attributes_for_keys(%w[target_url description coverage])
+
+        status.update(optional_attributes) if optional_attributes.any?
+        render_validation_error!(status) if status.invalid?
 
         begin
-          case params[:state].to_s
+          case params[:state]
           when 'pending'
             status.enqueue!
           when 'running'
@@ -87,6 +108,9 @@ module API
             render_api_error!('invalid state', 400)
           end
 
+          MergeRequest.where(source_project: @project, source_branch: ref)
+            .update_all(head_pipeline_id: pipeline) if pipeline.latest?
+
           present status, with: Entities::CommitStatus
         rescue StateMachines::InvalidTransition => e
           render_api_error!(e.message, 400)
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 14ddc8c9a62..c6fc17cc391 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -1,59 +1,68 @@
 require 'mime/types'
 
 module API
-  # Projects commits API
   class Commits < Grape::API
+    include PaginationParams
+
     before { authenticate! }
     before { authorize! :download_code, user_project }
 
-    resource :projects do
-      # Get a project repository commits
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   ref_name (optional) - The name of a repository branch or tag, if not given the default branch is used
-      #   since (optional) - Only commits after or in this date will be returned
-      #   until (optional) - Only commits before or in this date will be returned
-      # Example Request:
-      #   GET /projects/:id/repository/commits
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get a project repository commits' do
+        success Entities::RepoCommit
+      end
+      params do
+        optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used'
+        optional :since,    type: DateTime, desc: 'Only commits after or on this date will be returned'
+        optional :until,    type: DateTime, desc: 'Only commits before or on this date will be returned'
+        optional :path,     type: String, desc: 'The file path'
+        use :pagination
+      end
       get ":id/repository/commits" do
-        datetime_attributes! :since, :until
-
-        page = (params[:page] || 0).to_i
-        per_page = (params[:per_page] || 20).to_i
-        ref = params[:ref_name] || user_project.try(:default_branch) || 'master'
-        after = params[:since]
+        path   = params[:path]
         before = params[:until]
+        after  = params[:since]
+        ref    = params[:ref_name] || user_project.try(:default_branch) || 'master'
+        offset = (params[:page] - 1) * params[:per_page]
+
+        commits = user_project.repository.commits(ref,
+                                                  path: path,
+                                                  limit: params[:per_page],
+                                                  offset: offset,
+                                                  before: before,
+                                                  after: after)
+
+        commit_count =
+          if path || before || after
+            user_project.repository.count_commits(ref: ref, path: path, before: before, after: after)
+          else
+            # Cacheable commit count.
+            user_project.repository.commit_count_for_ref(ref)
+          end
+
+        paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count)
 
-        commits = user_project.repository.commits(ref, limit: per_page, offset: page * per_page, after: after, before: before)
-        present commits, with: Entities::RepoCommit
+        present paginate(paginated_commits), with: Entities::RepoCommit
       end
 
       desc 'Commit multiple file changes as one commit' do
+        success Entities::RepoCommitDetail
         detail 'This feature was introduced in GitLab 8.13'
       end
-
       params do
-        requires :id, type: Integer, desc: 'The project ID'
-        requires :branch_name, type: String, desc: 'The name of branch'
+        requires :branch, type: String, desc: 'The name of branch'
         requires :commit_message, type: String, desc: 'Commit message'
-        requires :actions, type: Array, desc: 'Actions to perform in commit'
+        requires :actions, type: Array[Hash], desc: 'Actions to perform in commit'
         optional :author_email, type: String, desc: 'Author email for commit'
         optional :author_name, type: String, desc: 'Author name for commit'
       end
-
       post ":id/repository/commits" do
         authorize! :push_code, user_project
 
-        attrs = declared(params)
-        attrs[:source_branch] = attrs[:branch_name]
-        attrs[:target_branch] = attrs[:branch_name]
-        attrs[:actions].map! do |action|
-          action[:action] = action[:action].to_sym
-          action[:file_path].slice!(0) if action[:file_path] && action[:file_path].start_with?('/')
-          action[:previous_path].slice!(0) if action[:previous_path] && action[:previous_path].start_with?('/')
-          action
-        end
+        attrs = declared_params.merge(start_branch: declared_params[:branch], branch_name: declared_params[:branch])
 
         result = ::Files::MultiService.new(user_project, current_user, attrs).execute
 
@@ -65,79 +74,114 @@ module API
         end
       end
 
-      # Get a specific commit of a project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   sha (required) - The commit hash or name of a repository branch or tag
-      # Example Request:
-      #   GET /projects/:id/repository/commits/:sha
+      desc 'Get a specific commit of a project' do
+        success Entities::RepoCommitDetail
+        failure [[404, 'Not Found']]
+      end
+      params do
+        requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag'
+      end
       get ":id/repository/commits/:sha" do
-        sha = params[:sha]
-        commit = user_project.commit(sha)
+        commit = user_project.commit(params[:sha])
+
         not_found! "Commit" unless commit
+
         present commit, with: Entities::RepoCommitDetail
       end
 
-      # Get the diff for a specific commit of a project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   sha (required) - The commit or branch name
-      # Example Request:
-      #   GET /projects/:id/repository/commits/:sha/diff
+      desc 'Get the diff for a specific commit of a project' do
+        failure [[404, 'Not Found']]
+      end
+      params do
+        requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag'
+      end
       get ":id/repository/commits/:sha/diff" do
-        sha = params[:sha]
-        commit = user_project.commit(sha)
+        commit = user_project.commit(params[:sha])
+
         not_found! "Commit" unless commit
+
         commit.raw_diffs.to_a
       end
 
-      # Get a commit's comments
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   sha (required) - The commit hash
-      # Examples:
-      #   GET /projects/:id/repository/commits/:sha/comments
+      desc "Get a commit's comments" do
+        success Entities::CommitNote
+        failure [[404, 'Not Found']]
+      end
+      params do
+        use :pagination
+        requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag'
+      end
       get ':id/repository/commits/:sha/comments' do
-        sha = params[:sha]
-        commit = user_project.commit(sha)
+        commit = user_project.commit(params[:sha])
+
         not_found! 'Commit' unless commit
-        notes = Note.where(commit_id: commit.id).order(:created_at)
+        notes = user_project.notes.where(commit_id: commit.id).order(:created_at)
+
         present paginate(notes), with: Entities::CommitNote
       end
 
-      # Post comment to commit
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   sha (required) - The commit hash
-      #   note (required) - Text of comment
-      #   path (optional) - The file path
-      #   line (optional) - The line number
-      #   line_type (optional) - The type of line (new or old)
-      # Examples:
-      #   POST /projects/:id/repository/commits/:sha/comments
-      post ':id/repository/commits/:sha/comments' do
-        required_attributes! [:note]
+      desc 'Cherry pick commit into a branch' do
+        detail 'This feature was introduced in GitLab 8.15'
+        success Entities::RepoCommit
+      end
+      params do
+        requires :sha, type: String, desc: 'A commit sha to be cherry picked'
+        requires :branch, type: String, desc: 'The name of the branch'
+      end
+      post ':id/repository/commits/:sha/cherry_pick' do
+        authorize! :push_code, user_project
+
+        commit = user_project.commit(params[:sha])
+        not_found!('Commit') unless commit
+
+        branch = user_project.repository.find_branch(params[:branch])
+        not_found!('Branch') unless branch
+
+        commit_params = {
+          commit: commit,
+          start_branch: params[:branch],
+          branch_name: params[:branch]
+        }
+
+        result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute
 
-        sha = params[:sha]
-        commit = user_project.commit(sha)
+        if result[:status] == :success
+          branch = user_project.repository.find_branch(params[:branch])
+          present user_project.repository.commit(branch.dereferenced_target), with: Entities::RepoCommit
+        else
+          render_api_error!(result[:message], 400)
+        end
+      end
+
+      desc 'Post comment to commit' do
+        success Entities::CommitNote
+      end
+      params do
+        requires :sha, type: String, regexp: /\A\h{6,40}\z/, desc: "The commit's SHA"
+        requires :note, type: String, desc: 'The text of the comment'
+        optional :path, type: String, desc: 'The file path'
+        given :path do
+          requires :line, type: Integer, desc: 'The line number'
+          requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line'
+        end
+      end
+      post ':id/repository/commits/:sha/comments' do
+        commit = user_project.commit(params[:sha])
         not_found! 'Commit' unless commit
+
         opts = {
           note: params[:note],
           noteable_type: 'Commit',
           commit_id: commit.id
         }
 
-        if params[:path] && params[:line] && params[:line_type]
-          commit.raw_diffs(all_diffs: true).each do |diff|
+        if params[:path]
+          commit.raw_diffs(limits: false).each do |diff|
             next unless diff.new_path == params[:path]
             lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line)
 
             lines.each do |line|
-              next unless line.new_pos == params[:line].to_i && line.type == params[:line_type]
+              next unless line.new_pos == params[:line] && line.type == params[:line_type]
               break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos)
             end
 
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index 825e05fbae3..d5c2f3d5094 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -1,116 +1,132 @@
 module API
-  # Projects API
   class DeployKeys < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
+    desc 'Return all deploy keys'
+    params do
+      use :pagination
+    end
     get "deploy_keys" do
       authenticated_as_admin!
 
-      keys = DeployKey.all
-      present keys, with: Entities::SSHKey
+      present paginate(DeployKey.all), with: Entities::SSHKey
     end
 
     params do
       requires :id, type: String, desc: 'The ID of the project'
     end
-    resource :projects do
+    resource :projects, requirements: { id: %r{[^/]+} } do
       before { authorize_admin_project }
 
-      # Routing "projects/:id/keys/..." is DEPRECATED and WILL BE REMOVED in version 9.0
-      # Use "projects/:id/deploy_keys/..." instead.
-      #
-      %w(keys deploy_keys).each do |path|
-        desc "Get a specific project's deploy keys" do
-          success Entities::SSHKey
-        end
-        get ":id/#{path}" do
-          present user_project.deploy_keys, with: Entities::SSHKey
-        end
+      desc "Get a specific project's deploy keys" do
+        success Entities::SSHKey
+      end
+      params do
+        use :pagination
+      end
+      get ":id/deploy_keys" do
+        present paginate(user_project.deploy_keys), with: Entities::SSHKey
+      end
 
-        desc 'Get single deploy key' do
-          success Entities::SSHKey
-        end
-        params do
-          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
-        end
-        get ":id/#{path}/:key_id" do
-          key = user_project.deploy_keys.find params[:key_id]
+      desc 'Get single deploy key' do
+        success Entities::SSHKey
+      end
+      params do
+        requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+      end
+      get ":id/deploy_keys/:key_id" do
+        key = user_project.deploy_keys.find params[:key_id]
+        present key, with: Entities::SSHKey
+      end
+
+      desc 'Add new deploy key to currently authenticated user' do
+        success Entities::SSHKey
+      end
+      params do
+        requires :key, type: String, desc: 'The new deploy key'
+        requires :title, type: String, desc: 'The name of the deploy key'
+        optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository"
+      end
+      post ":id/deploy_keys" do
+        params[:key].strip!
+
+        # Check for an existing key joined to this project
+        key = user_project.deploy_keys.find_by(key: params[:key])
+        if key
           present key, with: Entities::SSHKey
+          break
         end
 
-        # TODO: for 9.0 we should check if params are there with the params block
-        # grape provides, at this point we'd change behaviour so we can't
-        # Behaviour now if you don't provide all required params: it renders a
-        # validation error or two.
-        desc 'Add new deploy key to currently authenticated user' do
-          success Entities::SSHKey
-        end
-        post ":id/#{path}" do
-          attrs = attributes_for_keys [:title, :key]
-          attrs[:key].strip! if attrs[:key]
-
-          key = user_project.deploy_keys.find_by(key: attrs[:key])
-          present key, with: Entities::SSHKey if key
-
-          # Check for available deploy keys in other projects
-          key = current_user.accessible_deploy_keys.find_by(key: attrs[:key])
-          if key
-            user_project.deploy_keys << key
-            present key, with: Entities::SSHKey
-          end
-
-          key = DeployKey.new attrs
-
-          if key.valid? && user_project.deploy_keys << key
-            present key, with: Entities::SSHKey
-          else
-            render_validation_error!(key)
-          end
+        # Check for available deploy keys in other projects
+        key = current_user.accessible_deploy_keys.find_by(key: params[:key])
+        if key
+          user_project.deploy_keys << key
+          present key, with: Entities::SSHKey
+          break
         end
 
-        desc 'Enable a deploy key for a project' do
-          detail 'This feature was added in GitLab 8.11'
-          success Entities::SSHKey
-        end
-        params do
-          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
-        end
-        post ":id/#{path}/:key_id/enable" do
-          key = ::Projects::EnableDeployKeyService.new(user_project,
-                                                        current_user, declared(params)).execute
-
-          if key
-            present key, with: Entities::SSHKey
-          else
-            not_found!('Deploy Key')
-          end
+        # Create a new deploy key
+        key = DeployKey.new(declared_params(include_missing: false))
+        if key.valid? && user_project.deploy_keys << key
+          present key, with: Entities::SSHKey
+        else
+          render_validation_error!(key)
         end
+      end
 
-        desc 'Disable a deploy key for a project' do
-          detail 'This feature was added in GitLab 8.11'
-          success Entities::SSHKey
-        end
-        params do
-          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
-        end
-        delete ":id/#{path}/:key_id/disable" do
-          key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id])
-          key.destroy
+      desc 'Update an existing deploy key for a project' do
+        success Entities::SSHKey
+      end
+      params do
+        requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+        optional :title, type: String, desc: 'The name of the deploy key'
+        optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository"
+        at_least_one_of :title, :can_push
+      end
+      put ":id/deploy_keys/:key_id" do
+        key = DeployKey.find(params.delete(:key_id))
 
-          present key.deploy_key, with: Entities::SSHKey
-        end
+        authorize!(:update_deploy_key, key)
 
-        desc 'Delete existing deploy key of currently authenticated user' do
-          success Key
-        end
-        params do
-          requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+        if key.update_attributes(declared_params(include_missing: false))
+          present key, with: Entities::SSHKey
+        else
+          render_validation_error!(key)
         end
-        delete ":id/#{path}/:key_id" do
-          key = user_project.deploy_keys.find(params[:key_id])
-          key.destroy
+      end
+
+      desc 'Enable a deploy key for a project' do
+        detail 'This feature was added in GitLab 8.11'
+        success Entities::SSHKey
+      end
+      params do
+        requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+      end
+      post ":id/deploy_keys/:key_id/enable" do
+        key = ::Projects::EnableDeployKeyService.new(user_project,
+                                                      current_user, declared_params).execute
+
+        if key
+          present key, with: Entities::SSHKey
+        else
+          not_found!('Deploy Key')
         end
       end
+
+      desc 'Delete deploy key for a project' do
+        success Key
+      end
+      params do
+        requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+      end
+      delete ":id/deploy_keys/:key_id" do
+        key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id])
+        not_found!('Deploy Key') unless key
+
+        key.destroy
+      end
     end
   end
 end
diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb
index f782bcaf7e9..46b936897f6 100644
--- a/lib/api/deployments.rb
+++ b/lib/api/deployments.rb
@@ -1,19 +1,20 @@
 module API
-  # Deployments RESTfull API endpoints
+  # Deployments RESTful API endpoints
   class Deployments < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
     params do
       requires :id, type: String, desc: 'The project ID'
     end
-    resource :projects do
+    resource :projects, requirements: { id: %r{[^/]+} } do
       desc 'Get all deployments of the project' do
         detail 'This feature was introduced in GitLab 8.11.'
         success Entities::Deployment
       end
       params do
-        optional :page,     type: Integer, desc: 'Page number of the current request'
-        optional :per_page, type: Integer, desc: 'Number of items per page'
+        use :pagination
       end
       get ':id/deployments' do
         authorize! :read_deployment, user_project
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index c84a7ef19db..980b391c155 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -5,7 +5,10 @@ module API
     end
 
     class UserBasic < UserSafe
-      expose :id, :state, :avatar_url
+      expose :id, :state
+      expose :avatar_url do |user, options|
+        user.avatar_url(only_path: false)
+      end
 
       expose :web_url do |user, options|
         Gitlab::Routing.url_helpers.user_url(user)
@@ -14,19 +17,25 @@ module API
 
     class User < UserBasic
       expose :created_at
-      expose :is_admin?, as: :is_admin
       expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization
     end
 
+    class UserActivity < Grape::Entity
+      expose :username
+      expose :last_activity_on
+      expose :last_activity_on, as: :last_activity_at # Back-compat
+    end
+
     class Identity < Grape::Entity
       expose :provider, :extern_uid
     end
 
-    class UserFull < User
+    class UserPublic < User
       expose :last_sign_in_at
       expose :confirmed_at
+      expose :last_activity_on
       expose :email
-      expose :theme_id, :color_scheme_id, :projects_limit, :current_sign_in_at
+      expose :color_scheme_id, :projects_limit, :current_sign_in_at
       expose :identities, using: Entities::Identity
       expose :can_create_group?, as: :can_create_group
       expose :can_create_project?, as: :can_create_project
@@ -34,7 +43,11 @@ module API
       expose :external
     end
 
-    class UserLogin < UserFull
+    class UserWithAdmin < UserPublic
+      expose :admin?, as: :is_admin
+    end
+
+    class UserWithPrivateDetails < UserWithAdmin
       expose :private_token
     end
 
@@ -43,14 +56,14 @@ module API
     end
 
     class Hook < Grape::Entity
-      expose :id, :url, :created_at
+      expose :id, :url, :created_at, :push_events, :tag_push_events, :repository_update_events
+      expose :enable_ssl_verification
     end
 
     class ProjectHook < Hook
-      expose :project_id, :push_events
-      expose :issues_events, :merge_requests_events, :tag_push_events
-      expose :note_events, :build_events, :pipeline_events, :wiki_page_events
-      expose :enable_ssl_verification
+      expose :project_id, :issues_events, :merge_requests_events
+      expose :note_events, :pipeline_events, :wiki_page_events
+      expose :job_events
     end
 
     class BasicProjectDetails < Grape::Entity
@@ -70,38 +83,53 @@ module API
 
     class Project < Grape::Entity
       expose :id, :description, :default_branch, :tag_list
-      expose :public?, as: :public
       expose :archived?, as: :archived
-      expose :visibility_level, :ssh_url_to_repo, :http_url_to_repo, :web_url
+      expose :visibility, :ssh_url_to_repo, :http_url_to_repo, :web_url
       expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group }
       expose :name, :name_with_namespace
       expose :path, :path_with_namespace
       expose :container_registry_enabled
 
       # Expose old field names with the new permissions methods to keep API compatible
-      expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:user]) }
-      expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:user]) }
-      expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:user]) }
-      expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:user]) }
-      expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) }
+      expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) }
+      expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) }
+      expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) }
+      expose(:jobs_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) }
+      expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) }
 
       expose :created_at, :last_activity_at
       expose :shared_runners_enabled
       expose :lfs_enabled?, as: :lfs_enabled
       expose :creator_id
-      expose :namespace
+      expose :namespace, using: 'API::Entities::Namespace'
       expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? }
-      expose :avatar_url
+      expose :import_status
+      expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] }
+      expose :avatar_url do |user, options|
+        user.avatar_url(only_path: false)
+      end
       expose :star_count, :forks_count
-      expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:user]) && project.default_issues_tracker? }
+      expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
-      expose :public_builds
+      expose :public_builds, as: :public_jobs
       expose :ci_config_file
       expose :shared_with_groups do |project, options|
         SharedGroup.represent(project.project_group_links.all, options)
       end
-      expose :only_allow_merge_if_build_succeeds
+      expose :only_allow_merge_if_pipeline_succeeds
       expose :request_access_enabled
+      expose :only_allow_merge_if_all_discussions_are_resolved
+      expose :printing_merge_request_link_enabled
+
+      expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics
+    end
+
+    class ProjectStatistics < Grape::Entity
+      expose :commit_count
+      expose :storage_size
+      expose :repository_size
+      expose :lfs_objects_size
+      expose :build_artifacts_size, as: :job_artifacts_size
     end
 
     class Member < UserBasic
@@ -123,11 +151,27 @@ module API
     end
 
     class Group < Grape::Entity
-      expose :id, :name, :path, :description, :visibility_level
+      expose :id, :name, :path, :description, :visibility
       expose :lfs_enabled?, as: :lfs_enabled
-      expose :avatar_url
+      expose :avatar_url do |user, options|
+        user.avatar_url(only_path: false)
+      end
       expose :web_url
       expose :request_access_enabled
+      expose :full_name, :full_path
+
+      if ::Group.supports_nested_groups?
+        expose :parent_id
+      end
+
+      expose :statistics, if: :statistics do
+        with_options format_with: -> (value) { value.to_i } do
+          expose :storage_size
+          expose :repository_size
+          expose :lfs_objects_size
+          expose :build_artifacts_size, as: :job_artifacts_size
+        end
+      end
     end
 
     class GroupDetail < Group
@@ -135,66 +179,78 @@ module API
       expose :shared_projects, using: Entities::Project
     end
 
+    class RepoCommit < Grape::Entity
+      expose :id, :short_id, :title, :created_at
+      expose :parent_ids
+      expose :safe_message, as: :message
+      expose :author_name, :author_email, :authored_date
+      expose :committer_name, :committer_email, :committed_date
+    end
+
+    class RepoCommitStats < Grape::Entity
+      expose :additions, :deletions, :total
+    end
+
+    class RepoCommitDetail < RepoCommit
+      expose :stats, using: Entities::RepoCommitStats
+      expose :status
+    end
+
     class RepoBranch < Grape::Entity
       expose :name
 
-      expose :commit do |repo_branch, options|
-        options[:project].repository.commit(repo_branch.target)
+      expose :commit, using: Entities::RepoCommit do |repo_branch, options|
+        options[:project].repository.commit(repo_branch.dereferenced_target)
+      end
+
+      expose :merged do |repo_branch, options|
+        options[:project].repository.merged_to_root_ref?(repo_branch.name)
       end
 
       expose :protected do |repo_branch, options|
-        options[:project].protected_branch? repo_branch.name
+        ProtectedBranch.protected?(options[:project], repo_branch.name)
       end
 
       expose :developers_can_push do |repo_branch, options|
-        project = options[:project]
-        access_levels = project.protected_branches.matching(repo_branch.name).map(&:push_access_levels).flatten
-        access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER }
+        options[:project].protected_branches.developers_can?(:push, repo_branch.name)
       end
 
       expose :developers_can_merge do |repo_branch, options|
-        project = options[:project]
-        access_levels = project.protected_branches.matching(repo_branch.name).map(&:merge_access_levels).flatten
-        access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER }
+        options[:project].protected_branches.developers_can?(:merge, repo_branch.name)
       end
     end
 
     class RepoTreeObject < Grape::Entity
-      expose :id, :name, :type
+      expose :id, :name, :type, :path
 
       expose :mode do |obj, options|
-        filemode = obj.mode.to_s(8)
+        filemode = obj.mode
         filemode = "0" + filemode if filemode.length < 6
         filemode
       end
     end
 
-    class RepoCommit < Grape::Entity
-      expose :id, :short_id, :title, :author_name, :author_email, :created_at
-      expose :safe_message, as: :message
-    end
-
-    class RepoCommitStats < Grape::Entity
-      expose :additions, :deletions, :total
-    end
+    class ProjectSnippet < Grape::Entity
+      expose :id, :title, :file_name, :description
+      expose :author, using: Entities::UserBasic
+      expose :updated_at, :created_at
 
-    class RepoCommitDetail < RepoCommit
-      expose :parent_ids, :committed_date, :authored_date
-      expose :stats, using: Entities::RepoCommitStats
-      expose :status
+      expose :web_url do |snippet, options|
+        Gitlab::UrlBuilder.build(snippet)
+      end
     end
 
-    class ProjectSnippet < Grape::Entity
-      expose :id, :title, :file_name
+    class PersonalSnippet < Grape::Entity
+      expose :id, :title, :file_name, :description
       expose :author, using: Entities::UserBasic
       expose :updated_at, :created_at
 
-      # TODO (rspeicher): Deprecated; remove in 9.0
-      expose(:expires_at) { |snippet| nil }
-
-      expose :web_url do |snippet, options|
+      expose :web_url do |snippet|
         Gitlab::UrlBuilder.build(snippet)
       end
+      expose :raw_url do |snippet|
+        Gitlab::UrlBuilder.build(snippet) + "/raw"
+      end
     end
 
     class ProjectEntity < Grape::Entity
@@ -206,21 +262,25 @@ module API
 
     class RepoDiff < Grape::Entity
       expose :old_path, :new_path, :a_mode, :b_mode, :diff
-      expose :new_file, :renamed_file, :deleted_file
+      expose :new_file?, as: :new_file
+      expose :renamed_file?, as: :renamed_file
+      expose :deleted_file?, as: :deleted_file
     end
 
     class Milestone < ProjectEntity
       expose :due_date
+      expose :start_date
     end
 
-    class Issue < ProjectEntity
+    class IssueBasic < ProjectEntity
       expose :label_names, as: :labels
       expose :milestone, using: Entities::Milestone
-      expose :assignee, :author, using: Entities::UserBasic
+      expose :assignees, :author, using: Entities::UserBasic
 
-      expose :subscribed do |issue, options|
-        issue.subscribed?(options[:current_user])
+      expose :assignee, using: ::API::Entities::UserBasic do |issue, options|
+        issue.assignees.first
       end
+
       expose :user_notes_count
       expose :upvotes, :downvotes
       expose :due_date
@@ -231,12 +291,25 @@ module API
       end
     end
 
+    class Issue < IssueBasic
+      expose :subscribed do |issue, options|
+        issue.subscribed?(options[:current_user], options[:project] || issue.project)
+      end
+    end
+
+    class IssuableTimeStats < Grape::Entity
+      expose :time_estimate
+      expose :total_time_spent
+      expose :human_time_estimate
+      expose :human_total_time_spent
+    end
+
     class ExternalIssue < Grape::Entity
       expose :title
       expose :id
     end
 
-    class MergeRequest < ProjectEntity
+    class MergeRequestBasic < ProjectEntity
       expose :target_branch, :source_branch
       expose :upvotes, :downvotes
       expose :author, :assignee, using: Entities::UserBasic
@@ -244,13 +317,10 @@ module API
       expose :label_names, as: :labels
       expose :work_in_progress?, as: :work_in_progress
       expose :milestone, using: Entities::Milestone
-      expose :merge_when_build_succeeds
+      expose :merge_when_pipeline_succeeds
       expose :merge_status
       expose :diff_head_sha, as: :sha
       expose :merge_commit_sha
-      expose :subscribed do |merge_request, options|
-        merge_request.subscribed?(options[:current_user])
-      end
       expose :user_notes_count
       expose :should_remove_source_branch?, as: :should_remove_source_branch
       expose :force_remove_source_branch?, as: :force_remove_source_branch
@@ -260,9 +330,15 @@ module API
       end
     end
 
+    class MergeRequest < MergeRequestBasic
+      expose :subscribed do |merge_request, options|
+        merge_request.subscribed?(options[:current_user], options[:project])
+      end
+    end
+
     class MergeRequestChanges < MergeRequest
       expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _|
-        compare.raw_diffs(all_diffs: true).to_a
+        compare.raw_diffs(limits: false).to_a
       end
     end
 
@@ -275,16 +351,16 @@ module API
       expose :commits, using: Entities::RepoCommit
 
       expose :diffs, using: Entities::RepoDiff do |compare, _|
-        compare.raw_diffs(all_diffs: true).to_a
+        compare.raw_diffs(limits: false).to_a
       end
     end
 
     class SSHKey < Grape::Entity
-      expose :id, :title, :key, :created_at
+      expose :id, :title, :key, :created_at, :can_push
     end
 
     class SSHKeyWithUser < SSHKey
-      expose :user, using: Entities::UserFull
+      expose :user, using: Entities::UserPublic
     end
 
     class Note < Grape::Entity
@@ -295,9 +371,6 @@ module API
       expose :created_at, :updated_at
       expose :system?, as: :system
       expose :noteable_id, :noteable_type
-      # upvote? and downvote? are deprecated, always return false
-      expose(:upvote?)    { |note| false }
-      expose(:downvote?)  { |note| false }
     end
 
     class AwardEmoji < Grape::Entity
@@ -324,7 +397,7 @@ module API
 
     class CommitStatus < Grape::Entity
       expose :id, :sha, :ref, :status, :name, :target_url, :description,
-             :created_at, :started_at, :finished_at, :allow_failure
+             :created_at, :started_at, :finished_at, :allow_failure, :coverage
       expose :author, using: Entities::UserBasic
     end
 
@@ -337,9 +410,7 @@ module API
       expose :author, using: Entities::UserBasic, if: ->(event, options) { event.author }
 
       expose :author_username do |event, options|
-        if event.author
-          event.author.username
-        end
+        event.author&.username
       end
     end
 
@@ -355,7 +426,8 @@ module API
       expose :target_type
 
       expose :target do |todo, options|
-        Entities.const_get(todo.target_type).represent(todo.target, options)
+        target = todo.target_type == 'Commit' ? 'RepoCommit' : todo.target_type
+        Entities.const_get(target).represent(todo.target, options)
       end
 
       expose :target_url do |todo, options|
@@ -373,7 +445,7 @@ module API
     end
 
     class Namespace < Grape::Entity
-      expose :id, :path, :kind
+      expose :id, :name, :path, :kind, :full_path
     end
 
     class MemberAccess < Grape::Entity
@@ -409,12 +481,13 @@ module API
     class ProjectService < Grape::Entity
       expose :id, :title, :created_at, :updated_at, :active
       expose :push_events, :issues_events, :merge_requests_events
-      expose :tag_push_events, :note_events, :build_events, :pipeline_events
+      expose :tag_push_events, :note_events, :pipeline_events
+      expose :job_events
       # Expose serialized properties
       expose :properties do |service, options|
-        field_names = service.fields.
-          select { |field| options[:include_passwords] || field[:type] != 'password' }.
-          map { |field| field[:name] }
+        field_names = service.fields
+          .select { |field| options[:include_passwords] || field[:type] != 'password' }
+          .map { |field| field[:name] }
         service.properties.slice(*field_names)
       end
     end
@@ -422,26 +495,40 @@ module API
     class ProjectWithAccess < Project
       expose :permissions do
         expose :project_access, using: Entities::ProjectAccess do |project, options|
-          project.project_members.find_by(user_id: options[:user].id)
+          project.project_members.find_by(user_id: options[:current_user].id)
         end
 
         expose :group_access, using: Entities::GroupAccess do |project, options|
           if project.group
-            project.group.group_members.find_by(user_id: options[:user].id)
+            project.group.group_members.find_by(user_id: options[:current_user].id)
           end
         end
       end
     end
 
     class LabelBasic < Grape::Entity
-      expose :name, :color, :description
+      expose :id, :name, :color, :description
     end
 
     class Label < LabelBasic
-      expose :open_issues_count, :closed_issues_count, :open_merge_requests_count
+      expose :open_issues_count do |label, options|
+        label.open_issues_count(options[:current_user])
+      end
+
+      expose :closed_issues_count do |label, options|
+        label.closed_issues_count(options[:current_user])
+      end
+
+      expose :open_merge_requests_count do |label, options|
+        label.open_merge_requests_count(options[:current_user])
+      end
+
+      expose :priority do |label, options|
+        label.priority(options[:project])
+      end
 
       expose :subscribed do |label, options|
-        label.subscribed?(options[:current_user])
+        label.subscribed?(options[:current_user], options[:project])
       end
     end
 
@@ -468,7 +555,7 @@ module API
       end
 
       expose :diffs, using: Entities::RepoDiff do |compare, options|
-        compare.diffs(all_diffs: true).to_a
+        compare.diffs(limits: false).to_a
       end
 
       expose :compare_timeout do |compare, options|
@@ -498,12 +585,15 @@ module API
       expose :updated_at
       expose :home_page_url
       expose :default_branch_protection
-      expose :restricted_visibility_levels
+      expose(:restricted_visibility_levels) do |setting, _options|
+        setting.restricted_visibility_levels.map { |level| Gitlab::VisibilityLevel.string_level(level) }
+      end
       expose :max_attachment_size
       expose :session_expire_delay
-      expose :default_project_visibility
-      expose :default_snippet_visibility
-      expose :default_group_visibility
+      expose(:default_project_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_project_visibility) }
+      expose(:default_snippet_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_snippet_visibility) }
+      expose(:default_group_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_group_visibility) }
+      expose :default_artifacts_expire_in
       expose :domain_whitelist
       expose :domain_blacklist_enabled
       expose :domain_blacklist
@@ -511,8 +601,16 @@ module API
       expose :after_sign_out_path
       expose :container_registry_token_expire_delay
       expose :repository_storage
+      expose :repository_storages
       expose :koding_enabled
       expose :koding_url
+      expose :plantuml_enabled
+      expose :plantuml_url
+      expose :terminal_max_session_time
+      expose :polling_interval_multiplier
+      expose :help_page_hide_commercial_content
+      expose :help_page_text
+      expose :help_page_support_url
     end
 
     class Release < Grape::Entity
@@ -524,7 +622,7 @@ module API
       expose :name, :message
 
       expose :commit do |repo_tag, options|
-        options[:project].repository.commit(repo_tag.target)
+        options[:project].repository.commit(repo_tag.dereferenced_target)
       end
 
       expose :release, using: Entities::Release do |repo_tag, options|
@@ -532,10 +630,6 @@ module API
       end
     end
 
-    class TriggerRequest < Grape::Entity
-      expose :id, :variables
-    end
-
     class Runner < Grape::Entity
       expose :id
       expose :description
@@ -550,9 +644,9 @@ module API
       expose :locked
       expose :version, :revision, :platform, :architecture
       expose :contacted_at
-      expose :token, if: lambda { |runner, options| options[:current_user].is_admin? || !runner.is_shared? }
+      expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.is_shared? }
       expose :projects, with: Entities::BasicProjectDetails do |runner, options|
-        if options[:current_user].is_admin?
+        if options[:current_user].admin?
           runner.projects
         else
           options[:current_user].authorized_projects.where(id: runner.projects)
@@ -560,7 +654,11 @@ module API
       end
     end
 
-    class BuildArtifactFile < Grape::Entity
+    class RunnerRegistrationDetails < Grape::Entity
+      expose :id, :token
+    end
+
+    class JobArtifactFile < Grape::Entity
       expose :filename, :size
     end
 
@@ -568,22 +666,26 @@ module API
       expose :id, :sha, :ref, :status
     end
 
-    class Build < Grape::Entity
+    class Job < Grape::Entity
       expose :id, :status, :stage, :name, :ref, :tag, :coverage
       expose :created_at, :started_at, :finished_at
       expose :user, with: User
-      expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? }
+      expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? }
       expose :commit, with: RepoCommit
       expose :runner, with: Runner
       expose :pipeline, with: PipelineBasic
     end
 
     class Trigger < Grape::Entity
-      expose :token, :created_at, :updated_at, :deleted_at, :last_used
+      expose :id
+      expose :token, :description
+      expose :created_at, :updated_at, :deleted_at, :last_used
+      expose :owner, using: Entities::UserBasic
     end
 
     class Variable < Grape::Entity
       expose :key, :value
+      expose :protected?, as: :protected
     end
 
     class Pipeline < PipelineBasic
@@ -592,21 +694,33 @@ module API
       expose :user, with: Entities::UserBasic
       expose :created_at, :updated_at, :started_at, :finished_at, :committed_at
       expose :duration
+      expose :coverage
+    end
+
+    class PipelineSchedule < Grape::Entity
+      expose :id
+      expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active
+      expose :created_at, :updated_at
+      expose :owner, using: Entities::UserBasic
+    end
+
+    class PipelineScheduleDetails < PipelineSchedule
+      expose :last_pipeline, using: Entities::PipelineBasic
     end
 
     class EnvironmentBasic < Grape::Entity
-      expose :id, :name, :external_url
+      expose :id, :name, :slug, :external_url
     end
 
     class Environment < EnvironmentBasic
-      expose :project, using: Entities::Project
+      expose :project, using: Entities::BasicProjectDetails
     end
 
     class Deployment < Grape::Entity
       expose :id, :iid, :ref, :sha, :created_at
       expose :user,        using: Entities::UserBasic
       expose :environment, using: Entities::EnvironmentBasic
-      expose :deployable,  using: Entities::Build
+      expose :deployable,  using: Entities::Job
     end
 
     class RepoLicense < Grape::Entity
@@ -633,5 +747,125 @@ module API
       expose :id, :message, :starts_at, :ends_at, :color, :font
       expose :active?, as: :active
     end
+
+    class PersonalAccessToken < Grape::Entity
+      expose :id, :name, :revoked, :created_at, :scopes
+      expose :active?, as: :active
+      expose :expires_at do |personal_access_token|
+        personal_access_token.expires_at ? personal_access_token.expires_at.strftime("%Y-%m-%d") : nil
+      end
+    end
+
+    class PersonalAccessTokenWithToken < PersonalAccessToken
+      expose :token
+    end
+
+    class ImpersonationToken < PersonalAccessTokenWithToken
+      expose :impersonation
+    end
+
+    class FeatureGate < Grape::Entity
+      expose :key
+      expose :value
+    end
+
+    class Feature < Grape::Entity
+      expose :name
+      expose :state
+      expose :gates, using: FeatureGate do |model|
+        model.gates.map do |gate|
+          value = model.gate_values[gate.key]
+
+          # By default all gate values are populated. Only show relevant ones.
+          if (value.is_a?(Integer) && value.zero?) || (value.is_a?(Set) && value.empty?)
+            next
+          end
+
+          { key: gate.key, value: value }
+        end.compact
+      end
+    end
+
+    module JobRequest
+      class JobInfo < Grape::Entity
+        expose :name, :stage
+        expose :project_id, :project_name
+      end
+
+      class GitInfo < Grape::Entity
+        expose :repo_url, :ref, :sha, :before_sha
+        expose :ref_type do |model|
+          if model.tag
+            'tag'
+          else
+            'branch'
+          end
+        end
+      end
+
+      class RunnerInfo < Grape::Entity
+        expose :timeout
+      end
+
+      class Step < Grape::Entity
+        expose :name, :script, :timeout, :when, :allow_failure
+      end
+
+      class Image < Grape::Entity
+        expose :name, :entrypoint
+      end
+
+      class Service < Image
+        expose :alias, :command
+      end
+
+      class Artifacts < Grape::Entity
+        expose :name, :untracked, :paths, :when, :expire_in
+      end
+
+      class Cache < Grape::Entity
+        expose :key, :untracked, :paths
+      end
+
+      class Credentials < Grape::Entity
+        expose :type, :url, :username, :password
+      end
+
+      class ArtifactFile < Grape::Entity
+        expose :filename, :size
+      end
+
+      class Dependency < Grape::Entity
+        expose :id, :name, :token
+        expose :artifacts_file, using: ArtifactFile, if: ->(job, _) { job.artifacts? }
+      end
+
+      class Response < Grape::Entity
+        expose :id
+        expose :token
+        expose :allow_git_fetch
+
+        expose :job_info, using: JobInfo do |model|
+          model
+        end
+
+        expose :git_info, using: GitInfo do |model|
+          model
+        end
+
+        expose :runner_info, using: RunnerInfo do |model|
+          model
+        end
+
+        expose :variables
+        expose :steps, using: Step
+        expose :image, using: Image
+        expose :services, using: Service
+        expose :artifacts, using: Artifacts
+        expose :cache, using: Cache
+        expose :credentials, using: Credentials
+        expose :dependencies, using: Dependency
+      end
+    end
   end
 end
diff --git a/lib/api/environments.rb b/lib/api/environments.rb
index 819f80d8365..945771d46f3 100644
--- a/lib/api/environments.rb
+++ b/lib/api/environments.rb
@@ -1,19 +1,21 @@
 module API
   # Environments RESTfull API endpoints
   class Environments < Grape::API
+    include ::API::Helpers::CustomValidators
+    include PaginationParams
+
     before { authenticate! }
 
     params do
       requires :id, type: String, desc: 'The project ID'
     end
-    resource :projects do
+    resource :projects, requirements: { id: %r{[^/]+} } do
       desc 'Get all environments of the project' do
         detail 'This feature was introduced in GitLab 8.11.'
         success Entities::Environment
       end
       params do
-        optional :page,     type: Integer, desc: 'Page number of the current request'
-        optional :per_page, type: Integer, desc: 'Number of items per page'
+        use :pagination
       end
       get ':id/environments' do
         authorize! :read_environment, user_project
@@ -28,12 +30,12 @@ module API
       params do
         requires :name,           type: String,   desc: 'The name of the environment to be created'
         optional :external_url,   type: String,   desc: 'URL on which this deployment is viewable'
+        optional :slug, absence: { message: "is automatically generated and cannot be changed" }
       end
       post ':id/environments' do
         authorize! :create_environment, user_project
 
-        create_params = declared(params, include_parent_namespaces: false).to_h
-        environment = user_project.environments.create(create_params)
+        environment = user_project.environments.create(declared_params)
 
         if environment.persisted?
           present environment, with: Entities::Environment
@@ -50,13 +52,14 @@ module API
         requires :environment_id, type: Integer,  desc: 'The environment ID'
         optional :name,           type: String,   desc: 'The new environment name'
         optional :external_url,   type: String,   desc: 'The new URL on which this deployment is viewable'
+        optional :slug, absence: { message: "is automatically generated and cannot be changed" }
       end
       put ':id/environments/:environment_id' do
         authorize! :update_environment, user_project
 
         environment = user_project.environments.find(params[:environment_id])
-        
-        update_params = declared(params, include_missing: false).extract!(:name, :external_url).to_h
+
+        update_params = declared_params(include_missing: false).extract!(:name, :external_url)
         if environment.update(update_params)
           present environment, with: Entities::Environment
         else
@@ -76,7 +79,24 @@ module API
 
         environment = user_project.environments.find(params[:environment_id])
 
-        present environment.destroy, with: Entities::Environment
+        environment.destroy
+      end
+
+      desc 'Stops an existing environment' do
+        success Entities::Environment
+      end
+      params do
+        requires :environment_id, type: Integer,  desc: 'The environment ID'
+      end
+      post ':id/environments/:environment_id/stop' do
+        authorize! :create_deployment, user_project
+
+        environment = user_project.environments.find(params[:environment_id])
+
+        environment.stop_with_action!(current_user)
+
+        status 200
+        present environment, with: Entities::Environment
       end
     end
   end
diff --git a/lib/api/events.rb b/lib/api/events.rb
new file mode 100644
index 00000000000..dabdf579119
--- /dev/null
+++ b/lib/api/events.rb
@@ -0,0 +1,86 @@
+module API
+  class Events < Grape::API
+    include PaginationParams
+
+    helpers do
+      params :event_filter_params do
+        optional :action, type: String, values: Event.actions, desc: 'Event action to filter on'
+        optional :target_type, type: String, values: Event.target_types, desc: 'Event target type to filter on'
+        optional :before, type: Date, desc: 'Include only events created before this date'
+        optional :after, type: Date, desc: 'Include only events created after this date'
+      end
+
+      params :sort_params do
+        optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                        desc: 'Return events sorted in ascending and descending order'
+      end
+
+      def present_events(events)
+        events = events.reorder(created_at: params[:sort])
+
+        present paginate(events), with: Entities::Event
+      end
+    end
+
+    resource :events do
+      desc "List currently authenticated user's events" do
+        detail 'This feature was introduced in GitLab 9.3.'
+        success Entities::Event
+      end
+      params do
+        use :pagination
+        use :event_filter_params
+        use :sort_params
+      end
+      get do
+        authenticate!
+
+        events = EventsFinder.new(params.merge(source: current_user, current_user: current_user)).execute.preload(:author, :target)
+
+        present_events(events)
+      end
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID or Username of the user'
+    end
+    resource :users do
+      desc 'Get the contribution events of a specified user' do
+        detail 'This feature was introduced in GitLab 8.13.'
+        success Entities::Event
+      end
+      params do
+        use :pagination
+        use :event_filter_params
+        use :sort_params
+      end
+      get ':id/events' do
+        user = find_user(params[:id])
+        not_found!('User') unless user
+
+        events = EventsFinder.new(params.merge(source: user, current_user: current_user)).execute.preload(:author, :target)
+
+        present_events(events)
+      end
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc "List a Project's visible events" do
+        success Entities::Event
+      end
+      params do
+        use :pagination
+        use :event_filter_params
+        use :sort_params
+      end
+      get ":id/events" do
+        events = EventsFinder.new(params.merge(source: user_project, current_user: current_user)).execute.preload(:author, :target)
+
+        present_events(events)
+      end
+    end
+  end
+end
diff --git a/lib/api/features.rb b/lib/api/features.rb
new file mode 100644
index 00000000000..cff0ba2ddff
--- /dev/null
+++ b/lib/api/features.rb
@@ -0,0 +1,36 @@
+module API
+  class Features < Grape::API
+    before { authenticated_as_admin! }
+
+    resource :features do
+      desc 'Get a list of all features' do
+        success Entities::Feature
+      end
+      get do
+        features = Feature.all
+
+        present features, with: Entities::Feature, current_user: current_user
+      end
+
+      desc 'Set the gate value for the given feature' do
+        success Entities::Feature
+      end
+      params do
+        requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time'
+      end
+      post ':name' do
+        feature = Feature.get(params[:name])
+
+        if %w(0 false).include?(params[:value])
+          feature.disable
+        elsif params[:value] == 'true'
+          feature.enable
+        else
+          feature.enable_percentage_of_time(params[:value].to_i)
+        end
+
+        present feature, with: Entities::Feature, current_user: current_user
+      end
+    end
+  end
+end
diff --git a/lib/api/files.rb b/lib/api/files.rb
index 96510e651a3..521287ee2b4 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -1,163 +1,145 @@
 module API
-  # Projects API
   class Files < Grape::API
-    before { authenticate! }
-
     helpers do
       def commit_params(attrs)
         {
           file_path: attrs[:file_path],
-          source_branch: attrs[:branch_name],
-          target_branch: attrs[:branch_name],
+          start_branch: attrs[:branch],
+          branch_name: attrs[:branch],
           commit_message: attrs[:commit_message],
           file_content: attrs[:content],
           file_content_encoding: attrs[:encoding],
           author_email: attrs[:author_email],
-          author_name: attrs[:author_name]
+          author_name: attrs[:author_name],
+          last_commit_sha: attrs[:last_commit_id]
         }
       end
 
+      def assign_file_vars!
+        authorize! :download_code, user_project
+
+        @commit = user_project.commit(params[:ref])
+        not_found!('Commit') unless @commit
+
+        @repo = user_project.repository
+        @blob = @repo.blob_at(@commit.sha, params[:file_path])
+
+        not_found!('File') unless @blob
+        @blob.load_all_data!
+      end
+
       def commit_response(attrs)
         {
           file_path: attrs[:file_path],
-          branch_name: attrs[:branch_name]
+          branch: attrs[:branch]
         }
       end
-    end
 
-    resource :projects do
-      # Get file from repository
-      # File content is Base64 encoded
-      #
-      # Parameters:
-      #   file_path (required) - The path to the file. Ex. lib/class.rb
-      #   ref (required) - The name of branch, tag or commit
-      #
-      # Example Request:
-      #   GET /projects/:id/repository/files
-      #
-      # Example response:
-      # {
-      #   "file_name": "key.rb",
-      #   "file_path": "app/models/key.rb",
-      #   "size": 1476,
-      #   "encoding": "base64",
-      #   "content": "IyA9PSBTY2hlbWEgSW5mb3...",
-      #   "ref": "master",
-      #   "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83",
-      #   "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50",
-      #   "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d",
-      # }
-      #
-      get ":id/repository/files" do
-        authorize! :download_code, user_project
+      params :simple_file_params do
+        requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb'
+        requires :branch, type: String, desc: 'The name of branch'
+        requires :commit_message, type: String, desc: 'Commit Message'
+        optional :author_email, type: String, desc: 'The email of the author'
+        optional :author_name, type: String, desc: 'The name of the author'
+      end
 
-        required_attributes! [:file_path, :ref]
-        attrs = attributes_for_keys [:file_path, :ref]
-        ref = attrs.delete(:ref)
-        file_path = attrs.delete(:file_path)
+      params :extended_file_params do
+        use :simple_file_params
+        requires :content, type: String, desc: 'File content'
+        optional :encoding, type: String, values: %w[base64], desc: 'File encoding'
+        optional :last_commit_id, type: String, desc: 'Last known commit id for this file'
+      end
+    end
 
-        commit = user_project.commit(ref)
-        not_found! 'Commit' unless commit
+    params do
+      requires :id, type: String, desc: 'The project ID'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get raw file contents from the repository'
+      params do
+        requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb'
+        requires :ref, type: String, desc: 'The name of branch, tag commit'
+      end
+      get ":id/repository/files/:file_path/raw" do
+        assign_file_vars!
 
-        repo = user_project.repository
-        blob = repo.blob_at(commit.sha, file_path)
+        send_git_blob @repo, @blob
+      end
 
-        if blob
-          blob.load_all_data!(repo)
-          status(200)
+      desc 'Get a file from the repository'
+      params do
+        requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb'
+        requires :ref, type: String, desc: 'The name of branch, tag or commit'
+      end
+      get ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do
+        assign_file_vars!
 
-          {
-            file_name: blob.name,
-            file_path: blob.path,
-            size: blob.size,
-            encoding: "base64",
-            content: Base64.strict_encode64(blob.data),
-            ref: ref,
-            blob_id: blob.id,
-            commit_id: commit.id,
-            last_commit_id: repo.last_commit_for_path(commit.sha, file_path).id
-          }
-        else
-          not_found! 'File'
-        end
+        {
+          file_name: @blob.name,
+          file_path: @blob.path,
+          size: @blob.size,
+          encoding: "base64",
+          content: Base64.strict_encode64(@blob.data),
+          ref: params[:ref],
+          blob_id: @blob.id,
+          commit_id: @commit.id,
+          last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path])
+        }
       end
 
-      # Create new file in repository
-      #
-      # Parameters:
-      #   file_path (required) - The path to new file. Ex. lib/class.rb
-      #   branch_name (required) - The name of branch
-      #   content (required) - File content
-      #   commit_message (required) - Commit message
-      #
-      # Example Request:
-      #   POST /projects/:id/repository/files
-      #
-      post ":id/repository/files" do
+      desc 'Create new file in repository'
+      params do
+        use :extended_file_params
+      end
+      post ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do
         authorize! :push_code, user_project
 
-        required_attributes! [:file_path, :branch_name, :content, :commit_message]
-        attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name]
-        result = ::Files::CreateService.new(user_project, current_user, commit_params(attrs)).execute
+        file_params = declared_params(include_missing: false)
+        result = ::Files::CreateService.new(user_project, current_user, commit_params(file_params)).execute
 
         if result[:status] == :success
           status(201)
-          commit_response(attrs)
+          commit_response(file_params)
         else
           render_api_error!(result[:message], 400)
         end
       end
 
-      # Update existing file in repository
-      #
-      # Parameters:
-      #   file_path (optional) - The path to file. Ex. lib/class.rb
-      #   branch_name (required) - The name of branch
-      #   content (required) - File content
-      #   commit_message (required) - Commit message
-      #
-      # Example Request:
-      #   PUT /projects/:id/repository/files
-      #
-      put ":id/repository/files" do
+      desc 'Update existing file in repository'
+      params do
+        use :extended_file_params
+      end
+      put ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do
         authorize! :push_code, user_project
 
-        required_attributes! [:file_path, :branch_name, :content, :commit_message]
-        attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name]
-        result = ::Files::UpdateService.new(user_project, current_user, commit_params(attrs)).execute
+        file_params = declared_params(include_missing: false)
+
+        begin
+          result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute
+        rescue ::Files::UpdateService::FileChangedError => e
+          render_api_error!(e.message, 400)
+        end
 
         if result[:status] == :success
           status(200)
-          commit_response(attrs)
+          commit_response(file_params)
         else
           http_status = result[:http_status] || 400
           render_api_error!(result[:message], http_status)
         end
       end
 
-      # Delete existing file in repository
-      #
-      # Parameters:
-      #   file_path (optional) - The path to file. Ex. lib/class.rb
-      #   branch_name (required) - The name of branch
-      #   content (required) - File content
-      #   commit_message (required) - Commit message
-      #
-      # Example Request:
-      #   DELETE /projects/:id/repository/files
-      #
-      delete ":id/repository/files" do
+      desc 'Delete an existing file in repository'
+      params do
+        use :simple_file_params
+      end
+      delete ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do
         authorize! :push_code, user_project
 
-        required_attributes! [:file_path, :branch_name, :commit_message]
-        attrs = attributes_for_keys [:file_path, :branch_name, :commit_message, :author_email, :author_name]
-        result = ::Files::DeleteService.new(user_project, current_user, commit_params(attrs)).execute
+        file_params = declared_params(include_missing: false)
+        result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute
 
-        if result[:status] == :success
-          status(200)
-          commit_response(attrs)
-        else
+        if result[:status] != :success
           render_api_error!(result[:message], 400)
         end
       end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index bfb89475025..ebbaed0cbb7 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -1,127 +1,174 @@
 module API
-  # groups API
   class Groups < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
+    helpers do
+      params :optional_params_ce do
+        optional :description, type: String, desc: 'The description of the group'
+        optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group'
+        optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
+        optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
+        optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group'
+      end
+
+      params :optional_params do
+        use :optional_params_ce
+      end
+
+      params :statistics_params do
+        optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
+      end
+
+      def present_groups(groups, options = {})
+        options = options.reverse_merge(
+          with: Entities::Group,
+          current_user: current_user
+        )
+
+        groups = groups.with_statistics if options[:statistics]
+        present paginate(groups), options
+      end
+    end
+
     resource :groups do
-      # Get a groups list
-      #
-      # Parameters:
-      #   skip_groups (optional) - Array of group ids to exclude from list
-      # Example Request:
-      #  GET /groups
+      desc 'Get a groups list' do
+        success Entities::Group
+      end
+      params do
+        use :statistics_params
+        optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list'
+        optional :all_available, type: Boolean, desc: 'Show all group that you have access to'
+        optional :search, type: String, desc: 'Search for a specific group'
+        optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
+        optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path'
+        optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)'
+        use :pagination
+      end
       get do
-        @groups = if current_user.admin
-                    Group.all
-                  else
-                    current_user.groups
-                  end
-
-        @groups = @groups.search(params[:search]) if params[:search].present?
-        @groups = @groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
-        @groups = paginate @groups
-        present @groups, with: Entities::Group
-      end
-
-      # Create group. Available only for users who can create groups.
-      #
-      # Parameters:
-      #   name (required)                   - The name of the group
-      #   path (required)                   - The path of the group
-      #   description (optional)            - The description of the group
-      #   visibility_level (optional)       - The visibility level of the group
-      #   lfs_enabled (optional)            - Enable/disable LFS for the projects in this group
-      #   request_access_enabled (optional) - Allow users to request member access
-      # Example Request:
-      #   POST /groups
+        groups = if params[:owned]
+                   current_user.owned_groups
+                 elsif current_user.admin
+                   Group.all
+                 elsif params[:all_available]
+                   GroupsFinder.new(current_user).execute
+                 else
+                   current_user.groups
+                 end
+
+        groups = groups.search(params[:search]) if params[:search].present?
+        groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
+        groups = groups.reorder(params[:order_by] => params[:sort])
+
+        present_groups groups, statistics: params[:statistics] && current_user.admin?
+      end
+
+      desc 'Create a group. Available only for users who can create groups.' do
+        success Entities::Group
+      end
+      params do
+        requires :name, type: String, desc: 'The name of the group'
+        requires :path, type: String, desc: 'The path of the group'
+
+        if ::Group.supports_nested_groups?
+          optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group'
+        end
+
+        use :optional_params
+      end
       post do
         authorize! :create_group
-        required_attributes! [:name, :path]
 
-        attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled]
-        @group = Group.new(attrs)
+        group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute
 
-        if @group.save
-          @group.add_owner(current_user)
-          present @group, with: Entities::Group
+        if group.persisted?
+          present group, with: Entities::GroupDetail, current_user: current_user
         else
-          render_api_error!("Failed to save group #{@group.errors.messages}", 400)
+          render_api_error!("Failed to save group #{group.errors.messages}", 400)
         end
       end
+    end
 
-      # Update group. Available only for users who can administrate groups.
-      #
-      # Parameters:
-      #   id (required)                     - The ID of a group
-      #   path (optional)                   - The path of the group
-      #   description (optional)            - The description of the group
-      #   visibility_level (optional)       - The visibility level of the group
-      #   lfs_enabled (optional)            - Enable/disable LFS for the projects in this group
-      #   request_access_enabled (optional) - Allow users to request member access
-      # Example Request:
-      #   PUT /groups/:id
+    params do
+      requires :id, type: String, desc: 'The ID of a group'
+    end
+    resource :groups, requirements: { id: %r{[^/]+} } do
+      desc 'Update a group. Available only for users who can administrate groups.' do
+        success Entities::Group
+      end
+      params do
+        optional :name, type: String, desc: 'The name of the group'
+        optional :path, type: String, desc: 'The path of the group'
+        use :optional_params
+      end
       put ':id' do
-        group = find_group(params[:id])
+        group = find_group!(params[:id])
         authorize! :admin_group, group
 
-        attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled]
-
-        if ::Groups::UpdateService.new(group, current_user, attrs).execute
-          present group, with: Entities::GroupDetail
+        if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute
+          present group, with: Entities::GroupDetail, current_user: current_user
         else
           render_validation_error!(group)
         end
       end
 
-      # Get a single group, with containing projects
-      #
-      # Parameters:
-      #   id (required) - The ID of a group
-      # Example Request:
-      #   GET /groups/:id
+      desc 'Get a single group, with containing projects.' do
+        success Entities::GroupDetail
+      end
       get ":id" do
-        group = find_group(params[:id])
-        present group, with: Entities::GroupDetail
+        group = find_group!(params[:id])
+        present group, with: Entities::GroupDetail, current_user: current_user
       end
 
-      # Remove group
-      #
-      # Parameters:
-      #   id (required) - The ID of a group
-      # Example Request:
-      #   DELETE /groups/:id
+      desc 'Remove a group.'
       delete ":id" do
-        group = find_group(params[:id])
+        group = find_group!(params[:id])
         authorize! :admin_group, group
-        DestroyGroupService.new(group, current_user).execute
+        ::Groups::DestroyService.new(group, current_user).execute
+      end
+
+      desc 'Get a list of projects in this group.' do
+        success Entities::Project
       end
+      params do
+        optional :archived, type: Boolean, default: false, desc: 'Limit by archived status'
+        optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values,
+                              desc: 'Limit by visibility'
+        optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
+        optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
+                            default: 'created_at', desc: 'Return projects ordered by field'
+        optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                        desc: 'Return projects sorted in ascending and descending order'
+        optional :simple, type: Boolean, default: false,
+                          desc: 'Return only the ID, URL, name, and path of each project'
+        optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
+        optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
 
-      # Get a list of projects in this group
-      #
-      # Example Request:
-      #   GET /groups/:id/projects
+        use :pagination
+      end
       get ":id/projects" do
-        group = find_group(params[:id])
-        projects = GroupProjectsFinder.new(group).execute(current_user)
-        projects = paginate projects
-        present projects, with: Entities::Project, user: current_user
-      end
-
-      # Transfer a project to the Group namespace
-      #
-      # Parameters:
-      #   id - group id
-      #   project_id  - project id
-      # Example Request:
-      #   POST /groups/:id/projects/:project_id
-      post ":id/projects/:project_id" do
+        group = find_group!(params[:id])
+        projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute
+        projects = reorder_projects(projects)
+        entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
+        present paginate(projects), with: entity, current_user: current_user
+      end
+
+      desc 'Transfer a project to the group namespace. Available only for admin.' do
+        success Entities::GroupDetail
+      end
+      params do
+        requires :project_id, type: String, desc: 'The ID or path of the project'
+      end
+      post ":id/projects/:project_id", requirements: { project_id: /.+/ } do
         authenticated_as_admin!
-        group = Group.find_by(id: params[:id])
-        project = Project.find(params[:project_id])
+        group = find_group!(params[:id])
+        project = find_project!(params[:project_id])
         result = ::Projects::TransferService.new(project, current_user).execute(group)
 
         if result
-          present group
+          present group, with: Entities::GroupDetail, current_user: current_user
         else
           render_api_error!("Failed to transfer project #{project.errors.messages}", 400)
         end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 67473f300c9..2c73a6fdc4e 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -1,78 +1,56 @@
 module API
   module Helpers
-    PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN"
-    PRIVATE_TOKEN_PARAM = :private_token
-    SUDO_HEADER = "HTTP_SUDO"
-    SUDO_PARAM = :sudo
-
-    def to_boolean(value)
-      return true if value =~ /^(true|t|yes|y|1|on)$/i
-      return false if value =~ /^(false|f|no|n|0|off)$/i
+    include Gitlab::Utils
+    include Helpers::Pagination
 
-      nil
-    end
+    SUDO_HEADER = "HTTP_SUDO".freeze
+    SUDO_PARAM = :sudo
 
-    def private_token
-      params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]
+    def declared_params(options = {})
+      options = { include_parent_namespaces: false }.merge(options)
+      declared(params, options).to_h.symbolize_keys
     end
 
-    def warden
-      env['warden']
-    end
+    def current_user
+      return @current_user if defined?(@current_user)
 
-    # Check the Rails session for valid authentication details
-    #
-    # Until CSRF protection is added to the API, disallow this method for
-    # state-changing endpoints
-    def find_user_from_warden
-      warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD'])
-    end
+      @current_user = initial_current_user
 
-    def find_user_by_private_token
-      token = private_token
-      return nil unless token.present?
+      sudo!
 
-      User.find_by_authentication_token(token) || User.find_by_personal_access_token(token)
+      @current_user
     end
 
-    def current_user
-      @current_user ||= find_user_by_private_token
-      @current_user ||= doorkeeper_guard
-      @current_user ||= find_user_from_warden
+    def sudo?
+      initial_current_user != current_user
+    end
 
-      unless @current_user && Gitlab::UserAccess.new(@current_user).allowed?
-        return nil
-      end
+    def user_project
+      @project ||= find_project!(params[:id])
+    end
 
-      identifier = sudo_identifier()
+    def available_labels
+      @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute
+    end
 
-      # If the sudo is the current user do nothing
-      if identifier && !(@current_user.id == identifier || @current_user.username == identifier)
-        forbidden!('Must be admin to use sudo') unless @current_user.is_admin?
-        @current_user = User.by_username_or_id(identifier)
-        not_found!("No user id or username for: #{identifier}") if @current_user.nil?
+    def find_user(id)
+      if id =~ /^\d+$/
+        User.find_by(id: id)
+      else
+        User.find_by(username: id)
       end
-
-      @current_user
     end
 
-    def sudo_identifier
-      identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER]
-
-      # Regex for integers
-      if !!(identifier =~ /\A[0-9]+\z/)
-        identifier.to_i
+    def find_project(id)
+      if id =~ /^\d+$/
+        Project.find_by(id: id)
       else
-        identifier
+        Project.find_by_full_path(id)
       end
     end
 
-    def user_project
-      @project ||= find_project(params[:id])
-    end
-
-    def find_project(id)
-      project = Project.find_with_namespace(id) || Project.find_by(id: id)
+    def find_project!(id)
+      project = find_project(id)
 
       if can?(current_user, :read_project, project)
         project
@@ -81,34 +59,16 @@ module API
       end
     end
 
-    def project_service
-      @project_service ||= begin
-        underscored_service = params[:service_slug].underscore
-
-        if Service.available_services_names.include?(underscored_service)
-          user_project.build_missing_services
-
-          service_method = "#{underscored_service}_service"
-
-          send_service(service_method)
-        end
-      end
-
-      @project_service || not_found!("Service")
-    end
-
-    def send_service(service_method)
-      user_project.send(service_method)
-    end
-
-    def service_attributes
-      @service_attributes ||= project_service.fields.inject([]) do |arr, hash|
-        arr << hash[:name].to_sym
+    def find_group(id)
+      if id =~ /^\d+$/
+        Group.find_by(id: id)
+      else
+        Group.find_by_full_path(id)
       end
     end
 
-    def find_group(id)
-      group = Group.find_by(path: id) || Group.find_by(id: id)
+    def find_group!(id)
+      group = find_group(id)
 
       if can?(current_user, :read_group, group)
         group
@@ -118,24 +78,35 @@ module API
     end
 
     def find_project_label(id)
-      label = user_project.labels.find_by_id(id) || user_project.labels.find_by_title(id)
+      label = available_labels.find_by_id(id) || available_labels.find_by_title(id)
       label || not_found!('Label')
     end
 
-    def find_project_issue(id)
-      issue = user_project.issues.find(id)
-      not_found! unless can?(current_user, :read_issue, issue)
-      issue
+    def find_project_issue(iid)
+      IssuesFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid)
     end
 
-    def paginate(relation)
-      relation.page(params[:page]).per(params[:per_page].to_i).tap do |data|
-        add_pagination_headers(data)
-      end
+    def find_project_merge_request(iid)
+      MergeRequestsFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid)
+    end
+
+    def find_project_snippet(id)
+      finder_params = { project: user_project }
+      SnippetsFinder.new(current_user, finder_params).execute.find(id)
+    end
+
+    def find_merge_request_with_access(iid, access_level = :read_merge_request)
+      merge_request = user_project.merge_requests.find_by!(iid: iid)
+      authorize! access_level, merge_request
+      merge_request
     end
 
     def authenticate!
-      unauthorized! unless current_user
+      unauthorized! unless current_user && can?(initial_current_user, :access_api)
+    end
+
+    def authenticate_non_get!
+      authenticate! unless %w[GET HEAD].include?(route.request_method)
     end
 
     def authenticate_by_gitlab_shell_token!
@@ -146,10 +117,11 @@ module API
     end
 
     def authenticated_as_admin!
-      forbidden! unless current_user.is_admin?
+      authenticate!
+      forbidden! unless current_user.admin?
     end
 
-    def authorize!(action, subject = nil)
+    def authorize!(action, subject = :global)
       forbidden! unless can?(current_user, action, subject)
     end
 
@@ -167,7 +139,7 @@ module API
       end
     end
 
-    def can?(object, action, subject)
+    def can?(object, action, subject = :global)
       Ability.allowed?(object, action, subject)
     end
 
@@ -186,68 +158,21 @@ module API
       params_hash = custom_params || params
       attrs = {}
       keys.each do |key|
-        if params_hash[key].present? or (params_hash.has_key?(key) and params_hash[key] == false)
+        if params_hash[key].present? || (params_hash.key?(key) && params_hash[key] == false)
           attrs[key] = params_hash[key]
         end
       end
       ActionController::Parameters.new(attrs).permit!
     end
 
-    # Helper method for validating all labels against its names
-    def validate_label_params(params)
-      errors = {}
-
-      if params[:labels].present?
-        params[:labels].split(',').each do |label_name|
-          label = user_project.labels.create_with(
-            color: Label::DEFAULT_COLOR).find_or_initialize_by(
-              title: label_name.strip)
-
-          if label.invalid?
-            errors[label.title] = label.errors
-          end
-        end
-      end
-
-      errors
-    end
-
-    # Checks the occurrences of datetime attributes, each attribute if present in the params hash must be in ISO 8601
-    # format (YYYY-MM-DDTHH:MM:SSZ) or a Bad Request error is invoked.
-    #
-    # Parameters:
-    #   keys (required) - An array consisting of elements that must be parseable as dates from the params hash
-    def datetime_attributes!(*keys)
-      keys.each do |key|
-        begin
-          params[key] = Time.xmlschema(params[key]) if params[key].present?
-        rescue ArgumentError
-          message = "\"" + key.to_s + "\" must be a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ"
-          render_api_error!(message, 400)
-        end
-      end
-    end
-
-    def issuable_order_by
-      if params["order_by"] == 'updated_at'
-        'updated_at'
-      else
-        'created_at'
-      end
-    end
-
-    def issuable_sort
-      if params["sort"] == 'asc'
-        :asc
-      else
-        :desc
-      end
-    end
-
     def filter_by_iid(items, iid)
       items.where(iid: iid)
     end
 
+    def filter_by_search(items, text)
+      items.search(text)
+    end
+
     # error helpers
 
     def forbidden!(reason = nil)
@@ -293,14 +218,22 @@ module API
       render_api_error!('204 No Content', 204)
     end
 
+    def accepted!
+      render_api_error!('202 Accepted', 202)
+    end
+
     def render_validation_error!(model)
       if model.errors.any?
         render_api_error!(model.errors.messages || '400 Bad Request', 400)
       end
     end
 
+    def render_spam_error!
+      render_api_error!({ error: 'Spam detected' }, 400)
+    end
+
     def render_api_error!(message, status)
-      error!({ 'message' => message }, status)
+      error!({ 'message' => message }, status, header)
     end
 
     def handle_api_exception(exception)
@@ -321,41 +254,21 @@ module API
       rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500)
     end
 
-    # Projects helpers
-
-    def filter_projects(projects)
-      # If the archived parameter is passed, limit results accordingly
-      if params[:archived].present?
-        projects = projects.where(archived: to_boolean(params[:archived]))
-      end
-
-      if params[:search].present?
-        projects = projects.search(params[:search])
-      end
-
-      if params[:visibility].present?
-        projects = projects.search_by_visibility(params[:visibility])
-      end
-
-      projects.reorder(project_order_by => project_sort)
-    end
-
-    def project_order_by
-      order_fields = %w(id name path created_at updated_at last_activity_at)
+    # project helpers
 
-      if order_fields.include?(params['order_by'])
-        params['order_by']
-      else
-        'created_at'
-      end
+    def reorder_projects(projects)
+      projects.reorder(params[:order_by] => params[:sort])
     end
 
-    def project_sort
-      if params["sort"] == 'asc'
-        :asc
-      else
-        :desc
-      end
+    def project_finder_params
+      finder_params = {}
+      finder_params[:owned] = true if params[:owned].present?
+      finder_params[:non_public] = true if params[:membership].present?
+      finder_params[:starred] = true if params[:starred].present?
+      finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility]
+      finder_params[:archived] = params[:archived]
+      finder_params[:search] = params[:search] if params[:search]
+      finder_params
     end
 
     # file helpers
@@ -378,7 +291,7 @@ module API
       UploadedFile.new(
         file_path,
         params["#{field}.name"],
-        params["#{field}.type"] || 'application/octet-stream',
+        params["#{field}.type"] || 'application/octet-stream'
       )
     end
 
@@ -394,42 +307,77 @@ module API
         header['X-Sendfile'] = path
         body
       else
-        file FileStreamer.new(path)
+        file path
+      end
+    end
+
+    def present_artifacts!(artifacts_file)
+      return not_found! unless artifacts_file.exists?
+  
+      if artifacts_file.file_storage?
+        present_file!(artifacts_file.path, artifacts_file.filename)
+      else
+        redirect_to(artifacts_file.url)
       end
     end
 
     private
 
-    def add_pagination_headers(paginated_data)
-      header 'X-Total',       paginated_data.total_count.to_s
-      header 'X-Total-Pages', paginated_data.total_pages.to_s
-      header 'X-Per-Page',    paginated_data.limit_value.to_s
-      header 'X-Page',        paginated_data.current_page.to_s
-      header 'X-Next-Page',   paginated_data.next_page.to_s
-      header 'X-Prev-Page',   paginated_data.prev_page.to_s
-      header 'Link',          pagination_links(paginated_data)
+    def private_token
+      params[APIGuard::PRIVATE_TOKEN_PARAM] || env[APIGuard::PRIVATE_TOKEN_HEADER]
     end
 
-    def pagination_links(paginated_data)
-      request_url = request.url.split('?').first
-      request_params = params.clone
-      request_params[:per_page] = paginated_data.limit_value
+    def warden
+      env['warden']
+    end
 
-      links = []
+    # Check the Rails session for valid authentication details
+    #
+    # Until CSRF protection is added to the API, disallow this method for
+    # state-changing endpoints
+    def find_user_from_warden
+      warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD'])
+    end
 
-      request_params[:page] = paginated_data.current_page - 1
-      links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page?
+    def initial_current_user
+      return @initial_current_user if defined?(@initial_current_user)
+      Gitlab::Auth::UniqueIpsLimiter.limit_user! do
+        @initial_current_user ||= find_user_by_private_token(scopes: @scopes)
+        @initial_current_user ||= doorkeeper_guard(scopes: @scopes)
+        @initial_current_user ||= find_user_from_warden
 
-      request_params[:page] = paginated_data.current_page + 1
-      links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page?
+        unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed?
+          @initial_current_user = nil
+        end
 
-      request_params[:page] = 1
-      links << %(<#{request_url}?#{request_params.to_query}>; rel="first")
+        @initial_current_user
+      end
+    end
+
+    def sudo!
+      return unless sudo_identifier
+      return unless initial_current_user
+
+      unless initial_current_user.admin?
+        forbidden!('Must be admin to use sudo')
+      end
 
-      request_params[:page] = paginated_data.total_pages
-      links << %(<#{request_url}?#{request_params.to_query}>; rel="last")
+      # Only private tokens should be used for the SUDO feature
+      unless private_token == initial_current_user.private_token
+        forbidden!('Private token must be specified in order to use sudo')
+      end
+
+      sudoed_user = find_user(sudo_identifier)
 
-      links.join(', ')
+      if sudoed_user
+        @current_user = sudoed_user
+      else
+        not_found!("No user id or username for: #{sudo_identifier}")
+      end
+    end
+
+    def sudo_identifier
+      @sudo_identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER]
     end
 
     def secret_token
@@ -446,14 +394,6 @@ module API
       header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format))
     end
 
-    def issue_entity(project)
-      if project.has_external_issue_tracker?
-        Entities::ExternalIssue
-      else
-        Entities::Issue
-      end
-    end
-
     # The Grape Error Middleware only has access to env but no params. We workaround this by
     # defining a method that returns the right value.
     def define_params_for_grape_middleware
diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb
new file mode 100644
index 00000000000..322624c6092
--- /dev/null
+++ b/lib/api/helpers/common_helpers.rb
@@ -0,0 +1,13 @@
+module API
+  module Helpers
+    module CommonHelpers
+      def convert_parameters_from_legacy_format(params)
+        params.tap do |params|
+          if params[:assignee_id].present?
+            params[:assignee_ids] = [params.delete(:assignee_id)]
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb
new file mode 100644
index 00000000000..0a8f3073a50
--- /dev/null
+++ b/lib/api/helpers/custom_validators.rb
@@ -0,0 +1,14 @@
+module API
+  module Helpers
+    module CustomValidators
+      class Absence < Grape::Validations::Base
+        def validate_param!(attr_name, params)
+          return if params.respond_to?(:key?) && !params.key?(attr_name)
+          raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence)
+        end
+      end
+    end
+  end
+end
+
+Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence)
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
new file mode 100644
index 00000000000..5e9cf5e68b1
--- /dev/null
+++ b/lib/api/helpers/internal_helpers.rb
@@ -0,0 +1,68 @@
+module API
+  module Helpers
+    module InternalHelpers
+      def wiki?
+        set_project unless defined?(@wiki)
+        @wiki
+      end
+
+      def project
+        set_project unless defined?(@project)
+        @project
+      end
+      
+      def redirected_path
+        @redirected_path
+      end
+
+      def ssh_authentication_abilities
+        [
+          :read_project,
+          :download_code,
+          :push_code
+        ]
+      end
+
+      def parse_env
+        return {} if params[:env].blank?
+
+        JSON.parse(params[:env])
+      rescue JSON::ParserError
+        {}
+      end
+
+      def log_user_activity(actor)
+        commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS
+
+        ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action])
+      end
+
+      private
+
+      def set_project
+        if params[:gl_repository]
+          @project, @wiki = Gitlab::GlRepository.parse(params[:gl_repository])
+          @redirected_path = nil
+        else
+          @project, @wiki, @redirected_path = Gitlab::RepoPath.parse(params[:project])
+        end
+      end
+
+      # Project id to pass between components that don't share/don't have
+      # access to the same filesystem mounts
+      def gl_repository
+        Gitlab::GlRepository.gl_repository(project, wiki?)
+      end
+
+      # Return the repository full path so that gitlab-shell has it when
+      # handling ssh commands
+      def repository_path
+        if wiki?
+          project.wiki.repository.path_to_repo
+        else
+          project.repository.path_to_repo
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb
index 90114f6f667..d9cae1501f8 100644
--- a/lib/api/helpers/members_helpers.rb
+++ b/lib/api/helpers/members_helpers.rb
@@ -2,7 +2,7 @@ module API
   module Helpers
     module MembersHelpers
       def find_source(source_type, id)
-        public_send("find_#{source_type}", id)
+        public_send("find_#{source_type}!", id)
       end
 
       def authorize_admin_source!(source_type, source)
diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb
new file mode 100644
index 00000000000..0764b58fb4c
--- /dev/null
+++ b/lib/api/helpers/pagination.rb
@@ -0,0 +1,45 @@
+module API
+  module Helpers
+    module Pagination
+      def paginate(relation)
+        relation.page(params[:page]).per(params[:per_page]).tap do |data|
+          add_pagination_headers(data)
+        end
+      end
+
+      private
+
+      def add_pagination_headers(paginated_data)
+        header 'X-Total',       paginated_data.total_count.to_s
+        header 'X-Total-Pages', paginated_data.total_pages.to_s
+        header 'X-Per-Page',    paginated_data.limit_value.to_s
+        header 'X-Page',        paginated_data.current_page.to_s
+        header 'X-Next-Page',   paginated_data.next_page.to_s
+        header 'X-Prev-Page',   paginated_data.prev_page.to_s
+        header 'Link',          pagination_links(paginated_data)
+      end
+
+      def pagination_links(paginated_data)
+        request_url = request.url.split('?').first
+        request_params = params.clone
+        request_params[:per_page] = paginated_data.limit_value
+
+        links = []
+
+        request_params[:page] = paginated_data.current_page - 1
+        links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page?
+
+        request_params[:page] = paginated_data.current_page + 1
+        links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page?
+
+        request_params[:page] = 1
+        links << %(<#{request_url}?#{request_params.to_query}>; rel="first")
+
+        request_params[:page] = paginated_data.total_pages
+        links << %(<#{request_url}?#{request_params.to_query}>; rel="last")
+
+        links.join(', ')
+      end
+    end
+  end
+end
diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
new file mode 100644
index 00000000000..1369b021ea4
--- /dev/null
+++ b/lib/api/helpers/runner.rb
@@ -0,0 +1,73 @@
+module API
+  module Helpers
+    module Runner
+      JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze
+      JOB_TOKEN_PARAM = :token
+      UPDATE_RUNNER_EVERY = 10 * 60
+
+      def runner_registration_token_valid?
+        ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token],
+                                                                  current_application_settings.runners_registration_token)
+      end
+
+      def get_runner_version_from_params
+        return unless params['info'].present?
+        attributes_for_keys(%w(name version revision platform architecture), params['info'])
+      end
+
+      def authenticate_runner!
+        forbidden! unless current_runner
+      end
+
+      def current_runner
+        @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s)
+      end
+
+      def update_runner_info
+        return unless update_runner?
+
+        current_runner.contacted_at = Time.now
+        current_runner.assign_attributes(get_runner_version_from_params)
+        current_runner.save if current_runner.changed?
+      end
+
+      def update_runner?
+        # Use a random threshold to prevent beating DB updates.
+        # It generates a distribution between [40m, 80m].
+        #
+        contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
+
+        current_runner.contacted_at.nil? ||
+          (Time.now - current_runner.contacted_at) >= contacted_at_max_age
+      end
+
+      def validate_job!(job)
+        not_found! unless job
+
+        yield if block_given?
+
+        forbidden!('Project has been deleted!') unless job.project
+        forbidden!('Job has been erased!') if job.erased?
+      end
+
+      def authenticate_job!
+        job = Ci::Build.find_by_id(params[:id])
+
+        validate_job!(job) do
+          forbidden! unless job_token_valid?(job)
+        end
+
+        job
+      end
+
+      def job_token_valid?(job)
+        token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s
+        token && job.valid_token?(token)
+      end
+
+      def max_artifacts_size
+        current_application_settings.max_artifacts_size.megabytes.to_i
+      end
+    end
+  end
+end
diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 9a5d1ece070..f1c79970ba4 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -3,51 +3,24 @@ module API
   class Internal < Grape::API
     before { authenticate_by_gitlab_shell_token! }
 
+    helpers ::API::Helpers::InternalHelpers
+
     namespace 'internal' do
       # Check if git command is allowed to project
       #
       # Params:
       #   key_id - ssh key id for Git over SSH
       #   user_id - user id for Git over HTTP
+      #   protocol - Git access protocol being used, e.g. HTTP or SSH
       #   project - project path with namespace
       #   action - git action (git-upload-pack or git-receive-pack)
-      #   ref - branch name
-      #   forced_push - forced_push
-      #   protocol - Git access protocol being used, e.g. HTTP or SSH
-      #
-
-      helpers do
-        def wiki?
-          @wiki ||= params[:project].end_with?('.wiki') &&
-            !Project.find_with_namespace(params[:project])
-        end
-
-        def project
-          @project ||= begin
-            project_path = params[:project]
-
-            # Check for *.wiki repositories.
-            # Strip out the .wiki from the pathname before finding the
-            # project. This applies the correct project permissions to
-            # the wiki repository as well.
-            project_path.chomp!('.wiki') if wiki?
-
-            Project.find_with_namespace(project_path)
-          end
-        end
-
-        def ssh_authentication_abilities
-          [
-            :read_project,
-            :download_code,
-            :push_code
-          ]
-        end
-      end
-
+      #   changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
       post "/allowed" do
         status 200
 
+        # Stores some Git-specific env thread-safely
+        Gitlab::Git::Env.set(parse_env)
+
         actor =
           if params[:key_id]
             Key.find_by(id: params[:key_id])
@@ -57,35 +30,33 @@ module API
 
         protocol = params[:protocol]
 
-        access =
-          if wiki?
-            Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities)
-          else
-            Gitlab::GitAccess.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities)
-          end
-
-        access_status = access.check(params[:action], params[:changes])
+        actor.update_last_used_at if actor.is_a?(Key)
 
-        response = { status: access_status.status, message: access_status.message }
+        access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess
+        access_checker = access_checker_klass
+          .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities, redirected_path: redirected_path)
 
-        if access_status.status
-          # Return the repository full path so that gitlab-shell has it when
-          # handling ssh commands
-          response[:repository_path] =
-            if wiki?
-              project.wiki.repository.path_to_repo
-            else
-              project.repository.path_to_repo
-            end
+        begin
+          access_checker.check(params[:action], params[:changes])
+        rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e
+          return { status: false, message: e.message }
         end
 
-        response
+        log_user_activity(actor)
+
+        {
+          status: true,
+          gl_repository: gl_repository,
+          repository_path: repository_path
+        }
       end
 
       post "/lfs_authenticate" do
         status 200
 
         key = Key.find(params[:key_id])
+        key.update_last_used_at
+
         token_handler = Gitlab::LfsToken.new(key)
 
         {
@@ -100,23 +71,36 @@ module API
       end
 
       #
-      # Discover user by ssh key
+      # Discover user by ssh key or user id
       #
       get "/discover" do
-        key = Key.find(params[:key_id])
-        present key.user, with: Entities::UserSafe
+        if params[:key_id]
+          key = Key.find(params[:key_id])
+          user = key.user
+        elsif params[:user_id]
+          user = User.find_by(id: params[:user_id])
+        end
+        present user, with: Entities::UserSafe
       end
 
       get "/check" do
         {
           api_version: API.version,
           gitlab_version: Gitlab::VERSION,
-          gitlab_rev: Gitlab::REVISION,
+          gitlab_rev: Gitlab::REVISION
         }
       end
 
+      get "/broadcast_messages" do
+        if messages = BroadcastMessage.current
+          present messages, with: Entities::BroadcastMessage
+        else
+          []
+        end
+      end
+
       get "/broadcast_message" do
-        if message = BroadcastMessage.current
+        if message = BroadcastMessage.current.last
           present message, with: Entities::BroadcastMessage
         else
           {}
@@ -128,7 +112,9 @@ module API
 
         key = Key.find_by(id: params[:key_id])
 
-        unless key
+        if key
+          key.update_last_used_at
+        else
           return { 'success' => false, 'message' => 'Could not find the given key' }
         end
 
@@ -146,11 +132,28 @@ module API
           return { success: false, message: 'Two-factor authentication is not enabled for this user' }
         end
 
-        codes = user.generate_otp_backup_codes!
-        user.save!
+        codes = nil
+
+        ::Users::UpdateService.new(user).execute! do |user|
+          codes = user.generate_otp_backup_codes!
+        end
 
         { success: true, recovery_codes: codes }
       end
+
+      post "/notify_post_receive" do
+        status 200
+
+        # TODO: Re-enable when Gitaly is processing the post-receive notification
+        # return unless Gitlab::GitalyClient.enabled?
+        #
+        # begin
+        #   repository = wiki? ? project.wiki.repository : project.repository
+        #   Gitlab::GitalyClient::Notifications.new(repository.raw_repository).post_receive
+        # rescue GRPC::Unavailable => e
+        #   render_api_error!(e, 500)
+        # end
+      end
     end
   end
 end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index c9689e6f8ef..09dca0dff8b 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -1,264 +1,246 @@
 module API
-  # Issues API
   class Issues < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
     helpers do
-      def filter_issues_state(issues, state)
-        case state
-        when 'opened' then issues.opened
-        when 'closed' then issues.closed
-        else issues
-        end
+      def find_issues(args = {})
+        args = params.merge(args)
+
+        args.delete(:id)
+        args[:milestone_title] = args.delete(:milestone)
+        args[:label_name] = args.delete(:labels)
+
+        issues = IssuesFinder.new(current_user, args).execute
+
+        issues.reorder(args[:order_by] => args[:sort])
       end
 
-      def filter_issues_labels(issues, labels)
-        issues.includes(:labels).where('labels.title' => labels.split(','))
+      params :issues_params do
+        optional :labels, type: String, desc: 'Comma-separated list of label names'
+        optional :milestone, type: String, desc: 'Milestone title'
+        optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at',
+                            desc: 'Return issues ordered by `created_at` or `updated_at` fields.'
+        optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                        desc: 'Return issues sorted in `asc` or `desc` order.'
+        optional :milestone, type: String, desc: 'Return issues for a specific milestone'
+        optional :iids, type: Array[Integer], desc: 'The IID array of issues'
+        optional :search, type: String, desc: 'Search issues for text present in the title or description'
+        optional :created_after, type: DateTime, desc: 'Return issues created after the specified time'
+        optional :created_before, type: DateTime, desc: 'Return issues created before the specified time'
+        use :pagination
       end
 
-      def filter_issues_milestone(issues, milestone)
-        issues.includes(:milestone).where('milestones.title' => milestone)
+      params :issue_params_ce do
+        optional :description, type: String, desc: 'The description of an issue'
+        optional :assignee_ids, type: Array[Integer], desc: 'The array of user IDs to assign issue'
+        optional :assignee_id,  type: Integer, desc: '[Deprecated] The ID of a user to assign issue'
+        optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue'
+        optional :labels, type: String, desc: 'Comma-separated list of label names'
+        optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY'
+        optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential'
+      end
+
+      params :issue_params do
+        use :issue_params_ce
       end
     end
 
     resource :issues do
-      # Get currently authenticated user's issues
-      #
-      # Parameters:
-      #   state (optional) - Return "opened" or "closed" issues
-      #   labels (optional) - Comma-separated list of label names
-      #   order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at`
-      #   sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc`
-      #
-      # Example Requests:
-      #   GET /issues
-      #   GET /issues?state=opened
-      #   GET /issues?state=closed
-      #   GET /issues?labels=foo
-      #   GET /issues?labels=foo,bar
-      #   GET /issues?labels=foo,bar&state=opened
+      desc "Get currently authenticated user's issues" do
+        success Entities::IssueBasic
+      end
+      params do
+        optional :state, type: String, values: %w[opened closed all], default: 'all',
+                         desc: 'Return opened, closed, or all issues'
+        use :issues_params
+      end
       get do
-        issues = current_user.issues.inc_notes_with_associations
-        issues = filter_issues_state(issues, params[:state]) unless params[:state].nil?
-        issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil?
-        issues = issues.reorder(issuable_order_by => issuable_sort)
+        issues = find_issues(scope: 'authored')
 
-        present paginate(issues), with: Entities::Issue, current_user: current_user
+        present paginate(issues), with: Entities::IssueBasic, current_user: current_user
       end
     end
 
-    resource :groups do
-      # Get a list of group issues
-      #
-      # Parameters:
-      #   id (required) - The ID of a group
-      #   state (optional) - Return "opened" or "closed" issues
-      #   labels (optional) - Comma-separated list of label names
-      #   milestone (optional) - Milestone title
-      #   order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at`
-      #   sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc`
-      #
-      # Example Requests:
-      #   GET /groups/:id/issues
-      #   GET /groups/:id/issues?state=opened
-      #   GET /groups/:id/issues?state=closed
-      #   GET /groups/:id/issues?labels=foo
-      #   GET /groups/:id/issues?labels=foo,bar
-      #   GET /groups/:id/issues?labels=foo,bar&state=opened
-      #   GET /groups/:id/issues?milestone=1.0.0
-      #   GET /groups/:id/issues?milestone=1.0.0&state=closed
+    params do
+      requires :id, type: String, desc: 'The ID of a group'
+    end
+    resource :groups, requirements: { id: %r{[^/]+} } do
+      desc 'Get a list of group issues' do
+        success Entities::IssueBasic
+      end
+      params do
+        optional :state, type: String, values: %w[opened closed all], default: 'all',
+                         desc: 'Return opened, closed, or all issues'
+        use :issues_params
+      end
       get ":id/issues" do
-        group = find_group(params[:id])
-
-        params[:state] ||= 'opened'
-        params[:group_id] = group.id
-        params[:milestone_title] = params.delete(:milestone)
-        params[:label_name] = params.delete(:labels)
-
-        if params[:order_by] || params[:sort]
-          # The Sortable concern takes 'created_desc', not 'created_at_desc' (for example)
-          params[:sort] = "#{issuable_order_by.sub('_at', '')}_#{issuable_sort}"
-        end
+        group = find_group!(params[:id])
 
-        issues = IssuesFinder.new(current_user, params).execute
+        issues = find_issues(group_id: group.id)
 
-        present paginate(issues), with: Entities::Issue, current_user: current_user
+        present paginate(issues), with: Entities::IssueBasic, current_user: current_user
       end
     end
 
-    resource :projects do
-      # Get a list of project issues
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   iid (optional) - Return the project issue having the given `iid`
-      #   state (optional) - Return "opened" or "closed" issues
-      #   labels (optional) - Comma-separated list of label names
-      #   milestone (optional) - Milestone title
-      #   order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at`
-      #   sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc`
-      #
-      # Example Requests:
-      #   GET /projects/:id/issues
-      #   GET /projects/:id/issues?state=opened
-      #   GET /projects/:id/issues?state=closed
-      #   GET /projects/:id/issues?labels=foo
-      #   GET /projects/:id/issues?labels=foo,bar
-      #   GET /projects/:id/issues?labels=foo,bar&state=opened
-      #   GET /projects/:id/issues?milestone=1.0.0
-      #   GET /projects/:id/issues?milestone=1.0.0&state=closed
-      #   GET /issues?iid=42
-      get ":id/issues" do
-        issues = user_project.issues.inc_notes_with_associations.visible_to_user(current_user)
-        issues = filter_issues_state(issues, params[:state]) unless params[:state].nil?
-        issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil?
-        issues = filter_by_iid(issues, params[:iid]) unless params[:iid].nil?
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      include TimeTrackingEndpoints
 
-        unless params[:milestone].nil?
-          issues = filter_issues_milestone(issues, params[:milestone])
-        end
+      desc 'Get a list of project issues' do
+        success Entities::IssueBasic
+      end
+      params do
+        optional :state, type: String, values: %w[opened closed all], default: 'all',
+                         desc: 'Return opened, closed, or all issues'
+        use :issues_params
+      end
+      get ":id/issues" do
+        project = find_project!(params[:id])
 
-        issues = issues.reorder(issuable_order_by => issuable_sort)
+        issues = find_issues(project_id: project.id)
 
-        present paginate(issues), with: Entities::Issue, current_user: current_user
+        present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project
       end
 
-      # Get a single project issue
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   issue_id (required) - The ID of a project issue
-      # Example Request:
-      #   GET /projects/:id/issues/:issue_id
-      get ":id/issues/:issue_id" do
-        @issue = find_project_issue(params[:issue_id])
-        present @issue, with: Entities::Issue, current_user: current_user
+      desc 'Get a single project issue' do
+        success Entities::Issue
+      end
+      params do
+        requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
+      end
+      get ":id/issues/:issue_iid" do
+        issue = find_project_issue(params[:issue_iid])
+        present issue, with: Entities::Issue, current_user: current_user, project: user_project
       end
 
-      # Create a new project issue
-      #
-      # Parameters:
-      #   id (required)           - The ID of a project
-      #   title (required)        - The title of an issue
-      #   description (optional)  - The description of an issue
-      #   assignee_id (optional)  - The ID of a user to assign issue
-      #   milestone_id (optional) - The ID of a milestone to assign issue
-      #   labels (optional)       - The labels of an issue
-      #   created_at (optional)   - Date time string, ISO 8601 formatted
-      #   due_date (optional)     - Date time string in the format YEAR-MONTH-DAY
-      #   confidential (optional) - Boolean parameter if the issue should be confidential
-      # Example Request:
-      #   POST /projects/:id/issues
+      desc 'Create a new project issue' do
+        success Entities::Issue
+      end
+      params do
+        requires :title, type: String, desc: 'The title of an issue'
+        optional :created_at, type: DateTime,
+                              desc: 'Date time when the issue was created. Available only for admins and project owners.'
+        optional :merge_request_to_resolve_discussions_of, type: Integer,
+                                                           desc: 'The IID of a merge request for which to resolve discussions'
+        optional :discussion_to_resolve, type: String,
+                                         desc: 'The ID of a discussion to resolve, also pass `merge_request_to_resolve_discussions_of`'
+        use :issue_params
+      end
       post ':id/issues' do
-        required_attributes! [:title]
-
-        keys = [:title, :description, :assignee_id, :milestone_id, :due_date, :confidential]
-        keys << :created_at if current_user.admin? || user_project.owner == current_user
-        attrs = attributes_for_keys(keys)
-
-        # Validate label names in advance
-        if (errors = validate_label_params(params)).any?
-          render_api_error!({ labels: errors }, 400)
+        # Setting created_at time only allowed for admins and project owners
+        unless current_user.admin? || user_project.owner == current_user
+          params.delete(:created_at)
         end
 
-        attrs[:labels] = params[:labels] if params[:labels]
-
-        # Convert and filter out invalid confidential flags
-        attrs['confidential'] = to_boolean(attrs['confidential'])
-        attrs.delete('confidential') if attrs['confidential'].nil?
+        issue_params = declared_params(include_missing: false)
 
-        issue = ::Issues::CreateService.new(user_project, current_user, attrs.merge(request: request, api: true)).execute
+        issue_params = convert_parameters_from_legacy_format(issue_params)
 
+        issue = ::Issues::CreateService.new(user_project,
+                                            current_user,
+                                            issue_params.merge(request: request, api: true)).execute
         if issue.spam?
           render_api_error!({ error: 'Spam detected' }, 400)
         end
 
         if issue.valid?
-          present issue, with: Entities::Issue, current_user: current_user
+          present issue, with: Entities::Issue, current_user: current_user, project: user_project
         else
           render_validation_error!(issue)
         end
       end
 
-      # Update an existing issue
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   issue_id (required) - The ID of a project issue
-      #   title (optional) - The title of an issue
-      #   description (optional) - The description of an issue
-      #   assignee_id (optional) - The ID of a user to assign issue
-      #   milestone_id (optional) - The ID of a milestone to assign issue
-      #   labels (optional) - The labels of an issue
-      #   state_event (optional) - The state event of an issue (close|reopen)
-      #   updated_at (optional) - Date time string, ISO 8601 formatted
-      #   due_date (optional)     - Date time string in the format YEAR-MONTH-DAY
-      #   confidential (optional) - Boolean parameter if the issue should be confidential
-      # Example Request:
-      #   PUT /projects/:id/issues/:issue_id
-      put ':id/issues/:issue_id' do
-        issue = user_project.issues.find(params[:issue_id])
+      desc 'Update an existing issue' do
+        success Entities::Issue
+      end
+      params do
+        requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
+        optional :title, type: String, desc: 'The title of an issue'
+        optional :updated_at, type: DateTime,
+                              desc: 'Date time when the issue was updated. Available only for admins and project owners.'
+        optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue'
+        use :issue_params
+        at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id,
+                        :labels, :created_at, :due_date, :confidential, :state_event
+      end
+      put ':id/issues/:issue_iid' do
+        issue = user_project.issues.find_by!(iid: params.delete(:issue_iid))
         authorize! :update_issue, issue
-        keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date, :confidential]
-        keys << :updated_at if current_user.admin? || user_project.owner == current_user
-        attrs = attributes_for_keys(keys)
 
-        # Validate label names in advance
-        if (errors = validate_label_params(params)).any?
-          render_api_error!({ labels: errors }, 400)
+        # Setting created_at time only allowed for admins and project owners
+        unless current_user.admin? || user_project.owner == current_user
+          params.delete(:updated_at)
         end
 
-        attrs[:labels] = params[:labels] if params[:labels]
+        update_params = declared_params(include_missing: false).merge(request: request, api: true)
 
-        # Convert and filter out invalid confidential flags
-        attrs['confidential'] = to_boolean(attrs['confidential'])
-        attrs.delete('confidential') if attrs['confidential'].nil?
+        update_params = convert_parameters_from_legacy_format(update_params)
 
-        issue = ::Issues::UpdateService.new(user_project, current_user, attrs).execute(issue)
+        issue = ::Issues::UpdateService.new(user_project,
+                                            current_user,
+                                            update_params).execute(issue)
+
+        render_spam_error! if issue.spam?
 
         if issue.valid?
-          present issue, with: Entities::Issue, current_user: current_user
+          present issue, with: Entities::Issue, current_user: current_user, project: user_project
         else
           render_validation_error!(issue)
         end
       end
 
-      # Move an existing issue
-      #
-      # Parameters:
-      #  id (required)            - The ID of a project
-      #  issue_id (required)      - The ID of a project issue
-      #  to_project_id (required) - The ID of the new project
-      # Example Request:
-      #   POST /projects/:id/issues/:issue_id/move
-      post ':id/issues/:issue_id/move' do
-        required_attributes! [:to_project_id]
+      desc 'Move an existing issue' do
+        success Entities::Issue
+      end
+      params do
+        requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
+        requires :to_project_id, type: Integer, desc: 'The ID of the new project'
+      end
+      post ':id/issues/:issue_iid/move' do
+        issue = user_project.issues.find_by(iid: params[:issue_iid])
+        not_found!('Issue') unless issue
 
-        issue = user_project.issues.find(params[:issue_id])
-        new_project = Project.find(params[:to_project_id])
+        new_project = Project.find_by(id: params[:to_project_id])
+        not_found!('Project') unless new_project
 
         begin
           issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project)
-          present issue, with: Entities::Issue, current_user: current_user
+          present issue, with: Entities::Issue, current_user: current_user, project: user_project
         rescue ::Issues::MoveService::MoveError => error
           render_api_error!(error.message, 400)
         end
       end
 
-      #
-      # Delete a project issue
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   issue_id (required) - The ID of a project issue
-      # Example Request:
-      #   DELETE /projects/:id/issues/:issue_id
-      delete ":id/issues/:issue_id" do
-        issue = user_project.issues.find_by(id: params[:issue_id])
+      desc 'Delete a project issue'
+      params do
+        requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
+      end
+      delete ":id/issues/:issue_iid" do
+        issue = user_project.issues.find_by(iid: params[:issue_iid])
+        not_found!('Issue') unless issue
 
         authorize!(:destroy_issue, issue)
         issue.destroy
       end
+
+      desc 'List merge requests closing issue'  do
+        success Entities::MergeRequestBasic
+      end
+      params do
+        requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue'
+      end
+      get ':id/issues/:issue_iid/closed_by' do
+        issue = find_project_issue(params[:issue_iid])
+
+        merge_request_ids = MergeRequestsClosingIssues.where(issue_id: issue).select(:merge_request_id)
+        merge_requests = MergeRequestsFinder.new(current_user, project_id: user_project.id).execute.where(id: merge_request_ids)
+
+        present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project
+      end
     end
   end
 end
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
new file mode 100644
index 00000000000..8a67de10bca
--- /dev/null
+++ b/lib/api/jobs.rb
@@ -0,0 +1,247 @@
+module API
+  class Jobs < Grape::API
+    include PaginationParams
+
+    before { authenticate! }
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      helpers do
+        params :optional_scope do
+          optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show',
+                           values: ::CommitStatus::AVAILABLE_STATUSES,
+                           coerce_with: ->(scope) {
+                             case scope
+                             when String
+                               [scope]
+                             when Hashie::Mash
+                               scope.values
+                             when Hashie::Array
+                               scope
+                             else
+                               ['unknown']
+                             end
+                           }
+        end
+      end
+
+      desc 'Get a projects jobs' do
+        success Entities::Job
+      end
+      params do
+        use :optional_scope
+        use :pagination
+      end
+      get ':id/jobs' do
+        builds = user_project.builds.order('id DESC')
+        builds = filter_builds(builds, params[:scope])
+
+        present paginate(builds), with: Entities::Job
+      end
+
+      desc 'Get pipeline jobs' do
+        success Entities::Job
+      end
+      params do
+        requires :pipeline_id, type: Integer,  desc: 'The pipeline ID'
+        use :optional_scope
+        use :pagination
+      end
+      get ':id/pipelines/:pipeline_id/jobs' do
+        pipeline = user_project.pipelines.find(params[:pipeline_id])
+        builds = pipeline.builds
+        builds = filter_builds(builds, params[:scope])
+
+        present paginate(builds), with: Entities::Job
+      end
+
+      desc 'Get a specific job of a project' do
+        success Entities::Job
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a job'
+      end
+      get ':id/jobs/:job_id' do
+        authorize_read_builds!
+
+        build = get_build!(params[:job_id])
+
+        present build, with: Entities::Job
+      end
+
+      desc 'Download the artifacts file from a job' do
+        detail 'This feature was introduced in GitLab 8.5'
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a job'
+      end
+      get ':id/jobs/:job_id/artifacts' do
+        authorize_read_builds!
+
+        build = get_build!(params[:job_id])
+
+        present_artifacts!(build.artifacts_file)
+      end
+
+      desc 'Download the artifacts file from a job' do
+        detail 'This feature was introduced in GitLab 8.10'
+      end
+      params do
+        requires :ref_name, type: String, desc: 'The ref from repository'
+        requires :job,      type: String, desc: 'The name for the job'
+      end
+      get ':id/jobs/artifacts/:ref_name/download',
+        requirements: { ref_name: /.+/ } do
+        authorize_read_builds!
+
+        builds = user_project.latest_successful_builds_for(params[:ref_name])
+        latest_build = builds.find_by!(name: params[:job])
+
+        present_artifacts!(latest_build.artifacts_file)
+      end
+
+      # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace
+      #       is saved in the DB instead of file). But before that, we need to consider how to replace the value of
+      #       `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse.
+      desc 'Get a trace of a specific job of a project'
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a job'
+      end
+      get ':id/jobs/:job_id/trace' do
+        authorize_read_builds!
+
+        build = get_build!(params[:job_id])
+
+        header 'Content-Disposition', "infile; filename=\"#{build.id}.log\""
+        content_type 'text/plain'
+        env['api.format'] = :binary
+
+        trace = build.trace.raw
+        body trace
+      end
+
+      desc 'Cancel a specific job of a project' do
+        success Entities::Job
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a job'
+      end
+      post ':id/jobs/:job_id/cancel' do
+        authorize_update_builds!
+
+        build = get_build!(params[:job_id])
+        authorize!(:update_build, build)
+
+        build.cancel
+
+        present build, with: Entities::Job
+      end
+
+      desc 'Retry a specific build of a project' do
+        success Entities::Job
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a build'
+      end
+      post ':id/jobs/:job_id/retry' do
+        authorize_update_builds!
+
+        build = get_build!(params[:job_id])
+        authorize!(:update_build, build)
+        return forbidden!('Job is not retryable') unless build.retryable?
+
+        build = Ci::Build.retry(build, current_user)
+
+        present build, with: Entities::Job
+      end
+
+      desc 'Erase job (remove artifacts and the trace)' do
+        success Entities::Job
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a build'
+      end
+      post ':id/jobs/:job_id/erase' do
+        authorize_update_builds!
+
+        build = get_build!(params[:job_id])
+        authorize!(:update_build, build)
+        return forbidden!('Job is not erasable!') unless build.erasable?
+
+        build.erase(erased_by: current_user)
+        present build, with: Entities::Job
+      end
+
+      desc 'Keep the artifacts to prevent them from being deleted' do
+        success Entities::Job
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a job'
+      end
+      post ':id/jobs/:job_id/artifacts/keep' do
+        authorize_update_builds!
+
+        build = get_build!(params[:job_id])
+        authorize!(:update_build, build)
+        return not_found!(build) unless build.artifacts?
+
+        build.keep_artifacts!
+
+        status 200
+        present build, with: Entities::Job
+      end
+
+      desc 'Trigger a manual job' do
+        success Entities::Job
+        detail 'This feature was added in GitLab 8.11'
+      end
+      params do
+        requires :job_id, type: Integer, desc: 'The ID of a Job'
+      end
+      post ":id/jobs/:job_id/play" do
+        authorize_read_builds!
+
+        build = get_build!(params[:job_id])
+
+        authorize!(:update_build, build)
+        bad_request!("Unplayable Job") unless build.playable?
+
+        build.play(current_user)
+
+        status 200
+        present build, with: Entities::Job
+      end
+    end
+
+    helpers do
+      def find_build(id)
+        user_project.builds.find_by(id: id.to_i)
+      end
+
+      def get_build!(id)
+        find_build(id) || not_found!
+      end
+
+      def filter_builds(builds, scope)
+        return builds if scope.nil? || scope.empty?
+
+        available_statuses = ::CommitStatus::AVAILABLE_STATUSES
+
+        unknown = scope - available_statuses
+        render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty?
+
+        builds.where(status: available_statuses && scope)
+      end
+
+      def authorize_read_builds!
+        authorize! :read_build, user_project
+      end
+
+      def authorize_update_builds!
+        authorize! :update_build, user_project
+      end
+    end
+  end
+end
diff --git a/lib/api/labels.rb b/lib/api/labels.rb
index c806829d69e..20b25529d0c 100644
--- a/lib/api/labels.rb
+++ b/lib/api/labels.rb
@@ -1,99 +1,99 @@
 module API
-  # Labels API
   class Labels < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
-    resource :projects do
-      # Get all labels of the project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   GET /projects/:id/labels
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get all labels of the project' do
+        success Entities::Label
+      end
+      params do
+        use :pagination
+      end
       get ':id/labels' do
-        present user_project.labels, with: Entities::Label, current_user: current_user
+        present paginate(available_labels), with: Entities::Label, current_user: current_user, project: user_project
       end
 
-      # Creates a new label
-      #
-      # Parameters:
-      #   id    (required)       - The ID of a project
-      #   name  (required)       - The name of the label to be created
-      #   color (required)       - Color of the label given in 6-digit hex
-      #                            notation with leading '#' sign (e.g. #FFAABB)
-      #   description (optional) - The description of label to be created
-      # Example Request:
-      #   POST /projects/:id/labels
+      desc 'Create a new label' do
+        success Entities::Label
+      end
+      params do
+        requires :name, type: String, desc: 'The name of the label to be created'
+        requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names"
+        optional :description, type: String, desc: 'The description of label to be created'
+        optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true
+      end
       post ':id/labels' do
         authorize! :admin_label, user_project
-        required_attributes! [:name, :color]
-
-        attrs = attributes_for_keys [:name, :color, :description]
-        label = user_project.find_label(attrs[:name])
 
+        label = available_labels.find_by(title: params[:name])
         conflict!('Label already exists') if label
 
-        label = user_project.labels.create(attrs)
+        priority = params.delete(:priority)
+        label = ::Labels::CreateService.new(declared_params(include_missing: false)).execute(project: user_project)
 
         if label.valid?
-          present label, with: Entities::Label, current_user: current_user
+          label.prioritize!(user_project, priority) if priority
+          present label, with: Entities::Label, current_user: current_user, project: user_project
         else
           render_validation_error!(label)
         end
       end
 
-      # Deletes an existing label
-      #
-      # Parameters:
-      #   id    (required) - The ID of a project
-      #   name  (required) - The name of the label to be deleted
-      #
-      # Example Request:
-      #   DELETE /projects/:id/labels
+      desc 'Delete an existing label' do
+        success Entities::Label
+      end
+      params do
+        requires :name, type: String, desc: 'The name of the label to be deleted'
+      end
       delete ':id/labels' do
         authorize! :admin_label, user_project
-        required_attributes! [:name]
 
-        label = user_project.find_label(params[:name])
+        label = user_project.labels.find_by(title: params[:name])
         not_found!('Label') unless label
 
         label.destroy
       end
 
-      # Updates an existing label. At least one optional parameter is required.
-      #
-      # Parameters:
-      #   id        (required)   - The ID of a project
-      #   name      (required)   - The name of the label to be deleted
-      #   new_name  (optional)   - The new name of the label
-      #   color     (optional)   - Color of the label given in 6-digit hex
-      #                            notation with leading '#' sign (e.g. #FFAABB)
-      #   description (optional) - The description of label to be created
-      # Example Request:
-      #   PUT /projects/:id/labels
+      desc 'Update an existing label. At least one optional parameter is required.' do
+        success Entities::Label
+      end
+      params do
+        requires :name,  type: String, desc: 'The name of the label to be updated'
+        optional :new_name, type: String, desc: 'The new name of the label'
+        optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names"
+        optional :description, type: String, desc: 'The new description of label'
+        optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true
+        at_least_one_of :new_name, :color, :description, :priority
+      end
       put ':id/labels' do
         authorize! :admin_label, user_project
-        required_attributes! [:name]
 
-        label = user_project.find_label(params[:name])
+        label = user_project.labels.find_by(title: params[:name])
         not_found!('Label not found') unless label
 
-        attrs = attributes_for_keys [:new_name, :color, :description]
-
-        if attrs.empty?
-          render_api_error!('Required parameters "new_name" or "color" ' \
-                            'missing',
-                            400)
-        end
-
+        update_priority = params.key?(:priority)
+        priority = params.delete(:priority)
+        label_params = declared_params(include_missing: false)
         # Rename new name to the actual label attribute name
-        attrs[:name] = attrs.delete(:new_name) if attrs.key?(:new_name)
+        label_params[:name] = label_params.delete(:new_name) if label_params.key?(:new_name)
 
-        if label.update(attrs)
-          present label, with: Entities::Label, current_user: current_user
-        else
-          render_validation_error!(label)
+        label = ::Labels::UpdateService.new(label_params).execute(label)
+        render_validation_error!(label) unless label.valid?
+
+        if update_priority
+          if priority.nil?
+            label.unprioritize!(user_project)
+          else
+            label.prioritize!(user_project, priority)
+          end
         end
+
+        present label, with: Entities::Label, current_user: current_user, project: user_project
       end
     end
   end
diff --git a/lib/api/license_templates.rb b/lib/api/license_templates.rb
deleted file mode 100644
index d0552299ed0..00000000000
--- a/lib/api/license_templates.rb
+++ /dev/null
@@ -1,58 +0,0 @@
-module API
-  # License Templates API
-  class LicenseTemplates < Grape::API
-    PROJECT_TEMPLATE_REGEX =
-      /[\<\{\[]
-        (project|description|
-        one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here
-      [\>\}\]]/xi.freeze
-    YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze
-    FULLNAME_TEMPLATE_REGEX =
-      /[\<\{\[]
-        (fullname|name\sof\s(author|copyright\sowner))
-      [\>\}\]]/xi.freeze
-
-    # Get the list of the available license templates
-    #
-    # Parameters:
-    #   popular - Filter licenses to only the popular ones
-    #
-    # Example Request:
-    #   GET /licenses
-    #   GET /licenses?popular=1
-    get 'licenses' do
-      options = {
-        featured: params[:popular].present? ? true : nil
-      }
-      present Licensee::License.all(options), with: Entities::RepoLicense
-    end
-
-    # Get text for specific license
-    #
-    # Parameters:
-    #   key (required) - The key of a license
-    #   project        - Copyrighted project name
-    #   fullname       - Full name of copyright holder
-    #
-    # Example Request:
-    #   GET /licenses/mit
-    #
-    get 'licenses/:key', requirements: { key: /[\w\.-]+/ } do
-      required_attributes! [:key]
-
-      not_found!('License') unless Licensee::License.find(params[:key])
-
-      # We create a fresh Licensee::License object since we'll modify its
-      # content in place below.
-      license = Licensee::License.new(params[:key])
-
-      license.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s)
-      license.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present?
-
-      fullname = params[:fullname].presence || current_user.try(:name)
-      license.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname
-
-      present license, with: Entities::RepoLicense
-    end
-  end
-end
diff --git a/lib/api/members.rb b/lib/api/members.rb
index b80818f0eb6..c200e46a328 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -1,5 +1,7 @@
 module API
   class Members < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
     helpers ::API::Helpers::MembersHelpers
@@ -8,21 +10,21 @@ module API
       params do
         requires :id, type: String, desc: "The #{source_type} ID"
       end
-      resource source_type.pluralize do
+      resource source_type.pluralize, requirements: { id: %r{[^/]+} } do
         desc 'Gets a list of group or project members viewable by the authenticated user.' do
           success Entities::Member
         end
         params do
           optional :query, type: String, desc: 'A query string to search for members'
+          use :pagination
         end
         get ":id/members" do
           source = find_source(source_type, params[:id])
 
           users = source.users
           users = users.merge(User.search(params[:query])) if params[:query]
-          users = paginate(users)
 
-          present users, with: Entities::Member, source: source
+          present paginate(users), with: Entities::Member, source: source
         end
 
         desc 'Gets a member of a group or project.' do
@@ -53,24 +55,13 @@ module API
           authorize_admin_source!(source_type, source)
 
           member = source.members.find_by(user_id: params[:user_id])
+          conflict!('Member already exists') if member
 
-          # We need this explicit check because `source.add_user` doesn't
-          # currently return the member created so it would return 201 even if
-          # the member already existed...
-          # The `source_type == 'group'` check is to ensure back-compatibility
-          # but 409 behavior should be used for both project and group members in 9.0!
-          conflict!('Member already exists') if source_type == 'group' && member
-
-          unless member
-            member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at])
-          end
+          member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at])
 
           if member.persisted? && member.valid?
             present member.user, with: Entities::Member, member: member
           else
-            # This is to ensure back-compatibility but 400 behavior should be used
-            # for all validation errors in 9.0!
-            render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level)
             render_validation_error!(member)
           end
         end
@@ -84,18 +75,14 @@ module API
           optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
         end
         put ":id/members/:user_id" do
-          source = find_source(source_type, params[:id])
+          source = find_source(source_type, params.delete(:id))
           authorize_admin_source!(source_type, source)
 
-          member = source.members.find_by!(user_id: params[:user_id])
-          attrs = attributes_for_keys [:access_level, :expires_at]
+          member = source.members.find_by!(user_id: params.delete(:user_id))
 
-          if member.update_attributes(attrs)
+          if member.update_attributes(declared_params(include_missing: false))
             present member.user, with: Entities::Member, member: member
           else
-            # This is to ensure back-compatibility but 400 behavior should be used
-            # for all validation errors in 9.0!
-            render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level)
             render_validation_error!(member)
           end
         end
@@ -106,24 +93,10 @@ module API
         end
         delete ":id/members/:user_id" do
           source = find_source(source_type, params[:id])
+          # Ensure that memeber exists
+          source.members.find_by!(user_id: params[:user_id])
 
-          # This is to ensure back-compatibility but find_by! should be used
-          # in that casse in 9.0!
-          member = source.members.find_by(user_id: params[:user_id])
-
-          # This is to ensure back-compatibility but this should be removed in
-          # favor of find_by! in 9.0!
-          not_found!("Member: user_id:#{params[:user_id]}") if source_type == 'group' && member.nil?
-
-          # This is to ensure back-compatibility but 204 behavior should be used
-          # for all DELETE endpoints in 9.0!
-          if member.nil?
-            { message: "Access revoked", id: params[:user_id].to_i }
-          else
-            ::Members::DestroyService.new(source, current_user, declared(params)).execute
-
-            present member.user, with: Entities::Member, member: member
-          end
+          ::Members::DestroyService.new(source, current_user, declared_params).execute
         end
       end
     end
diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb
index 07435d78468..4b79eac2b8b 100644
--- a/lib/api/merge_request_diffs.rb
+++ b/lib/api/merge_request_diffs.rb
@@ -1,25 +1,27 @@
 module API
   # MergeRequestDiff API
   class MergeRequestDiffs < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
-    resource :projects do
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
       desc 'Get a list of merge request diff versions' do
         detail 'This feature was introduced in GitLab 8.12.'
         success Entities::MergeRequestDiff
       end
 
       params do
-        requires :id, type: String, desc: 'The ID of a project'
-        requires :merge_request_id, type: Integer, desc: 'The ID of a merge request'
+        requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request'
+        use :pagination
       end
+      get ":id/merge_requests/:merge_request_iid/versions" do
+        merge_request = find_merge_request_with_access(params[:merge_request_iid])
 
-      get ":id/merge_requests/:merge_request_id/versions" do
-        merge_request = user_project.merge_requests.
-          find(params[:merge_request_id])
-
-        authorize! :read_merge_request, merge_request
-        present merge_request.merge_request_diffs, with: Entities::MergeRequestDiff
+        present paginate(merge_request.merge_request_diffs), with: Entities::MergeRequestDiff
       end
 
       desc 'Get a single merge request diff version' do
@@ -28,16 +30,13 @@ module API
       end
 
       params do
-        requires :id, type: String, desc: 'The ID of a project'
-        requires :merge_request_id, type: Integer, desc: 'The ID of a merge request'
+        requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request'
         requires :version_id, type: Integer, desc: 'The ID of a merge request diff version'
       end
 
-      get ":id/merge_requests/:merge_request_id/versions/:version_id" do
-        merge_request = user_project.merge_requests.
-          find(params[:merge_request_id])
+      get ":id/merge_requests/:merge_request_iid/versions/:version_id" do
+        merge_request = find_merge_request_with_access(params[:merge_request_iid])
 
-        authorize! :read_merge_request, merge_request
         present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull
       end
     end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 2b685621da9..1118fc7465b 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -1,9 +1,15 @@
 module API
-  # MergeRequest API
   class MergeRequests < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
-    resource :projects do
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      include TimeTrackingEndpoints
+
       helpers do
         def handle_merge_request_errors!(errors)
           if errors[:project_access].any?
@@ -14,323 +20,245 @@ module API
             error!(errors[:validate_fork], 422)
           elsif errors[:validate_branches].any?
             conflict!(errors[:validate_branches])
+          elsif errors[:base].any?
+            error!(errors[:base], 422)
           end
 
           render_api_error!(errors, 400)
         end
+
+        def issue_entity(project)
+          if project.has_external_issue_tracker?
+            Entities::ExternalIssue
+          else
+            Entities::IssueBasic
+          end
+        end
+
+        def find_merge_requests(args = {})
+          args = params.merge(args)
+
+          args[:milestone_title] = args.delete(:milestone)
+          args[:label_name] = args.delete(:labels)
+
+          merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations
+
+          merge_requests.reorder(args[:order_by] => args[:sort])
+        end
+
+        params :optional_params_ce do
+          optional :description, type: String, desc: 'The description of the merge request'
+          optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request'
+          optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request'
+          optional :labels, type: String, desc: 'Comma-separated list of label names'
+          optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging'
+        end
+
+        params :optional_params do
+          use :optional_params_ce
+        end
       end
 
-      # List merge requests
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   iid (optional) - Return the project MR having the given `iid`
-      #   state (optional) - Return requests "merged", "opened" or "closed"
-      #   order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at`
-      #   sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc`
-      #
-      # Example:
-      #   GET /projects/:id/merge_requests
-      #   GET /projects/:id/merge_requests?state=opened
-      #   GET /projects/:id/merge_requests?state=closed
-      #   GET /projects/:id/merge_requests?order_by=created_at
-      #   GET /projects/:id/merge_requests?order_by=updated_at
-      #   GET /projects/:id/merge_requests?sort=desc
-      #   GET /projects/:id/merge_requests?sort=asc
-      #   GET /projects/:id/merge_requests?iid=42
-      #
+      desc 'List merge requests' do
+        success Entities::MergeRequestBasic
+      end
+      params do
+        optional :state, type: String, values: %w[opened closed merged all], default: 'all',
+                         desc: 'Return opened, closed, merged, or all merge requests'
+        optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at',
+                            desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.'
+        optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                        desc: 'Return merge requests sorted in `asc` or `desc` order.'
+        optional :iids, type: Array[Integer], desc: 'The IID array of merge requests'
+        optional :milestone, type: String, desc: 'Return merge requests for a specific milestone'
+        optional :labels, type: String, desc: 'Comma-separated list of label names'
+        optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time'
+        optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time'
+        use :pagination
+      end
       get ":id/merge_requests" do
         authorize! :read_merge_request, user_project
-        merge_requests = user_project.merge_requests.inc_notes_with_associations
 
-        unless params[:iid].nil?
-          merge_requests = filter_by_iid(merge_requests, params[:iid])
-        end
-
-        merge_requests =
-          case params["state"]
-          when "opened" then merge_requests.opened
-          when "closed" then merge_requests.closed
-          when "merged" then merge_requests.merged
-          else merge_requests
-          end
+        merge_requests = find_merge_requests(project_id: user_project.id)
 
-        merge_requests = merge_requests.reorder(issuable_order_by => issuable_sort)
-        present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user
+        present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project
       end
 
-      # Create MR
-      #
-      # Parameters:
-      #
-      #   id (required)            - The ID of a project - this will be the source of the merge request
-      #   source_branch (required) - The source branch
-      #   target_branch (required) - The target branch
-      #   target_project_id        - The target project of the merge request defaults to the :id of the project
-      #   assignee_id              - Assignee user ID
-      #   title (required)         - Title of MR
-      #   description              - Description of MR
-      #   labels (optional)        - Labels for MR as a comma-separated list
-      #   milestone_id (optional)   - Milestone ID
-      #
-      # Example:
-      #   POST /projects/:id/merge_requests
-      #
+      desc 'Create a merge request' do
+        success Entities::MergeRequest
+      end
+      params do
+        requires :title, type: String, desc: 'The title of the merge request'
+        requires :source_branch, type: String, desc: 'The source branch'
+        requires :target_branch, type: String, desc: 'The target branch'
+        optional :target_project_id, type: Integer,
+                                     desc: 'The target project of the merge request defaults to the :id of the project'
+        use :optional_params
+      end
       post ":id/merge_requests" do
         authorize! :create_merge_request, user_project
-        required_attributes! [:source_branch, :target_branch, :title]
-        attrs = attributes_for_keys [:source_branch, :target_branch, :assignee_id, :title, :target_project_id, :description, :milestone_id]
 
-        # Validate label names in advance
-        if (errors = validate_label_params(params)).any?
-          render_api_error!({ labels: errors }, 400)
-        end
+        mr_params = declared_params(include_missing: false)
+        mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch)
 
-        merge_request = ::MergeRequests::CreateService.new(user_project, current_user, attrs).execute
+        merge_request = ::MergeRequests::CreateService.new(user_project, current_user, mr_params).execute
 
         if merge_request.valid?
-          # Find or create labels and attach to issue
-          if params[:labels].present?
-            merge_request.add_labels_by_names(params[:labels].split(","))
-          end
-
-          present merge_request, with: Entities::MergeRequest, current_user: current_user
+          present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project
         else
           handle_merge_request_errors! merge_request.errors
         end
       end
 
-      # Delete a MR
-      #
-      # Parameters:
-      # id (required)               - The ID of the project
-      # merge_request_id (required) - The MR id
-      delete ":id/merge_requests/:merge_request_id" do
-        merge_request = user_project.merge_requests.find_by(id: params[:merge_request_id])
+      desc 'Delete a merge request'
+      params do
+        requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request'
+      end
+      delete ":id/merge_requests/:merge_request_iid" do
+        merge_request = find_project_merge_request(params[:merge_request_iid])
 
         authorize!(:destroy_merge_request, merge_request)
         merge_request.destroy
       end
 
-      # Routing "merge_request/:merge_request_id/..." is DEPRECATED and WILL BE REMOVED in version 9.0
-      # Use "merge_requests/:merge_request_id/..." instead.
-      #
-      [":id/merge_request/:merge_request_id", ":id/merge_requests/:merge_request_id"].each do |path|
-        # Show MR
-        #
-        # Parameters:
-        #   id (required)               - The ID of a project
-        #   merge_request_id (required) - The ID of MR
-        #
-        # Example:
-        #   GET /projects/:id/merge_requests/:merge_request_id
-        #
-        get path do
-          merge_request = user_project.merge_requests.find(params[:merge_request_id])
-
-          authorize! :read_merge_request, merge_request
-
-          present merge_request, with: Entities::MergeRequest, current_user: current_user
-        end
+      params do
+        requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request'
+      end
+      desc 'Get a single merge request' do
+        success Entities::MergeRequest
+      end
+      get ':id/merge_requests/:merge_request_iid' do
+        merge_request = find_merge_request_with_access(params[:merge_request_iid])
 
-        # Show MR commits
-        #
-        # Parameters:
-        #   id (required)               - The ID of a project
-        #   merge_request_id (required) - The ID of MR
-        #
-        # Example:
-        #   GET /projects/:id/merge_requests/:merge_request_id/commits
-        #
-        get "#{path}/commits" do
-          merge_request = user_project.merge_requests.
-            find(params[:merge_request_id])
-          authorize! :read_merge_request, merge_request
-          present merge_request.commits, with: Entities::RepoCommit
-        end
+        present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project
+      end
 
-        # Show MR changes
-        #
-        # Parameters:
-        #   id (required)               - The ID of a project
-        #   merge_request_id (required) - The ID of MR
-        #
-        # Example:
-        #   GET /projects/:id/merge_requests/:merge_request_id/changes
-        #
-        get "#{path}/changes" do
-          merge_request = user_project.merge_requests.
-            find(params[:merge_request_id])
-          authorize! :read_merge_request, merge_request
-          present merge_request, with: Entities::MergeRequestChanges, current_user: current_user
-        end
+      desc 'Get the commits of a merge request' do
+        success Entities::RepoCommit
+      end
+      get ':id/merge_requests/:merge_request_iid/commits' do
+        merge_request = find_merge_request_with_access(params[:merge_request_iid])
+        commits = ::Kaminari.paginate_array(merge_request.commits)
 
-        # Update MR
-        #
-        # Parameters:
-        #   id (required)               - The ID of a project
-        #   merge_request_id (required) - ID of MR
-        #   target_branch               - The target branch
-        #   assignee_id                 - Assignee user ID
-        #   title                       - Title of MR
-        #   state_event                 - Status of MR. (close|reopen|merge)
-        #   description                 - Description of MR
-        #   labels (optional)           - Labels for a MR as a comma-separated list
-        #   milestone_id (optional)     - Milestone ID
-        # Example:
-        #   PUT /projects/:id/merge_requests/:merge_request_id
-        #
-        put path do
-          attrs = attributes_for_keys [:target_branch, :assignee_id, :title, :state_event, :description, :milestone_id]
-          merge_request = user_project.merge_requests.find(params[:merge_request_id])
-          authorize! :update_merge_request, merge_request
-
-          # Ensure source_branch is not specified
-          if params[:source_branch].present?
-            render_api_error!('Source branch cannot be changed', 400)
-          end
+        present paginate(commits), with: Entities::RepoCommit
+      end
 
-          # Validate label names in advance
-          if (errors = validate_label_params(params)).any?
-            render_api_error!({ labels: errors }, 400)
-          end
+      desc 'Show the merge request changes' do
+        success Entities::MergeRequestChanges
+      end
+      get ':id/merge_requests/:merge_request_iid/changes' do
+        merge_request = find_merge_request_with_access(params[:merge_request_iid])
 
-          merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, attrs).execute(merge_request)
+        present merge_request, with: Entities::MergeRequestChanges, current_user: current_user
+      end
 
-          if merge_request.valid?
-            # Find or create labels and attach to issue
-            unless params[:labels].nil?
-              merge_request.remove_labels
-              merge_request.add_labels_by_names(params[:labels].split(","))
-            end
+      desc 'Update a merge request' do
+        success Entities::MergeRequest
+      end
+      params do
+        # CE
+        at_least_one_of_ce = [
+          :assignee_id,
+          :description,
+          :labels,
+          :milestone_id,
+          :remove_source_branch,
+          :state_event,
+          :target_branch,
+          :title
+        ]
+        optional :title, type: String, allow_blank: false, desc: 'The title of the merge request'
+        optional :target_branch, type: String, allow_blank: false, desc: 'The target branch'
+        optional :state_event, type: String, values: %w[close reopen],
+                               desc: 'Status of the merge request'
+
+        use :optional_params
+        at_least_one_of(*at_least_one_of_ce)
+      end
+      put ':id/merge_requests/:merge_request_iid' do
+        merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request)
 
-            present merge_request, with: Entities::MergeRequest, current_user: current_user
-          else
-            handle_merge_request_errors! merge_request.errors
-          end
-        end
+        mr_params = declared_params(include_missing: false)
+        mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present?
 
-        # Merge MR
-        #
-        # Parameters:
-        #   id (required)                           - The ID of a project
-        #   merge_request_id (required)             - ID of MR
-        #   merge_commit_message (optional)         - Custom merge commit message
-        #   should_remove_source_branch (optional)  - When true, the source branch will be deleted if possible
-        #   merge_when_build_succeeds (optional)    - When true, this MR will be merged when the build succeeds
-        #   sha (optional)                          - When present, must have the HEAD SHA of the source branch
-        # Example:
-        #   PUT /projects/:id/merge_requests/:merge_request_id/merge
-        #
-        put "#{path}/merge" do
-          merge_request = user_project.merge_requests.find(params[:merge_request_id])
-
-          # Merge request can not be merged
-          # because user dont have permissions to push into target branch
-          unauthorized! unless merge_request.can_be_merged_by?(current_user)
-
-          not_allowed! unless merge_request.mergeable_state?
-
-          render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?
-
-          if params[:sha] && merge_request.diff_head_sha != params[:sha]
-            render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409)
-          end
+        merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request)
 
-          merge_params = {
-            commit_message: params[:merge_commit_message],
-            should_remove_source_branch: params[:should_remove_source_branch]
-          }
+        if merge_request.valid?
+          present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project
+        else
+          handle_merge_request_errors! merge_request.errors
+        end
+      end
 
-          if to_boolean(params[:merge_when_build_succeeds]) && merge_request.pipeline && merge_request.pipeline.active?
-            ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params).
-              execute(merge_request)
-          else
-            ::MergeRequests::MergeService.new(merge_request.target_project, current_user, merge_params).
-              execute(merge_request)
-          end
+      desc 'Merge a merge request' do
+        success Entities::MergeRequest
+      end
+      params do
+        # CE
+        optional :merge_commit_message, type: String, desc: 'Custom merge commit message'
+        optional :should_remove_source_branch, type: Boolean,
+                                               desc: 'When true, the source branch will be deleted if possible'
+        optional :merge_when_pipeline_succeeds, type: Boolean,
+                                                desc: 'When true, this merge request will be merged when the pipeline succeeds'
+        optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch'
+      end
+      put ':id/merge_requests/:merge_request_iid/merge' do
+        merge_request = find_project_merge_request(params[:merge_request_iid])
+        merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds])
 
-          present merge_request, with: Entities::MergeRequest, current_user: current_user
-        end
+        # Merge request can not be merged
+        # because user dont have permissions to push into target branch
+        unauthorized! unless merge_request.can_be_merged_by?(current_user)
 
-        # Cancel Merge if Merge When build succeeds is enabled
-        # Parameters:
-        #   id (required)                         - The ID of a project
-        #   merge_request_id (required)           - ID of MR
-        #
-        post "#{path}/cancel_merge_when_build_succeeds" do
-          merge_request = user_project.merge_requests.find(params[:merge_request_id])
+        not_allowed! unless merge_request.mergeable_state?(skip_ci_check: merge_when_pipeline_succeeds)
 
-          unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user)
+        render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: merge_when_pipeline_succeeds)
 
-          ::MergeRequest::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user).cancel(merge_request)
+        if params[:sha] && merge_request.diff_head_sha != params[:sha]
+          render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409)
         end
 
-        # Duplicate. DEPRECATED and WILL BE REMOVED in 9.0.
-        # Use GET "/projects/:id/merge_requests/:merge_request_id/notes" instead
-        #
-        # Get a merge request's comments
-        #
-        # Parameters:
-        #   id (required)               - The ID of a project
-        #   merge_request_id (required) - ID of MR
-        # Examples:
-        #   GET /projects/:id/merge_requests/:merge_request_id/comments
-        #
-        get "#{path}/comments" do
-          merge_request = user_project.merge_requests.find(params[:merge_request_id])
-
-          authorize! :read_merge_request, merge_request
-
-          present paginate(merge_request.notes.fresh), with: Entities::MRNote
-        end
+        merge_params = {
+          commit_message: params[:merge_commit_message],
+          should_remove_source_branch: params[:should_remove_source_branch]
+        }
 
-        # Duplicate. DEPRECATED and WILL BE REMOVED in 9.0.
-        # Use POST "/projects/:id/merge_requests/:merge_request_id/notes" instead
-        #
-        # Post comment to merge request
-        #
-        # Parameters:
-        #   id (required)               - The ID of a project
-        #   merge_request_id (required) - ID of MR
-        #   note (required)             - Text of comment
-        # Examples:
-        #   POST /projects/:id/merge_requests/:merge_request_id/comments
-        #
-        post "#{path}/comments" do
-          required_attributes! [:note]
-
-          merge_request = user_project.merge_requests.find(params[:merge_request_id])
-
-          authorize! :create_note, merge_request
-
-          opts = {
-            note: params[:note],
-            noteable_type: 'MergeRequest',
-            noteable_id: merge_request.id
-          }
-
-          note = ::Notes::CreateService.new(user_project, current_user, opts).execute
-
-          if note.save
-            present note, with: Entities::MRNote
-          else
-            render_api_error!("Failed to save note #{note.errors.messages}", 400)
-          end
+        if merge_when_pipeline_succeeds && merge_request.head_pipeline && merge_request.head_pipeline.active?
+          ::MergeRequests::MergeWhenPipelineSucceedsService
+            .new(merge_request.target_project, current_user, merge_params)
+            .execute(merge_request)
+        else
+          ::MergeRequests::MergeService
+            .new(merge_request.target_project, current_user, merge_params)
+            .execute(merge_request)
         end
 
-        # List issues that will close on merge
-        #
-        # Parameters:
-        #   id (required)               - The ID of a project
-        #   merge_request_id (required) - ID of MR
-        # Examples:
-        #   GET /projects/:id/merge_requests/:merge_request_id/closes_issues
-        get "#{path}/closes_issues" do
-          merge_request = user_project.merge_requests.find(params[:merge_request_id])
-          issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user))
-          present paginate(issues), with: issue_entity(user_project), current_user: current_user
-        end
+        present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project
+      end
+
+      desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do
+        success Entities::MergeRequest
+      end
+      post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds' do
+        merge_request = find_project_merge_request(params[:merge_request_iid])
+
+        unauthorized! unless merge_request.can_cancel_merge_when_pipeline_succeeds?(current_user)
+
+        ::MergeRequest::MergeWhenPipelineSucceedsService
+          .new(merge_request.target_project, current_user)
+          .cancel(merge_request)
+      end
+
+      desc 'List issues that will be closed on merge' do
+        success Entities::MRNote
+      end
+      params do
+        use :pagination
+      end
+      get ':id/merge_requests/:merge_request_iid/closes_issues' do
+        merge_request = find_merge_request_with_access(params[:merge_request_iid])
+        issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user))
+        present paginate(issues), with: issue_entity(user_project), current_user: current_user
       end
     end
   end
diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb
index 9b73f6826cf..3541d3c95fb 100644
--- a/lib/api/milestones.rb
+++ b/lib/api/milestones.rb
@@ -1,6 +1,7 @@
 module API
-  # Milestones API
   class Milestones < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
     helpers do
@@ -11,57 +12,63 @@ module API
         else milestones
         end
       end
+
+      params :optional_params do
+        optional :description, type: String, desc: 'The description of the milestone'
+        optional :due_date, type: String, desc: 'The due date of the milestone. The ISO 8601 date format (%Y-%m-%d)'
+        optional :start_date, type: String, desc: 'The start date of the milestone. The ISO 8601 date format (%Y-%m-%d)'
+      end
     end
 
-    resource :projects do
-      # Get a list of project milestones
-      #
-      # Parameters:
-      #   id (required)    - The ID of a project
-      #   state (optional) - Return "active" or "closed" milestones
-      # Example Request:
-      #   GET /projects/:id/milestones
-      #   GET /projects/:id/milestones?iid=42
-      #   GET /projects/:id/milestones?state=active
-      #   GET /projects/:id/milestones?state=closed
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get a list of project milestones' do
+        success Entities::Milestone
+      end
+      params do
+        optional :state, type: String, values: %w[active closed all], default: 'all',
+                         desc: 'Return "active", "closed", or "all" milestones'
+        optional :iids, type: Array[Integer], desc: 'The IIDs of the milestones'
+        optional :search, type: String, desc: 'The search criteria for the title or description of the milestone'
+        use :pagination
+      end
       get ":id/milestones" do
         authorize! :read_milestone, user_project
 
         milestones = user_project.milestones
         milestones = filter_milestones_state(milestones, params[:state])
-        milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present?
+        milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present?
+        milestones = filter_by_search(milestones, params[:search]) if params[:search]
 
         present paginate(milestones), with: Entities::Milestone
       end
 
-      # Get a single project milestone
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   milestone_id (required) - The ID of a project milestone
-      # Example Request:
-      #   GET /projects/:id/milestones/:milestone_id
+      desc 'Get a single project milestone' do
+        success Entities::Milestone
+      end
+      params do
+        requires :milestone_id, type: Integer, desc: 'The ID of a project milestone'
+      end
       get ":id/milestones/:milestone_id" do
         authorize! :read_milestone, user_project
 
-        @milestone = user_project.milestones.find(params[:milestone_id])
-        present @milestone, with: Entities::Milestone
+        milestone = user_project.milestones.find(params[:milestone_id])
+        present milestone, with: Entities::Milestone
       end
 
-      # Create a new project milestone
-      #
-      # Parameters:
-      #   id (required) - The ID of the project
-      #   title (required) - The title of the milestone
-      #   description (optional) - The description of the milestone
-      #   due_date (optional) - The due date of the milestone
-      # Example Request:
-      #   POST /projects/:id/milestones
+      desc 'Create a new project milestone' do
+        success Entities::Milestone
+      end
+      params do
+        requires :title, type: String, desc: 'The title of the milestone'
+        use :optional_params
+      end
       post ":id/milestones" do
         authorize! :admin_milestone, user_project
-        required_attributes! [:title]
-        attrs = attributes_for_keys [:title, :description, :due_date]
-        milestone = ::Milestones::CreateService.new(user_project, current_user, attrs).execute
+
+        milestone = ::Milestones::CreateService.new(user_project, current_user, declared_params).execute
 
         if milestone.valid?
           present milestone, with: Entities::Milestone
@@ -70,22 +77,23 @@ module API
         end
       end
 
-      # Update an existing project milestone
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   milestone_id (required) - The ID of a project milestone
-      #   title (optional) - The title of a milestone
-      #   description (optional) - The description of a milestone
-      #   due_date (optional) - The due date of a milestone
-      #   state_event (optional) - The state event of the milestone (close|activate)
-      # Example Request:
-      #   PUT /projects/:id/milestones/:milestone_id
+      desc 'Update an existing project milestone' do
+        success Entities::Milestone
+      end
+      params do
+        requires :milestone_id, type: Integer, desc: 'The ID of a project milestone'
+        optional :title, type: String, desc: 'The title of the milestone'
+        optional :state_event, type: String, values: %w[close activate],
+                               desc: 'The state event of the milestone '
+        use :optional_params
+        at_least_one_of :title, :description, :due_date, :state_event
+      end
       put ":id/milestones/:milestone_id" do
         authorize! :admin_milestone, user_project
-        attrs = attributes_for_keys [:title, :description, :due_date, :state_event]
-        milestone = user_project.milestones.find(params[:milestone_id])
-        milestone = ::Milestones::UpdateService.new(user_project, current_user, attrs).execute(milestone)
+        milestone = user_project.milestones.find(params.delete(:milestone_id))
+
+        milestone_params = declared_params(include_missing: false)
+        milestone = ::Milestones::UpdateService.new(user_project, current_user, milestone_params).execute(milestone)
 
         if milestone.valid?
           present milestone, with: Entities::Milestone
@@ -94,25 +102,52 @@ module API
         end
       end
 
-      # Get all issues for a single project milestone
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   milestone_id (required) - The ID of a project milestone
-      # Example Request:
-      #   GET /projects/:id/milestones/:milestone_id/issues
+      desc 'Get all issues for a single project milestone' do
+        success Entities::IssueBasic
+      end
+      params do
+        requires :milestone_id, type: Integer, desc: 'The ID of a project milestone'
+        use :pagination
+      end
       get ":id/milestones/:milestone_id/issues" do
         authorize! :read_milestone, user_project
 
-        @milestone = user_project.milestones.find(params[:milestone_id])
+        milestone = user_project.milestones.find(params[:milestone_id])
 
         finder_params = {
           project_id: user_project.id,
-          milestone_title: @milestone.title
+          milestone_title: milestone.title,
+          sort: 'label_priority'
         }
 
         issues = IssuesFinder.new(current_user, finder_params).execute
-        present paginate(issues), with: Entities::Issue, current_user: current_user
+        present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project
+      end
+
+      desc 'Get all merge requests for a single project milestone' do
+        detail 'This feature was introduced in GitLab 9.'
+        success Entities::MergeRequestBasic
+      end
+      params do
+        requires :milestone_id, type: Integer, desc: 'The ID of a project milestone'
+        use :pagination
+      end
+      get ':id/milestones/:milestone_id/merge_requests' do
+        authorize! :read_milestone, user_project
+
+        milestone = user_project.milestones.find(params[:milestone_id])
+
+        finder_params = {
+          project_id: user_project.id,
+          milestone_title: milestone.title,
+          sort: 'label_priority'
+        }
+
+        merge_requests = MergeRequestsFinder.new(current_user, finder_params).execute
+        present paginate(merge_requests),
+          with: Entities::MergeRequestBasic,
+          current_user: current_user,
+          project: user_project
       end
     end
   end
diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb
index fe981d7b9fa..30761cb9b55 100644
--- a/lib/api/namespaces.rb
+++ b/lib/api/namespaces.rb
@@ -1,6 +1,7 @@
 module API
-  # namespaces API
   class Namespaces < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
     resource :namespaces do
@@ -9,6 +10,7 @@ module API
       end
       params do
         optional :search, type: String, desc: "Search query for namespaces"
+        use :pagination
       end
       get do
         namespaces = current_user.admin ? Namespace.all : current_user.namespaces
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index c5c214d4d13..01ca62b593f 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -1,27 +1,29 @@
 module API
-  # Notes API
   class Notes < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
-    NOTEABLE_TYPES = [Issue, MergeRequest, Snippet]
+    NOTEABLE_TYPES = [Issue, MergeRequest, Snippet].freeze
 
-    resource :projects do
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
       NOTEABLE_TYPES.each do |noteable_type|
         noteables_str = noteable_type.to_s.underscore.pluralize
-        noteable_id_str = "#{noteable_type.to_s.underscore}_id"
-
-        # Get a list of project +noteable+ notes
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   noteable_id (required) - The ID of an issue or snippet
-        # Example Request:
-        #   GET /projects/:id/issues/:noteable_id/notes
-        #   GET /projects/:id/snippets/:noteable_id/notes
-        get ":id/#{noteables_str}/:#{noteable_id_str}/notes" do
-          @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym])
-
-          if can?(current_user, noteable_read_ability_name(@noteable), @noteable)
+
+        desc 'Get a list of project +noteable+ notes' do
+          success Entities::Note
+        end
+        params do
+          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          use :pagination
+        end
+        get ":id/#{noteables_str}/:noteable_id/notes" do
+          noteable = find_project_noteable(noteables_str, params[:noteable_id])
+
+          if can?(current_user, noteable_read_ability_name(noteable), noteable)
             # We exclude notes that are cross-references and that cannot be viewed
             # by the current user. By doing this exclusion at this level and not
             # at the DB query level (which we cannot in that case), the current
@@ -31,80 +33,76 @@ module API
               # paginate() only works with a relation. This could lead to a
               # mismatch between the pagination headers info and the actual notes
               # array returned, but this is really a edge-case.
-              paginate(@noteable.notes).
-              reject { |n| n.cross_reference_not_visible_for?(current_user) }
+              paginate(noteable.notes)
+              .reject { |n| n.cross_reference_not_visible_for?(current_user) }
             present notes, with: Entities::Note
           else
             not_found!("Notes")
           end
         end
 
-        # Get a single +noteable+ note
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   noteable_id (required) - The ID of an issue or snippet
-        #   note_id (required) - The ID of a note
-        # Example Request:
-        #   GET /projects/:id/issues/:noteable_id/notes/:note_id
-        #   GET /projects/:id/snippets/:noteable_id/notes/:note_id
-        get ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do
-          @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym])
-          @note = @noteable.notes.find(params[:note_id])
-          can_read_note = can?(current_user, noteable_read_ability_name(@noteable), @noteable) && !@note.cross_reference_not_visible_for?(current_user)
+        desc 'Get a single +noteable+ note' do
+          success Entities::Note
+        end
+        params do
+          requires :note_id, type: Integer, desc: 'The ID of a note'
+          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+        end
+        get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do
+          noteable = find_project_noteable(noteables_str, params[:noteable_id])
+          note = noteable.notes.find(params[:note_id])
+          can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user)
 
           if can_read_note
-            present @note, with: Entities::Note
+            present note, with: Entities::Note
           else
             not_found!("Note")
           end
         end
 
-        # Create a new +noteable+ note
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   noteable_id (required) - The ID of an issue or snippet
-        #   body (required) - The content of a note
-        #   created_at (optional) - The date
-        # Example Request:
-        #   POST /projects/:id/issues/:noteable_id/notes
-        #   POST /projects/:id/snippets/:noteable_id/notes
-        post ":id/#{noteables_str}/:#{noteable_id_str}/notes" do
-          required_attributes! [:body]
+        desc 'Create a new +noteable+ note' do
+          success Entities::Note
+        end
+        params do
+          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          requires :body, type: String, desc: 'The content of a note'
+          optional :created_at, type: String, desc: 'The creation date of the note'
+        end
+        post ":id/#{noteables_str}/:noteable_id/notes" do
+          noteable = find_project_noteable(noteables_str, params[:noteable_id])
 
           opts = {
-           note: params[:body],
-           noteable_type: noteables_str.classify,
-           noteable_id: params[noteable_id_str]
+            note: params[:body],
+            noteable_type: noteables_str.classify,
+            noteable_id: noteable.id
           }
 
-          if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user)
-            opts[:created_at] = params[:created_at]
-          end
+          if can?(current_user, noteable_read_ability_name(noteable), noteable)
+            if params[:created_at] && (current_user.admin? || user_project.owner == current_user)
+              opts[:created_at] = params[:created_at]
+            end
 
-          note = ::Notes::CreateService.new(user_project, current_user, opts).execute
+            note = ::Notes::CreateService.new(user_project, current_user, opts).execute
 
-          if note.valid?
-            present note, with: Entities::const_get(note.class.name)
+            if note.valid?
+              present note, with: Entities.const_get(note.class.name)
+            else
+              not_found!("Note #{note.errors.messages}")
+            end
           else
-            not_found!("Note #{note.errors.messages}")
+            not_found!("Note")
           end
         end
 
-        # Modify existing +noteable+ note
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   noteable_id (required) - The ID of an issue or snippet
-        #   node_id (required) - The ID of a note
-        #   body (required) - New content of a note
-        # Example Request:
-        #   PUT /projects/:id/issues/:noteable_id/notes/:note_id
-        #   PUT /projects/:id/snippets/:noteable_id/notes/:node_id
-        put ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do
-          required_attributes! [:body]
-
+        desc 'Update an existing +noteable+ note' do
+          success Entities::Note
+        end
+        params do
+          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          requires :note_id, type: Integer, desc: 'The ID of a note'
+          requires :body, type: String, desc: 'The content of a note'
+        end
+        put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do
           note = user_project.notes.find(params[:note_id])
 
           authorize! :admin_note, note
@@ -113,36 +111,36 @@ module API
             note: params[:body]
           }
 
-          @note = ::Notes::UpdateService.new(user_project, current_user, opts).execute(note)
+          note = ::Notes::UpdateService.new(user_project, current_user, opts).execute(note)
 
-          if @note.valid?
-            present @note, with: Entities::Note
+          if note.valid?
+            present note, with: Entities::Note
           else
             render_api_error!("Failed to save note #{note.errors.messages}", 400)
           end
         end
 
-        # Delete a +noteable+ note
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   noteable_id (required) - The ID of an issue, MR, or snippet
-        #   node_id (required) - The ID of a note
-        # Example Request:
-        #   DELETE /projects/:id/issues/:noteable_id/notes/:note_id
-        #   DELETE /projects/:id/snippets/:noteable_id/notes/:node_id
-        delete ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do
+        desc 'Delete a +noteable+ note' do
+          success Entities::Note
+        end
+        params do
+          requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          requires :note_id, type: Integer, desc: 'The ID of a note'
+        end
+        delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do
           note = user_project.notes.find(params[:note_id])
           authorize! :admin_note, note
 
-          ::Notes::DeleteService.new(user_project, current_user).execute(note)
-
-          present note, with: Entities::Note
+          ::Notes::DestroyService.new(user_project, current_user).execute(note)
         end
       end
     end
 
     helpers do
+      def find_project_noteable(noteables_str, noteable_id)
+        public_send("find_project_#{noteables_str.singularize}", noteable_id)
+      end
+
       def noteable_read_ability_name(noteable)
         "read_#{noteable.class.to_s.underscore}".to_sym
       end
diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb
index a70a7e71073..5d113c94b22 100644
--- a/lib/api/notification_settings.rb
+++ b/lib/api/notification_settings.rb
@@ -33,10 +33,12 @@ module API
         begin
           notification_setting.transaction do
             new_notification_email = params.delete(:notification_email)
-            declared_params = declared(params, include_missing: false).to_h
 
-            current_user.update(notification_email: new_notification_email) if new_notification_email
-            notification_setting.update(declared_params)
+            if new_notification_email
+              ::Users::UpdateService.new(current_user, notification_email: new_notification_email).execute
+            end
+
+            notification_setting.update(declared_params(include_missing: false))
           end
         rescue ArgumentError => e # catch level enum error
           render_api_error! e.to_s, 400
@@ -49,14 +51,14 @@ module API
     end
 
     %w[group project].each do |source_type|
-      resource source_type.pluralize do
+      params do
+        requires :id, type: String, desc: "The #{source_type} ID"
+      end
+      resource source_type.pluralize, requirements: { id: %r{[^/]+} } do
         desc "Get #{source_type} level notification level settings, defaults to Global" do
           detail 'This feature was introduced in GitLab 8.12'
           success Entities::NotificationSetting
         end
-        params do
-          requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME'
-        end
         get ":id/notification_settings" do
           source = find_source(source_type, params[:id])
 
@@ -70,7 +72,6 @@ module API
           success Entities::NotificationSetting
         end
         params do
-          requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME'
           optional :level, type: String, desc: "The #{source_type} notification level"
           NotificationSetting::EMAIL_EVENTS.each do |event|
             optional event, type: Boolean, desc: 'Enable/disable this notification'
@@ -81,9 +82,7 @@ module API
           notification_setting = current_user.notification_settings_for(source)
 
           begin
-            declared_params = declared(params, include_missing: false).to_h
-
-            notification_setting.update(declared_params)
+            notification_setting.update(declared_params(include_missing: false))
           rescue ArgumentError => e # catch level enum error
             render_api_error! e.to_s, 400
           end
diff --git a/lib/api/pagination_params.rb b/lib/api/pagination_params.rb
new file mode 100644
index 00000000000..f566eb3ed2b
--- /dev/null
+++ b/lib/api/pagination_params.rb
@@ -0,0 +1,24 @@
+module API
+  # Concern for declare pagination params.
+  #
+  # @example
+  #   class CustomApiResource < Grape::API
+  #     include PaginationParams
+  #
+  #     params do
+  #       use :pagination
+  #     end
+  #   end
+  module PaginationParams
+    extend ActiveSupport::Concern
+
+    included do
+      helpers do
+        params :pagination do
+          optional :page, type: Integer, default: 1, desc: 'Current page number'
+          optional :per_page, type: Integer, default: 20, desc: 'Number of items per page'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb
new file mode 100644
index 00000000000..93d89209934
--- /dev/null
+++ b/lib/api/pipeline_schedules.rb
@@ -0,0 +1,131 @@
+module API
+  class PipelineSchedules < Grape::API
+    include PaginationParams
+
+    before { authenticate! }
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get all pipeline schedules' do
+        success Entities::PipelineSchedule
+      end
+      params do
+        use :pagination
+        optional :scope,    type: String, values: %w[active inactive],
+                            desc: 'The scope of pipeline schedules'
+      end
+      get ':id/pipeline_schedules' do
+        authorize! :read_pipeline_schedule, user_project
+
+        schedules = PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope])
+          .preload([:owner, :last_pipeline])
+        present paginate(schedules), with: Entities::PipelineSchedule
+      end
+
+      desc 'Get a single pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+      end
+      get ':id/pipeline_schedules/:pipeline_schedule_id' do
+        authorize! :read_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        present pipeline_schedule, with: Entities::PipelineScheduleDetails
+      end
+
+      desc 'Create a new pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :description, type: String, desc: 'The description of pipeline schedule'
+        requires :ref, type: String, desc: 'The branch/tag name will be triggered'
+        requires :cron, type: String, desc: 'The cron'
+        optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone'
+        optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule'
+      end
+      post ':id/pipeline_schedules' do
+        authorize! :create_pipeline_schedule, user_project
+
+        pipeline_schedule = Ci::CreatePipelineScheduleService
+          .new(user_project, current_user, declared_params(include_missing: false))
+          .execute
+
+        if pipeline_schedule.persisted?
+          present pipeline_schedule, with: Entities::PipelineScheduleDetails
+        else
+          render_validation_error!(pipeline_schedule)
+        end
+      end
+
+      desc 'Edit a pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+        optional :description, type: String, desc: 'The description of pipeline schedule'
+        optional :ref, type: String, desc: 'The branch/tag name will be triggered'
+        optional :cron, type: String, desc: 'The cron'
+        optional :cron_timezone, type: String, desc: 'The timezone'
+        optional :active, type: Boolean, desc: 'The activation of pipeline schedule'
+      end
+      put ':id/pipeline_schedules/:pipeline_schedule_id' do
+        authorize! :update_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        if pipeline_schedule.update(declared_params(include_missing: false))
+          present pipeline_schedule, with: Entities::PipelineScheduleDetails
+        else
+          render_validation_error!(pipeline_schedule)
+        end
+      end
+
+      desc 'Take ownership of a pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+      end
+      post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do
+        authorize! :update_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        if pipeline_schedule.own!(current_user)
+          present pipeline_schedule, with: Entities::PipelineScheduleDetails
+        else
+          render_validation_error!(pipeline_schedule)
+        end
+      end
+
+      desc 'Delete a pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+      end
+      delete ':id/pipeline_schedules/:pipeline_schedule_id' do
+        authorize! :admin_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        status :accepted
+        present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails
+      end
+    end
+
+    helpers do
+      def pipeline_schedule
+        @pipeline_schedule ||=
+          user_project.pipeline_schedules
+                      .preload(:owner, :last_pipeline)
+                      .find_by(id: params.delete(:pipeline_schedule_id))
+      end
+    end
+  end
+end
diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb
index 2a0c8e1f2c0..e505cae3992 100644
--- a/lib/api/pipelines.rb
+++ b/lib/api/pipelines.rb
@@ -1,26 +1,58 @@
 module API
   class Pipelines < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
     params do
       requires :id, type: String, desc: 'The project ID'
     end
-    resource :projects do
+    resource :projects, requirements: { id: %r{[^/]+} } do
       desc 'Get all Pipelines of the project' do
         detail 'This feature was introduced in GitLab 8.11.'
-        success Entities::Pipeline
+        success Entities::PipelineBasic
       end
       params do
-        optional :page,     type: Integer, desc: 'Page number of the current request'
-        optional :per_page, type: Integer, desc: 'Number of items per page'
-        optional :scope,    type: String, values: ['running', 'branches', 'tags'],
-                            desc: 'Either running, branches, or tags'
+        use :pagination
+        optional :scope,    type: String, values: %w[running pending finished branches tags],
+                            desc: 'The scope of pipelines'
+        optional :status,   type: String, values: HasStatus::AVAILABLE_STATUSES,
+                            desc: 'The status of pipelines'
+        optional :ref,      type: String, desc: 'The ref of pipelines'
+        optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations'
+        optional :name,     type: String, desc: 'The name of the user who triggered pipelines'
+        optional :username, type: String, desc: 'The username of the user who triggered pipelines'
+        optional :order_by, type: String, values: PipelinesFinder::ALLOWED_INDEXED_COLUMNS, default: 'id',
+                            desc: 'Order pipelines'
+        optional :sort,     type: String, values: %w[asc desc], default: 'desc',
+                            desc: 'Sort pipelines'
       end
       get ':id/pipelines' do
         authorize! :read_pipeline, user_project
 
-        pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope])
-        present paginate(pipelines), with: Entities::Pipeline
+        pipelines = PipelinesFinder.new(user_project, params).execute
+        present paginate(pipelines), with: Entities::PipelineBasic
+      end
+
+      desc 'Create a new pipeline' do
+        detail 'This feature was introduced in GitLab 8.14'
+        success Entities::Pipeline
+      end
+      params do
+        requires :ref, type: String,  desc: 'Reference'
+      end
+      post ':id/pipeline' do
+        authorize! :create_pipeline, user_project
+
+        new_pipeline = Ci::CreatePipelineService.new(user_project,
+                                                     current_user,
+                                                     declared_params(include_missing: false))
+                           .execute(:api, ignore_skip_ci: true, save_on_errors: false)
+        if new_pipeline.persisted?
+          present new_pipeline, with: Entities::Pipeline
+        else
+          render_validation_error!(new_pipeline)
+        end
       end
 
       desc 'Gets a specific pipeline for the project' do
@@ -36,7 +68,7 @@ module API
         present pipeline, with: Entities::Pipeline
       end
 
-      desc 'Retry failed builds in the pipeline' do
+      desc 'Retry builds in the pipeline' do
         detail 'This feature was introduced in GitLab 8.11.'
         success Entities::Pipeline
       end
diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb
index 14f5be3b5f6..7a345289617 100644
--- a/lib/api/project_hooks.rb
+++ b/lib/api/project_hooks.rb
@@ -1,116 +1,102 @@
 module API
-  # Projects API
   class ProjectHooks < Grape::API
+    include PaginationParams
+
     before { authenticate! }
     before { authorize_admin_project }
 
-    resource :projects do
-      # Get project hooks
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   GET /projects/:id/hooks
+    helpers do
+      params :project_hook_properties do
+        requires :url, type: String, desc: "The URL to send the request to"
+        optional :push_events, type: Boolean, desc: "Trigger hook on push events"
+        optional :issues_events, type: Boolean, desc: "Trigger hook on issues events"
+        optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events"
+        optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events"
+        optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events"
+        optional :job_events, type: Boolean, desc: "Trigger hook on job events"
+        optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events"
+        optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events"
+        optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook"
+        optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response"
+      end
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get project hooks' do
+        success Entities::ProjectHook
+      end
+      params do
+        use :pagination
+      end
       get ":id/hooks" do
-        @hooks = paginate user_project.hooks
-        present @hooks, with: Entities::ProjectHook
+        present paginate(user_project.hooks), with: Entities::ProjectHook
       end
 
-      # Get a project hook
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   hook_id (required) - The ID of a project hook
-      # Example Request:
-      #   GET /projects/:id/hooks/:hook_id
+      desc 'Get a project hook' do
+        success Entities::ProjectHook
+      end
+      params do
+        requires :hook_id, type: Integer, desc: 'The ID of a project hook'
+      end
       get ":id/hooks/:hook_id" do
-        @hook = user_project.hooks.find(params[:hook_id])
-        present @hook, with: Entities::ProjectHook
+        hook = user_project.hooks.find(params[:hook_id])
+        present hook, with: Entities::ProjectHook
       end
 
-      # Add hook to project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   url (required) - The hook URL
-      # Example Request:
-      #   POST /projects/:id/hooks
+      desc 'Add hook to project' do
+        success Entities::ProjectHook
+      end
+      params do
+        use :project_hook_properties
+      end
       post ":id/hooks" do
-        required_attributes! [:url]
-        attrs = attributes_for_keys [
-          :url,
-          :push_events,
-          :issues_events,
-          :merge_requests_events,
-          :tag_push_events,
-          :note_events,
-          :build_events,
-          :pipeline_events,
-          :wiki_page_events,
-          :enable_ssl_verification
-        ]
-        @hook = user_project.hooks.new(attrs)
+        hook_params = declared_params(include_missing: false)
+
+        hook = user_project.hooks.new(hook_params)
 
-        if @hook.save
-          present @hook, with: Entities::ProjectHook
+        if hook.save
+          present hook, with: Entities::ProjectHook
         else
-          if @hook.errors[:url].present?
-            error!("Invalid url given", 422)
-          end
-          not_found!("Project hook #{@hook.errors.messages}")
+          error!("Invalid url given", 422) if hook.errors[:url].present?
+
+          not_found!("Project hook #{hook.errors.messages}")
         end
       end
 
-      # Update an existing project hook
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   hook_id (required) - The ID of a project hook
-      #   url (required) - The hook URL
-      # Example Request:
-      #   PUT /projects/:id/hooks/:hook_id
+      desc 'Update an existing project hook' do
+        success Entities::ProjectHook
+      end
+      params do
+        requires :hook_id, type: Integer, desc: "The ID of the hook to update"
+        use :project_hook_properties
+      end
       put ":id/hooks/:hook_id" do
-        @hook = user_project.hooks.find(params[:hook_id])
-        required_attributes! [:url]
-        attrs = attributes_for_keys [
-          :url,
-          :push_events,
-          :issues_events,
-          :merge_requests_events,
-          :tag_push_events,
-          :note_events,
-          :build_events,
-          :pipeline_events,
-          :wiki_page_events,
-          :enable_ssl_verification
-        ]
+        hook = user_project.hooks.find(params.delete(:hook_id))
 
-        if @hook.update_attributes attrs
-          present @hook, with: Entities::ProjectHook
+        update_params = declared_params(include_missing: false)
+
+        if hook.update_attributes(update_params)
+          present hook, with: Entities::ProjectHook
         else
-          if @hook.errors[:url].present?
-            error!("Invalid url given", 422)
-          end
-          not_found!("Project hook #{@hook.errors.messages}")
+          error!("Invalid url given", 422) if hook.errors[:url].present?
+
+          not_found!("Project hook #{hook.errors.messages}")
         end
       end
 
-      # Deletes project hook. This is an idempotent function.
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   hook_id (required) - The ID of hook to delete
-      # Example Request:
-      #   DELETE /projects/:id/hooks/:hook_id
+      desc 'Deletes project hook' do
+        success Entities::ProjectHook
+      end
+      params do
+        requires :hook_id, type: Integer, desc: 'The ID of the hook to delete'
+      end
       delete ":id/hooks/:hook_id" do
-        required_attributes! [:hook_id]
+        hook = user_project.hooks.find(params.delete(:hook_id))
 
-        begin
-          @hook = user_project.hooks.destroy(params[:hook_id])
-        rescue
-          # ProjectHook can raise Error if hook_id not found
-          not_found!("Error deleting hook #{params[:hook_id]}")
-        end
+        hook.destroy
       end
     end
   end
diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb
index ce1bf0d26d2..64efe82a937 100644
--- a/lib/api/project_snippets.rb
+++ b/lib/api/project_snippets.rb
@@ -1,9 +1,13 @@
 module API
-  # Projects API
   class ProjectSnippets < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
-    resource :projects do
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
       helpers do
         def handle_project_member_errors(errors)
           if errors[:project_access].any?
@@ -13,116 +17,119 @@ module API
         end
 
         def snippets_for_current_user
-          finder_params = { filter: :by_project, project: user_project }
-          SnippetsFinder.new.execute(current_user, finder_params)
+          SnippetsFinder.new(current_user, project: user_project).execute
         end
       end
 
-      # Get a project snippets
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   GET /projects/:id/snippets
+      desc 'Get all project snippets' do
+        success Entities::ProjectSnippet
+      end
+      params do
+        use :pagination
+      end
       get ":id/snippets" do
         present paginate(snippets_for_current_user), with: Entities::ProjectSnippet
       end
 
-      # Get a project snippet
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   snippet_id (required) - The ID of a project snippet
-      # Example Request:
-      #   GET /projects/:id/snippets/:snippet_id
+      desc 'Get a single project snippet' do
+        success Entities::ProjectSnippet
+      end
+      params do
+        requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
+      end
       get ":id/snippets/:snippet_id" do
-        @snippet = snippets_for_current_user.find(params[:snippet_id])
-        present @snippet, with: Entities::ProjectSnippet
-      end
-
-      # Create a new project snippet
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   title (required) - The title of a snippet
-      #   file_name (required) - The name of a snippet file
-      #   code (required) - The content of a snippet
-      #   visibility_level (required) - The snippet's visibility
-      # Example Request:
-      #   POST /projects/:id/snippets
+        snippet = snippets_for_current_user.find(params[:snippet_id])
+        present snippet, with: Entities::ProjectSnippet
+      end
+
+      desc 'Create a new project snippet' do
+        success Entities::ProjectSnippet
+      end
+      params do
+        requires :title, type: String, desc: 'The title of the snippet'
+        requires :file_name, type: String, desc: 'The file name of the snippet'
+        requires :code, type: String, desc: 'The content of the snippet'
+        optional :description, type: String, desc: 'The description of a snippet'
+        requires :visibility, type: String,
+                              values: Gitlab::VisibilityLevel.string_values,
+                              desc: 'The visibility of the snippet'
+      end
       post ":id/snippets" do
         authorize! :create_project_snippet, user_project
-        required_attributes! [:title, :file_name, :code, :visibility_level]
+        snippet_params = declared_params.merge(request: request, api: true)
+        snippet_params[:content] = snippet_params.delete(:code)
+
+        snippet = CreateSnippetService.new(user_project, current_user, snippet_params).execute
 
-        attrs = attributes_for_keys [:title, :file_name, :visibility_level]
-        attrs[:content] = params[:code] if params[:code].present?
-        @snippet = CreateSnippetService.new(user_project, current_user,
-                                            attrs).execute
+        render_spam_error! if snippet.spam?
 
-        if @snippet.errors.any?
-          render_validation_error!(@snippet)
+        if snippet.persisted?
+          present snippet, with: Entities::ProjectSnippet
         else
-          present @snippet, with: Entities::ProjectSnippet
+          render_validation_error!(snippet)
         end
       end
 
-      # Update an existing project snippet
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   snippet_id (required) - The ID of a project snippet
-      #   title (optional) - The title of a snippet
-      #   file_name (optional) - The name of a snippet file
-      #   code (optional) - The content of a snippet
-      #   visibility_level (optional) - The snippet's visibility
-      # Example Request:
-      #   PUT /projects/:id/snippets/:snippet_id
+      desc 'Update an existing project snippet' do
+        success Entities::ProjectSnippet
+      end
+      params do
+        requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
+        optional :title, type: String, desc: 'The title of the snippet'
+        optional :file_name, type: String, desc: 'The file name of the snippet'
+        optional :code, type: String, desc: 'The content of the snippet'
+        optional :description, type: String, desc: 'The description of a snippet'
+        optional :visibility, type: String,
+                              values: Gitlab::VisibilityLevel.string_values,
+                              desc: 'The visibility of the snippet'
+        at_least_one_of :title, :file_name, :code, :visibility_level
+      end
       put ":id/snippets/:snippet_id" do
-        @snippet = snippets_for_current_user.find(params[:snippet_id])
-        authorize! :update_project_snippet, @snippet
+        snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id))
+        not_found!('Snippet') unless snippet
+
+        authorize! :update_project_snippet, snippet
+
+        snippet_params = declared_params(include_missing: false)
+          .merge(request: request, api: true)
+
+        snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present?
 
-        attrs = attributes_for_keys [:title, :file_name, :visibility_level]
-        attrs[:content] = params[:code] if params[:code].present?
+        UpdateSnippetService.new(user_project, current_user, snippet,
+                                 snippet_params).execute
 
-        UpdateSnippetService.new(user_project, current_user, @snippet,
-                                 attrs).execute
-        if @snippet.errors.any?
-          render_validation_error!(@snippet)
+        render_spam_error! if snippet.spam?
+
+        if snippet.valid?
+          present snippet, with: Entities::ProjectSnippet
         else
-          present @snippet, with: Entities::ProjectSnippet
+          render_validation_error!(snippet)
         end
       end
 
-      # Delete a project snippet
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   snippet_id (required) - The ID of a project snippet
-      # Example Request:
-      #   DELETE /projects/:id/snippets/:snippet_id
+      desc 'Delete a project snippet'
+      params do
+        requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
+      end
       delete ":id/snippets/:snippet_id" do
-        begin
-          @snippet = snippets_for_current_user.find(params[:snippet_id])
-          authorize! :update_project_snippet, @snippet
-          @snippet.destroy
-        rescue
-          not_found!('Snippet')
-        end
+        snippet = snippets_for_current_user.find_by(id: params[:snippet_id])
+        not_found!('Snippet') unless snippet
+
+        authorize! :admin_project_snippet, snippet
+        snippet.destroy
       end
 
-      # Get a raw project snippet
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   snippet_id (required) - The ID of a project snippet
-      # Example Request:
-      #   GET /projects/:id/snippets/:snippet_id/raw
+      desc 'Get a raw project snippet'
+      params do
+        requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
+      end
       get ":id/snippets/:snippet_id/raw" do
-        @snippet = snippets_for_current_user.find(params[:snippet_id])
+        snippet = snippets_for_current_user.find_by(id: params[:snippet_id])
+        not_found!('Snippet') unless snippet
 
         env['api.format'] = :txt
         content_type 'text/plain'
-        present @snippet.content
+        present snippet.content
       end
     end
   end
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 291e7b689bf..20707e97e53 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -1,311 +1,262 @@
 module API
   # Projects API
   class Projects < Grape::API
-    before { authenticate! }
+    include PaginationParams
+
+    before { authenticate_non_get! }
+
+    helpers do
+      params :optional_params_ce do
+        optional :description, type: String, desc: 'The description of the project'
+        optional :ci_config_file, type: String, desc: 'The path to CI config file. Default to `.gitlab-ci.yml`'
+        optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled'
+        optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled'
+        optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled'
+        optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled'
+        optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled'
+        optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project'
+        optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project'
+        optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project'
+        optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.'
+        optional :public_builds, type: Boolean, desc: 'Perform public builds'
+        optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
+        optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed'
+        optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved'
+        optional :tag_list, type: Array[String], desc: 'The list of tags for a project'
+        optional :avatar, type: File, desc: 'Avatar image for project'
+        optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line'
+      end
+
+      params :optional_params do
+        use :optional_params_ce
+      end
+
+      params :statistics_params do
+        optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
+      end
+    end
 
-    resource :projects, requirements: { id: /[^\/]+/ } do
+    resource :projects do
       helpers do
-        def map_public_to_visibility_level(attrs)
-          publik = attrs.delete(:public)
-          if publik.present? && !attrs[:visibility_level].present?
-            publik = to_boolean(publik)
-            # Since setting the public attribute to private could mean either
-            # private or internal, use the more conservative option, private.
-            attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE
-          end
-          attrs
+        params :collection_params do
+          use :sort_params
+          use :filter_params
+          use :pagination
+
+          optional :simple, type: Boolean, default: false,
+                            desc: 'Return only the ID, URL, name, and path of each project'
+        end
+
+        params :sort_params do
+          optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
+                              default: 'created_at', desc: 'Return projects ordered by field'
+          optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                          desc: 'Return projects sorted in ascending and descending order'
+        end
+
+        params :filter_params do
+          optional :archived, type: Boolean, default: false, desc: 'Limit by archived status'
+          optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values,
+                                desc: 'Limit by visibility'
+          optional :search, type: String, desc: 'Return list of projects matching the search criteria'
+          optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
+          optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
+          optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of'
+          optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
+          optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
+        end
+
+        params :create_params do
+          optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
+          optional :import_url, type: String, desc: 'URL from which the project is imported'
+        end
+
+        def present_projects(options = {})
+          projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
+          projects = reorder_projects(projects)
+          projects = projects.with_statistics if params[:statistics]
+          projects = projects.with_issues_enabled if params[:with_issues_enabled]
+          projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
+
+          options = options.reverse_merge(
+            with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
+            statistics: params[:statistics],
+            current_user: current_user
+          )
+          options[:with] = Entities::BasicProjectDetails if params[:simple]
+
+          present paginate(projects), options
         end
       end
 
-      # Get a projects list for authenticated user
-      #
-      # Example Request:
-      #   GET /projects
+      desc 'Get a list of visible projects for authenticated user' do
+        success Entities::BasicProjectDetails
+      end
+      params do
+        use :collection_params
+        use :statistics_params
+      end
       get do
-        projects = current_user.authorized_projects
-        projects = filter_projects(projects)
-        projects = paginate projects
-        entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present projects, with: entity, user: current_user
-      end
-
-      # Get a list of visible projects for authenticated user
-      #
-      # Example Request:
-      #   GET /projects/visible
-      get '/visible' do
-        projects = ProjectsFinder.new.execute(current_user)
-        projects = filter_projects(projects)
-        projects = paginate projects
-        entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present projects, with: entity, user: current_user
-      end
-
-      # Get an owned projects list for authenticated user
-      #
-      # Example Request:
-      #   GET /projects/owned
-      get '/owned' do
-        projects = current_user.owned_projects
-        projects = filter_projects(projects)
-        projects = paginate projects
-        present projects, with: Entities::ProjectWithAccess, user: current_user
-      end
-
-      # Gets starred project for the authenticated user
-      #
-      # Example Request:
-      #   GET /projects/starred
-      get '/starred' do
-        projects = current_user.viewable_starred_projects
-        projects = filter_projects(projects)
-        projects = paginate projects
-        present projects, with: Entities::Project, user: current_user
-      end
-
-      # Get all projects for admin user
-      #
-      # Example Request:
-      #   GET /projects/all
-      get '/all' do
-        authenticated_as_admin!
-        projects = Project.all
-        projects = filter_projects(projects)
-        projects = paginate projects
-        present projects, with: Entities::ProjectWithAccess, user: current_user
-      end
-
-      # Get a single project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   GET /projects/:id
-      get ":id" do
-        present user_project, with: Entities::ProjectWithAccess, user: current_user,
-                              user_can_admin_project: can?(current_user, :admin_project, user_project)
-      end
-
-      # Get events for a single project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   GET /projects/:id/events
-      get ":id/events" do
-        events = paginate user_project.events.recent
-        present events, with: Entities::Event
-      end
-
-      # Create new project
-      #
-      # Parameters:
-      #   name (required)                   - name for new project
-      #   description (optional)            - short project description
-      #   issues_enabled (optional)
-      #   merge_requests_enabled (optional)
-      #   builds_enabled (optional)
-      #   wiki_enabled (optional)
-      #   snippets_enabled (optional)
-      #   container_registry_enabled (optional)
-      #   shared_runners_enabled (optional)
-      #   namespace_id (optional)           - defaults to user namespace
-      #   public (optional)                 - if true same as setting visibility_level = 20
-      #   visibility_level (optional)       - 0 by default
-      #   import_url (optional)
-      #   public_builds (optional)
-      #   lfs_enabled (optional)
-      #   request_access_enabled (optional) - Allow users to request member access
-      #   ci_config_file (optional)
-      # Example Request
-      #   POST /projects
+        present_projects
+      end
+
+      desc 'Create new project' do
+        success Entities::Project
+      end
+      params do
+        optional :name, type: String, desc: 'The name of the project'
+        optional :path, type: String, desc: 'The path of the repository'
+        at_least_one_of :name, :path
+        use :optional_params
+        use :create_params
+      end
       post do
-        required_attributes! [:name]
-        attrs = attributes_for_keys [:builds_enabled,
-                                     :container_registry_enabled,
-                                     :ci_config_file,
-                                     :description,
-                                     :import_url,
-                                     :issues_enabled,
-                                     :lfs_enabled,
-                                     :merge_requests_enabled,
-                                     :name,
-                                     :namespace_id,
-                                     :only_allow_merge_if_build_succeeds,
-                                     :path,
-                                     :public,
-                                     :public_builds,
-                                     :request_access_enabled,
-                                     :shared_runners_enabled,
-                                     :snippets_enabled,
-                                     :visibility_level,
-                                     :wiki_enabled]
-        attrs = map_public_to_visibility_level(attrs)
-        @project = ::Projects::CreateService.new(current_user, attrs).execute
-        if @project.saved?
-          present @project, with: Entities::Project,
-                            user_can_admin_project: can?(current_user, :admin_project, @project)
+        attrs = declared_params(include_missing: false)
+        attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled)
+        project = ::Projects::CreateService.new(current_user, attrs).execute
+
+        if project.saved?
+          present project, with: Entities::Project,
+                           user_can_admin_project: can?(current_user, :admin_project, project)
         else
-          if @project.errors[:limit_reached].present?
-            error!(@project.errors[:limit_reached], 403)
+          if project.errors[:limit_reached].present?
+            error!(project.errors[:limit_reached], 403)
           end
-          render_validation_error!(@project)
+          render_validation_error!(project)
         end
       end
 
-      # Create new project for a specified user.  Only available to admin users.
-      #
-      # Parameters:
-      #   user_id (required)                - The ID of a user
-      #   name (required)                   - name for new project
-      #   description (optional)            - short project description
-      #   default_branch (optional)         - 'master' by default
-      #   issues_enabled (optional)
-      #   merge_requests_enabled (optional)
-      #   builds_enabled (optional)
-      #   wiki_enabled (optional)
-      #   snippets_enabled (optional)
-      #   container_registry_enabled (optional)
-      #   shared_runners_enabled (optional)
-      #   public (optional)                 - if true same as setting visibility_level = 20
-      #   visibility_level (optional)
-      #   import_url (optional)
-      #   public_builds (optional)
-      #   lfs_enabled (optional)
-      #   request_access_enabled (optional) - Allow users to request member access
-      #   ci_config_file (optional)
-      # Example Request
-      #   POST /projects/user/:user_id
+      desc 'Create new project for a specified user. Only available to admin users.' do
+        success Entities::Project
+      end
+      params do
+        requires :name, type: String, desc: 'The name of the project'
+        requires :user_id, type: Integer, desc: 'The ID of a user'
+        optional :path, type: String, desc: 'The path of the repository'
+        optional :default_branch, type: String, desc: 'The default branch of the project'
+        use :optional_params
+        use :create_params
+      end
       post "user/:user_id" do
         authenticated_as_admin!
-        user = User.find(params[:user_id])
-        attrs = attributes_for_keys [:builds_enabled,
-                                     :ci_config_file,
-                                     :default_branch,
-                                     :description,
-                                     :import_url,
-                                     :issues_enabled,
-                                     :lfs_enabled,
-                                     :merge_requests_enabled,
-                                     :name,
-                                     :only_allow_merge_if_build_succeeds,
-                                     :public,
-                                     :public_builds,
-                                     :request_access_enabled,
-                                     :shared_runners_enabled,
-                                     :snippets_enabled,
-                                     :visibility_level,
-                                     :wiki_enabled]
-        attrs = map_public_to_visibility_level(attrs)
-        @project = ::Projects::CreateService.new(user, attrs).execute
-        if @project.saved?
-          present @project, with: Entities::Project,
-                            user_can_admin_project: can?(current_user, :admin_project, @project)
+        user = User.find_by(id: params.delete(:user_id))
+        not_found!('User') unless user
+
+        attrs = declared_params(include_missing: false)
+        project = ::Projects::CreateService.new(user, attrs).execute
+
+        if project.saved?
+          present project, with: Entities::Project,
+                           user_can_admin_project: can?(current_user, :admin_project, project)
         else
-          render_validation_error!(@project)
+          render_validation_error!(project)
         end
       end
+    end
 
-      # Fork new project for the current user or provided namespace.
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   namespace (optional) - The ID or name of the namespace that the project will be forked into.
-      # Example Request
-      #   POST /projects/fork/:id
-      post 'fork/:id' do
-        attrs = {}
-        namespace_id = params[:namespace]
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get a single project' do
+        success Entities::ProjectWithAccess
+      end
+      params do
+        use :statistics_params
+      end
+      get ":id" do
+        entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
+        present user_project, with: entity, current_user: current_user,
+                              user_can_admin_project: can?(current_user, :admin_project, user_project), statistics: params[:statistics]
+      end
+
+      desc 'Fork new project for the current user or provided namespace.' do
+        success Entities::Project
+      end
+      params do
+        optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into'
+      end
+      post ':id/fork' do
+        fork_params = declared_params(include_missing: false)
+        namespace_id = fork_params[:namespace]
 
         if namespace_id.present?
-          namespace = Namespace.find_by(id: namespace_id) || Namespace.find_by_path_or_name(namespace_id)
+          fork_params[:namespace] = if namespace_id =~ /^\d+$/
+                                      Namespace.find_by(id: namespace_id)
+                                    else
+                                      Namespace.find_by_path_or_name(namespace_id)
+                                    end
 
-          unless namespace && can?(current_user, :create_projects, namespace)
+          unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace])
             not_found!('Target Namespace')
           end
-
-          attrs[:namespace] = namespace
         end
 
-        @forked_project =
-          ::Projects::ForkService.new(user_project,
-                                      current_user,
-                                      attrs).execute
+        forked_project = ::Projects::ForkService.new(user_project, current_user, fork_params).execute
 
-        if @forked_project.errors.any?
-          conflict!(@forked_project.errors.messages)
+        if forked_project.errors.any?
+          conflict!(forked_project.errors.messages)
         else
-          present @forked_project, with: Entities::Project,
-                                   user_can_admin_project: can?(current_user, :admin_project, @forked_project)
+          present forked_project, with: Entities::Project,
+                                  user_can_admin_project: can?(current_user, :admin_project, forked_project)
         end
       end
 
-      # Update an existing project
-      #
-      # Parameters:
-      #   id (required) - the id of a project
-      #   name (optional) - name of a project
-      #   path (optional) - path of a project
-      #   description (optional) - short project description
-      #   issues_enabled (optional)
-      #   merge_requests_enabled (optional)
-      #   builds_enabled (optional)
-      #   wiki_enabled (optional)
-      #   snippets_enabled (optional)
-      #   container_registry_enabled (optional)
-      #   shared_runners_enabled (optional)
-      #   public (optional) - if true same as setting visibility_level = 20
-      #   visibility_level (optional) - visibility level of a project
-      #   public_builds (optional)
-      #   lfs_enabled (optional)
-      #   ci_config_file (optional)
-      # Example Request
-      #   PUT /projects/:id
+      desc 'Update an existing project' do
+        success Entities::Project
+      end
+      params do
+        # CE
+        at_least_one_of_ce =
+          [
+            :jobs_enabled,
+            :container_registry_enabled,
+            :default_branch,
+            :description,
+            :issues_enabled,
+            :lfs_enabled,
+            :merge_requests_enabled,
+            :name,
+            :only_allow_merge_if_all_discussions_are_resolved,
+            :only_allow_merge_if_pipeline_succeeds,
+            :path,
+            :printing_merge_request_link_enabled,
+            :public_builds,
+            :request_access_enabled,
+            :shared_runners_enabled,
+            :snippets_enabled,
+            :tag_list,
+            :visibility,
+            :wiki_enabled
+          ]
+        optional :name, type: String, desc: 'The name of the project'
+        optional :default_branch, type: String, desc: 'The default branch of the project'
+        optional :path, type: String, desc: 'The path of the repository'
+
+        use :optional_params
+        at_least_one_of(*at_least_one_of_ce)
+      end
       put ':id' do
-        attrs = attributes_for_keys [:builds_enabled,
-                                     :container_registry_enabled,
-                                     :ci_config_file,
-                                     :default_branch,
-                                     :description,
-                                     :issues_enabled,
-                                     :lfs_enabled,
-                                     :merge_requests_enabled,
-                                     :name,
-                                     :only_allow_merge_if_build_succeeds,
-                                     :path,
-                                     :public,
-                                     :public_builds,
-                                     :request_access_enabled,
-                                     :shared_runners_enabled,
-                                     :snippets_enabled,
-                                     :visibility_level,
-                                     :wiki_enabled]
-        attrs = map_public_to_visibility_level(attrs)
         authorize_admin_project
+        attrs = declared_params(include_missing: false)
         authorize! :rename_project, user_project if attrs[:name].present?
-        if attrs[:visibility_level].present?
-          authorize! :change_visibility_level, user_project
-        end
+        authorize! :change_visibility_level, user_project if attrs[:visibility].present?
 
-        ::Projects::UpdateService.new(user_project,
-                                      current_user, attrs).execute
+        attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled)
 
-        if user_project.errors.any?
-          render_validation_error!(user_project)
-        else
+        result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute
+
+        if result[:status] == :success
           present user_project, with: Entities::Project,
                                 user_can_admin_project: can?(current_user, :admin_project, user_project)
+        else
+          render_validation_error!(user_project)
         end
       end
 
-      # Archive project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   PUT /projects/:id/archive
+      desc 'Archive a project' do
+        success Entities::Project
+      end
       post ':id/archive' do
         authorize!(:archive_project, user_project)
 
@@ -314,12 +265,9 @@ module API
         present user_project, with: Entities::Project
       end
 
-      # Unarchive project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   PUT /projects/:id/unarchive
+      desc 'Unarchive a project' do
+        success Entities::Project
+      end
       post ':id/unarchive' do
         authorize!(:archive_project, user_project)
 
@@ -328,12 +276,9 @@ module API
         present user_project, with: Entities::Project
       end
 
-      # Star project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   POST /projects/:id/star
+      desc 'Star a project' do
+        success Entities::Project
+      end
       post ':id/star' do
         if current_user.starred?(user_project)
           not_modified!
@@ -345,13 +290,10 @@ module API
         end
       end
 
-      # Unstar project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   DELETE /projects/:id/star
-      delete ':id/star' do
+      desc 'Unstar a project' do
+        success Entities::Project
+      end
+      post ':id/unstar' do
         if current_user.starred?(user_project)
           current_user.toggle_star(user_project)
           user_project.reload
@@ -362,71 +304,63 @@ module API
         end
       end
 
-      # Remove project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   DELETE /projects/:id
+      desc 'Remove a project'
       delete ":id" do
         authorize! :remove_project, user_project
         ::Projects::DestroyService.new(user_project, current_user, {}).async_execute
+
+        accepted!
       end
 
-      # Mark this project as forked from another
-      #
-      # Parameters:
-      #   id: (required) - The ID of the project being marked as a fork
-      #   forked_from_id: (required) - The ID of the project it was forked from
-      # Example Request:
-      #   POST /projects/:id/fork/:forked_from_id
+      desc 'Mark this project as forked from another'
+      params do
+        requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from'
+      end
       post ":id/fork/:forked_from_id" do
         authenticated_as_admin!
-        forked_from_project = find_project(params[:forked_from_id])
-        unless forked_from_project.nil?
-          if user_project.forked_from_project.nil?
-            user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id)
-          else
-            render_api_error!("Project already forked", 409)
-          end
+
+        forked_from_project = find_project!(params[:forked_from_id])
+        not_found!("Source Project") unless forked_from_project
+
+        if user_project.forked_from_project.nil?
+          user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id)
         else
-          not_found!("Source Project")
+          render_api_error!("Project already forked", 409)
         end
       end
 
-      # Remove a forked_from relationship
-      #
-      # Parameters:
-      #   id: (required) - The ID of the project being marked as a fork
-      # Example Request:
-      #  DELETE /projects/:id/fork
+      desc 'Remove a forked_from relationship'
       delete ":id/fork" do
         authorize! :remove_fork_project, user_project
+
         if user_project.forked?
           user_project.forked_project_link.destroy
+        else
+          not_modified!
         end
       end
 
-      # Share project with group
-      #
-      # Parameters:
-      #   id (required)           - The ID of a project
-      #   group_id (required)     - The ID of a group
-      #   group_access (required) - Level of permissions for sharing
-      #   expires_at (optional)   - Share expiration date
-      #
-      # Example Request:
-      #   POST /projects/:id/share
+      desc 'Share the project with a group' do
+        success Entities::ProjectGroupLink
+      end
+      params do
+        requires :group_id, type: Integer, desc: 'The ID of a group'
+        requires :group_access, type: Integer, values: Gitlab::Access.values, desc: 'The group access level'
+        optional :expires_at, type: Date, desc: 'Share expiration date'
+      end
       post ":id/share" do
         authorize! :admin_project, user_project
-        required_attributes! [:group_id, :group_access]
-        attrs = attributes_for_keys [:group_id, :group_access, :expires_at]
+        group = Group.find_by_id(params[:group_id])
+
+        unless group && can?(current_user, :read_group, group)
+          not_found!('Group')
+        end
 
         unless user_project.allowed_to_share_with_group?
           return render_api_error!("The project sharing with group is disabled", 400)
         end
 
-        link = user_project.project_group_links.new(attrs)
+        link = user_project.project_group_links.new(declared_params(include_missing: false))
 
         if link.save
           present link, with: Entities::ProjectGroupLink
@@ -435,40 +369,51 @@ module API
         end
       end
 
-      # Upload a file
-      #
-      # Parameters:
-      #   id: (required) - The ID of the project
-      #   file: (required) - The file to be uploaded
-      post ":id/uploads" do
-        ::Projects::UploadService.new(user_project, params[:file]).execute
+      params do
+        requires :group_id, type: Integer, desc: 'The ID of the group'
       end
+      delete ":id/share/:group_id" do
+        authorize! :admin_project, user_project
+
+        link = user_project.project_group_links.find_by(group_id: params[:group_id])
+        not_found!('Group Link') unless link
 
-      # search for projects current_user has access to
-      #
-      # Parameters:
-      #   query (required) - A string contained in the project name
-      #   per_page (optional) - number of projects to return per page
-      #   page (optional) - the page to retrieve
-      # Example Request:
-      #   GET /projects/search/:query
-      get "/search/:query" do
-        search_service = Search::GlobalService.new(current_user, search: params[:query]).execute
-        projects = search_service.objects('projects', params[:page])
-        projects = projects.reorder(project_order_by => project_sort)
+        link.destroy
+      end
 
-        present paginate(projects), with: Entities::Project
+      desc 'Upload a file'
+      params do
+        requires :file, type: File, desc: 'The file to be uploaded'
+      end
+      post ":id/uploads" do
+        UploadService.new(user_project, params[:file]).execute
       end
 
-      # Get a users list
-      #
-      # Example Request:
-      #  GET /users
+      desc 'Get the users list of a project' do
+        success Entities::UserBasic
+      end
+      params do
+        optional :search, type: String, desc: 'Return list of users matching the search criteria'
+        use :pagination
+      end
       get ':id/users' do
-        @users = User.where(id: user_project.team.users.map(&:id))
-        @users = @users.search(params[:search]) if params[:search].present?
-        @users = paginate @users
-        present @users, with: Entities::UserBasic
+        users = user_project.team.users
+        users = users.search(params[:search]) if params[:search].present?
+
+        present paginate(users), with: Entities::UserBasic
+      end
+
+      desc 'Start the housekeeping task for a project' do
+        detail 'This feature was introduced in GitLab 9.0.'
+      end
+      post ':id/housekeeping' do
+        authorize_admin_project
+
+        begin
+          ::Projects::HousekeepingService.new(user_project).execute
+        rescue ::Projects::HousekeepingService::LeaseTaken => error
+          conflict!(error.message)
+        end
       end
     end
   end
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index f55aceed92c..14d2bff9cb5 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -1,12 +1,15 @@
 require 'mime/types'
 
 module API
-  # Projects API
   class Repositories < Grape::API
-    before { authenticate! }
+    include PaginationParams
+
     before { authorize! :download_code, user_project }
 
-    resource :projects do
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
       helpers do
         def handle_project_member_errors(errors)
           if errors[:project_access].any?
@@ -14,85 +17,75 @@ module API
           end
           not_found!
         end
+
+        def assign_blob_vars!
+          authorize! :download_code, user_project
+
+          @repo = user_project.repository
+
+          begin
+            @blob = Gitlab::Git::Blob.raw(@repo, params[:sha])
+            @blob.load_all_data!(@repo)
+          rescue
+            not_found! 'Blob'
+          end
+
+          not_found! 'Blob' unless @blob
+        end
       end
 
-      # Get a project repository tree
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   ref_name (optional) - The name of a repository branch or tag, if not given the default branch is used
-      # Example Request:
-      #   GET /projects/:id/repository/tree
+      desc 'Get a project repository tree' do
+        success Entities::RepoTreeObject
+      end
+      params do
+        optional :ref, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used'
+        optional :path, type: String, desc: 'The path of the tree'
+        optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree'
+        use :pagination
+      end
       get ':id/repository/tree' do
-        ref = params[:ref_name] || user_project.try(:default_branch) || 'master'
+        ref = params[:ref] || user_project.try(:default_branch) || 'master'
         path = params[:path] || nil
 
         commit = user_project.commit(ref)
         not_found!('Tree') unless commit
 
-        tree = user_project.repository.tree(commit.id, path)
-
-        present tree.sorted_entries, with: Entities::RepoTreeObject
+        tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive])
+        entries = ::Kaminari.paginate_array(tree.sorted_entries)
+        present paginate(entries), with: Entities::RepoTreeObject
       end
 
-      # Get a raw file contents
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   sha (required) - The commit or branch name
-      #   filepath (required) - The path to the file to display
-      # Example Request:
-      #   GET /projects/:id/repository/blobs/:sha
-      get [ ":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob" ] do
-        required_attributes! [:filepath]
-
-        ref = params[:sha]
-
-        repo = user_project.repository
-
-        commit = repo.commit(ref)
-        not_found! "Commit" unless commit
-
-        blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath])
-        not_found! "File" unless blob
-
-        send_git_blob repo, blob
+      desc 'Get raw blob contents from the repository'
+      params do
+        requires :sha, type: String, desc: 'The commit, branch name, or tag name'
       end
+      get ':id/repository/blobs/:sha/raw' do
+        assign_blob_vars!
 
-      # Get a raw blob contents by blob sha
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   sha (required) - The blob's sha
-      # Example Request:
-      #   GET /projects/:id/repository/raw_blobs/:sha
-      get ':id/repository/raw_blobs/:sha' do
-        ref = params[:sha]
-
-        repo = user_project.repository
-
-        begin
-          blob = Gitlab::Git::Blob.raw(repo, ref)
-        rescue
-          not_found! 'Blob'
-        end
-
-        not_found! 'Blob' unless blob
-
-        send_git_blob repo, blob
+        send_git_blob @repo, @blob
       end
 
-      # Get a an archive of the repository
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   sha (optional) - the commit sha to download defaults to the tip of the default branch
-      # Example Request:
-      #   GET /projects/:id/repository/archive
-      get ':id/repository/archive',
-          requirements: { format: Gitlab::Regex.archive_formats_regex } do
-        authorize! :download_code, user_project
+      desc 'Get a blob from the repository'
+      params do
+        requires :sha, type: String, desc: 'The commit, branch name, or tag name'
+      end
+      get ':id/repository/blobs/:sha' do
+        assign_blob_vars!
+
+        {
+          size: @blob.size,
+          encoding: "base64",
+          content: Base64.strict_encode64(@blob.data),
+          sha: @blob.id
+        }
+      end
 
+      desc 'Get an archive of the repository'
+      params do
+        optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded'
+        optional :format, type: String, desc: 'The archive format'
+      end
+      get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do
         begin
           send_git_archive user_project.repository, ref: params[:sha], format: params[:format]
         rescue
@@ -100,33 +93,28 @@ module API
         end
       end
 
-      # Compare two branches, tags or commits
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   from (required) - the commit sha or branch name
-      #   to (required) - the commit sha or branch name
-      # Example Request:
-      #   GET /projects/:id/repository/compare?from=master&to=feature
+      desc 'Compare two branches, tags, or commits' do
+        success Entities::Compare
+      end
+      params do
+        requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison'
+        requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison'
+      end
       get ':id/repository/compare' do
-        authorize! :download_code, user_project
-        required_attributes! [:from, :to]
         compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to])
         present compare, with: Entities::Compare
       end
 
-      # Get repository contributors
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   GET /projects/:id/repository/contributors
+      desc 'Get repository contributors' do
+        success Entities::Contributor
+      end
+      params do
+        use :pagination
+      end
       get ':id/repository/contributors' do
-        authorize! :download_code, user_project
-
         begin
-          present user_project.repository.contributors,
-                  with: Entities::Contributor
+          contributors = ::Kaminari.paginate_array(user_project.repository.contributors)
+          present paginate(contributors), with: Entities::Contributor
         rescue
           not_found!
         end
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
new file mode 100644
index 00000000000..4552115b3e2
--- /dev/null
+++ b/lib/api/runner.rb
@@ -0,0 +1,248 @@
+module API
+  class Runner < Grape::API
+    helpers ::API::Helpers::Runner
+
+    resource :runners do
+      desc 'Registers a new Runner' do
+        success Entities::RunnerRegistrationDetails
+        http_codes [[201, 'Runner was created'], [403, 'Forbidden']]
+      end
+      params do
+        requires :token, type: String, desc: 'Registration token'
+        optional :description, type: String, desc: %q(Runner's description)
+        optional :info, type: Hash, desc: %q(Runner's metadata)
+        optional :locked, type: Boolean, desc: 'Should Runner be locked for current project'
+        optional :run_untagged, type: Boolean, desc: 'Should Runner handle untagged jobs'
+        optional :tag_list, type: Array[String], desc: %q(List of Runner's tags)
+      end
+      post '/' do
+        attributes = attributes_for_keys [:description, :locked, :run_untagged, :tag_list]
+
+        runner =
+          if runner_registration_token_valid?
+            # Create shared runner. Requires admin access
+            Ci::Runner.create(attributes.merge(is_shared: true))
+          elsif project = Project.find_by(runners_token: params[:token])
+            # Create a specific runner for project.
+            project.runners.create(attributes)
+          end
+
+        return forbidden! unless runner
+
+        if runner.id
+          runner.update(get_runner_version_from_params)
+          present runner, with: Entities::RunnerRegistrationDetails
+        else
+          not_found!
+        end
+      end
+
+      desc 'Deletes a registered Runner' do
+        http_codes [[204, 'Runner was deleted'], [403, 'Forbidden']]
+      end
+      params do
+        requires :token, type: String, desc: %q(Runner's authentication token)
+      end
+      delete '/' do
+        authenticate_runner!
+        Ci::Runner.find_by_token(params[:token]).destroy
+      end
+
+      desc 'Validates authentication credentials' do
+        http_codes [[200, 'Credentials are valid'], [403, 'Forbidden']]
+      end
+      params do
+        requires :token, type: String, desc: %q(Runner's authentication token)
+      end
+      post '/verify' do
+        authenticate_runner!
+        status 200
+      end
+    end
+
+    resource :jobs do
+      desc 'Request a job' do
+        success Entities::JobRequest::Response
+        http_codes [[201, 'Job was scheduled'],
+                    [204, 'No job for Runner'],
+                    [403, 'Forbidden']]
+      end
+      params do
+        requires :token, type: String, desc: %q(Runner's authentication token)
+        optional :last_update, type: String, desc: %q(Runner's queue last_update token)
+        optional :info, type: Hash, desc: %q(Runner's metadata)
+      end
+      post '/request' do
+        authenticate_runner!
+        no_content! unless current_runner.active?
+        update_runner_info
+
+        if current_runner.is_runner_queue_value_latest?(params[:last_update])
+          header 'X-GitLab-Last-Update', params[:last_update]
+          Gitlab::Metrics.add_event(:build_not_found_cached)
+          return no_content!
+        end
+
+        new_update = current_runner.ensure_runner_queue_value
+        result = ::Ci::RegisterJobService.new(current_runner).execute
+
+        if result.valid?
+          if result.build
+            Gitlab::Metrics.add_event(:build_found,
+                                      project: result.build.project.path_with_namespace)
+            present result.build, with: Entities::JobRequest::Response
+          else
+            Gitlab::Metrics.add_event(:build_not_found)
+            header 'X-GitLab-Last-Update', new_update
+            no_content!
+          end
+        else
+          # We received build that is invalid due to concurrency conflict
+          Gitlab::Metrics.add_event(:build_invalid)
+          conflict!
+        end
+      end
+
+      desc 'Updates a job' do
+        http_codes [[200, 'Job was updated'], [403, 'Forbidden']]
+      end
+      params do
+        requires :token, type: String, desc: %q(Runners's authentication token)
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :trace, type: String, desc: %q(Job's full trace)
+        optional :state, type: String, desc: %q(Job's status: success, failed)
+      end
+      put '/:id' do
+        job = authenticate_job!
+
+        job.trace.set(params[:trace]) if params[:trace]
+
+        Gitlab::Metrics.add_event(:update_build,
+                                  project: job.project.path_with_namespace)
+
+        case params[:state].to_s
+        when 'success'
+          job.success
+        when 'failed'
+          job.drop
+        end
+      end
+
+      desc 'Appends a patch to the job trace' do
+        http_codes [[202, 'Trace was patched'],
+                    [400, 'Missing Content-Range header'],
+                    [403, 'Forbidden'],
+                    [416, 'Range not satisfiable']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+      end
+      patch '/:id/trace' do
+        job = authenticate_job!
+
+        error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
+        content_range = request.headers['Content-Range']
+        content_range = content_range.split('-')
+
+        stream_size = job.trace.append(request.body.read, content_range[0].to_i)
+        if stream_size < 0
+          return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
+        end
+
+        status 202
+        header 'Job-Status', job.status
+        header 'Range', "0-#{stream_size}"
+      end
+
+      desc 'Authorize artifacts uploading for job' do
+        http_codes [[200, 'Upload allowed'],
+                    [403, 'Forbidden'],
+                    [405, 'Artifacts support not enabled'],
+                    [413, 'File too large']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+        optional :filesize, type: Integer, desc: %q(Artifacts filesize)
+      end
+      post '/:id/artifacts/authorize' do
+        not_allowed! unless Gitlab.config.artifacts.enabled
+        require_gitlab_workhorse!
+        Gitlab::Workhorse.verify_api_request!(headers)
+
+        job = authenticate_job!
+        forbidden!('Job is not running') unless job.running?
+
+        if params[:filesize]
+          file_size = params[:filesize].to_i
+          file_to_large! unless file_size < max_artifacts_size
+        end
+
+        status 200
+        content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
+        Gitlab::Workhorse.artifact_upload_ok
+      end
+
+      desc 'Upload artifacts for job' do
+        success Entities::JobRequest::Response
+        http_codes [[201, 'Artifact uploaded'],
+                    [400, 'Bad request'],
+                    [403, 'Forbidden'],
+                    [405, 'Artifacts support not enabled'],
+                    [413, 'File too large']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+        optional :expire_in, type: String, desc: %q(Specify when artifacts should expire)
+        optional :file, type: File, desc: %q(Artifact's file)
+        optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
+        optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
+        optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
+        optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
+        optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse))
+      end
+      post '/:id/artifacts' do
+        not_allowed! unless Gitlab.config.artifacts.enabled
+        require_gitlab_workhorse!
+
+        job = authenticate_job!
+        forbidden!('Job is not running!') unless job.running?
+
+        artifacts_upload_path = ArtifactUploader.artifacts_upload_path
+        artifacts = uploaded_file(:file, artifacts_upload_path)
+        metadata = uploaded_file(:metadata, artifacts_upload_path)
+
+        bad_request!('Missing artifacts file!') unless artifacts
+        file_to_large! unless artifacts.size < max_artifacts_size
+
+        job.artifacts_file = artifacts
+        job.artifacts_metadata = metadata
+        job.artifacts_expire_in = params['expire_in'] ||
+          Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
+
+        if job.save
+          present job, with: Entities::JobRequest::Response
+        else
+          render_validation_error!(job)
+        end
+      end
+
+      desc 'Download the artifacts file for job' do
+        http_codes [[200, 'Upload allowed'],
+                    [403, 'Forbidden'],
+                    [404, 'Artifact not found']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+      end
+      get '/:id/artifacts' do
+        job = authenticate_job!
+
+        present_artifacts!(job.artifacts_file)
+      end
+    end
+  end
+end
diff --git a/lib/api/runners.rb b/lib/api/runners.rb
index ecc8f2fc5a2..db6c7c59092 100644
--- a/lib/api/runners.rb
+++ b/lib/api/runners.rb
@@ -1,34 +1,43 @@
 module API
-  # Runners API
   class Runners < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
     resource :runners do
-      # Get runners available for user
-      #
-      # Example Request:
-      #   GET /runners
+      desc 'Get runners available for user' do
+        success Entities::Runner
+      end
+      params do
+        optional :scope, type: String, values: %w[active paused online],
+                         desc: 'The scope of specific runners to show'
+        use :pagination
+      end
       get do
-        runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: ['specific', 'shared'])
+        runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: %w(specific shared))
         present paginate(runners), with: Entities::Runner
       end
 
-      # Get all runners - shared and specific
-      #
-      # Example Request:
-      #   GET /runners/all
+      desc 'Get all runners - shared and specific' do
+        success Entities::Runner
+      end
+      params do
+        optional :scope, type: String, values: %w[active paused online specific shared],
+                         desc: 'The scope of specific runners to show'
+        use :pagination
+      end
       get 'all' do
         authenticated_as_admin!
         runners = filter_runners(Ci::Runner.all, params[:scope])
         present paginate(runners), with: Entities::Runner
       end
 
-      # Get runner's details
-      #
-      # Parameters:
-      #   id (required) - The ID of ther runner
-      # Example Request:
-      #   GET /runners/:id
+      desc "Get runner's details" do
+        success Entities::RunnerDetails
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the runner'
+      end
       get ':id' do
         runner = get_runner(params[:id])
         authenticate_show_runner!(runner)
@@ -36,64 +45,70 @@ module API
         present runner, with: Entities::RunnerDetails, current_user: current_user
       end
 
-      # Update runner's details
-      #
-      # Parameters:
-      #   id (required) - The ID of ther runner
-      #   description (optional) - Runner's description
-      #   active (optional) - Runner's status
-      #   tag_list (optional) - Array of tags for runner
-      # Example Request:
-      #   PUT /runners/:id
+      desc "Update runner's details" do
+        success Entities::RunnerDetails
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the runner'
+        optional :description, type: String, desc: 'The description of the runner'
+        optional :active, type: Boolean, desc: 'The state of a runner'
+        optional :tag_list, type: Array[String], desc: 'The list of tags for a runner'
+        optional :run_untagged, type: Boolean, desc: 'Flag indicating the runner can execute untagged jobs'
+        optional :locked, type: Boolean, desc: 'Flag indicating the runner is locked'
+        at_least_one_of :description, :active, :tag_list, :run_untagged, :locked
+      end
       put ':id' do
-        runner = get_runner(params[:id])
+        runner = get_runner(params.delete(:id))
         authenticate_update_runner!(runner)
+        update_service = Ci::UpdateRunnerService.new(runner)
 
-        attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged, :locked]
-        if runner.update(attrs)
+        if update_service.update(declared_params(include_missing: false))
           present runner, with: Entities::RunnerDetails, current_user: current_user
         else
           render_validation_error!(runner)
         end
       end
 
-      # Remove runner
-      #
-      # Parameters:
-      #   id (required) - The ID of ther runner
-      # Example Request:
-      #   DELETE /runners/:id
+      desc 'Remove a runner' do
+        success Entities::Runner
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the runner'
+      end
       delete ':id' do
         runner = get_runner(params[:id])
         authenticate_delete_runner!(runner)
-        runner.destroy!
 
-        present runner, with: Entities::Runner
+        runner.destroy!
       end
     end
 
-    resource :projects do
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
       before { authorize_admin_project }
 
-      # Get runners available for project
-      #
-      # Example Request:
-      #   GET /projects/:id/runners
+      desc 'Get runners available for project' do
+        success Entities::Runner
+      end
+      params do
+        optional :scope, type: String, values: %w[active paused online specific shared],
+                         desc: 'The scope of specific runners to show'
+        use :pagination
+      end
       get ':id/runners' do
         runners = filter_runners(Ci::Runner.owned_or_shared(user_project.id), params[:scope])
         present paginate(runners), with: Entities::Runner
       end
 
-      # Enable runner for project
-      #
-      # Parameters:
-      #   id (required) - The ID of the project
-      #   runner_id (required) - The ID of the runner
-      # Example Request:
-      #   POST /projects/:id/runners/:runner_id
+      desc 'Enable a runner for a project' do
+        success Entities::Runner
+      end
+      params do
+        requires :runner_id, type: Integer, desc: 'The ID of the runner'
+      end
       post ':id/runners' do
-        required_attributes! [:runner_id]
-
         runner = get_runner(params[:runner_id])
         authenticate_enable_runner!(runner)
 
@@ -106,13 +121,12 @@ module API
         end
       end
 
-      # Disable project's runner
-      #
-      # Parameters:
-      #   id (required) - The ID of the project
-      #   runner_id (required) - The ID of the runner
-      # Example Request:
-      #   DELETE /projects/:id/runners/:runner_id
+      desc "Disable project's runner" do
+        success Entities::Runner
+      end
+      params do
+        requires :runner_id, type: Integer, desc: 'The ID of the runner'
+      end
       delete ':id/runners/:runner_id' do
         runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id])
         not_found!('Runner') unless runner_project
@@ -121,8 +135,6 @@ module API
         forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1
 
         runner_project.destroy
-
-        present runner, with: Entities::Runner
       end
     end
 
@@ -149,18 +161,18 @@ module API
       end
 
       def authenticate_show_runner!(runner)
-        return if runner.is_shared || current_user.is_admin?
+        return if runner.is_shared || current_user.admin?
         forbidden!("No access granted") unless user_can_access_runner?(runner)
       end
 
       def authenticate_update_runner!(runner)
-        return if current_user.is_admin?
+        return if current_user.admin?
         forbidden!("Runner is shared") if runner.is_shared?
         forbidden!("No access granted") unless user_can_access_runner?(runner)
       end
 
       def authenticate_delete_runner!(runner)
-        return if current_user.is_admin?
+        return if current_user.admin?
         forbidden!("Runner is shared") if runner.is_shared?
         forbidden!("Runner associated with more than one project") if runner.projects.count > 1
         forbidden!("No access granted") unless user_can_access_runner?(runner)
@@ -169,7 +181,7 @@ module API
       def authenticate_enable_runner!(runner)
         forbidden!("Runner is shared") if runner.is_shared?
         forbidden!("Runner is locked") if runner.locked?
-        return if current_user.is_admin?
+        return if current_user.admin?
         forbidden!("No access granted") unless user_can_access_runner?(runner)
       end
 
diff --git a/lib/api/services.rb b/lib/api/services.rb
index fc8598daa32..7488f95a9b7 100644
--- a/lib/api/services.rb
+++ b/lib/api/services.rb
@@ -1,62 +1,725 @@
 module API
-  # Projects API
   class Services < Grape::API
-    before { authenticate! }
-    before { authorize_admin_project }
-
-    resource :projects do
-      # Set <service_slug> service for project
-      #
-      # Example Request:
-      #
-      #   PUT /projects/:id/services/gitlab-ci
-      #
-      put ':id/services/:service_slug' do
-        if project_service
-          validators = project_service.class.validators.select do |s|
-            s.class == ActiveRecord::Validations::PresenceValidator &&
-              s.attributes != [:project_id]
-          end
+    services = {
+      'asana' => [
+        {
+          required: true,
+          name: :api_key,
+          type: String,
+          desc: 'User API token'
+        },
+        {
+          required: false,
+          name: :restrict_to_branch,
+          type: String,
+          desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches'
+        }
+      ],
+      'assembla' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The authentication token'
+        },
+        {
+          required: false,
+          name: :subdomain,
+          type: String,
+          desc: 'Subdomain setting'
+        }
+      ],
+      'bamboo' => [
+        {
+          required: true,
+          name: :bamboo_url,
+          type: String,
+          desc: 'Bamboo root URL like https://bamboo.example.com'
+        },
+        {
+          required: true,
+          name: :build_key,
+          type: String,
+          desc: 'Bamboo build plan key like'
+        },
+        {
+          required: true,
+          name: :username,
+          type: String,
+          desc: 'A user with API access, if applicable'
+        },
+        {
+          required: true,
+          name: :password,
+          type: String,
+          desc: 'Passord of the user'
+        }
+      ],
+      'bugzilla' => [
+        {
+          required: true,
+          name: :new_issue_url,
+          type: String,
+          desc: 'New issue URL'
+        },
+        {
+          required: true,
+          name: :issues_url,
+          type: String,
+          desc: 'Issues URL'
+        },
+        {
+          required: true,
+          name: :project_url,
+          type: String,
+          desc: 'Project URL'
+        },
+        {
+          required: false,
+          name: :description,
+          type: String,
+          desc: 'Description'
+        },
+        {
+          required: false,
+          name: :title,
+          type: String,
+          desc: 'Title'
+        }
+      ],
+      'buildkite' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'Buildkite project GitLab token'
+        },
+        {
+          required: true,
+          name: :project_url,
+          type: String,
+          desc: 'The buildkite project URL'
+        },
+        {
+          required: false,
+          name: :enable_ssl_verification,
+          type: Boolean,
+          desc: 'Enable SSL verification for communication'
+        }
+      ],
+      'campfire' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'Campfire token'
+        },
+        {
+          required: false,
+          name: :subdomain,
+          type: String,
+          desc: 'Campfire subdomain'
+        },
+        {
+          required: false,
+          name: :room,
+          type: String,
+          desc: 'Campfire room'
+        }
+      ],
+      'custom-issue-tracker' => [
+        {
+          required: true,
+          name: :new_issue_url,
+          type: String,
+          desc: 'New issue URL'
+        },
+        {
+          required: true,
+          name: :issues_url,
+          type: String,
+          desc: 'Issues URL'
+        },
+        {
+          required: true,
+          name: :project_url,
+          type: String,
+          desc: 'Project URL'
+        },
+        {
+          required: false,
+          name: :description,
+          type: String,
+          desc: 'Description'
+        },
+        {
+          required: false,
+          name: :title,
+          type: String,
+          desc: 'Title'
+        }
+      ],
+      'drone-ci' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'Drone CI token'
+        },
+        {
+          required: true,
+          name: :drone_url,
+          type: String,
+          desc: 'Drone CI URL'
+        },
+        {
+          required: false,
+          name: :enable_ssl_verification,
+          type: Boolean,
+          desc: 'Enable SSL verification for communication'
+        }
+      ],
+      'emails-on-push' => [
+        {
+          required: true,
+          name: :recipients,
+          type: String,
+          desc: 'Comma-separated list of recipient email addresses'
+        },
+        {
+          required: false,
+          name: :disable_diffs,
+          type: Boolean,
+          desc: 'Disable code diffs'
+        },
+        {
+          required: false,
+          name: :send_from_committer_email,
+          type: Boolean,
+          desc: 'Send from committer'
+        }
+      ],
+      'external-wiki' => [
+        {
+          required: true,
+          name: :external_wiki_url,
+          type: String,
+          desc: 'The URL of the external Wiki'
+        }
+      ],
+      'flowdock' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'Flowdock token'
+        }
+      ],
+      'gemnasium' => [
+        {
+          required: true,
+          name: :api_key,
+          type: String,
+          desc: 'Your personal API key on gemnasium.com'
+        },
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: "The project's slug on gemnasium.com"
+        }
+      ],
+      'hipchat' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The room token'
+        },
+        {
+          required: false,
+          name: :room,
+          type: String,
+          desc: 'The room name or ID'
+        },
+        {
+          required: false,
+          name: :color,
+          type: String,
+          desc: 'The room color'
+        },
+        {
+          required: false,
+          name: :notify,
+          type: Boolean,
+          desc: 'Enable notifications'
+        },
+        {
+          required: false,
+          name: :api_version,
+          type: String,
+          desc: 'Leave blank for default (v2)'
+        },
+        {
+          required: false,
+          name: :server,
+          type: String,
+          desc: 'Leave blank for default. https://hipchat.example.com'
+        }
+      ],
+      'irker' => [
+        {
+          required: true,
+          name: :recipients,
+          type: String,
+          desc: 'Recipients/channels separated by whitespaces'
+        },
+        {
+          required: false,
+          name: :default_irc_uri,
+          type: String,
+          desc: 'Default: irc://irc.network.net:6697'
+        },
+        {
+          required: false,
+          name: :server_host,
+          type: String,
+          desc: 'Server host. Default localhost'
+        },
+        {
+          required: false,
+          name: :server_port,
+          type: Integer,
+          desc: 'Server port. Default 6659'
+        },
+        {
+          required: false,
+          name: :colorize_messages,
+          type: Boolean,
+          desc: 'Colorize messages'
+        }
+      ],
+      'jira' => [
+        {
+          required: true,
+          name: :url,
+          type: String,
+          desc: 'The base URL to the JIRA instance web interface which is being linked to this GitLab project. E.g., https://jira.example.com'
+        },
+        {
+          required: false,
+          name: :api_url,
+          type: String,
+          desc: 'The base URL to the JIRA instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com'
+        },
+        {
+          required: true,
+          name: :project_key,
+          type: String,
+          desc: 'The short identifier for your JIRA project, all uppercase, e.g., PROJ'
+        },
+        {
+          required: false,
+          name: :username,
+          type: String,
+          desc: 'The username of the user created to be used with GitLab/JIRA'
+        },
+        {
+          required: false,
+          name: :password,
+          type: String,
+          desc: 'The password of the user created to be used with GitLab/JIRA'
+        },
+        {
+          required: false,
+          name: :jira_issue_transition_id,
+          type: Integer,
+          desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`'
+        }
+      ],
 
-          required_attributes! validators.map(&:attributes).flatten.uniq
-          attrs = attributes_for_keys service_attributes
+      'kubernetes' => [
+        {
+          required: true,
+          name: :namespace,
+          type: String,
+          desc: 'The Kubernetes namespace to use'
+        },
+        {
+          required: true,
+          name: :api_url,
+          type: String,
+          desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com'
+        },
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The service token to authenticate against the Kubernetes cluster with'
+        },
+        {
+          required: false,
+          name: :ca_pem,
+          type: String,
+          desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)'
+        }
+      ],
+      'mattermost-slash-commands' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The Mattermost token'
+        }
+      ],
+      'slack-slash-commands' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The Slack token'
+        }
+      ],
+      'pipelines-email' => [
+        {
+          required: true,
+          name: :recipients,
+          type: String,
+          desc: 'Comma-separated list of recipient email addresses'
+        },
+        {
+          required: false,
+          name: :notify_only_broken_pipelines,
+          type: Boolean,
+          desc: 'Notify only broken pipelines'
+        }
+      ],
+      'pivotaltracker' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The Pivotaltracker token'
+        },
+        {
+          required: false,
+          name: :restrict_to_branch,
+          type: String,
+          desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.'
+        }
+      ],
+      'prometheus' => [
+        {
+          required: true,
+          name: :api_url,
+          type: String,
+          desc: 'Prometheus API Base URL, like http://prometheus.example.com/'
+        }
+      ],
+      'pushover' => [
+        {
+          required: true,
+          name: :api_key,
+          type: String,
+          desc: 'The application key'
+        },
+        {
+          required: true,
+          name: :user_key,
+          type: String,
+          desc: 'The user key'
+        },
+        {
+          required: true,
+          name: :priority,
+          type: String,
+          desc: 'The priority'
+        },
+        {
+          required: true,
+          name: :device,
+          type: String,
+          desc: 'Leave blank for all active devices'
+        },
+        {
+          required: true,
+          name: :sound,
+          type: String,
+          desc: 'The sound of the notification'
+        }
+      ],
+      'redmine' => [
+        {
+          required: true,
+          name: :new_issue_url,
+          type: String,
+          desc: 'The new issue URL'
+        },
+        {
+          required: true,
+          name: :project_url,
+          type: String,
+          desc: 'The project URL'
+        },
+        {
+          required: true,
+          name: :issues_url,
+          type: String,
+          desc: 'The issues URL'
+        },
+        {
+          required: false,
+          name: :description,
+          type: String,
+          desc: 'The description of the tracker'
+        }
+      ],
+      'slack' => [
+        {
+          required: true,
+          name: :webhook,
+          type: String,
+          desc: 'The Slack webhook. e.g. https://hooks.slack.com/services/...'
+        },
+        {
+          required: false,
+          name: :new_issue_url,
+          type: String,
+          desc: 'The user name'
+        },
+        {
+          required: false,
+          name: :channel,
+          type: String,
+          desc: 'The channel name'
+        }
+      ],
+      'microsoft-teams' => [
+        {
+          required: true,
+          name: :webhook,
+          type: String,
+          desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…'
+        }
+      ],
+      'mattermost' => [
+        {
+          required: true,
+          name: :webhook,
+          type: String,
+          desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...'
+        }
+      ],
+      'teamcity' => [
+        {
+          required: true,
+          name: :teamcity_url,
+          type: String,
+          desc: 'TeamCity root URL like https://teamcity.example.com'
+        },
+        {
+          required: true,
+          name: :build_type,
+          type: String,
+          desc: 'Build configuration ID'
+        },
+        {
+          required: true,
+          name: :username,
+          type: String,
+          desc: 'A user with permissions to trigger a manual build'
+        },
+        {
+          required: true,
+          name: :password,
+          type: String,
+          desc: 'The password of the user'
+        }
+      ]
+    }
 
-          if project_service.update_attributes(attrs.merge(active: true))
-            true
-          else
-            not_found!
+    service_classes = [
+      AsanaService,
+      AssemblaService,
+      BambooService,
+      BugzillaService,
+      BuildkiteService,
+      CampfireService,
+      CustomIssueTrackerService,
+      DroneCiService,
+      EmailsOnPushService,
+      ExternalWikiService,
+      FlowdockService,
+      GemnasiumService,
+      HipchatService,
+      IrkerService,
+      JiraService,
+      KubernetesService,
+      MattermostSlashCommandsService,
+      SlackSlashCommandsService,
+      PipelinesEmailService,
+      PivotaltrackerService,
+      PrometheusService,
+      PushoverService,
+      RedmineService,
+      SlackService,
+      MattermostService,
+      MicrosoftTeamsService,
+      TeamcityService
+    ]
+
+    if Rails.env.development?
+      services['mock-ci'] = [
+        {
+          required: true,
+          name: :mock_service_url,
+          type: String,
+          desc: 'URL to the mock service'
+        }
+      ]
+      services['mock-deployment'] = []
+      services['mock-monitoring'] = []
+
+      service_classes += [
+        MockCiService,
+        MockDeploymentService,
+        MockMonitoringService
+      ]
+    end
+
+    trigger_services = {
+      'mattermost-slash-commands' => [
+        {
+          name: :token,
+          type: String,
+          desc: 'The Mattermost token'
+        }
+      ],
+      'slack-slash-commands' => [
+        {
+          name: :token,
+          type: String,
+          desc: 'The Slack token'
+        }
+      ]
+    }.freeze
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      before { authenticate! }
+      before { authorize_admin_project }
+
+      helpers do
+        def service_attributes(service)
+          service.fields.inject([]) do |arr, hash|
+            arr << hash[:name].to_sym
           end
         end
       end
 
-      # Delete <service_slug> service for project
-      #
-      # Example Request:
-      #
-      #   DELETE /project/:id/services/gitlab-ci
-      #
-      delete ':id/services/:service_slug' do
-        if project_service
-          attrs = service_attributes.inject({}) do |hash, key|
-            hash.merge!(key => nil)
+      services.each do |service_slug, settings|
+        desc "Set #{service_slug} service for project"
+        params do
+          service_classes.each do |service|
+            event_names = service.try(:event_names) || next
+            event_names.each do |event_name|
+              services[service.to_param.tr("_", "-")] << {
+                required: false,
+                name: event_name.to_sym,
+                type: String,
+                desc: ServicesHelper.service_event_description(event_name)
+              }
+            end
           end
+          services.freeze
 
-          if project_service.update_attributes(attrs.merge(active: false))
-            true
+          settings.each do |setting|
+            if setting[:required]
+              requires setting[:name], type: setting[:type], desc: setting[:desc]
+            else
+              optional setting[:name], type: setting[:type], desc: setting[:desc]
+            end
+          end
+        end
+        put ":id/services/#{service_slug}" do
+          service = user_project.find_or_initialize_service(service_slug.underscore)
+          service_params = declared_params(include_missing: false).merge(active: true)
+
+          if service.update_attributes(service_params)
+            present service, with: Entities::ProjectService, include_passwords: current_user.admin?
           else
-            not_found!
+            render_api_error!('400 Bad Request', 400)
+          end
+        end
+      end
+
+      desc "Delete a service for project"
+      params do
+        requires :service_slug, type: String, values: services.keys, desc: 'The name of the service'
+      end
+      delete ":id/services/:service_slug" do
+        service = user_project.find_or_initialize_service(params[:service_slug].underscore)
+
+        attrs = service_attributes(service).inject({}) do |hash, key|
+          hash.merge!(key => nil)
+        end
+
+        unless service.update_attributes(attrs.merge(active: false))
+          render_api_error!('400 Bad Request', 400)
+        end
+      end
+
+      desc 'Get the service settings for project' do
+        success Entities::ProjectService
+      end
+      params do
+        requires :service_slug, type: String, values: services.keys, desc: 'The name of the service'
+      end
+      get ":id/services/:service_slug" do
+        service = user_project.find_or_initialize_service(params[:service_slug].underscore)
+        present service, with: Entities::ProjectService, include_passwords: current_user.admin?
+      end
+    end
+
+    trigger_services.each do |service_slug, settings|
+      helpers do
+        def slash_command_service(project, service_slug, params)
+          project.services.active.where(template: false).find do |service|
+            service.try(:token) == params[:token] && service.to_param == service_slug.underscore
           end
         end
       end
 
-      # Get <service_slug> service settings for project
-      #
-      # Example Request:
-      #
-      #   GET /project/:id/services/gitlab-ci
-      #
-      get ':id/services/:service_slug' do
-        present project_service, with: Entities::ProjectService, include_passwords: current_user.is_admin?
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc "Trigger a slash command for #{service_slug}" do
+          detail 'Added in GitLab 8.13'
+        end
+        params do
+          settings.each do |setting|
+            requires setting[:name], type: setting[:type], desc: setting[:desc]
+          end
+        end
+        post ":id/services/#{service_slug.underscore}/trigger" do
+          project = find_project(params[:id])
+
+          # This is not accurate, but done to prevent leakage of the project names
+          not_found!('Service') unless project
+
+          service = slash_command_service(project, service_slug, params)
+          result = service.try(:trigger, params)
+
+          if result
+            status result[:status] || 200
+            present result
+          else
+            not_found!('Service')
+          end
+        end
       end
     end
   end
diff --git a/lib/api/session.rb b/lib/api/session.rb
index 55ec66a6d67..016415c3023 100644
--- a/lib/api/session.rb
+++ b/lib/api/session.rb
@@ -1,21 +1,20 @@
 module API
-  # Users API
   class Session < Grape::API
-    # Login to get token
-    #
-    # Parameters:
-    #   login (*required) - user login
-    #   email (*required) - user email
-    #   password (required) - user password
-    #
-    # Example Request:
-    #  POST /session
+    desc 'Login to get token' do
+      success Entities::UserWithPrivateDetails
+    end
+    params do
+      optional :login, type: String, desc: 'The username'
+      optional :email, type: String, desc: 'The email of the user'
+      requires :password, type: String, desc: 'The password of the user'
+      at_least_one_of :login, :email
+    end
     post "/session" do
       user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password])
 
       return unauthorized! unless user
       return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled?
-      present user, with: Entities::UserLogin
+      present user, with: Entities::UserWithPrivateDetails
     end
   end
 end
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index c885fcd7ea3..d598f9a62a2 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -9,21 +9,172 @@ module API
       end
     end
 
-    # Get current applicaiton settings
-    #
-    # Example Request:
-    #   GET /application/settings
+    desc 'Get the current application settings' do
+      success Entities::ApplicationSetting
+    end
     get "application/settings" do
       present current_settings, with: Entities::ApplicationSetting
     end
 
-    # Modify applicaiton settings
-    #
-    # Example Request:
-    #   PUT /application/settings
+    desc 'Modify application settings' do
+      success Entities::ApplicationSetting
+    end
+    params do
+      # CE
+      at_least_one_of_ce = [
+        :admin_notification_email,
+        :after_sign_out_path,
+        :after_sign_up_text,
+        :akismet_enabled,
+        :container_registry_token_expire_delay,
+        :default_artifacts_expire_in,
+        :default_branch_protection,
+        :default_group_visibility,
+        :default_project_visibility,
+        :default_projects_limit,
+        :default_snippet_visibility,
+        :disabled_oauth_sign_in_sources,
+        :domain_blacklist_enabled,
+        :domain_whitelist,
+        :email_author_in_body,
+        :enabled_git_access_protocol,
+        :gravatar_enabled,
+        :help_page_hide_commercial_content,
+        :help_page_text,
+        :help_page_support_url,
+        :home_page_url,
+        :housekeeping_enabled,
+        :html_emails_enabled,
+        :import_sources,
+        :koding_enabled,
+        :max_artifacts_size,
+        :max_attachment_size,
+        :max_pages_size,
+        :metrics_enabled,
+        :plantuml_enabled,
+        :polling_interval_multiplier,
+        :recaptcha_enabled,
+        :repository_checks_enabled,
+        :repository_storage,
+        :require_two_factor_authentication,
+        :restricted_visibility_levels,
+        :send_user_confirmation_email,
+        :sentry_enabled,
+        :clientside_sentry_enabled,
+        :session_expire_delay,
+        :shared_runners_enabled,
+        :sidekiq_throttling_enabled,
+        :sign_in_text,
+        :signin_enabled,
+        :signup_enabled,
+        :terminal_max_session_time,
+        :user_default_external,
+        :user_oauth_applications,
+        :version_check_enabled
+      ]
+      optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master'
+      optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility'
+      optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility'
+      optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility'
+      optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.'
+      optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project],
+                                desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com'
+      optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources'
+      optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.'
+      optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled'
+      optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects'
+      optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB'
+      optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.'
+      optional :user_oauth_applications, type: Boolean, desc: 'Allow users to register any application to use GitLab as an OAuth provider'
+      optional :user_default_external, type: Boolean, desc: 'Newly registered users will by default be external'
+      optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled'
+      optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up'
+      optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
+      optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups'
+      given domain_blacklist_enabled: ->(val) { val } do
+        requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
+      end
+      optional :after_sign_up_text, type: String, desc: 'Text shown after sign up'
+      optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled'
+      optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication'
+      given require_two_factor_authentication: ->(val) { val } do
+        requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication'
+      end
+      optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page'
+      optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out'
+      optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application'
+      optional :help_page_hide_commercial_content, type: Boolean, desc: 'Hide marketing-related entries from help'
+      optional :help_page_text, type: String, desc: 'Custom text displayed on the help page'
+      optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page'
+      optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects'
+      given shared_runners_enabled: ->(val) { val } do
+        requires :shared_runners_text, type: String, desc: 'Shared runners text '
+      end
+      optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts"
+      optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts"
+      optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB'
+      optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)'
+      optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics'
+      optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics'
+      given metrics_enabled: ->(val) { val } do
+        requires :metrics_host, type: String, desc: 'The InfluxDB host'
+        requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB'
+        requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open'
+        requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out'
+        requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.'
+        requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds'
+        requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet'
+      end
+      optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling'
+      given sidekiq_throttling_enabled: ->(val) { val } do
+        requires :sidekiq_throttling_queus, type: Array[String], desc: 'Choose which queues you wish to throttle'
+        requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.'
+      end
+      optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts'
+      given recaptcha_enabled: ->(val) { val } do
+        requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha'
+        requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha'
+      end
+      optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues'
+      given akismet_enabled: ->(val) { val } do
+        requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com'
+      end
+      optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.'
+      optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com'
+      given sentry_enabled: ->(val) { val } do
+        requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name'
+      end
+      optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/'
+      given clientside_sentry_enabled: ->(val) { val } do
+        requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name'
+      end
+      optional :repository_storage, type: String, desc: 'Storage paths for new projects'
+      optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues."
+      optional :koding_enabled, type: Boolean, desc: 'Enable Koding'
+      given koding_enabled: ->(val) { val } do
+        requires :koding_url, type: String, desc: 'The Koding team URL'
+      end
+      optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML'
+      given plantuml_enabled: ->(val) { val } do
+        requires :plantuml_url, type: String, desc: 'The PlantUML server URL'
+      end
+      optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.'
+      optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.'
+      optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.'
+      optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)'
+      given housekeeping_enabled: ->(val) { val } do
+        requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance."
+        requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run."
+        requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run."
+        requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run."
+      end
+      optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.'
+      optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.'
+
+      at_least_one_of(*at_least_one_of_ce)
+    end
     put "application/settings" do
-      attributes = current_settings.attributes.keys - ["id"]
-      attrs = attributes_for_keys(attributes)
+      attrs = declared_params(include_missing: false)
 
       if current_settings.update_attributes(attrs)
         present current_settings, with: Entities::ApplicationSetting
diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb
index d3d6827dc54..11f2b40269a 100644
--- a/lib/api/sidekiq_metrics.rb
+++ b/lib/api/sidekiq_metrics.rb
@@ -39,50 +39,22 @@ module API
       end
     end
 
-    # Get Sidekiq Queue metrics
-    #
-    # Parameters:
-    #   None
-    #
-    # Example:
-    #   GET /sidekiq/queue_metrics
-    #
+    desc 'Get the Sidekiq queue metrics'
     get 'sidekiq/queue_metrics' do
       { queues: queue_metrics }
     end
 
-    # Get Sidekiq Process metrics
-    #
-    # Parameters:
-    #   None
-    #
-    # Example:
-    #   GET /sidekiq/process_metrics
-    #
+    desc 'Get the Sidekiq process metrics'
     get 'sidekiq/process_metrics' do
       { processes: process_metrics }
     end
 
-    # Get Sidekiq Job statistics
-    #
-    # Parameters:
-    #   None
-    #
-    # Example:
-    #   GET /sidekiq/job_stats
-    #
+    desc 'Get the Sidekiq job statistics'
     get 'sidekiq/job_stats' do
       { jobs: job_stats }
     end
 
-    # Get Sidekiq Compound metrics. Includes all previous metrics
-    #
-    # Parameters:
-    #   None
-    #
-    # Example:
-    #   GET /sidekiq/compound_metrics
-    #
+    desc 'Get the Sidekiq Compound metrics. Includes queue, process, and job statistics'
     get 'sidekiq/compound_metrics' do
       { queues: queue_metrics, processes: process_metrics, jobs: job_stats }
     end
diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb
new file mode 100644
index 00000000000..c630c24c339
--- /dev/null
+++ b/lib/api/snippets.rb
@@ -0,0 +1,145 @@
+module API
+  # Snippets API
+  class Snippets < Grape::API
+    include PaginationParams
+
+    before { authenticate! }
+
+    resource :snippets do
+      helpers do
+        def snippets_for_current_user
+          SnippetsFinder.new(current_user, author: current_user).execute
+        end
+
+        def public_snippets
+          SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute
+        end
+      end
+
+      desc 'Get a snippets list for authenticated user' do
+        detail 'This feature was introduced in GitLab 8.15.'
+        success Entities::PersonalSnippet
+      end
+      params do
+        use :pagination
+      end
+      get do
+        present paginate(snippets_for_current_user), with: Entities::PersonalSnippet
+      end
+
+      desc 'List all public snippets current_user has access to' do
+        detail 'This feature was introduced in GitLab 8.15.'
+        success Entities::PersonalSnippet
+      end
+      params do
+        use :pagination
+      end
+      get 'public' do
+        present paginate(public_snippets), with: Entities::PersonalSnippet
+      end
+
+      desc 'Get a single snippet' do
+        detail 'This feature was introduced in GitLab 8.15.'
+        success Entities::PersonalSnippet
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of a snippet'
+      end
+      get ':id' do
+        snippet = snippets_for_current_user.find(params[:id])
+        present snippet, with: Entities::PersonalSnippet
+      end
+
+      desc 'Create new snippet' do
+        detail 'This feature was introduced in GitLab 8.15.'
+        success Entities::PersonalSnippet
+      end
+      params do
+        requires :title, type: String, desc: 'The title of a snippet'
+        requires :file_name, type: String, desc: 'The name of a snippet file'
+        requires :content, type: String, desc: 'The content of a snippet'
+        optional :description, type: String, desc: 'The description of a snippet'
+        optional :visibility, type: String,
+                              values: Gitlab::VisibilityLevel.string_values,
+                              default: 'internal',
+                              desc: 'The visibility of the snippet'
+      end
+      post do
+        attrs = declared_params(include_missing: false).merge(request: request, api: true)
+        snippet = CreateSnippetService.new(nil, current_user, attrs).execute
+
+        render_spam_error! if snippet.spam?
+
+        if snippet.persisted?
+          present snippet, with: Entities::PersonalSnippet
+        else
+          render_validation_error!(snippet)
+        end
+      end
+
+      desc 'Update an existing snippet' do
+        detail 'This feature was introduced in GitLab 8.15.'
+        success Entities::PersonalSnippet
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of a snippet'
+        optional :title, type: String, desc: 'The title of a snippet'
+        optional :file_name, type: String, desc: 'The name of a snippet file'
+        optional :content, type: String, desc: 'The content of a snippet'
+        optional :description, type: String, desc: 'The description of a snippet'
+        optional :visibility, type: String,
+                              values: Gitlab::VisibilityLevel.string_values,
+                              desc: 'The visibility of the snippet'
+        at_least_one_of :title, :file_name, :content, :visibility
+      end
+      put ':id' do
+        snippet = snippets_for_current_user.find_by(id: params.delete(:id))
+        return not_found!('Snippet') unless snippet
+        authorize! :update_personal_snippet, snippet
+
+        attrs = declared_params(include_missing: false).merge(request: request, api: true)
+
+        UpdateSnippetService.new(nil, current_user, snippet, attrs).execute
+
+        render_spam_error! if snippet.spam?
+
+        if snippet.persisted?
+          present snippet, with: Entities::PersonalSnippet
+        else
+          render_validation_error!(snippet)
+        end
+      end
+
+      desc 'Remove snippet' do
+        detail 'This feature was introduced in GitLab 8.15.'
+        success Entities::PersonalSnippet
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of a snippet'
+      end
+      delete ':id' do
+        snippet = snippets_for_current_user.find_by(id: params.delete(:id))
+        return not_found!('Snippet') unless snippet
+
+        authorize! :destroy_personal_snippet, snippet
+
+        snippet.destroy
+      end
+
+      desc 'Get a raw snippet' do
+        detail 'This feature was introduced in GitLab 8.15.'
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of a snippet'
+      end
+      get ":id/raw" do
+        snippet = snippets_for_current_user.find_by(id: params.delete(:id))
+        return not_found!('Snippet') unless snippet
+
+        env['api.format'] = :txt
+        content_type 'text/plain'
+        present snippet.content
+      end
+    end
+  end
+end
diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb
index c49e2a21b82..91567909998 100644
--- a/lib/api/subscriptions.rb
+++ b/lib/api/subscriptions.rb
@@ -3,55 +3,45 @@ module API
     before { authenticate! }
 
     subscribable_types = {
-      'merge_request' => proc { |id| user_project.merge_requests.find(id) },
-      'merge_requests' => proc { |id| user_project.merge_requests.find(id) },
+      'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) },
       'issues' => proc { |id| find_project_issue(id) },
-      'labels' => proc { |id| find_project_label(id) },
+      'labels' => proc { |id| find_project_label(id) }
     }
 
-    resource :projects do
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+      requires :subscribable_id, type: String, desc: 'The ID of a resource'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
       subscribable_types.each do |type, finder|
         type_singularized = type.singularize
-        type_id_str = :"#{type_singularized}_id"
         entity_class = Entities.const_get(type_singularized.camelcase)
 
-        # Subscribe to a resource
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   subscribable_id (required) - The ID of a resource
-        # Example Request:
-        #   POST /projects/:id/labels/:subscribable_id/subscription
-        #   POST /projects/:id/issues/:subscribable_id/subscription
-        #   POST /projects/:id/merge_requests/:subscribable_id/subscription
-        post ":id/#{type}/:#{type_id_str}/subscription" do
-          resource = instance_exec(params[type_id_str], &finder)
+        desc 'Subscribe to a resource' do
+          success entity_class
+        end
+        post ":id/#{type}/:subscribable_id/subscribe" do
+          resource = instance_exec(params[:subscribable_id], &finder)
 
-          if resource.subscribed?(current_user)
+          if resource.subscribed?(current_user, user_project)
             not_modified!
           else
-            resource.subscribe(current_user)
-            present resource, with: entity_class, current_user: current_user
+            resource.subscribe(current_user, user_project)
+            present resource, with: entity_class, current_user: current_user, project: user_project
           end
         end
 
-        # Unsubscribe from a resource
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   subscribable_id (required) - The ID of a resource
-        # Example Request:
-        #   DELETE /projects/:id/labels/:subscribable_id/subscription
-        #   DELETE /projects/:id/issues/:subscribable_id/subscription
-        #   DELETE /projects/:id/merge_requests/:subscribable_id/subscription
-        delete ":id/#{type}/:#{type_id_str}/subscription" do
-          resource = instance_exec(params[type_id_str], &finder)
+        desc 'Unsubscribe from a resource' do
+          success entity_class
+        end
+        post ":id/#{type}/:subscribable_id/unsubscribe" do
+          resource = instance_exec(params[:subscribable_id], &finder)
 
-          if !resource.subscribed?(current_user)
+          if !resource.subscribed?(current_user, user_project)
             not_modified!
           else
-            resource.unsubscribe(current_user)
-            present resource, with: entity_class, current_user: current_user
+            resource.unsubscribe(current_user, user_project)
+            present resource, with: entity_class, current_user: current_user, project: user_project
           end
         end
       end
diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb
index 22b8f90dc5c..ed7b23b474a 100644
--- a/lib/api/system_hooks.rb
+++ b/lib/api/system_hooks.rb
@@ -1,44 +1,49 @@
 module API
-  # Hooks API
   class SystemHooks < Grape::API
+    include PaginationParams
+
     before do
       authenticate!
       authenticated_as_admin!
     end
 
     resource :hooks do
-      # Get the list of system hooks
-      #
-      # Example Request:
-      #   GET /hooks
+      desc 'Get the list of system hooks' do
+        success Entities::Hook
+      end
+      params do
+        use :pagination
+      end
       get do
-        @hooks = SystemHook.all
-        present @hooks, with: Entities::Hook
+        present paginate(SystemHook.all), with: Entities::Hook
       end
 
-      # Create new system hook
-      #
-      # Parameters:
-      #   url (required) - url for system hook
-      # Example Request
-      #   POST /hooks
+      desc 'Create a new system hook' do
+        success Entities::Hook
+      end
+      params do
+        requires :url, type: String, desc: "The URL to send the request to"
+        optional :token, type: String, desc: 'The token used to validate payloads'
+        optional :push_events, type: Boolean, desc: "Trigger hook on push events"
+        optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events"
+        optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook"
+      end
       post do
-        attrs = attributes_for_keys [:url]
-        required_attributes! [:url]
-        @hook = SystemHook.new attrs
-        if @hook.save
-          present @hook, with: Entities::Hook
+        hook = SystemHook.new(declared_params(include_missing: false))
+
+        if hook.save
+          present hook, with: Entities::Hook
         else
-          not_found!
+          render_validation_error!(hook)
         end
       end
 
-      # Test a hook
-      #
-      # Example Request
-      #   GET /hooks/:id
+      desc 'Test a hook'
+      params do
+        requires :id, type: Integer, desc: 'The ID of the system hook'
+      end
       get ":id" do
-        @hook = SystemHook.find(params[:id])
+        hook = SystemHook.find(params[:id])
         data = {
           event_name: "project_create",
           name: "Ruby",
@@ -47,23 +52,21 @@ module API
           owner_name: "Someone",
           owner_email: "example@gitlabhq.com"
         }
-        @hook.execute(data, 'system_hooks')
+        hook.execute(data, 'system_hooks')
         data
       end
 
-      # Delete a hook. This is an idempotent function.
-      #
-      # Parameters:
-      #   id (required) - ID of the hook
-      # Example Request:
-      #   DELETE /hooks/:id
+      desc 'Delete a hook' do
+        success Entities::Hook
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the system hook'
+      end
       delete ":id" do
-        begin
-          @hook = SystemHook.find(params[:id])
-          @hook.destroy
-        rescue
-          # SystemHook raises an Error if no hook with id found
-        end
+        hook = SystemHook.find_by(id: params[:id])
+        not_found!('System hook') unless hook
+
+        hook.destroy
       end
     end
   end
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index 7b675e05fbb..633a858f8c7 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -1,28 +1,30 @@
 module API
-  # Git Tags API
   class Tags < Grape::API
-    before { authenticate! }
+    include PaginationParams
+
     before { authorize! :download_code, user_project }
 
-    resource :projects do
-      # Get a project repository tags
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   GET /projects/:id/repository/tags
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get a project repository tags' do
+        success Entities::RepoTag
+      end
+      params do
+        use :pagination
+      end
       get ":id/repository/tags" do
-        present user_project.repository.tags.sort_by(&:name).reverse,
-                with: Entities::RepoTag, project: user_project
+        tags = ::Kaminari.paginate_array(user_project.repository.tags.sort_by(&:name).reverse)
+        present paginate(tags), with: Entities::RepoTag, project: user_project
       end
 
-      # Get a single repository tag
-      #
-      # Parameters:
-      #   id (required)       - The ID of a project
-      #   tag_name (required) - The name of the tag
-      # Example Request:
-      #   GET /projects/:id/repository/tags/:tag_name
+      desc 'Get a single repository tag' do
+        success Entities::RepoTag
+      end
+      params do
+        requires :tag_name, type: String, desc: 'The name of the tag'
+      end
       get ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do
         tag = user_project.repository.find_tag(params[:tag_name])
         not_found!('Tag') unless tag
@@ -30,20 +32,20 @@ module API
         present tag, with: Entities::RepoTag, project: user_project
       end
 
-      # Create tag
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   tag_name (required) - The name of the tag
-      #   ref (required) - Create tag from commit sha or branch
-      #   message (optional) - Specifying a message creates an annotated tag.
-      # Example Request:
-      #   POST /projects/:id/repository/tags
+      desc 'Create a new repository tag' do
+        success Entities::RepoTag
+      end
+      params do
+        requires :tag_name,            type: String, desc: 'The name of the tag'
+        requires :ref,                 type: String, desc: 'The commit sha or branch name'
+        optional :message,             type: String, desc: 'Specifying a message creates an annotated tag'
+        optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database'
+      end
       post ':id/repository/tags' do
         authorize_push_project
-        message = params[:message] || nil
-        result = CreateTagService.new(user_project, current_user).
-          execute(params[:tag_name], params[:ref], message, params[:release_description])
+
+        result = ::Tags::CreateService.new(user_project, current_user)
+          .execute(params[:tag_name], params[:ref], params[:message], params[:release_description])
 
         if result[:status] == :success
           present result[:tag],
@@ -54,40 +56,33 @@ module API
         end
       end
 
-      # Delete tag
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   tag_name (required) - The name of the tag
-      # Example Request:
-      #   DELETE /projects/:id/repository/tags/:tag
+      desc 'Delete a repository tag'
+      params do
+        requires :tag_name, type: String, desc: 'The name of the tag'
+      end
       delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do
         authorize_push_project
-        result = DeleteTagService.new(user_project, current_user).
-          execute(params[:tag_name])
 
-        if result[:status] == :success
-          {
-            tag_name: params[:tag_name]
-          }
-        else
+        result = ::Tags::DestroyService.new(user_project, current_user)
+          .execute(params[:tag_name])
+
+        if result[:status] != :success
           render_api_error!(result[:message], result[:return_code])
         end
       end
 
-      # Add release notes to tag
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   tag_name (required) - The name of the tag
-      #   description (required) - Release notes with markdown support
-      # Example Request:
-      #   POST /projects/:id/repository/tags/:tag_name/release
+      desc 'Add a release note to a tag' do
+        success Entities::Release
+      end
+      params do
+        requires :tag_name,    type: String, desc: 'The name of the tag'
+        requires :description, type: String, desc: 'Release notes with markdown support'
+      end
       post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do
         authorize_push_project
-        required_attributes! [:description]
-        result = CreateReleaseService.new(user_project, current_user).
-          execute(params[:tag_name], params[:description])
+
+        result = CreateReleaseService.new(user_project, current_user)
+          .execute(params[:tag_name], params[:description])
 
         if result[:status] == :success
           present result[:release], with: Entities::Release
@@ -96,19 +91,18 @@ module API
         end
       end
 
-      # Updates a release notes of a tag
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   tag_name (required) - The name of the tag
-      #   description (required) - Release notes with markdown support
-      # Example Request:
-      #   PUT /projects/:id/repository/tags/:tag_name/release
+      desc "Update a tag's release note" do
+        success Entities::Release
+      end
+      params do
+        requires :tag_name,    type: String, desc: 'The name of the tag'
+        requires :description, type: String, desc: 'Release notes with markdown support'
+      end
       put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do
         authorize_push_project
-        required_attributes! [:description]
-        result = UpdateReleaseService.new(user_project, current_user).
-          execute(params[:tag_name], params[:description])
+
+        result = UpdateReleaseService.new(user_project, current_user)
+          .execute(params[:tag_name], params[:description])
 
         if result[:status] == :success
           present result[:release], with: Entities::Release
diff --git a/lib/api/templates.rb b/lib/api/templates.rb
index b9e718147e1..0fc13b35d5b 100644
--- a/lib/api/templates.rb
+++ b/lib/api/templates.rb
@@ -1,38 +1,109 @@
 module API
   class Templates < Grape::API
+    include PaginationParams
+
     GLOBAL_TEMPLATE_TYPES = {
-      gitignores:     Gitlab::Template::GitignoreTemplate,
-      gitlab_ci_ymls: Gitlab::Template::GitlabCiYmlTemplate
+      gitignores: {
+        klass: Gitlab::Template::GitignoreTemplate,
+        gitlab_version: 8.8
+      },
+      gitlab_ci_ymls: {
+        klass: Gitlab::Template::GitlabCiYmlTemplate,
+        gitlab_version: 8.9
+      },
+      dockerfiles: {
+        klass: Gitlab::Template::DockerfileTemplate,
+        gitlab_version: 8.15
+      }
     }.freeze
+    PROJECT_TEMPLATE_REGEX =
+      /[\<\{\[]
+        (project|description|
+        one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here
+      [\>\}\]]/xi.freeze
+    YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze
+    FULLNAME_TEMPLATE_REGEX =
+      /[\<\{\[]
+        (fullname|name\sof\s(author|copyright\sowner))
+      [\>\}\]]/xi.freeze
 
     helpers do
+      def parsed_license_template
+        # We create a fresh Licensee::License object since we'll modify its
+        # content in place below.
+        template = Licensee::License.new(params[:name])
+
+        template.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s)
+        template.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present?
+
+        fullname = params[:fullname].presence || current_user.try(:name)
+        template.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname
+        template
+      end
+
       def render_response(template_type, template)
         not_found!(template_type.to_s.singularize) unless template
         present template, with: Entities::Template
       end
     end
 
-    GLOBAL_TEMPLATE_TYPES.each do |template_type, klass|
-      # Get the list of the available template
-      #
-      # Example Request:
-      #   GET /gitignores
-      #   GET /gitlab_ci_ymls
-      get template_type.to_s do
-        present klass.all, with: Entities::TemplatesList
-      end
-
-      # Get the text for a specific template present in local filesystem
-      #
-      # Parameters:
-      #   name (required) - The name of a template
-      #
-      # Example Request:
-      #   GET /gitignores/Elixir
-      #   GET /gitlab_ci_ymls/Ruby
-      get "#{template_type}/:name" do
-        required_attributes! [:name]
-        new_template = klass.find(params[:name])
+    desc 'Get the list of the available license template' do
+      detail 'This feature was introduced in GitLab 8.7.'
+      success ::API::Entities::RepoLicense
+    end
+    params do
+      optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses'
+      use :pagination
+    end
+    get "templates/licenses" do
+      options = {
+        featured: declared(params).popular.present? ? true : nil
+      }
+      licences = ::Kaminari.paginate_array(Licensee::License.all(options))
+      present paginate(licences), with: Entities::RepoLicense
+    end
+
+    desc 'Get the text for a specific license' do
+      detail 'This feature was introduced in GitLab 8.7.'
+      success ::API::Entities::RepoLicense
+    end
+    params do
+      requires :name, type: String, desc: 'The name of the template'
+    end
+    get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ } do
+      not_found!('License') unless Licensee::License.find(declared(params).name)
+
+      template = parsed_license_template
+
+      present template, with: ::API::Entities::RepoLicense
+    end
+
+    GLOBAL_TEMPLATE_TYPES.each do |template_type, properties|
+      klass = properties[:klass]
+      gitlab_version = properties[:gitlab_version]
+
+      desc 'Get the list of the available template' do
+        detail "This feature was introduced in GitLab #{gitlab_version}."
+        success Entities::TemplatesList
+      end
+      params do
+        use :pagination
+      end
+      get "templates/#{template_type}" do
+        templates = ::Kaminari.paginate_array(klass.all)
+        present paginate(templates), with: Entities::TemplatesList
+      end
+
+      desc 'Get the text for a specific template present in local filesystem' do
+        detail "This feature was introduced in GitLab #{gitlab_version}."
+        success Entities::Template
+      end
+      params do
+        requires :name, type: String, desc: 'The name of the template'
+      end
+      get "templates/#{template_type}/:name" do
+        new_template = klass.find(declared(params).name)
+
         render_response(template_type, new_template)
       end
     end
diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb
new file mode 100644
index 00000000000..df4632346dd
--- /dev/null
+++ b/lib/api/time_tracking_endpoints.rb
@@ -0,0 +1,114 @@
+module API
+  module TimeTrackingEndpoints
+    extend ActiveSupport::Concern
+
+    included do
+      helpers do
+        def issuable_name
+          declared_params.key?(:issue_iid) ? 'issue' : 'merge_request'
+        end
+
+        def issuable_key
+          "#{issuable_name}_iid".to_sym
+        end
+
+        def update_issuable_key
+          "update_#{issuable_name}".to_sym
+        end
+
+        def read_issuable_key
+          "read_#{issuable_name}".to_sym
+        end
+
+        def load_issuable
+          @issuable ||= begin
+            case issuable_name
+            when 'issue'
+              find_project_issue(params.delete(issuable_key))
+            when 'merge_request'
+              find_project_merge_request(params.delete(issuable_key))
+            end
+          end
+        end
+
+        def update_issuable(attrs)
+          custom_params = declared_params(include_missing: false)
+          custom_params.merge!(attrs)
+
+          issuable = update_service.new(user_project, current_user, custom_params).execute(load_issuable)
+          if issuable.valid?
+            present issuable, with: Entities::IssuableTimeStats
+          else
+            render_validation_error!(issuable)
+          end
+        end
+
+        def update_service
+          issuable_name == 'issue' ? ::Issues::UpdateService : ::MergeRequests::UpdateService
+        end
+      end
+
+      issuable_name            = name.end_with?('Issues') ? 'issue' : 'merge_request'
+      issuable_collection_name = issuable_name.pluralize
+      issuable_key             = "#{issuable_name}_iid".to_sym
+
+      desc "Set a time estimate for a project #{issuable_name}"
+      params do
+        requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}"
+        requires :duration, type: String, desc: 'The duration to be parsed'
+      end
+      post ":id/#{issuable_collection_name}/:#{issuable_key}/time_estimate" do
+        authorize! update_issuable_key, load_issuable
+
+        status :ok
+        update_issuable(time_estimate: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)))
+      end
+
+      desc "Reset the time estimate for a project #{issuable_name}"
+      params do
+        requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}"
+      end
+      post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_time_estimate" do
+        authorize! update_issuable_key, load_issuable
+
+        status :ok
+        update_issuable(time_estimate: 0)
+      end
+
+      desc "Add spent time for a project #{issuable_name}"
+      params do
+        requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}"
+        requires :duration, type: String, desc: 'The duration to be parsed'
+      end
+      post ":id/#{issuable_collection_name}/:#{issuable_key}/add_spent_time" do
+        authorize! update_issuable_key, load_issuable
+
+        update_issuable(spend_time: {
+          duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)),
+          user: current_user
+        })
+      end
+
+      desc "Reset spent time for a project #{issuable_name}"
+      params do
+        requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}"
+      end
+      post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_spent_time" do
+        authorize! update_issuable_key, load_issuable
+
+        status :ok
+        update_issuable(spend_time: { duration: :reset, user: current_user })
+      end
+
+      desc "Show time stats for a project #{issuable_name}"
+      params do
+        requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}"
+      end
+      get ":id/#{issuable_collection_name}/:#{issuable_key}/time_stats" do
+        authorize! read_issuable_key, load_issuable
+
+        present load_issuable, with: Entities::IssuableTimeStats
+      end
+    end
+  end
+end
diff --git a/lib/api/todos.rb b/lib/api/todos.rb
index 19df13d8aac..d1f7e364029 100644
--- a/lib/api/todos.rb
+++ b/lib/api/todos.rb
@@ -1,25 +1,27 @@
 module API
-  # Todos API
   class Todos < Grape::API
+    include PaginationParams
+
     before { authenticate! }
 
     ISSUABLE_TYPES = {
-      'merge_requests' => ->(id) { user_project.merge_requests.find(id) },
-      'issues' => ->(id) { find_project_issue(id) }
-    }
+      'merge_requests' => ->(iid) { find_merge_request_with_access(iid) },
+      'issues' => ->(iid) { find_project_issue(iid) }
+    }.freeze
 
-    resource :projects do
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
       ISSUABLE_TYPES.each do |type, finder|
-        type_id_str = "#{type.singularize}_id".to_sym
+        type_id_str = "#{type.singularize}_iid".to_sym
 
-        # Create a todo on an issuable
-        #
-        # Parameters:
-        #   id (required) - The ID of a project
-        #   issuable_id (required) - The ID of an issuable
-        # Example Request:
-        #   POST /projects/:id/issues/:issuable_id/todo
-        #   POST /projects/:id/merge_requests/:issuable_id/todo
+        desc 'Create a todo on an issuable' do
+          success Entities::Todo
+        end
+        params do
+          requires type_id_str, type: Integer, desc: 'The IID of an issuable'
+        end
         post ":id/#{type}/:#{type_id_str}/todo" do
           issuable = instance_exec(params[type_id_str], &finder)
           todo = TodoService.new.mark_todo(issuable, current_user).first
@@ -40,40 +42,35 @@ module API
         end
       end
 
-      # Get a todo list
-      #
-      # Example Request:
-      #  GET /todos
-      #
+      desc 'Get a todo list' do
+        success Entities::Todo
+      end
+      params do
+        use :pagination
+      end
       get do
-        todos = find_todos
-
-        present paginate(todos), with: Entities::Todo, current_user: current_user
+        present paginate(find_todos), with: Entities::Todo, current_user: current_user
       end
 
-      # Mark a todo as done
-      #
-      # Parameters:
-      #   id: (required) - The ID of the todo being marked as done
-      #
-      # Example Request:
-      #  DELETE /todos/:id
-      #
-      delete ':id' do
+      desc 'Mark a todo as done' do
+        success Entities::Todo
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the todo being marked as done'
+      end
+      post ':id/mark_as_done' do
         todo = current_user.todos.find(params[:id])
         TodoService.new.mark_todos_as_done([todo], current_user)
 
         present todo.reload, with: Entities::Todo, current_user: current_user
       end
 
-      # Mark all todos as done
-      #
-      # Example Request:
-      #  DELETE /todos
-      #
-      delete do
+      desc 'Mark all todos as done'
+      post '/mark_as_done' do
         todos = find_todos
         TodoService.new.mark_todos_as_done(todos, current_user)
+
+        no_content!
       end
     end
   end
diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb
index d1d07394e92..a9f2ca2608e 100644
--- a/lib/api/triggers.rb
+++ b/lib/api/triggers.rb
@@ -1,116 +1,148 @@
 module API
-  # Triggers API
   class Triggers < Grape::API
-    resource :projects do
-      # Trigger a GitLab project build
-      #
-      # Parameters:
-      #   id (required) - The ID of a CI project
-      #   ref (required) - The name of project's branch or tag
-      #   token (required) - The uniq token of trigger
-      #   variables (optional) - The list of variables to be injected into build
-      # Example Request:
-      #   POST /projects/:id/trigger/builds
-      post ":id/trigger/builds" do
-        required_attributes! [:ref, :token]
-
-        project = Project.find_with_namespace(params[:id]) || Project.find_by(id: params[:id])
+    include PaginationParams
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Trigger a GitLab project pipeline' do
+        success Entities::Pipeline
+      end
+      params do
+        requires :ref, type: String, desc: 'The commit sha or name of a branch or tag'
+        requires :token, type: String, desc: 'The unique token of trigger'
+        optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
+      end
+      post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do
+        project = find_project(params[:id])
         trigger = Ci::Trigger.find_by_token(params[:token].to_s)
         not_found! unless project && trigger
         unauthorized! unless trigger.project == project
 
         # validate variables
-        variables = params[:variables]
-        if variables
-          unless variables.is_a?(Hash)
-            render_api_error!('variables needs to be a hash', 400)
-          end
-
-          unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) }
-            render_api_error!('variables needs to be a map of key-valued strings', 400)
-          end
-
-          # convert variables from Mash to Hash
-          variables = variables.to_h
+        variables = params[:variables].to_h
+        unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) }
+          render_api_error!('variables needs to be a map of key-valued strings', 400)
         end
 
         # create request and trigger builds
         trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables)
         if trigger_request
-          present trigger_request, with: Entities::TriggerRequest
+          present trigger_request.pipeline, with: Entities::Pipeline
         else
-          errors = 'No builds created'
+          errors = 'No pipeline created'
           render_api_error!(errors, 400)
         end
       end
 
-      # Get triggers list
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   page (optional) - The page number for pagination
-      #   per_page (optional) - The value of items per page to show
-      # Example Request:
-      #   GET /projects/:id/triggers
+      desc 'Get triggers list' do
+        success Entities::Trigger
+      end
+      params do
+        use :pagination
+      end
       get ':id/triggers' do
         authenticate!
         authorize! :admin_build, user_project
 
         triggers = user_project.triggers.includes(:trigger_requests)
-        triggers = paginate(triggers)
 
-        present triggers, with: Entities::Trigger
+        present paginate(triggers), with: Entities::Trigger
       end
 
-      # Get specific trigger of a project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   token (required) - The `token` of a trigger
-      # Example Request:
-      #   GET /projects/:id/triggers/:token
-      get ':id/triggers/:token' do
+      desc 'Get specific trigger of a project' do
+        success Entities::Trigger
+      end
+      params do
+        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
+      end
+      get ':id/triggers/:trigger_id' do
         authenticate!
         authorize! :admin_build, user_project
 
-        trigger = user_project.triggers.find_by(token: params[:token].to_s)
+        trigger = user_project.triggers.find(params.delete(:trigger_id))
         return not_found!('Trigger') unless trigger
 
         present trigger, with: Entities::Trigger
       end
 
-      # Create trigger
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      # Example Request:
-      #   POST /projects/:id/triggers
+      desc 'Create a trigger' do
+        success Entities::Trigger
+      end
+      params do
+        requires :description, type: String,  desc: 'The trigger description'
+      end
       post ':id/triggers' do
         authenticate!
         authorize! :admin_build, user_project
 
-        trigger = user_project.triggers.create
+        trigger = user_project.triggers.create(
+          declared_params(include_missing: false).merge(owner: current_user))
 
-        present trigger, with: Entities::Trigger
+        if trigger.valid?
+          present trigger, with: Entities::Trigger
+        else
+          render_validation_error!(trigger)
+        end
       end
 
-      # Delete trigger
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   token (required) - The `token` of a trigger
-      # Example Request:
-      #   DELETE /projects/:id/triggers/:token
-      delete ':id/triggers/:token' do
+      desc 'Update a trigger' do
+        success Entities::Trigger
+      end
+      params do
+        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
+        optional :description, type: String,  desc: 'The trigger description'
+      end
+      put ':id/triggers/:trigger_id' do
         authenticate!
         authorize! :admin_build, user_project
 
-        trigger = user_project.triggers.find_by(token: params[:token].to_s)
+        trigger = user_project.triggers.find(params.delete(:trigger_id))
         return not_found!('Trigger') unless trigger
 
-        trigger.destroy
+        if trigger.update(declared_params(include_missing: false))
+          present trigger, with: Entities::Trigger
+        else
+          render_validation_error!(trigger)
+        end
+      end
 
-        present trigger, with: Entities::Trigger
+      desc 'Take ownership of trigger' do
+        success Entities::Trigger
+      end
+      params do
+        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
+      end
+      post ':id/triggers/:trigger_id/take_ownership' do
+        authenticate!
+        authorize! :admin_build, user_project
+
+        trigger = user_project.triggers.find(params.delete(:trigger_id))
+        return not_found!('Trigger') unless trigger
+
+        if trigger.update(owner: current_user)
+          status :ok
+          present trigger, with: Entities::Trigger
+        else
+          render_validation_error!(trigger)
+        end
+      end
+
+      desc 'Delete a trigger' do
+        success Entities::Trigger
+      end
+      params do
+        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
+      end
+      delete ':id/triggers/:trigger_id' do
+        authenticate!
+        authorize! :admin_build, user_project
+
+        trigger = user_project.triggers.find(params.delete(:trigger_id))
+        return not_found!('Trigger') unless trigger
+
+        trigger.destroy
       end
     end
   end
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 18c4cad09ae..f9555842daf 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -1,147 +1,151 @@
 module API
-  # Users API
   class Users < Grape::API
-    before { authenticate! }
+    include PaginationParams
+
+    before do
+      allow_access_with_scope :read_user if request.get?
+      authenticate!
+    end
 
     resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do
-      # Get a users list
-      #
-      # Example Request:
-      #  GET /users
-      #  GET /users?search=Admin
-      #  GET /users?username=root
-      get do
-        unless can?(current_user, :read_users_list, nil)
-          render_api_error!("Not authorized.", 403)
+      helpers do
+        def find_user(params)
+          id = params[:user_id] || params[:id]
+          User.find_by(id: id) || not_found!('User')
         end
 
-        if params[:username].present?
-          @users = User.where(username: params[:username])
-        else
-          @users = User.all
-          @users = @users.active if params[:active].present?
-          @users = @users.search(params[:search]) if params[:search].present?
-          @users = paginate @users
+        params :optional_attributes do
+          optional :skype, type: String, desc: 'The Skype username'
+          optional :linkedin, type: String, desc: 'The LinkedIn username'
+          optional :twitter, type: String, desc: 'The Twitter username'
+          optional :website_url, type: String, desc: 'The website of the user'
+          optional :organization, type: String, desc: 'The organization of the user'
+          optional :projects_limit, type: Integer, desc: 'The number of projects a user can create'
+          optional :extern_uid, type: String, desc: 'The external authentication provider UID'
+          optional :provider, type: String, desc: 'The external provider'
+          optional :bio, type: String, desc: 'The biography of the user'
+          optional :location, type: String, desc: 'The location of the user'
+          optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator'
+          optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups'
+          optional :skip_confirmation, type: Boolean, default: false, desc: 'Flag indicating the account is confirmed'
+          optional :external, type: Boolean, desc: 'Flag indicating the user is an external user'
+          optional :avatar, type: File, desc: 'Avatar image for user'
+          all_or_none_of :extern_uid, :provider
         end
+      end
 
-        if current_user.is_admin?
-          present @users, with: Entities::UserFull
-        else
-          present @users, with: Entities::UserBasic
+      desc 'Get the list of users' do
+        success Entities::UserBasic
+      end
+      params do
+        # CE
+        optional :username, type: String, desc: 'Get a single user with a specific username'
+        optional :extern_uid, type: String, desc: 'Get a single user with a specific external authentication provider UID'
+        optional :provider, type: String, desc: 'The external provider'
+        optional :search, type: String, desc: 'Search for a username'
+        optional :active, type: Boolean, default: false, desc: 'Filters only active users'
+        optional :external, type: Boolean, default: false, desc: 'Filters only external users'
+        optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users'
+        all_or_none_of :extern_uid, :provider
+
+        use :pagination
+      end
+      get do
+        unless can?(current_user, :read_users_list)
+          render_api_error!("Not authorized.", 403)
         end
+
+        authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?)
+
+        users = UsersFinder.new(current_user, params).execute
+
+        entity = current_user.admin? ? Entities::UserWithAdmin : Entities::UserBasic
+        present paginate(users), with: entity
       end
 
-      # Get a single user
-      #
-      # Parameters:
-      #   id (required) - The ID of a user
-      # Example Request:
-      #   GET /users/:id
+      desc 'Get a single user' do
+        success Entities::UserBasic
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+      end
       get ":id" do
-        @user = User.find(params[:id])
+        user = User.find_by(id: params[:id])
+        not_found!('User') unless user
 
-        if current_user && current_user.is_admin?
-          present @user, with: Entities::UserFull
-        elsif can?(current_user, :read_user, @user)
-          present @user, with: Entities::User
+        if current_user && current_user.admin?
+          present user, with: Entities::UserPublic
+        elsif can?(current_user, :read_user, user)
+          present user, with: Entities::User
         else
           render_api_error!("User not found.", 404)
         end
       end
 
-      # Create user. Available only for admin
-      #
-      # Parameters:
-      #   email (required)                  - Email
-      #   password (required)               - Password
-      #   name (required)                   - Name
-      #   username (required)               - Name
-      #   skype                             - Skype ID
-      #   linkedin                          - Linkedin
-      #   twitter                           - Twitter account
-      #   website_url                       - Website url
-      #   organization                      - Organization
-      #   projects_limit                    - Number of projects user can create
-      #   extern_uid                        - External authentication provider UID
-      #   provider                          - External provider
-      #   bio                               - Bio
-      #   location                          - Location of the user
-      #   admin                             - User is admin - true or false (default)
-      #   can_create_group                  - User can create groups - true or false
-      #   confirm                           - Require user confirmation - true (default) or false
-      #   external                          - Flags the user as external - true or false(default)
-      # Example Request:
-      #   POST /users
+      desc 'Create a user. Available only for admins.' do
+        success Entities::UserPublic
+      end
+      params do
+        requires :email, type: String, desc: 'The email of the user'
+        optional :password, type: String, desc: 'The password of the new user'
+        optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token'
+        at_least_one_of :password, :reset_password
+        requires :name, type: String, desc: 'The name of the user'
+        requires :username, type: String, desc: 'The username of the user'
+        use :optional_attributes
+      end
       post do
         authenticated_as_admin!
-        required_attributes! [:email, :password, :name, :username]
-        attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external, :organization]
-        admin = attrs.delete(:admin)
-        confirm = !(attrs.delete(:confirm) =~ /(false|f|no|0)$/i)
-        user = User.build_user(attrs)
-        user.admin = admin unless admin.nil?
-        user.skip_confirmation! unless confirm
-        identity_attrs = attributes_for_keys [:provider, :extern_uid]
 
-        if identity_attrs.any?
-          user.identities.build(identity_attrs)
-        end
+        params = declared_params(include_missing: false)
+        user = ::Users::CreateService.new(current_user, params).execute(skip_authorization: true)
 
-        if user.save
-          present user, with: Entities::UserFull
+        if user.persisted?
+          present user, with: Entities::UserPublic
         else
-          conflict!('Email has already been taken') if User.
-              where(email: user.email).
-              count > 0
+          conflict!('Email has already been taken') if User
+              .where(email: user.email)
+              .count > 0
 
-          conflict!('Username has already been taken') if User.
-              where(username: user.username).
-              count > 0
+          conflict!('Username has already been taken') if User
+              .where(username: user.username)
+              .count > 0
 
           render_validation_error!(user)
         end
       end
 
-      # Update user. Available only for admin
-      #
-      # Parameters:
-      #   email                             - Email
-      #   name                              - Name
-      #   password                          - Password
-      #   skype                             - Skype ID
-      #   linkedin                          - Linkedin
-      #   twitter                           - Twitter account
-      #   website_url                       - Website url
-      #   organization                      - Organization
-      #   projects_limit                    - Limit projects each user can create
-      #   bio                               - Bio
-      #   location                          - Location of the user
-      #   admin                             - User is admin - true or false (default)
-      #   can_create_group                  - User can create groups - true or false
-      #   external                          - Flags the user as external - true or false(default)
-      # Example Request:
-      #   PUT /users/:id
+      desc 'Update a user. Available only for admins.' do
+        success Entities::UserPublic
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+        optional :email, type: String, desc: 'The email of the user'
+        optional :password, type: String, desc: 'The password of the new user'
+        optional :name, type: String, desc: 'The name of the user'
+        optional :username, type: String, desc: 'The username of the user'
+        use :optional_attributes
+      end
       put ":id" do
         authenticated_as_admin!
 
-        attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external, :organization]
-        user = User.find(params[:id])
+        user = User.find_by(id: params.delete(:id))
         not_found!('User') unless user
 
-        admin = attrs.delete(:admin)
-        user.admin = admin unless admin.nil?
+        conflict!('Email has already been taken') if params[:email] &&
+            User.where(email: params[:email])
+                .where.not(id: user.id).count > 0
 
-        conflict!('Email has already been taken') if attrs[:email] &&
-            User.where(email: attrs[:email]).
-                where.not(id: user.id).count > 0
+        conflict!('Username has already been taken') if params[:username] &&
+            User.where(username: params[:username])
+                .where.not(id: user.id).count > 0
 
-        conflict!('Username has already been taken') if attrs[:username] &&
-            User.where(username: attrs[:username]).
-                where.not(id: user.id).count > 0
+        user_params = declared_params(include_missing: false)
+        identity_attrs = user_params.slice(:provider, :extern_uid)
 
-        identity_attrs = attributes_for_keys [:provider, :extern_uid]
         if identity_attrs.any?
           identity = user.identities.find_by(provider: identity_attrs[:provider])
+
           if identity
             identity.update_attributes(identity_attrs)
           else
@@ -150,28 +154,33 @@ module API
           end
         end
 
-        if user.update_attributes(attrs)
-          present user, with: Entities::UserFull
+        user_params[:password_expires_at] = Time.now if user_params[:password].present?
+
+        result = ::Users::UpdateService.new(user, user_params.except(:extern_uid, :provider)).execute
+
+        if result[:status] == :success
+          present user, with: Entities::UserPublic
         else
           render_validation_error!(user)
         end
       end
 
-      # Add ssh key to a specified user. Only available to admin users.
-      #
-      # Parameters:
-      #   id (required) - The ID of a user
-      #   key (required) - New SSH Key
-      #   title (required) - New SSH Key's title
-      # Example Request:
-      #   POST /users/:id/keys
+      desc 'Add an SSH key to a specified user. Available only for admins.' do
+        success Entities::SSHKey
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+        requires :key, type: String, desc: 'The new SSH key'
+        requires :title, type: String, desc: 'The title of the new SSH key'
+      end
       post ":id/keys" do
         authenticated_as_admin!
-        required_attributes! [:title, :key]
 
-        user = User.find(params[:id])
-        attrs = attributes_for_keys [:title, :key]
-        key = user.keys.new attrs
+        user = User.find_by(id: params.delete(:id))
+        not_found!('User') unless user
+
+        key = user.keys.new(declared_params(include_missing: false))
+
         if key.save
           present key, with: Entities::SSHKey
         else
@@ -179,56 +188,57 @@ module API
         end
       end
 
-      # Get ssh keys of a specified user. Only available to admin users.
-      #
-      # Parameters:
-      #   uid (required) - The ID of a user
-      # Example Request:
-      #   GET /users/:uid/keys
-      get ':uid/keys' do
+      desc 'Get the SSH keys of a specified user. Available only for admins.' do
+        success Entities::SSHKey
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+        use :pagination
+      end
+      get ':id/keys' do
         authenticated_as_admin!
-        user = User.find_by(id: params[:uid])
+
+        user = User.find_by(id: params[:id])
         not_found!('User') unless user
 
-        present user.keys, with: Entities::SSHKey
+        present paginate(user.keys), with: Entities::SSHKey
       end
 
-      # Delete existing ssh key of a specified user. Only available to admin
-      # users.
-      #
-      # Parameters:
-      #   uid (required) - The ID of a user
-      #   id (required) - SSH Key ID
-      # Example Request:
-      #   DELETE /users/:uid/keys/:id
-      delete ':uid/keys/:id' do
+      desc 'Delete an existing SSH key from a specified user. Available only for admins.' do
+        success Entities::SSHKey
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+        requires :key_id, type: Integer, desc: 'The ID of the SSH key'
+      end
+      delete ':id/keys/:key_id' do
         authenticated_as_admin!
-        user = User.find_by(id: params[:uid])
+
+        user = User.find_by(id: params[:id])
         not_found!('User') unless user
 
-        begin
-          key = user.keys.find params[:id]
-          key.destroy
-        rescue ActiveRecord::RecordNotFound
-          not_found!('Key')
-        end
+        key = user.keys.find_by(id: params[:key_id])
+        not_found!('Key') unless key
+
+        key.destroy
       end
 
-      # Add email to a specified user. Only available to admin users.
-      #
-      # Parameters:
-      #   id (required) - The ID of a user
-      #   email (required) - Email address
-      # Example Request:
-      #   POST /users/:id/emails
+      desc 'Add an email address to a specified user. Available only for admins.' do
+        success Entities::Email
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+        requires :email, type: String, desc: 'The email of the user'
+      end
       post ":id/emails" do
         authenticated_as_admin!
-        required_attributes! [:email]
 
-        user = User.find(params[:id])
-        attrs = attributes_for_keys [:email]
-        email = user.emails.new attrs
-        if email.save
+        user = User.find_by(id: params.delete(:id))
+        not_found!('User') unless user
+
+        email = Emails::CreateService.new(user, declared_params(include_missing: false)).execute
+
+        if email.errors.blank?
           NotificationService.new.new_email(email)
           present email, with: Entities::Email
         else
@@ -236,131 +246,198 @@ module API
         end
       end
 
-      # Get emails of a specified user. Only available to admin users.
-      #
-      # Parameters:
-      #   uid (required) - The ID of a user
-      # Example Request:
-      #   GET /users/:uid/emails
-      get ':uid/emails' do
+      desc 'Get the emails addresses of a specified user. Available only for admins.' do
+        success Entities::Email
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+        use :pagination
+      end
+      get ':id/emails' do
         authenticated_as_admin!
-        user = User.find_by(id: params[:uid])
+        user = User.find_by(id: params[:id])
         not_found!('User') unless user
 
-        present user.emails, with: Entities::Email
+        present paginate(user.emails), with: Entities::Email
       end
 
-      # Delete existing email of a specified user. Only available to admin
-      # users.
-      #
-      # Parameters:
-      #   uid (required) - The ID of a user
-      #   id (required) - Email ID
-      # Example Request:
-      #   DELETE /users/:uid/emails/:id
-      delete ':uid/emails/:id' do
+      desc 'Delete an email address of a specified user. Available only for admins.' do
+        success Entities::Email
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+        requires :email_id, type: Integer, desc: 'The ID of the email'
+      end
+      delete ':id/emails/:email_id' do
         authenticated_as_admin!
-        user = User.find_by(id: params[:uid])
+        user = User.find_by(id: params[:id])
         not_found!('User') unless user
 
-        begin
-          email = user.emails.find params[:id]
-          email.destroy
+        email = user.emails.find_by(id: params[:email_id])
+        not_found!('Email') unless email
 
-          user.update_secondary_emails!
-        rescue ActiveRecord::RecordNotFound
-          not_found!('Email')
-        end
+        Emails::DestroyService.new(user, email: email.email).execute
       end
 
-      # Delete user. Available only for admin
-      #
-      # Example Request:
-      #   DELETE /users/:id
+      desc 'Delete a user. Available only for admins.' do
+        success Entities::Email
+      end
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+        optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions"
+      end
       delete ":id" do
         authenticated_as_admin!
         user = User.find_by(id: params[:id])
+        not_found!('User') unless user
 
-        if user
-          DeleteUserService.new(current_user).execute(user)
-        else
-          not_found!('User')
-        end
+        user.delete_async(deleted_by: current_user, params: params)
       end
 
-      # Block user. Available only for admin
-      #
-      # Example Request:
-      #   PUT /users/:id/block
-      put ':id/block' do
+      desc 'Block a user. Available only for admins.'
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+      end
+      post ':id/block' do
         authenticated_as_admin!
         user = User.find_by(id: params[:id])
+        not_found!('User') unless user
 
-        if !user
-          not_found!('User')
-        elsif !user.ldap_blocked?
+        if !user.ldap_blocked?
           user.block
         else
           forbidden!('LDAP blocked users cannot be modified by the API')
         end
       end
 
-      # Unblock user. Available only for admin
-      #
-      # Example Request:
-      #   PUT /users/:id/unblock
-      put ':id/unblock' do
+      desc 'Unblock a user. Available only for admins.'
+      params do
+        requires :id, type: Integer, desc: 'The ID of the user'
+      end
+      post ':id/unblock' do
         authenticated_as_admin!
         user = User.find_by(id: params[:id])
+        not_found!('User') unless user
 
-        if !user
-          not_found!('User')
-        elsif user.ldap_blocked?
+        if user.ldap_blocked?
           forbidden!('LDAP blocked users cannot be unblocked by the API')
         else
           user.activate
         end
       end
+
+      params do
+        requires :user_id, type: Integer, desc: 'The ID of the user'
+      end
+      segment ':user_id' do
+        resource :impersonation_tokens do
+          helpers do
+            def finder(options = {})
+              user = find_user(params)
+              PersonalAccessTokensFinder.new({ user: user, impersonation: true }.merge(options))
+            end
+
+            def find_impersonation_token
+              finder.find_by(id: declared_params[:impersonation_token_id]) || not_found!('Impersonation Token')
+            end
+          end
+
+          before { authenticated_as_admin! }
+
+          desc 'Retrieve impersonation tokens. Available only for admins.' do
+            detail 'This feature was introduced in GitLab 9.0'
+            success Entities::ImpersonationToken
+          end
+          params do
+            use :pagination
+            optional :state, type: String, default: 'all', values: %w[all active inactive], desc: 'Filters (all|active|inactive) impersonation_tokens'
+          end
+          get { present paginate(finder(declared_params(include_missing: false)).execute), with: Entities::ImpersonationToken }
+
+          desc 'Create a impersonation token. Available only for admins.' do
+            detail 'This feature was introduced in GitLab 9.0'
+            success Entities::ImpersonationToken
+          end
+          params do
+            requires :name, type: String, desc: 'The name of the impersonation token'
+            optional :expires_at, type: Date, desc: 'The expiration date in the format YEAR-MONTH-DAY of the impersonation token'
+            optional :scopes, type: Array, desc: 'The array of scopes of the impersonation token'
+          end
+          post do
+            impersonation_token = finder.build(declared_params(include_missing: false))
+
+            if impersonation_token.save
+              present impersonation_token, with: Entities::ImpersonationToken
+            else
+              render_validation_error!(impersonation_token)
+            end
+          end
+
+          desc 'Retrieve impersonation token. Available only for admins.' do
+            detail 'This feature was introduced in GitLab 9.0'
+            success Entities::ImpersonationToken
+          end
+          params do
+            requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token'
+          end
+          get ':impersonation_token_id' do
+            present find_impersonation_token, with: Entities::ImpersonationToken
+          end
+
+          desc 'Revoke a impersonation token. Available only for admins.' do
+            detail 'This feature was introduced in GitLab 9.0'
+          end
+          params do
+            requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token'
+          end
+          delete ':impersonation_token_id' do
+            find_impersonation_token.revoke!
+          end
+        end
+      end
     end
 
     resource :user do
-      # Get currently authenticated user
-      #
-      # Example Request:
-      #   GET /user
+      desc 'Get the currently authenticated user' do
+        success Entities::UserPublic
+      end
       get do
-        present @current_user, with: Entities::UserFull
+        present current_user, with: sudo? ? Entities::UserWithPrivateDetails : Entities::UserPublic
       end
 
-      # Get currently authenticated user's keys
-      #
-      # Example Request:
-      #   GET /user/keys
+      desc "Get the currently authenticated user's SSH keys" do
+        success Entities::SSHKey
+      end
+      params do
+        use :pagination
+      end
       get "keys" do
-        present current_user.keys, with: Entities::SSHKey
+        present paginate(current_user.keys), with: Entities::SSHKey
       end
 
-      # Get single key owned by currently authenticated user
-      #
-      # Example Request:
-      #   GET /user/keys/:id
-      get "keys/:id" do
-        key = current_user.keys.find params[:id]
+      desc 'Get a single key owned by currently authenticated user' do
+        success Entities::SSHKey
+      end
+      params do
+        requires :key_id, type: Integer, desc: 'The ID of the SSH key'
+      end
+      get "keys/:key_id" do
+        key = current_user.keys.find_by(id: params[:key_id])
+        not_found!('Key') unless key
+
         present key, with: Entities::SSHKey
       end
 
-      # Add new ssh key to currently authenticated user
-      #
-      # Parameters:
-      #   key (required) - New SSH Key
-      #   title (required) - New SSH Key's title
-      # Example Request:
-      #   POST /user/keys
+      desc 'Add a new SSH key to the currently authenticated user' do
+        success Entities::SSHKey
+      end
+      params do
+        requires :key, type: String, desc: 'The new SSH key'
+        requires :title, type: String, desc: 'The title of the new SSH key'
+      end
       post "keys" do
-        required_attributes! [:title, :key]
+        key = current_user.keys.new(declared_params)
 
-        attrs = attributes_for_keys [:title, :key]
-        key = current_user.keys.new attrs
         if key.save
           present key, with: Entities::SSHKey
         else
@@ -368,49 +445,52 @@ module API
         end
       end
 
-      # Delete existing ssh key of currently authenticated user
-      #
-      # Parameters:
-      #   id (required) - SSH Key ID
-      # Example Request:
-      #   DELETE /user/keys/:id
-      delete "keys/:id" do
-        begin
-          key = current_user.keys.find params[:id]
-          key.destroy
-        rescue
-        end
+      desc 'Delete an SSH key from the currently authenticated user' do
+        success Entities::SSHKey
       end
+      params do
+        requires :key_id, type: Integer, desc: 'The ID of the SSH key'
+      end
+      delete "keys/:key_id" do
+        key = current_user.keys.find_by(id: params[:key_id])
+        not_found!('Key') unless key
 
-      # Get currently authenticated user's emails
-      #
-      # Example Request:
-      #   GET /user/emails
+        key.destroy
+      end
+
+      desc "Get the currently authenticated user's email addresses" do
+        success Entities::Email
+      end
+      params do
+        use :pagination
+      end
       get "emails" do
-        present current_user.emails, with: Entities::Email
+        present paginate(current_user.emails), with: Entities::Email
       end
 
-      # Get single email owned by currently authenticated user
-      #
-      # Example Request:
-      #   GET /user/emails/:id
-      get "emails/:id" do
-        email = current_user.emails.find params[:id]
+      desc 'Get a single email address owned by the currently authenticated user' do
+        success Entities::Email
+      end
+      params do
+        requires :email_id, type: Integer, desc: 'The ID of the email'
+      end
+      get "emails/:email_id" do
+        email = current_user.emails.find_by(id: params[:email_id])
+        not_found!('Email') unless email
+
         present email, with: Entities::Email
       end
 
-      # Add new email to currently authenticated user
-      #
-      # Parameters:
-      #   email (required) - Email address
-      # Example Request:
-      #   POST /user/emails
+      desc 'Add new email address to the currently authenticated user' do
+        success Entities::Email
+      end
+      params do
+        requires :email, type: String, desc: 'The new email'
+      end
       post "emails" do
-        required_attributes! [:email]
+        email = Emails::CreateService.new(current_user, declared_params).execute
 
-        attrs = attributes_for_keys [:email]
-        email = current_user.emails.new attrs
-        if email.save
+        if email.errors.blank?
           NotificationService.new.new_email(email)
           present email, with: Entities::Email
         else
@@ -418,20 +498,30 @@ module API
         end
       end
 
-      # Delete existing email of currently authenticated user
-      #
-      # Parameters:
-      #   id (required) - EMail ID
-      # Example Request:
-      #   DELETE /user/emails/:id
-      delete "emails/:id" do
-        begin
-          email = current_user.emails.find params[:id]
-          email.destroy
+      desc 'Delete an email address from the currently authenticated user'
+      params do
+        requires :email_id, type: Integer, desc: 'The ID of the email'
+      end
+      delete "emails/:email_id" do
+        email = current_user.emails.find_by(id: params[:email_id])
+        not_found!('Email') unless email
 
-          current_user.update_secondary_emails!
-        rescue
-        end
+        Emails::DestroyService.new(current_user, email: email.email).execute
+      end
+
+      desc 'Get a list of user activities'
+      params do
+        optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY'
+        use :pagination
+      end
+      get "activities" do
+        authenticated_as_admin!
+
+        activities = User
+          .where(User.arel_table[:last_activity_on].gteq(params[:from]))
+          .reorder(last_activity_on: :asc)
+
+        present paginate(activities), with: Entities::UserActivity
       end
     end
   end
diff --git a/lib/api/v3/award_emoji.rb b/lib/api/v3/award_emoji.rb
new file mode 100644
index 00000000000..b96b2d70b12
--- /dev/null
+++ b/lib/api/v3/award_emoji.rb
@@ -0,0 +1,130 @@
+module API
+  module V3
+    class AwardEmoji < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+      AWARDABLES = %w[issue merge_request snippet].freeze
+
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        AWARDABLES.each do |awardable_type|
+          awardable_string = awardable_type.pluralize
+          awardable_id_string = "#{awardable_type}_id"
+
+          params do
+            requires :id, type: String, desc: 'The ID of a project'
+            requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet"
+          end
+
+          [
+            ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji",
+            ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji"
+          ].each do |endpoint|
+
+            desc 'Get a list of project +awardable+ award emoji' do
+              detail 'This feature was introduced in 8.9'
+              success Entities::AwardEmoji
+            end
+            params do
+              use :pagination
+            end
+            get endpoint do
+              if can_read_awardable?
+                awards = awardable.award_emoji
+                present paginate(awards), with: Entities::AwardEmoji
+              else
+                not_found!("Award Emoji")
+              end
+            end
+
+            desc 'Get a specific award emoji' do
+              detail 'This feature was introduced in 8.9'
+              success Entities::AwardEmoji
+            end
+            params do
+              requires :award_id, type: Integer, desc: 'The ID of the award'
+            end
+            get "#{endpoint}/:award_id" do
+              if can_read_awardable?
+                present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji
+              else
+                not_found!("Award Emoji")
+              end
+            end
+
+            desc 'Award a new Emoji' do
+              detail 'This feature was introduced in 8.9'
+              success Entities::AwardEmoji
+            end
+            params do
+              requires :name, type: String, desc: 'The name of a award_emoji (without colons)'
+            end
+            post endpoint do
+              not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable?
+
+              award = awardable.create_award_emoji(params[:name], current_user)
+
+              if award.persisted?
+                present award, with: Entities::AwardEmoji
+              else
+                not_found!("Award Emoji #{award.errors.messages}")
+              end
+            end
+
+            desc 'Delete a +awardables+ award emoji' do
+              detail 'This feature was introduced in 8.9'
+              success Entities::AwardEmoji
+            end
+            params do
+              requires :award_id, type: Integer, desc: 'The ID of an award emoji'
+            end
+            delete "#{endpoint}/:award_id" do
+              award = awardable.award_emoji.find(params[:award_id])
+
+              unauthorized! unless award.user == current_user || current_user.admin?
+
+              award.destroy
+              present award, with: Entities::AwardEmoji
+            end
+          end
+        end
+      end
+
+      helpers do
+        def can_read_awardable?
+          can?(current_user, read_ability(awardable), awardable)
+        end
+
+        def can_award_awardable?
+          awardable.user_can_award?(current_user, params[:name])
+        end
+
+        def awardable
+          @awardable ||=
+            begin
+              if params.include?(:note_id)
+                note_id = params.delete(:note_id)
+
+                awardable.notes.find(note_id)
+              elsif params.include?(:issue_id)
+                user_project.issues.find(params[:issue_id])
+              elsif params.include?(:merge_request_id)
+                user_project.merge_requests.find(params[:merge_request_id])
+              else
+                user_project.snippets.find(params[:snippet_id])
+              end
+            end
+        end
+
+        def read_ability(awardable)
+          case awardable
+          when Note
+            read_ability(awardable.noteable)
+          else
+            :"read_#{awardable.class.to_s.underscore}"
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/boards.rb b/lib/api/v3/boards.rb
new file mode 100644
index 00000000000..94acc67171e
--- /dev/null
+++ b/lib/api/v3/boards.rb
@@ -0,0 +1,72 @@
+module API
+  module V3
+    class Boards < Grape::API
+      before { authenticate! }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get all project boards' do
+          detail 'This feature was introduced in 8.13'
+          success ::API::Entities::Board
+        end
+        get ':id/boards' do
+          authorize!(:read_board, user_project)
+          present user_project.boards, with: ::API::Entities::Board
+        end
+
+        params do
+          requires :board_id, type: Integer, desc: 'The ID of a board'
+        end
+        segment ':id/boards/:board_id' do
+          helpers do
+            def project_board
+              board = user_project.boards.first
+
+              if params[:board_id] == board.id
+                board
+              else
+                not_found!('Board')
+              end
+            end
+
+            def board_lists
+              project_board.lists.destroyable
+            end
+          end
+
+          desc 'Get the lists of a project board' do
+            detail 'Does not include `done` list. This feature was introduced in 8.13'
+            success ::API::Entities::List
+          end
+          get '/lists' do
+            authorize!(:read_board, user_project)
+            present board_lists, with: ::API::Entities::List
+          end
+
+          desc 'Delete a board list' do
+            detail 'This feature was introduced in 8.13'
+            success ::API::Entities::List
+          end
+          params do
+            requires :list_id, type: Integer, desc: 'The ID of a board list'
+          end
+          delete "/lists/:list_id" do
+            authorize!(:admin_list, user_project)
+
+            list = board_lists.find(params[:list_id])
+
+            service = ::Boards::Lists::DestroyService.new(user_project, current_user)
+
+            if service.execute(list)
+              present list, with: ::API::Entities::List
+            else
+              render_api_error!({ error: 'List could not be deleted!' }, 400)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb
new file mode 100644
index 00000000000..81b13249892
--- /dev/null
+++ b/lib/api/v3/branches.rb
@@ -0,0 +1,72 @@
+require 'mime/types'
+
+module API
+  module V3
+    class Branches < Grape::API
+      before { authenticate! }
+      before { authorize! :download_code, user_project }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get a project repository branches' do
+          success ::API::Entities::RepoBranch
+        end
+        get ":id/repository/branches" do
+          branches = user_project.repository.branches.sort_by(&:name)
+
+          present branches, with: ::API::Entities::RepoBranch, project: user_project
+        end
+
+        desc 'Delete a branch'
+        params do
+          requires :branch, type: String, desc: 'The name of the branch'
+        end
+        delete ":id/repository/branches/:branch", requirements: { branch: /.+/ } do
+          authorize_push_project
+
+          result = DeleteBranchService.new(user_project, current_user)
+                   .execute(params[:branch])
+
+          if result[:status] == :success
+            status(200)
+            {
+              branch_name: params[:branch]
+            }
+          else
+            render_api_error!(result[:message], result[:return_code])
+          end
+        end
+
+        desc 'Delete all merged branches'
+        delete ":id/repository/merged_branches" do
+          DeleteMergedBranchesService.new(user_project, current_user).async_execute
+
+          status(200)
+        end
+
+        desc 'Create branch' do
+          success ::API::Entities::RepoBranch
+        end
+        params do
+          requires :branch_name, type: String, desc: 'The name of the branch'
+          requires :ref, type: String, desc: 'Create branch from commit sha or existing branch'
+        end
+        post ":id/repository/branches" do
+          authorize_push_project
+          result = CreateBranchService.new(user_project, current_user)
+            .execute(params[:branch_name], params[:ref])
+
+          if result[:status] == :success
+            present result[:branch],
+              with: ::API::Entities::RepoBranch,
+              project: user_project
+          else
+            render_api_error!(result[:message], 400)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/broadcast_messages.rb b/lib/api/v3/broadcast_messages.rb
new file mode 100644
index 00000000000..417e4ad0b26
--- /dev/null
+++ b/lib/api/v3/broadcast_messages.rb
@@ -0,0 +1,31 @@
+module API
+  module V3
+    class BroadcastMessages < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+      before { authenticated_as_admin! }
+
+      resource :broadcast_messages do
+        helpers do
+          def find_message
+            BroadcastMessage.find(params[:id])
+          end
+        end
+
+        desc 'Delete a broadcast message' do
+          detail 'This feature was introduced in GitLab 8.12.'
+          success ::API::Entities::BroadcastMessage
+        end
+        params do
+          requires :id, type: Integer, desc: 'Broadcast message ID'
+        end
+        delete ':id' do
+          message = find_message
+
+          present message.destroy, with: ::API::Entities::BroadcastMessage
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb
new file mode 100644
index 00000000000..93ad9eb26b8
--- /dev/null
+++ b/lib/api/v3/builds.rb
@@ -0,0 +1,249 @@
+module API
+  module V3
+    class Builds < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects do
+        helpers do
+          params :optional_scope do
+            optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show',
+                             values:  %w(pending running failed success canceled skipped),
+                             coerce_with: ->(scope) {
+                                            if scope.is_a?(String)
+                                              [scope]
+                                            elsif   scope.is_a?(Hashie::Mash)
+                                              scope.values
+                                            else
+                                              ['unknown']
+                                            end
+                                          }
+          end
+        end
+
+        desc 'Get a project builds' do
+          success ::API::V3::Entities::Build
+        end
+        params do
+          use :optional_scope
+          use :pagination
+        end
+        get ':id/builds' do
+          builds = user_project.builds.order('id DESC')
+          builds = filter_builds(builds, params[:scope])
+
+          present paginate(builds), with: ::API::V3::Entities::Build
+        end
+
+        desc 'Get builds for a specific commit of a project' do
+          success ::API::V3::Entities::Build
+        end
+        params do
+          requires :sha, type: String, desc: 'The SHA id of a commit'
+          use :optional_scope
+          use :pagination
+        end
+        get ':id/repository/commits/:sha/builds' do
+          authorize_read_builds!
+
+          return not_found! unless user_project.commit(params[:sha])
+
+          pipelines = user_project.pipelines.where(sha: params[:sha])
+          builds = user_project.builds.where(pipeline: pipelines).order('id DESC')
+          builds = filter_builds(builds, params[:scope])
+
+          present paginate(builds), with: ::API::V3::Entities::Build
+        end
+
+        desc 'Get a specific build of a project' do
+          success ::API::V3::Entities::Build
+        end
+        params do
+          requires :build_id, type: Integer, desc: 'The ID of a build'
+        end
+        get ':id/builds/:build_id' do
+          authorize_read_builds!
+
+          build = get_build!(params[:build_id])
+
+          present build, with: ::API::V3::Entities::Build
+        end
+
+        desc 'Download the artifacts file from build' do
+          detail 'This feature was introduced in GitLab 8.5'
+        end
+        params do
+          requires :build_id, type: Integer, desc: 'The ID of a build'
+        end
+        get ':id/builds/:build_id/artifacts' do
+          authorize_read_builds!
+
+          build = get_build!(params[:build_id])
+
+          present_artifacts!(build.artifacts_file)
+        end
+
+        desc 'Download the artifacts file from build' do
+          detail 'This feature was introduced in GitLab 8.10'
+        end
+        params do
+          requires :ref_name, type: String, desc: 'The ref from repository'
+          requires :job,      type: String, desc: 'The name for the build'
+        end
+        get ':id/builds/artifacts/:ref_name/download',
+          requirements: { ref_name: /.+/ } do
+          authorize_read_builds!
+
+          builds = user_project.latest_successful_builds_for(params[:ref_name])
+          latest_build = builds.find_by!(name: params[:job])
+
+          present_artifacts!(latest_build.artifacts_file)
+        end
+
+        # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace
+        #       is saved in the DB instead of file). But before that, we need to consider how to replace the value of
+        #       `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse.
+        desc 'Get a trace of a specific build of a project'
+        params do
+          requires :build_id, type: Integer, desc: 'The ID of a build'
+        end
+        get ':id/builds/:build_id/trace' do
+          authorize_read_builds!
+
+          build = get_build!(params[:build_id])
+
+          header 'Content-Disposition', "infile; filename=\"#{build.id}.log\""
+          content_type 'text/plain'
+          env['api.format'] = :binary
+
+          trace = build.trace.raw
+          body trace
+        end
+
+        desc 'Cancel a specific build of a project' do
+          success ::API::V3::Entities::Build
+        end
+        params do
+          requires :build_id, type: Integer, desc: 'The ID of a build'
+        end
+        post ':id/builds/:build_id/cancel' do
+          authorize_update_builds!
+
+          build = get_build!(params[:build_id])
+          authorize!(:update_build, build)
+
+          build.cancel
+
+          present build, with: ::API::V3::Entities::Build
+        end
+
+        desc 'Retry a specific build of a project' do
+          success ::API::V3::Entities::Build
+        end
+        params do
+          requires :build_id, type: Integer, desc: 'The ID of a build'
+        end
+        post ':id/builds/:build_id/retry' do
+          authorize_update_builds!
+
+          build = get_build!(params[:build_id])
+          authorize!(:update_build, build)
+          return forbidden!('Build is not retryable') unless build.retryable?
+
+          build = Ci::Build.retry(build, current_user)
+
+          present build, with: ::API::V3::Entities::Build
+        end
+
+        desc 'Erase build (remove artifacts and build trace)' do
+          success ::API::V3::Entities::Build
+        end
+        params do
+          requires :build_id, type: Integer, desc: 'The ID of a build'
+        end
+        post ':id/builds/:build_id/erase' do
+          authorize_update_builds!
+
+          build = get_build!(params[:build_id])
+          authorize!(:update_build, build)
+          return forbidden!('Build is not erasable!') unless build.erasable?
+
+          build.erase(erased_by: current_user)
+          present build, with: ::API::V3::Entities::Build
+        end
+
+        desc 'Keep the artifacts to prevent them from being deleted' do
+          success ::API::V3::Entities::Build
+        end
+        params do
+          requires :build_id, type: Integer, desc: 'The ID of a build'
+        end
+        post ':id/builds/:build_id/artifacts/keep' do
+          authorize_update_builds!
+
+          build = get_build!(params[:build_id])
+          authorize!(:update_build, build)
+          return not_found!(build) unless build.artifacts?
+
+          build.keep_artifacts!
+
+          status 200
+          present build, with: ::API::V3::Entities::Build
+        end
+
+        desc 'Trigger a manual build' do
+          success ::API::V3::Entities::Build
+          detail 'This feature was added in GitLab 8.11'
+        end
+        params do
+          requires :build_id, type: Integer, desc: 'The ID of a Build'
+        end
+        post ":id/builds/:build_id/play" do
+          authorize_read_builds!
+
+          build = get_build!(params[:build_id])
+          authorize!(:update_build, build)
+          bad_request!("Unplayable Job") unless build.playable?
+
+          build.play(current_user)
+
+          status 200
+          present build, with: ::API::V3::Entities::Build
+        end
+      end
+
+      helpers do
+        def find_build(id)
+          user_project.builds.find_by(id: id.to_i)
+        end
+
+        def get_build!(id)
+          find_build(id) || not_found!
+        end
+
+        def filter_builds(builds, scope)
+          return builds if scope.nil? || scope.empty?
+
+          available_statuses = ::CommitStatus::AVAILABLE_STATUSES
+
+          unknown = scope - available_statuses
+          render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty?
+
+          builds.where(status: available_statuses && scope)
+        end
+
+        def authorize_read_builds!
+          authorize! :read_build, user_project
+        end
+
+        def authorize_update_builds!
+          authorize! :update_build, user_project
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb
new file mode 100644
index 00000000000..5936f4700aa
--- /dev/null
+++ b/lib/api/v3/commits.rb
@@ -0,0 +1,196 @@
+require 'mime/types'
+
+module API
+  module V3
+    class Commits < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+      before { authorize! :download_code, user_project }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get a project repository commits' do
+          success ::API::Entities::RepoCommit
+        end
+        params do
+          optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used'
+          optional :since,    type: DateTime, desc: 'Only commits after or in this date will be returned'
+          optional :until,    type: DateTime, desc: 'Only commits before or in this date will be returned'
+          optional :page,     type: Integer, default: 0, desc: 'The page for pagination'
+          optional :per_page, type: Integer, default: 20, desc: 'The number of results per page'
+          optional :path,     type: String, desc: 'The file path'
+        end
+        get ":id/repository/commits" do
+          ref = params[:ref_name] || user_project.try(:default_branch) || 'master'
+          offset = params[:page] * params[:per_page]
+
+          commits = user_project.repository.commits(ref,
+                                                    path: params[:path],
+                                                    limit: params[:per_page],
+                                                    offset: offset,
+                                                    after: params[:since],
+                                                    before: params[:until])
+
+          present commits, with: ::API::Entities::RepoCommit
+        end
+
+        desc 'Commit multiple file changes as one commit' do
+          success ::API::Entities::RepoCommitDetail
+          detail 'This feature was introduced in GitLab 8.13'
+        end
+        params do
+          requires :branch_name, type: String, desc: 'The name of branch'
+          requires :commit_message, type: String, desc: 'Commit message'
+          requires :actions, type: Array[Hash], desc: 'Actions to perform in commit'
+          optional :author_email, type: String, desc: 'Author email for commit'
+          optional :author_name, type: String, desc: 'Author name for commit'
+        end
+        post ":id/repository/commits" do
+          authorize! :push_code, user_project
+
+          attrs = declared_params.dup
+          branch = attrs.delete(:branch_name)
+          attrs.merge!(start_branch: branch, branch_name: branch)
+
+          result = ::Files::MultiService.new(user_project, current_user, attrs).execute
+
+          if result[:status] == :success
+            commit_detail = user_project.repository.commits(result[:result], limit: 1).first
+            present commit_detail, with: ::API::Entities::RepoCommitDetail
+          else
+            render_api_error!(result[:message], 400)
+          end
+        end
+
+        desc 'Get a specific commit of a project' do
+          success ::API::Entities::RepoCommitDetail
+          failure [[404, 'Not Found']]
+        end
+        params do
+          requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag'
+        end
+        get ":id/repository/commits/:sha" do
+          commit = user_project.commit(params[:sha])
+
+          not_found! "Commit" unless commit
+
+          present commit, with: ::API::Entities::RepoCommitDetail
+        end
+
+        desc 'Get the diff for a specific commit of a project' do
+          failure [[404, 'Not Found']]
+        end
+        params do
+          requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag'
+        end
+        get ":id/repository/commits/:sha/diff" do
+          commit = user_project.commit(params[:sha])
+
+          not_found! "Commit" unless commit
+
+          commit.raw_diffs.to_a
+        end
+
+        desc "Get a commit's comments" do
+          success ::API::Entities::CommitNote
+          failure [[404, 'Not Found']]
+        end
+        params do
+          use :pagination
+          requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag'
+        end
+        get ':id/repository/commits/:sha/comments' do
+          commit = user_project.commit(params[:sha])
+
+          not_found! 'Commit' unless commit
+          notes = Note.where(commit_id: commit.id).order(:created_at)
+
+          present paginate(notes), with: ::API::Entities::CommitNote
+        end
+
+        desc 'Cherry pick commit into a branch' do
+          detail 'This feature was introduced in GitLab 8.15'
+          success ::API::Entities::RepoCommit
+        end
+        params do
+          requires :sha, type: String, desc: 'A commit sha to be cherry picked'
+          requires :branch, type: String, desc: 'The name of the branch'
+        end
+        post ':id/repository/commits/:sha/cherry_pick' do
+          authorize! :push_code, user_project
+
+          commit = user_project.commit(params[:sha])
+          not_found!('Commit') unless commit
+
+          branch = user_project.repository.find_branch(params[:branch])
+          not_found!('Branch') unless branch
+
+          commit_params = {
+            commit: commit,
+            start_branch: params[:branch],
+            branch_name: params[:branch]
+          }
+
+          result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute
+
+          if result[:status] == :success
+            branch = user_project.repository.find_branch(params[:branch])
+            present user_project.repository.commit(branch.dereferenced_target), with: ::API::Entities::RepoCommit
+          else
+            render_api_error!(result[:message], 400)
+          end
+        end
+
+        desc 'Post comment to commit' do
+          success ::API::Entities::CommitNote
+        end
+        params do
+          requires :sha, type: String, regexp: /\A\h{6,40}\z/, desc: "The commit's SHA"
+          requires :note, type: String, desc: 'The text of the comment'
+          optional :path, type: String, desc: 'The file path'
+          given :path do
+            requires :line, type: Integer, desc: 'The line number'
+            requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line'
+          end
+        end
+        post ':id/repository/commits/:sha/comments' do
+          commit = user_project.commit(params[:sha])
+          not_found! 'Commit' unless commit
+
+          opts = {
+            note: params[:note],
+            noteable_type: 'Commit',
+            commit_id: commit.id
+          }
+
+          if params[:path]
+            commit.raw_diffs(limits: false).each do |diff|
+              next unless diff.new_path == params[:path]
+              lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line)
+
+              lines.each do |line|
+                next unless line.new_pos == params[:line] && line.type == params[:line_type]
+                break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos)
+              end
+
+              break if opts[:line_code]
+            end
+
+            opts[:type] = LegacyDiffNote.name if opts[:line_code]
+          end
+
+          note = ::Notes::CreateService.new(user_project, current_user, opts).execute
+
+          if note.save
+            present note, with: ::API::Entities::CommitNote
+          else
+            render_api_error!("Failed to save note #{note.errors.messages}", 400)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/deploy_keys.rb b/lib/api/v3/deploy_keys.rb
new file mode 100644
index 00000000000..b90e2061da3
--- /dev/null
+++ b/lib/api/v3/deploy_keys.rb
@@ -0,0 +1,123 @@
+module API
+  module V3
+    class DeployKeys < Grape::API
+      before { authenticate! }
+
+      get "deploy_keys" do
+        authenticated_as_admin!
+
+        keys = DeployKey.all
+        present keys, with: ::API::Entities::SSHKey
+      end
+
+      params do
+        requires :id, type: String, desc: 'The ID of the project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        before { authorize_admin_project }
+
+        %w(keys deploy_keys).each do |path|
+          desc "Get a specific project's deploy keys" do
+            success ::API::Entities::SSHKey
+          end
+          get ":id/#{path}" do
+            present user_project.deploy_keys, with: ::API::Entities::SSHKey
+          end
+
+          desc 'Get single deploy key' do
+            success ::API::Entities::SSHKey
+          end
+          params do
+            requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+          end
+          get ":id/#{path}/:key_id" do
+            key = user_project.deploy_keys.find params[:key_id]
+            present key, with: ::API::Entities::SSHKey
+          end
+
+          desc 'Add new deploy key to currently authenticated user' do
+            success ::API::Entities::SSHKey
+          end
+          params do
+            requires :key, type: String, desc: 'The new deploy key'
+            requires :title, type: String, desc: 'The name of the deploy key'
+            optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository"
+          end
+          post ":id/#{path}" do
+            params[:key].strip!
+
+            # Check for an existing key joined to this project
+            key = user_project.deploy_keys.find_by(key: params[:key])
+            if key
+              present key, with: ::API::Entities::SSHKey
+              break
+            end
+
+            # Check for available deploy keys in other projects
+            key = current_user.accessible_deploy_keys.find_by(key: params[:key])
+            if key
+              user_project.deploy_keys << key
+              present key, with: ::API::Entities::SSHKey
+              break
+            end
+
+            # Create a new deploy key
+            key = DeployKey.new(declared_params(include_missing: false))
+            if key.valid? && user_project.deploy_keys << key
+              present key, with: ::API::Entities::SSHKey
+            else
+              render_validation_error!(key)
+            end
+          end
+
+          desc 'Enable a deploy key for a project' do
+            detail 'This feature was added in GitLab 8.11'
+            success ::API::Entities::SSHKey
+          end
+          params do
+            requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+          end
+          post ":id/#{path}/:key_id/enable" do
+            key = ::Projects::EnableDeployKeyService.new(user_project,
+                                                          current_user, declared_params).execute
+
+            if key
+              present key, with: ::API::Entities::SSHKey
+            else
+              not_found!('Deploy Key')
+            end
+          end
+
+          desc 'Disable a deploy key for a project' do
+            detail 'This feature was added in GitLab 8.11'
+            success ::API::Entities::SSHKey
+          end
+          params do
+            requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+          end
+          delete ":id/#{path}/:key_id/disable" do
+            key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id])
+            key.destroy
+
+            present key.deploy_key, with: ::API::Entities::SSHKey
+          end
+
+          desc 'Delete deploy key for a project' do
+            success Key
+          end
+          params do
+            requires :key_id, type: Integer, desc: 'The ID of the deploy key'
+          end
+          delete ":id/#{path}/:key_id" do
+            key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id])
+            if key
+              key.destroy
+            else
+              not_found!('Deploy Key')
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/deployments.rb b/lib/api/v3/deployments.rb
new file mode 100644
index 00000000000..1d4972eda26
--- /dev/null
+++ b/lib/api/v3/deployments.rb
@@ -0,0 +1,43 @@
+module API
+  module V3
+    # Deployments RESTful API endpoints
+    class Deployments < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+
+      params do
+        requires :id, type: String, desc: 'The project ID'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get all deployments of the project' do
+          detail 'This feature was introduced in GitLab 8.11.'
+          success ::API::V3::Deployments
+        end
+        params do
+          use :pagination
+        end
+        get ':id/deployments' do
+          authorize! :read_deployment, user_project
+
+          present paginate(user_project.deployments), with: ::API::V3::Deployments
+        end
+
+        desc 'Gets a specific deployment' do
+          detail 'This feature was introduced in GitLab 8.11.'
+          success ::API::V3::Deployments
+        end
+        params do
+          requires :deployment_id, type: Integer,  desc: 'The deployment ID'
+        end
+        get ':id/deployments/:deployment_id' do
+          authorize! :read_deployment, user_project
+
+          deployment = user_project.deployments.find(params[:deployment_id])
+
+          present deployment, with: ::API::V3::Deployments
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb
new file mode 100644
index 00000000000..c848f52723b
--- /dev/null
+++ b/lib/api/v3/entities.rb
@@ -0,0 +1,269 @@
+module API
+  module V3
+    module Entities
+      class ProjectSnippet < Grape::Entity
+        expose :id, :title, :file_name
+        expose :author, using: ::API::Entities::UserBasic
+        expose :updated_at, :created_at
+        expose(:expires_at) { |snippet| nil }
+
+        expose :web_url do |snippet, options|
+          Gitlab::UrlBuilder.build(snippet)
+        end
+      end
+
+      class Note < Grape::Entity
+        expose :id
+        expose :note, as: :body
+        expose :attachment_identifier, as: :attachment
+        expose :author, using: ::API::Entities::UserBasic
+        expose :created_at, :updated_at
+        expose :system?, as: :system
+        expose :noteable_id, :noteable_type
+        # upvote? and downvote? are deprecated, always return false
+        expose(:upvote?)    { |note| false }
+        expose(:downvote?)  { |note| false }
+      end
+
+      class Event < Grape::Entity
+        expose :title, :project_id, :action_name
+        expose :target_id, :target_type, :author_id
+        expose :data, :target_title
+        expose :created_at
+        expose :note, using: Entities::Note, if: ->(event, options) { event.note? }
+        expose :author, using: ::API::Entities::UserBasic, if: ->(event, options) { event.author }
+
+        expose :author_username do |event, options|
+          event.author&.username
+        end
+      end
+
+      class AwardEmoji < Grape::Entity
+        expose :id
+        expose :name
+        expose :user, using: ::API::Entities::UserBasic
+        expose :created_at, :updated_at
+        expose :awardable_id, :awardable_type
+      end
+
+      class Project < Grape::Entity
+        expose :id, :description, :default_branch, :tag_list
+        expose :public?, as: :public
+        expose :archived?, as: :archived
+        expose :visibility_level, :ssh_url_to_repo, :http_url_to_repo, :web_url
+        expose :owner, using: ::API::Entities::UserBasic, unless: ->(project, options) { project.group }
+        expose :name, :name_with_namespace
+        expose :path, :path_with_namespace
+        expose :container_registry_enabled
+
+        # Expose old field names with the new permissions methods to keep API compatible
+        expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) }
+        expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) }
+        expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) }
+        expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) }
+        expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) }
+
+        expose :created_at, :last_activity_at
+        expose :shared_runners_enabled
+        expose :lfs_enabled?, as: :lfs_enabled
+        expose :creator_id
+        expose :namespace, using: 'API::Entities::Namespace'
+        expose :forked_from_project, using: ::API::Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? }
+        expose :avatar_url do |user, options|
+          user.avatar_url(only_path: false)
+        end
+        expose :star_count, :forks_count
+        expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? }
+        expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
+        expose :public_builds
+        expose :shared_with_groups do |project, options|
+          ::API::Entities::SharedGroup.represent(project.project_group_links.all, options)
+        end
+        expose :only_allow_merge_if_pipeline_succeeds, as: :only_allow_merge_if_build_succeeds
+        expose :request_access_enabled
+        expose :only_allow_merge_if_all_discussions_are_resolved
+
+        expose :statistics, using: '::API::V3::Entities::ProjectStatistics', if: :statistics
+      end
+
+      class ProjectWithAccess < Project
+        expose :permissions do
+          expose :project_access, using: ::API::Entities::ProjectAccess do |project, options|
+            project.project_members.find_by(user_id: options[:current_user].id)
+          end
+
+          expose :group_access, using: ::API::Entities::GroupAccess do |project, options|
+            if project.group
+              project.group.group_members.find_by(user_id: options[:current_user].id)
+            end
+          end
+        end
+      end
+
+      class MergeRequest < Grape::Entity
+        expose :id, :iid
+        expose(:project_id) { |entity| entity.project.id }
+        expose :title, :description
+        expose :state, :created_at, :updated_at
+        expose :target_branch, :source_branch
+        expose :upvotes, :downvotes
+        expose :author, :assignee, using: ::API::Entities::UserBasic
+        expose :source_project_id, :target_project_id
+        expose :label_names, as: :labels
+        expose :work_in_progress?, as: :work_in_progress
+        expose :milestone, using: ::API::Entities::Milestone
+        expose :merge_when_pipeline_succeeds, as: :merge_when_build_succeeds
+        expose :merge_status
+        expose :diff_head_sha, as: :sha
+        expose :merge_commit_sha
+        expose :subscribed do |merge_request, options|
+          merge_request.subscribed?(options[:current_user], options[:project])
+        end
+        expose :user_notes_count
+        expose :should_remove_source_branch?, as: :should_remove_source_branch
+        expose :force_remove_source_branch?, as: :force_remove_source_branch
+
+        expose :web_url do |merge_request, options|
+          Gitlab::UrlBuilder.build(merge_request)
+        end
+      end
+
+      class Group < Grape::Entity
+        expose :id, :name, :path, :description, :visibility_level
+        expose :lfs_enabled?, as: :lfs_enabled
+        expose :avatar_url do |user, options|
+          user.avatar_url(only_path: false)
+        end
+        expose :web_url
+        expose :request_access_enabled
+        expose :full_name, :full_path
+
+        if ::Group.supports_nested_groups?
+          expose :parent_id
+        end
+
+        expose :statistics, if: :statistics do
+          with_options format_with: -> (value) { value.to_i } do
+            expose :storage_size
+            expose :repository_size
+            expose :lfs_objects_size
+            expose :build_artifacts_size
+          end
+        end
+      end
+
+      class GroupDetail < Group
+        expose :projects, using: Entities::Project
+        expose :shared_projects, using: Entities::Project
+      end
+
+      class ApplicationSetting < Grape::Entity
+        expose :id
+        expose :default_projects_limit
+        expose :signup_enabled
+        expose :signin_enabled
+        expose :gravatar_enabled
+        expose :sign_in_text
+        expose :after_sign_up_text
+        expose :created_at
+        expose :updated_at
+        expose :home_page_url
+        expose :default_branch_protection
+        expose :restricted_visibility_levels
+        expose :max_attachment_size
+        expose :session_expire_delay
+        expose :default_project_visibility
+        expose :default_snippet_visibility
+        expose :default_group_visibility
+        expose :domain_whitelist
+        expose :domain_blacklist_enabled
+        expose :domain_blacklist
+        expose :user_oauth_applications
+        expose :after_sign_out_path
+        expose :container_registry_token_expire_delay
+        expose :repository_storage
+        expose :repository_storages
+        expose :koding_enabled
+        expose :koding_url
+        expose :plantuml_enabled
+        expose :plantuml_url
+        expose :terminal_max_session_time
+      end
+
+      class Environment < ::API::Entities::EnvironmentBasic
+        expose :project, using: Entities::Project
+      end
+
+      class Trigger < Grape::Entity
+        expose :token, :created_at, :updated_at, :deleted_at, :last_used
+        expose :owner, using: ::API::Entities::UserBasic
+      end
+
+      class TriggerRequest < Grape::Entity
+        expose :id, :variables
+      end
+
+      class Build < Grape::Entity
+        expose :id, :status, :stage, :name, :ref, :tag, :coverage
+        expose :created_at, :started_at, :finished_at
+        expose :user, with: ::API::Entities::User
+        expose :artifacts_file, using: ::API::Entities::JobArtifactFile, if: -> (build, opts) { build.artifacts? }
+        expose :commit, with: ::API::Entities::RepoCommit
+        expose :runner, with: ::API::Entities::Runner
+        expose :pipeline, with: ::API::Entities::PipelineBasic
+      end
+
+      class BuildArtifactFile < Grape::Entity
+        expose :filename, :size
+      end
+
+      class Deployment < Grape::Entity
+        expose :id, :iid, :ref, :sha, :created_at
+        expose :user,        using: ::API::Entities::UserBasic
+        expose :environment, using: ::API::Entities::EnvironmentBasic
+        expose :deployable,  using: Entities::Build
+      end
+
+      class MergeRequestChanges < MergeRequest
+        expose :diffs, as: :changes, using: ::API::Entities::RepoDiff do |compare, _|
+          compare.raw_diffs(limits: false).to_a
+        end
+      end
+
+      class ProjectStatistics < Grape::Entity
+        expose :commit_count
+        expose :storage_size
+        expose :repository_size
+        expose :lfs_objects_size
+        expose :build_artifacts_size
+      end
+
+      class ProjectService < Grape::Entity
+        expose :id, :title, :created_at, :updated_at, :active
+        expose :push_events, :issues_events, :merge_requests_events
+        expose :tag_push_events, :note_events, :pipeline_events
+        expose :job_events, as: :build_events
+        # Expose serialized properties
+        expose :properties do |service, options|
+          field_names = service.fields
+            .select { |field| options[:include_passwords] || field[:type] != 'password' }
+            .map { |field| field[:name] }
+          service.properties.slice(*field_names)
+        end
+      end
+
+      class ProjectHook < ::API::Entities::Hook
+        expose :project_id, :issues_events, :merge_requests_events
+        expose :note_events, :pipeline_events, :wiki_page_events
+        expose :job_events, as: :build_events
+      end
+
+      class Issue < ::API::Entities::Issue
+        unexpose :assignees
+        expose :assignee do |issue, options|
+          ::API::Entities::UserBasic.represent(issue.assignees.first, options)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/environments.rb b/lib/api/v3/environments.rb
new file mode 100644
index 00000000000..6bb4e016a01
--- /dev/null
+++ b/lib/api/v3/environments.rb
@@ -0,0 +1,87 @@
+module API
+  module V3
+    class Environments < Grape::API
+      include ::API::Helpers::CustomValidators
+      include PaginationParams
+
+      before { authenticate! }
+
+      params do
+        requires :id, type: String, desc: 'The project ID'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get all environments of the project' do
+          detail 'This feature was introduced in GitLab 8.11.'
+          success Entities::Environment
+        end
+        params do
+          use :pagination
+        end
+        get ':id/environments' do
+          authorize! :read_environment, user_project
+
+          present paginate(user_project.environments), with: Entities::Environment
+        end
+
+        desc 'Creates a new environment' do
+          detail 'This feature was introduced in GitLab 8.11.'
+          success Entities::Environment
+        end
+        params do
+          requires :name,           type: String,   desc: 'The name of the environment to be created'
+          optional :external_url,   type: String,   desc: 'URL on which this deployment is viewable'
+          optional :slug, absence: { message: "is automatically generated and cannot be changed" }
+        end
+        post ':id/environments' do
+          authorize! :create_environment, user_project
+
+          environment = user_project.environments.create(declared_params)
+
+          if environment.persisted?
+            present environment, with: Entities::Environment
+          else
+            render_validation_error!(environment)
+          end
+        end
+
+        desc 'Updates an existing environment' do
+          detail 'This feature was introduced in GitLab 8.11.'
+          success Entities::Environment
+        end
+        params do
+          requires :environment_id, type: Integer,  desc: 'The environment ID'
+          optional :name,           type: String,   desc: 'The new environment name'
+          optional :external_url,   type: String,   desc: 'The new URL on which this deployment is viewable'
+          optional :slug, absence: { message: "is automatically generated and cannot be changed" }
+        end
+        put ':id/environments/:environment_id' do
+          authorize! :update_environment, user_project
+
+          environment = user_project.environments.find(params[:environment_id])
+
+          update_params = declared_params(include_missing: false).extract!(:name, :external_url)
+          if environment.update(update_params)
+            present environment, with: Entities::Environment
+          else
+            render_validation_error!(environment)
+          end
+        end
+
+        desc 'Deletes an existing environment' do
+          detail 'This feature was introduced in GitLab 8.11.'
+          success Entities::Environment
+        end
+        params do
+          requires :environment_id, type: Integer,  desc: 'The environment ID'
+        end
+        delete ':id/environments/:environment_id' do
+          authorize! :update_environment, user_project
+
+          environment = user_project.environments.find(params[:environment_id])
+
+          present environment.destroy, with: Entities::Environment
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/files.rb b/lib/api/v3/files.rb
new file mode 100644
index 00000000000..7b4b3448b6d
--- /dev/null
+++ b/lib/api/v3/files.rb
@@ -0,0 +1,138 @@
+module API
+  module V3
+    class Files < Grape::API
+      helpers do
+        def commit_params(attrs)
+          {
+            file_path: attrs[:file_path],
+            start_branch: attrs[:branch],
+            branch_name: attrs[:branch],
+            commit_message: attrs[:commit_message],
+            file_content: attrs[:content],
+            file_content_encoding: attrs[:encoding],
+            author_email: attrs[:author_email],
+            author_name: attrs[:author_name]
+          }
+        end
+
+        def commit_response(attrs)
+          {
+            file_path: attrs[:file_path],
+            branch: attrs[:branch]
+          }
+        end
+
+        params :simple_file_params do
+          requires :file_path, type: String, desc: 'The path to new file. Ex. lib/class.rb'
+          requires :branch_name, type: String, desc: 'The name of branch'
+          requires :commit_message, type: String, desc: 'Commit Message'
+          optional :author_email, type: String, desc: 'The email of the author'
+          optional :author_name, type: String, desc: 'The name of the author'
+        end
+
+        params :extended_file_params do
+          use :simple_file_params
+          requires :content, type: String, desc: 'File content'
+          optional :encoding, type: String, values: %w[base64], desc: 'File encoding'
+        end
+      end
+
+      params do
+        requires :id, type: String, desc: 'The project ID'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get a file from repository'
+        params do
+          requires :file_path, type: String, desc: 'The path to the file. Ex. lib/class.rb'
+          requires :ref, type: String, desc: 'The name of branch, tag, or commit'
+        end
+        get ":id/repository/files" do
+          authorize! :download_code, user_project
+
+          commit = user_project.commit(params[:ref])
+          not_found!('Commit') unless commit
+
+          repo = user_project.repository
+          blob = repo.blob_at(commit.sha, params[:file_path])
+          not_found!('File') unless blob
+
+          blob.load_all_data!
+          status(200)
+
+          {
+            file_name: blob.name,
+            file_path: blob.path,
+            size: blob.size,
+            encoding: "base64",
+            content: Base64.strict_encode64(blob.data),
+            ref: params[:ref],
+            blob_id: blob.id,
+            commit_id: commit.id,
+            last_commit_id: repo.last_commit_id_for_path(commit.sha, params[:file_path])
+          }
+        end
+
+        desc 'Create new file in repository'
+        params do
+          use :extended_file_params
+        end
+        post ":id/repository/files" do
+          authorize! :push_code, user_project
+
+          file_params = declared_params(include_missing: false)
+          file_params[:branch] = file_params.delete(:branch_name)
+
+          result = ::Files::CreateService.new(user_project, current_user, commit_params(file_params)).execute
+
+          if result[:status] == :success
+            status(201)
+            commit_response(file_params)
+          else
+            render_api_error!(result[:message], 400)
+          end
+        end
+
+        desc 'Update existing file in repository'
+        params do
+          use :extended_file_params
+        end
+        put ":id/repository/files" do
+          authorize! :push_code, user_project
+
+          file_params = declared_params(include_missing: false)
+          file_params[:branch] = file_params.delete(:branch_name)
+
+          result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute
+
+          if result[:status] == :success
+            status(200)
+            commit_response(file_params)
+          else
+            http_status = result[:http_status] || 400
+            render_api_error!(result[:message], http_status)
+          end
+        end
+
+        desc 'Delete an existing file in repository'
+        params do
+          use :simple_file_params
+        end
+        delete ":id/repository/files" do
+          authorize! :push_code, user_project
+
+          file_params = declared_params(include_missing: false)
+          file_params[:branch] = file_params.delete(:branch_name)
+
+          result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute
+
+          if result[:status] == :success
+            status(200)
+            commit_response(file_params)
+          else
+            render_api_error!(result[:message], 400)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb
new file mode 100644
index 00000000000..2c52d21fa1c
--- /dev/null
+++ b/lib/api/v3/groups.rb
@@ -0,0 +1,185 @@
+module API
+  module V3
+    class Groups < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+
+      helpers do
+        params :optional_params do
+          optional :description, type: String, desc: 'The description of the group'
+          optional :visibility_level, type: Integer, desc: 'The visibility level of the group'
+          optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
+          optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
+        end
+
+        params :statistics_params do
+          optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
+        end
+
+        def present_groups(groups, options = {})
+          options = options.reverse_merge(
+            with: Entities::Group,
+            current_user: current_user
+          )
+
+          groups = groups.with_statistics if options[:statistics]
+          present paginate(groups), options
+        end
+      end
+
+      resource :groups do
+        desc 'Get a groups list' do
+          success Entities::Group
+        end
+        params do
+          use :statistics_params
+          optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list'
+          optional :all_available, type: Boolean, desc: 'Show all group that you have access to'
+          optional :search, type: String, desc: 'Search for a specific group'
+          optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path'
+          optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)'
+          use :pagination
+        end
+        get do
+          groups = if current_user.admin
+                     Group.all
+                   elsif params[:all_available]
+                     GroupsFinder.new(current_user).execute
+                   else
+                     current_user.groups
+                   end
+
+          groups = groups.search(params[:search]) if params[:search].present?
+          groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
+          groups = groups.reorder(params[:order_by] => params[:sort])
+
+          present_groups groups, statistics: params[:statistics] && current_user.admin?
+        end
+
+        desc 'Get list of owned groups for authenticated user' do
+          success Entities::Group
+        end
+        params do
+          use :pagination
+          use :statistics_params
+        end
+        get '/owned' do
+          present_groups current_user.owned_groups, statistics: params[:statistics]
+        end
+
+        desc 'Create a group. Available only for users who can create groups.' do
+          success Entities::Group
+        end
+        params do
+          requires :name, type: String, desc: 'The name of the group'
+          requires :path, type: String, desc: 'The path of the group'
+
+          if ::Group.supports_nested_groups?
+            optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group'
+          end
+
+          use :optional_params
+        end
+        post do
+          authorize! :create_group
+
+          group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute
+
+          if group.persisted?
+            present group, with: Entities::Group, current_user: current_user
+          else
+            render_api_error!("Failed to save group #{group.errors.messages}", 400)
+          end
+        end
+      end
+
+      params do
+        requires :id, type: String, desc: 'The ID of a group'
+      end
+      resource :groups, requirements: { id: %r{[^/]+} } do
+        desc 'Update a group. Available only for users who can administrate groups.' do
+          success Entities::Group
+        end
+        params do
+          optional :name, type: String, desc: 'The name of the group'
+          optional :path, type: String, desc: 'The path of the group'
+          use :optional_params
+          at_least_one_of :name, :path, :description, :visibility_level,
+                          :lfs_enabled, :request_access_enabled
+        end
+        put ':id' do
+          group = find_group!(params[:id])
+          authorize! :admin_group, group
+
+          if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute
+            present group, with: Entities::GroupDetail, current_user: current_user
+          else
+            render_validation_error!(group)
+          end
+        end
+
+        desc 'Get a single group, with containing projects.' do
+          success Entities::GroupDetail
+        end
+        get ":id" do
+          group = find_group!(params[:id])
+          present group, with: Entities::GroupDetail, current_user: current_user
+        end
+
+        desc 'Remove a group.'
+        delete ":id" do
+          group = find_group!(params[:id])
+          authorize! :admin_group, group
+          present ::Groups::DestroyService.new(group, current_user).execute, with: Entities::GroupDetail, current_user: current_user
+        end
+
+        desc 'Get a list of projects in this group.' do
+          success Entities::Project
+        end
+        params do
+          optional :archived, type: Boolean, default: false, desc: 'Limit by archived status'
+          optional :visibility, type: String, values: %w[public internal private],
+                                desc: 'Limit by visibility'
+          optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
+          optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
+                              default: 'created_at', desc: 'Return projects ordered by field'
+          optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                          desc: 'Return projects sorted in ascending and descending order'
+          optional :simple, type: Boolean, default: false,
+                            desc: 'Return only the ID, URL, name, and path of each project'
+          optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
+          optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
+
+          use :pagination
+        end
+        get ":id/projects" do
+          group = find_group!(params[:id])
+          projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute
+          projects = filter_projects(projects)
+          entity = params[:simple] ? ::API::Entities::BasicProjectDetails : Entities::Project
+          present paginate(projects), with: entity, current_user: current_user
+        end
+
+        desc 'Transfer a project to the group namespace. Available only for admin.' do
+          success Entities::GroupDetail
+        end
+        params do
+          requires :project_id, type: String, desc: 'The ID or path of the project'
+        end
+        post ":id/projects/:project_id", requirements: { project_id: /.+/ } do
+          authenticated_as_admin!
+          group = find_group!(params[:id])
+          project = find_project!(params[:project_id])
+          result = ::Projects::TransferService.new(project, current_user).execute(group)
+
+          if result
+            present group, with: Entities::GroupDetail, current_user: current_user
+          else
+            render_api_error!("Failed to transfer project #{project.errors.messages}", 400)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb
new file mode 100644
index 00000000000..4e63aa01c1a
--- /dev/null
+++ b/lib/api/v3/helpers.rb
@@ -0,0 +1,49 @@
+module API
+  module V3
+    module Helpers
+      def find_project_issue(id)
+        IssuesFinder.new(current_user, project_id: user_project.id).find(id)
+      end
+
+      def find_project_merge_request(id)
+        MergeRequestsFinder.new(current_user, project_id: user_project.id).find(id)
+      end
+
+      def find_merge_request_with_access(id, access_level = :read_merge_request)
+        merge_request = user_project.merge_requests.find(id)
+        authorize! access_level, merge_request
+        merge_request
+      end
+
+      # project helpers
+
+      def filter_projects(projects)
+        if params[:membership]
+          projects = projects.merge(current_user.authorized_projects)
+        end
+
+        if params[:owned]
+          projects = projects.merge(current_user.owned_projects)
+        end
+
+        if params[:starred]
+          projects = projects.merge(current_user.starred_projects)
+        end
+
+        if params[:search].present?
+          projects = projects.search(params[:search])
+        end
+
+        if params[:visibility].present?
+          projects = projects.where(visibility_level: Gitlab::VisibilityLevel.level_value(params[:visibility]))
+        end
+
+        unless params[:archived].nil?
+          projects = projects.where(archived: to_boolean(params[:archived]))
+        end
+
+        projects.reorder(params[:order_by] => params[:sort])
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/issues.rb b/lib/api/v3/issues.rb
new file mode 100644
index 00000000000..cb371fdbab8
--- /dev/null
+++ b/lib/api/v3/issues.rb
@@ -0,0 +1,234 @@
+module API
+  module V3
+    class Issues < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+
+      helpers do
+        def find_issues(args = {})
+          args = params.merge(args)
+          args = convert_parameters_from_legacy_format(args)
+
+          args.delete(:id)
+          args[:milestone_title] = args.delete(:milestone)
+
+          match_all_labels = args.delete(:match_all_labels)
+          labels = args.delete(:labels)
+          args[:label_name] = labels if match_all_labels
+
+          # IssuesFinder expects iids
+          args[:iids] = args.delete(:iid) if args.key?(:iid)
+
+          issues = IssuesFinder.new(current_user, args).execute.inc_notes_with_associations
+
+          if !match_all_labels && labels.present?
+            issues = issues.includes(:labels).where('labels.title' => labels.split(','))
+          end
+
+          issues.reorder(args[:order_by] => args[:sort])
+        end
+
+        params :issues_params do
+          optional :labels, type: String, desc: 'Comma-separated list of label names'
+          optional :milestone, type: String, desc: 'Milestone title'
+          optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at',
+                              desc: 'Return issues ordered by `created_at` or `updated_at` fields.'
+          optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                          desc: 'Return issues sorted in `asc` or `desc` order.'
+          optional :milestone, type: String, desc: 'Return issues for a specific milestone'
+          use :pagination
+        end
+
+        params :issue_params do
+          optional :description, type: String, desc: 'The description of an issue'
+          optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue'
+          optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue'
+          optional :labels, type: String, desc: 'Comma-separated list of label names'
+          optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY'
+          optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential'
+        end
+      end
+
+      resource :issues do
+        desc "Get currently authenticated user's issues" do
+          success ::API::V3::Entities::Issue
+        end
+        params do
+          optional :state, type: String, values: %w[opened closed all], default: 'all',
+                           desc: 'Return opened, closed, or all issues'
+          use :issues_params
+        end
+        get do
+          issues = find_issues(scope: 'authored')
+
+          present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user
+        end
+      end
+
+      params do
+        requires :id, type: String, desc: 'The ID of a group'
+      end
+      resource :groups, requirements: { id: %r{[^/]+} } do
+        desc 'Get a list of group issues' do
+          success ::API::V3::Entities::Issue
+        end
+        params do
+          optional :state, type: String, values: %w[opened closed all], default: 'all',
+                           desc: 'Return opened, closed, or all issues'
+          use :issues_params
+        end
+        get ":id/issues" do
+          group = find_group!(params[:id])
+
+          issues = find_issues(group_id: group.id, match_all_labels: true)
+
+          present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user
+        end
+      end
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        include TimeTrackingEndpoints
+
+        desc 'Get a list of project issues' do
+          detail 'iid filter is deprecated have been removed on V4'
+          success ::API::V3::Entities::Issue
+        end
+        params do
+          optional :state, type: String, values: %w[opened closed all], default: 'all',
+                           desc: 'Return opened, closed, or all issues'
+          optional :iid, type: Integer, desc: 'Return the issue having the given `iid`'
+          use :issues_params
+        end
+        get ":id/issues" do
+          project = find_project!(params[:id])
+
+          issues = find_issues(project_id: project.id)
+
+          present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project
+        end
+
+        desc 'Get a single project issue' do
+          success ::API::V3::Entities::Issue
+        end
+        params do
+          requires :issue_id, type: Integer, desc: 'The ID of a project issue'
+        end
+        get ":id/issues/:issue_id" do
+          issue = find_project_issue(params[:issue_id])
+          present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project
+        end
+
+        desc 'Create a new project issue' do
+          success ::API::V3::Entities::Issue
+        end
+        params do
+          requires :title, type: String, desc: 'The title of an issue'
+          optional :created_at, type: DateTime,
+                                desc: 'Date time when the issue was created. Available only for admins and project owners.'
+          optional :merge_request_for_resolving_discussions, type: Integer,
+                                                             desc: 'The IID of a merge request for which to resolve discussions'
+          use :issue_params
+        end
+        post ':id/issues' do
+          # Setting created_at time only allowed for admins and project owners
+          unless current_user.admin? || user_project.owner == current_user
+            params.delete(:created_at)
+          end
+
+          issue_params = declared_params(include_missing: false)
+          issue_params = issue_params.merge(merge_request_to_resolve_discussions_of: issue_params.delete(:merge_request_for_resolving_discussions))
+          issue_params = convert_parameters_from_legacy_format(issue_params)
+
+          issue = ::Issues::CreateService.new(user_project,
+                                              current_user,
+                                              issue_params.merge(request: request, api: true)).execute
+          render_spam_error! if issue.spam?
+
+          if issue.valid?
+            present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project
+          else
+            render_validation_error!(issue)
+          end
+        end
+
+        desc 'Update an existing issue' do
+          success ::API::V3::Entities::Issue
+        end
+        params do
+          requires :issue_id, type: Integer, desc: 'The ID of a project issue'
+          optional :title, type: String, desc: 'The title of an issue'
+          optional :updated_at, type: DateTime,
+                                desc: 'Date time when the issue was updated. Available only for admins and project owners.'
+          optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue'
+          use :issue_params
+          at_least_one_of :title, :description, :assignee_id, :milestone_id,
+                          :labels, :created_at, :due_date, :confidential, :state_event
+        end
+        put ':id/issues/:issue_id' do
+          issue = user_project.issues.find(params.delete(:issue_id))
+          authorize! :update_issue, issue
+
+          # Setting created_at time only allowed for admins and project owners
+          unless current_user.admin? || user_project.owner == current_user
+            params.delete(:updated_at)
+          end
+
+          update_params = declared_params(include_missing: false).merge(request: request, api: true)
+          update_params = convert_parameters_from_legacy_format(update_params)
+
+          issue = ::Issues::UpdateService.new(user_project,
+                                              current_user,
+                                              update_params).execute(issue)
+
+          render_spam_error! if issue.spam?
+
+          if issue.valid?
+            present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project
+          else
+            render_validation_error!(issue)
+          end
+        end
+
+        desc 'Move an existing issue' do
+          success ::API::V3::Entities::Issue
+        end
+        params do
+          requires :issue_id, type: Integer, desc: 'The ID of a project issue'
+          requires :to_project_id, type: Integer, desc: 'The ID of the new project'
+        end
+        post ':id/issues/:issue_id/move' do
+          issue = user_project.issues.find_by(id: params[:issue_id])
+          not_found!('Issue') unless issue
+
+          new_project = Project.find_by(id: params[:to_project_id])
+          not_found!('Project') unless new_project
+
+          begin
+            issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project)
+            present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project
+          rescue ::Issues::MoveService::MoveError => error
+            render_api_error!(error.message, 400)
+          end
+        end
+
+        desc 'Delete a project issue'
+        params do
+          requires :issue_id, type: Integer, desc: 'The ID of a project issue'
+        end
+        delete ":id/issues/:issue_id" do
+          issue = user_project.issues.find_by(id: params[:issue_id])
+          not_found!('Issue') unless issue
+
+          authorize!(:destroy_issue, issue)
+
+          status(200)
+          issue.destroy
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/labels.rb b/lib/api/v3/labels.rb
new file mode 100644
index 00000000000..bd5eb2175e8
--- /dev/null
+++ b/lib/api/v3/labels.rb
@@ -0,0 +1,34 @@
+module API
+  module V3
+    class Labels < Grape::API
+      before { authenticate! }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get all labels of the project' do
+          success ::API::Entities::Label
+        end
+        get ':id/labels' do
+          present available_labels, with: ::API::Entities::Label, current_user: current_user, project: user_project
+        end
+
+        desc 'Delete an existing label' do
+          success ::API::Entities::Label
+        end
+        params do
+          requires :name, type: String, desc: 'The name of the label to be deleted'
+        end
+        delete ':id/labels' do
+          authorize! :admin_label, user_project
+
+          label = user_project.labels.find_by(title: params[:name])
+          not_found!('Label') unless label
+
+          present label.destroy, with: ::API::Entities::Label, current_user: current_user, project: user_project
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/members.rb b/lib/api/v3/members.rb
new file mode 100644
index 00000000000..684860b553e
--- /dev/null
+++ b/lib/api/v3/members.rb
@@ -0,0 +1,134 @@
+module API
+  module V3
+    class Members < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+
+      helpers ::API::Helpers::MembersHelpers
+
+      %w[group project].each do |source_type|
+        params do
+          requires :id, type: String, desc: "The #{source_type} ID"
+        end
+        resource source_type.pluralize, requirements: { id: %r{[^/]+} } do
+          desc 'Gets a list of group or project members viewable by the authenticated user.' do
+            success ::API::Entities::Member
+          end
+          params do
+            optional :query, type: String, desc: 'A query string to search for members'
+            use :pagination
+          end
+          get ":id/members" do
+            source = find_source(source_type, params[:id])
+
+            users = source.users
+            users = users.merge(User.search(params[:query])) if params[:query]
+
+            present paginate(users), with: ::API::Entities::Member, source: source
+          end
+
+          desc 'Gets a member of a group or project.' do
+            success ::API::Entities::Member
+          end
+          params do
+            requires :user_id, type: Integer, desc: 'The user ID of the member'
+          end
+          get ":id/members/:user_id" do
+            source = find_source(source_type, params[:id])
+
+            members = source.members
+            member = members.find_by!(user_id: params[:user_id])
+
+            present member.user, with: ::API::Entities::Member, member: member
+          end
+
+          desc 'Adds a member to a group or project.' do
+            success ::API::Entities::Member
+          end
+          params do
+            requires :user_id, type: Integer, desc: 'The user ID of the new member'
+            requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)'
+            optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
+          end
+          post ":id/members" do
+            source = find_source(source_type, params[:id])
+            authorize_admin_source!(source_type, source)
+
+            member = source.members.find_by(user_id: params[:user_id])
+
+            # We need this explicit check because `source.add_user` doesn't
+            # currently return the member created so it would return 201 even if
+            # the member already existed...
+            # The `source_type == 'group'` check is to ensure back-compatibility
+            # but 409 behavior should be used for both project and group members in 9.0!
+            conflict!('Member already exists') if source_type == 'group' && member
+
+            unless member
+              member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at])
+            end
+            if member.persisted? && member.valid?
+              present member.user, with: ::API::Entities::Member, member: member
+            else
+              # This is to ensure back-compatibility but 400 behavior should be used
+              # for all validation errors in 9.0!
+              render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level)
+              render_validation_error!(member)
+            end
+          end
+
+          desc 'Updates a member of a group or project.' do
+            success ::API::Entities::Member
+          end
+          params do
+            requires :user_id, type: Integer, desc: 'The user ID of the new member'
+            requires :access_level, type: Integer, desc: 'A valid access level'
+            optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
+          end
+          put ":id/members/:user_id" do
+            source = find_source(source_type, params.delete(:id))
+            authorize_admin_source!(source_type, source)
+
+            member = source.members.find_by!(user_id: params.delete(:user_id))
+
+            if member.update_attributes(declared_params(include_missing: false))
+              present member.user, with: ::API::Entities::Member, member: member
+            else
+              # This is to ensure back-compatibility but 400 behavior should be used
+              # for all validation errors in 9.0!
+              render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level)
+              render_validation_error!(member)
+            end
+          end
+
+          desc 'Removes a user from a group or project.'
+          params do
+            requires :user_id, type: Integer, desc: 'The user ID of the member'
+          end
+          delete ":id/members/:user_id" do
+            source = find_source(source_type, params[:id])
+
+            # This is to ensure back-compatibility but find_by! should be used
+            # in that casse in 9.0!
+            member = source.members.find_by(user_id: params[:user_id])
+
+            # This is to ensure back-compatibility but this should be removed in
+            # favor of find_by! in 9.0!
+            not_found!("Member: user_id:#{params[:user_id]}") if source_type == 'group' && member.nil?
+
+            # This is to ensure back-compatibility but 204 behavior should be used
+            # for all DELETE endpoints in 9.0!
+            if member.nil?
+              status(200  )
+              { message: "Access revoked", id: params[:user_id].to_i }
+            else
+              ::Members::DestroyService.new(source, current_user, declared_params).execute
+
+              present member.user, with: ::API::Entities::Member, member: member
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/merge_request_diffs.rb b/lib/api/v3/merge_request_diffs.rb
new file mode 100644
index 00000000000..35f462e907b
--- /dev/null
+++ b/lib/api/v3/merge_request_diffs.rb
@@ -0,0 +1,44 @@
+module API
+  module V3
+    # MergeRequestDiff API
+    class MergeRequestDiffs < Grape::API
+      before { authenticate! }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get a list of merge request diff versions' do
+          detail 'This feature was introduced in GitLab 8.12.'
+          success ::API::Entities::MergeRequestDiff
+        end
+
+        params do
+          requires :merge_request_id, type: Integer, desc: 'The ID of a merge request'
+        end
+
+        get ":id/merge_requests/:merge_request_id/versions" do
+          merge_request = find_merge_request_with_access(params[:merge_request_id])
+
+          present merge_request.merge_request_diffs, with: ::API::Entities::MergeRequestDiff
+        end
+
+        desc 'Get a single merge request diff version' do
+          detail 'This feature was introduced in GitLab 8.12.'
+          success ::API::Entities::MergeRequestDiffFull
+        end
+
+        params do
+          requires :merge_request_id, type: Integer, desc: 'The ID of a merge request'
+          requires :version_id, type: Integer, desc: 'The ID of a merge request diff version'
+        end
+
+        get ":id/merge_requests/:merge_request_id/versions/:version_id" do
+          merge_request = find_merge_request_with_access(params[:merge_request_id])
+
+          present merge_request.merge_request_diffs.find(params[:version_id]), with: ::API::Entities::MergeRequestDiffFull
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb
new file mode 100644
index 00000000000..b6b7254ae29
--- /dev/null
+++ b/lib/api/v3/merge_requests.rb
@@ -0,0 +1,292 @@
+module API
+  module V3
+    class MergeRequests < Grape::API
+      include PaginationParams
+
+      DEPRECATION_MESSAGE = 'This endpoint is deprecated and has been removed on V4'.freeze
+
+      before { authenticate! }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        include TimeTrackingEndpoints
+
+        helpers do
+          def handle_merge_request_errors!(errors)
+            if errors[:project_access].any?
+              error!(errors[:project_access], 422)
+            elsif errors[:branch_conflict].any?
+              error!(errors[:branch_conflict], 422)
+            elsif errors[:validate_fork].any?
+              error!(errors[:validate_fork], 422)
+            elsif errors[:validate_branches].any?
+              conflict!(errors[:validate_branches])
+            elsif errors[:base].any?
+              error!(errors[:base], 422)
+            end
+
+            render_api_error!(errors, 400)
+          end
+
+          def issue_entity(project)
+            if project.has_external_issue_tracker?
+              ::API::Entities::ExternalIssue
+            else
+              ::API::V3::Entities::Issue
+            end
+          end
+
+          params :optional_params do
+            optional :description, type: String, desc: 'The description of the merge request'
+            optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request'
+            optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request'
+            optional :labels, type: String, desc: 'Comma-separated list of label names'
+            optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging'
+          end
+        end
+
+        desc 'List merge requests' do
+          detail 'iid filter is deprecated have been removed on V4'
+          success ::API::V3::Entities::MergeRequest
+        end
+        params do
+          optional :state, type: String, values: %w[opened closed merged all], default: 'all',
+                           desc: 'Return opened, closed, merged, or all merge requests'
+          optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at',
+                              desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.'
+          optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                          desc: 'Return merge requests sorted in `asc` or `desc` order.'
+          optional :iid, type: Array[Integer], desc: 'The IID of the merge requests'
+          use :pagination
+        end
+        get ":id/merge_requests" do
+          authorize! :read_merge_request, user_project
+
+          merge_requests = user_project.merge_requests.inc_notes_with_associations
+          merge_requests = filter_by_iid(merge_requests, params[:iid]) if params[:iid].present?
+
+          merge_requests =
+            case params[:state]
+            when 'opened' then merge_requests.opened
+            when 'closed' then merge_requests.closed
+            when 'merged' then merge_requests.merged
+            else merge_requests
+            end
+
+          merge_requests = merge_requests.reorder(params[:order_by] => params[:sort])
+          present paginate(merge_requests), with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project
+        end
+
+        desc 'Create a merge request' do
+          success ::API::V3::Entities::MergeRequest
+        end
+        params do
+          requires :title, type: String, desc: 'The title of the merge request'
+          requires :source_branch, type: String, desc: 'The source branch'
+          requires :target_branch, type: String, desc: 'The target branch'
+          optional :target_project_id, type: Integer,
+                                       desc: 'The target project of the merge request defaults to the :id of the project'
+          use :optional_params
+        end
+        post ":id/merge_requests" do
+          authorize! :create_merge_request, user_project
+
+          mr_params = declared_params(include_missing: false)
+          mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present?
+
+          merge_request = ::MergeRequests::CreateService.new(user_project, current_user, mr_params).execute
+
+          if merge_request.valid?
+            present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project
+          else
+            handle_merge_request_errors! merge_request.errors
+          end
+        end
+
+        desc 'Delete a merge request'
+        params do
+          requires :merge_request_id, type: Integer, desc: 'The ID of a merge request'
+        end
+        delete ":id/merge_requests/:merge_request_id" do
+          merge_request = find_project_merge_request(params[:merge_request_id])
+
+          authorize!(:destroy_merge_request, merge_request)
+
+          status(200)
+          merge_request.destroy
+        end
+
+        params do
+          requires :merge_request_id, type: Integer, desc: 'The ID of a merge request'
+        end
+        { ":id/merge_request/:merge_request_id" => :deprecated, ":id/merge_requests/:merge_request_id" => :ok }.each do |path, status|
+          desc 'Get a single merge request' do
+            if status == :deprecated
+              detail DEPRECATION_MESSAGE
+            end
+            success ::API::V3::Entities::MergeRequest
+          end
+          get path do
+            merge_request = find_merge_request_with_access(params[:merge_request_id])
+
+            present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project
+          end
+
+          desc 'Get the commits of a merge request' do
+            success ::API::Entities::RepoCommit
+          end
+          get "#{path}/commits" do
+            merge_request = find_merge_request_with_access(params[:merge_request_id])
+
+            present merge_request.commits, with: ::API::Entities::RepoCommit
+          end
+
+          desc 'Show the merge request changes' do
+            success ::API::Entities::MergeRequestChanges
+          end
+          get "#{path}/changes" do
+            merge_request = find_merge_request_with_access(params[:merge_request_id])
+
+            present merge_request, with: ::API::Entities::MergeRequestChanges, current_user: current_user
+          end
+
+          desc 'Update a merge request' do
+            success ::API::V3::Entities::MergeRequest
+          end
+          params do
+            optional :title, type: String, allow_blank: false, desc: 'The title of the merge request'
+            optional :target_branch, type: String, allow_blank: false, desc: 'The target branch'
+            optional :state_event, type: String, values: %w[close reopen merge],
+                                   desc: 'Status of the merge request'
+            use :optional_params
+            at_least_one_of :title, :target_branch, :description, :assignee_id,
+                            :milestone_id, :labels, :state_event,
+                            :remove_source_branch
+          end
+          put path do
+            merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request)
+
+            mr_params = declared_params(include_missing: false)
+            mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present?
+
+            merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request)
+
+            if merge_request.valid?
+              present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project
+            else
+              handle_merge_request_errors! merge_request.errors
+            end
+          end
+
+          desc 'Merge a merge request' do
+            success ::API::V3::Entities::MergeRequest
+          end
+          params do
+            optional :merge_commit_message, type: String, desc: 'Custom merge commit message'
+            optional :should_remove_source_branch, type: Boolean,
+                                                   desc: 'When true, the source branch will be deleted if possible'
+            optional :merge_when_build_succeeds, type: Boolean,
+                                                 desc: 'When true, this merge request will be merged when the build succeeds'
+            optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch'
+          end
+          put "#{path}/merge" do
+            merge_request = find_project_merge_request(params[:merge_request_id])
+
+            # Merge request can not be merged
+            # because user dont have permissions to push into target branch
+            unauthorized! unless merge_request.can_be_merged_by?(current_user)
+
+            not_allowed! unless merge_request.mergeable_state?
+
+            render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?
+
+            if params[:sha] && merge_request.diff_head_sha != params[:sha]
+              render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409)
+            end
+
+            merge_params = {
+              commit_message: params[:merge_commit_message],
+              should_remove_source_branch: params[:should_remove_source_branch]
+            }
+
+            if params[:merge_when_build_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active?
+              ::MergeRequests::MergeWhenPipelineSucceedsService
+                .new(merge_request.target_project, current_user, merge_params)
+                .execute(merge_request)
+            else
+              ::MergeRequests::MergeService
+                .new(merge_request.target_project, current_user, merge_params)
+                .execute(merge_request)
+            end
+
+            present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project
+          end
+
+          desc 'Cancel merge if "Merge When Build succeeds" is enabled' do
+            success ::API::V3::Entities::MergeRequest
+          end
+          post "#{path}/cancel_merge_when_build_succeeds" do
+            merge_request = find_project_merge_request(params[:merge_request_id])
+
+            unauthorized! unless merge_request.can_cancel_merge_when_pipeline_succeeds?(current_user)
+
+            ::MergeRequest::MergeWhenPipelineSucceedsService
+              .new(merge_request.target_project, current_user)
+              .cancel(merge_request)
+          end
+
+          desc 'Get the comments of a merge request' do
+            detail 'Duplicate. DEPRECATED and HAS BEEN REMOVED in V4'
+            success ::API::Entities::MRNote
+          end
+          params do
+            use :pagination
+          end
+          get "#{path}/comments" do
+            merge_request = find_merge_request_with_access(params[:merge_request_id])
+            present paginate(merge_request.notes.fresh), with: ::API::Entities::MRNote
+          end
+
+          desc 'Post a comment to a merge request' do
+            detail 'Duplicate. DEPRECATED and HAS BEEN REMOVED in V4'
+            success ::API::Entities::MRNote
+          end
+          params do
+            requires :note, type: String, desc: 'The text of the comment'
+          end
+          post "#{path}/comments" do
+            merge_request = find_merge_request_with_access(params[:merge_request_id], :create_note)
+
+            opts = {
+              note: params[:note],
+              noteable_type: 'MergeRequest',
+              noteable_id: merge_request.id
+            }
+
+            note = ::Notes::CreateService.new(user_project, current_user, opts).execute
+
+            if note.save
+              present note, with: ::API::Entities::MRNote
+            else
+              render_api_error!("Failed to save note #{note.errors.messages}", 400)
+            end
+          end
+
+          desc 'List issues that will be closed on merge' do
+            success ::API::Entities::MRNote
+          end
+          params do
+            use :pagination
+          end
+          get "#{path}/closes_issues" do
+            merge_request = find_merge_request_with_access(params[:merge_request_id])
+            issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user))
+            present paginate(issues), with: issue_entity(user_project), current_user: current_user
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/milestones.rb b/lib/api/v3/milestones.rb
new file mode 100644
index 00000000000..4c7061d4939
--- /dev/null
+++ b/lib/api/v3/milestones.rb
@@ -0,0 +1,64 @@
+module API
+  module V3
+    class Milestones < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+
+      helpers do
+        def filter_milestones_state(milestones, state)
+          case state
+          when 'active' then milestones.active
+          when 'closed' then milestones.closed
+          else milestones
+          end
+        end
+      end
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get a list of project milestones' do
+          success ::API::Entities::Milestone
+        end
+        params do
+          optional :state, type: String, values: %w[active closed all], default: 'all',
+                           desc: 'Return "active", "closed", or "all" milestones'
+          optional :iid, type: Array[Integer], desc: 'The IID of the milestone'
+          use :pagination
+        end
+        get ":id/milestones" do
+          authorize! :read_milestone, user_project
+
+          milestones = user_project.milestones
+          milestones = filter_milestones_state(milestones, params[:state])
+          milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present?
+
+          present paginate(milestones), with: ::API::Entities::Milestone
+        end
+
+        desc 'Get all issues for a single project milestone' do
+          success ::API::V3::Entities::Issue
+        end
+        params do
+          requires :milestone_id, type: Integer, desc: 'The ID of a project milestone'
+          use :pagination
+        end
+        get ':id/milestones/:milestone_id/issues' do
+          authorize! :read_milestone, user_project
+
+          milestone = user_project.milestones.find(params[:milestone_id])
+
+          finder_params = {
+            project_id: user_project.id,
+            milestone_title: milestone.title
+          }
+
+          issues = IssuesFinder.new(current_user, finder_params).execute
+          present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/notes.rb b/lib/api/v3/notes.rb
new file mode 100644
index 00000000000..23fe95e42e4
--- /dev/null
+++ b/lib/api/v3/notes.rb
@@ -0,0 +1,148 @@
+module API
+  module V3
+    class Notes < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+
+      NOTEABLE_TYPES = [Issue, MergeRequest, Snippet].freeze
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        NOTEABLE_TYPES.each do |noteable_type|
+          noteables_str = noteable_type.to_s.underscore.pluralize
+
+          desc 'Get a list of project +noteable+ notes' do
+            success ::API::V3::Entities::Note
+          end
+          params do
+            requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+            use :pagination
+          end
+          get ":id/#{noteables_str}/:noteable_id/notes" do
+            noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id])
+
+            if can?(current_user, noteable_read_ability_name(noteable), noteable)
+              # We exclude notes that are cross-references and that cannot be viewed
+              # by the current user. By doing this exclusion at this level and not
+              # at the DB query level (which we cannot in that case), the current
+              # page can have less elements than :per_page even if
+              # there's more than one page.
+              notes =
+                # paginate() only works with a relation. This could lead to a
+                # mismatch between the pagination headers info and the actual notes
+                # array returned, but this is really a edge-case.
+                paginate(noteable.notes)
+                .reject { |n| n.cross_reference_not_visible_for?(current_user) }
+              present notes, with: ::API::V3::Entities::Note
+            else
+              not_found!("Notes")
+            end
+          end
+
+          desc 'Get a single +noteable+ note' do
+            success ::API::V3::Entities::Note
+          end
+          params do
+            requires :note_id, type: Integer, desc: 'The ID of a note'
+            requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+          end
+          get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do
+            noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id])
+            note = noteable.notes.find(params[:note_id])
+            can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user)
+
+            if can_read_note
+              present note, with: ::API::V3::Entities::Note
+            else
+              not_found!("Note")
+            end
+          end
+
+          desc 'Create a new +noteable+ note' do
+            success ::API::V3::Entities::Note
+          end
+          params do
+            requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+            requires :body, type: String, desc: 'The content of a note'
+            optional :created_at, type: String, desc: 'The creation date of the note'
+          end
+          post ":id/#{noteables_str}/:noteable_id/notes" do
+            opts = {
+              note: params[:body],
+              noteable_type: noteables_str.classify,
+              noteable_id: params[:noteable_id]
+            }
+
+            noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id])
+
+            if can?(current_user, noteable_read_ability_name(noteable), noteable)
+              if params[:created_at] && (current_user.admin? || user_project.owner == current_user)
+                opts[:created_at] = params[:created_at]
+              end
+
+              note = ::Notes::CreateService.new(user_project, current_user, opts).execute
+              if note.valid?
+                present note, with: ::API::V3::Entities.const_get(note.class.name)
+              else
+                not_found!("Note #{note.errors.messages}")
+              end
+            else
+              not_found!("Note")
+            end
+          end
+
+          desc 'Update an existing +noteable+ note' do
+            success ::API::V3::Entities::Note
+          end
+          params do
+            requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+            requires :note_id, type: Integer, desc: 'The ID of a note'
+            requires :body, type: String, desc: 'The content of a note'
+          end
+          put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do
+            note = user_project.notes.find(params[:note_id])
+
+            authorize! :admin_note, note
+
+            opts = {
+              note: params[:body]
+            }
+
+            note = ::Notes::UpdateService.new(user_project, current_user, opts).execute(note)
+
+            if note.valid?
+              present note, with: ::API::V3::Entities::Note
+            else
+              render_api_error!("Failed to save note #{note.errors.messages}", 400)
+            end
+          end
+
+          desc 'Delete a +noteable+ note' do
+            success ::API::V3::Entities::Note
+          end
+          params do
+            requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
+            requires :note_id, type: Integer, desc: 'The ID of a note'
+          end
+          delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do
+            note = user_project.notes.find(params[:note_id])
+            authorize! :admin_note, note
+
+            ::Notes::DestroyService.new(user_project, current_user).execute(note)
+
+            present note, with: ::API::V3::Entities::Note
+          end
+        end
+      end
+
+      helpers do
+        def noteable_read_ability_name(noteable)
+          "read_#{noteable.class.to_s.underscore}".to_sym
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb
new file mode 100644
index 00000000000..c48cbd2b765
--- /dev/null
+++ b/lib/api/v3/pipelines.rb
@@ -0,0 +1,36 @@
+module API
+  module V3
+    class Pipelines < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+
+      params do
+        requires :id, type: String, desc: 'The project ID'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get all Pipelines of the project' do
+          detail 'This feature was introduced in GitLab 8.11.'
+          success ::API::Entities::Pipeline
+        end
+        params do
+          use :pagination
+          optional :scope,    type: String, values: %w(running branches tags),
+                              desc: 'Either running, branches, or tags'
+        end
+        get ':id/pipelines' do
+          authorize! :read_pipeline, user_project
+
+          pipelines = PipelinesFinder.new(user_project, scope: params[:scope]).execute
+          present paginate(pipelines), with: ::API::Entities::Pipeline
+        end
+      end
+
+      helpers do
+        def pipeline
+          @pipeline ||= user_project.pipelines.find(params[:pipeline_id])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/project_hooks.rb b/lib/api/v3/project_hooks.rb
new file mode 100644
index 00000000000..94614bfc8b6
--- /dev/null
+++ b/lib/api/v3/project_hooks.rb
@@ -0,0 +1,106 @@
+module API
+  module V3
+    class ProjectHooks < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+      before { authorize_admin_project }
+
+      helpers do
+        params :project_hook_properties do
+          requires :url, type: String, desc: "The URL to send the request to"
+          optional :push_events, type: Boolean, desc: "Trigger hook on push events"
+          optional :issues_events, type: Boolean, desc: "Trigger hook on issues events"
+          optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events"
+          optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events"
+          optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events"
+          optional :build_events, type: Boolean, desc: "Trigger hook on build events"
+          optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events"
+          optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events"
+          optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook"
+          optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response"
+        end
+      end
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get project hooks' do
+          success ::API::V3::Entities::ProjectHook
+        end
+        params do
+          use :pagination
+        end
+        get ":id/hooks" do
+          hooks = paginate user_project.hooks
+
+          present hooks, with: ::API::V3::Entities::ProjectHook
+        end
+
+        desc 'Get a project hook' do
+          success ::API::V3::Entities::ProjectHook
+        end
+        params do
+          requires :hook_id, type: Integer, desc: 'The ID of a project hook'
+        end
+        get ":id/hooks/:hook_id" do
+          hook = user_project.hooks.find(params[:hook_id])
+          present hook, with: ::API::V3::Entities::ProjectHook
+        end
+
+        desc 'Add hook to project' do
+          success ::API::V3::Entities::ProjectHook
+        end
+        params do
+          use :project_hook_properties
+        end
+        post ":id/hooks" do
+          hook = user_project.hooks.new(declared_params(include_missing: false))
+
+          if hook.save
+            present hook, with: ::API::V3::Entities::ProjectHook
+          else
+            error!("Invalid url given", 422) if hook.errors[:url].present?
+
+            not_found!("Project hook #{hook.errors.messages}")
+          end
+        end
+
+        desc 'Update an existing project hook' do
+          success ::API::V3::Entities::ProjectHook
+        end
+        params do
+          requires :hook_id, type: Integer, desc: "The ID of the hook to update"
+          use :project_hook_properties
+        end
+        put ":id/hooks/:hook_id" do
+          hook = user_project.hooks.find(params.delete(:hook_id))
+
+          if hook.update_attributes(declared_params(include_missing: false))
+            present hook, with: ::API::V3::Entities::ProjectHook
+          else
+            error!("Invalid url given", 422) if hook.errors[:url].present?
+
+            not_found!("Project hook #{hook.errors.messages}")
+          end
+        end
+
+        desc 'Deletes project hook' do
+          success ::API::V3::Entities::ProjectHook
+        end
+        params do
+          requires :hook_id, type: Integer, desc: 'The ID of the hook to delete'
+        end
+        delete ":id/hooks/:hook_id" do
+          begin
+            present user_project.hooks.destroy(params[:hook_id]), with: ::API::V3::Entities::ProjectHook
+          rescue
+            # ProjectHook can raise Error if hook_id not found
+            not_found!("Error deleting hook #{params[:hook_id]}")
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/project_snippets.rb b/lib/api/v3/project_snippets.rb
new file mode 100644
index 00000000000..c41fee32610
--- /dev/null
+++ b/lib/api/v3/project_snippets.rb
@@ -0,0 +1,142 @@
+module API
+  module V3
+    class ProjectSnippets < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        helpers do
+          def handle_project_member_errors(errors)
+            if errors[:project_access].any?
+              error!(errors[:project_access], 422)
+            end
+            not_found!
+          end
+
+          def snippets_for_current_user
+            SnippetsFinder.new(current_user, project: user_project).execute
+          end
+        end
+
+        desc 'Get all project snippets' do
+          success ::API::V3::Entities::ProjectSnippet
+        end
+        params do
+          use :pagination
+        end
+        get ":id/snippets" do
+          present paginate(snippets_for_current_user), with: ::API::V3::Entities::ProjectSnippet
+        end
+
+        desc 'Get a single project snippet' do
+          success ::API::V3::Entities::ProjectSnippet
+        end
+        params do
+          requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
+        end
+        get ":id/snippets/:snippet_id" do
+          snippet = snippets_for_current_user.find(params[:snippet_id])
+          present snippet, with: ::API::V3::Entities::ProjectSnippet
+        end
+
+        desc 'Create a new project snippet' do
+          success ::API::V3::Entities::ProjectSnippet
+        end
+        params do
+          requires :title, type: String, desc: 'The title of the snippet'
+          requires :file_name, type: String, desc: 'The file name of the snippet'
+          requires :code, type: String, desc: 'The content of the snippet'
+          requires :visibility_level, type: Integer,
+                                      values: [Gitlab::VisibilityLevel::PRIVATE,
+                                               Gitlab::VisibilityLevel::INTERNAL,
+                                               Gitlab::VisibilityLevel::PUBLIC],
+                                      desc: 'The visibility level of the snippet'
+        end
+        post ":id/snippets" do
+          authorize! :create_project_snippet, user_project
+          snippet_params = declared_params.merge(request: request, api: true)
+          snippet_params[:content] = snippet_params.delete(:code)
+
+          snippet = CreateSnippetService.new(user_project, current_user, snippet_params).execute
+
+          render_spam_error! if snippet.spam?
+
+          if snippet.persisted?
+            present snippet, with: ::API::V3::Entities::ProjectSnippet
+          else
+            render_validation_error!(snippet)
+          end
+        end
+
+        desc 'Update an existing project snippet' do
+          success ::API::V3::Entities::ProjectSnippet
+        end
+        params do
+          requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
+          optional :title, type: String, desc: 'The title of the snippet'
+          optional :file_name, type: String, desc: 'The file name of the snippet'
+          optional :code, type: String, desc: 'The content of the snippet'
+          optional :visibility_level, type: Integer,
+                                      values: [Gitlab::VisibilityLevel::PRIVATE,
+                                               Gitlab::VisibilityLevel::INTERNAL,
+                                               Gitlab::VisibilityLevel::PUBLIC],
+                                      desc: 'The visibility level of the snippet'
+          at_least_one_of :title, :file_name, :code, :visibility_level
+        end
+        put ":id/snippets/:snippet_id" do
+          snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id))
+          not_found!('Snippet') unless snippet
+
+          authorize! :update_project_snippet, snippet
+
+          snippet_params = declared_params(include_missing: false)
+            .merge(request: request, api: true)
+
+          snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present?
+
+          UpdateSnippetService.new(user_project, current_user, snippet,
+                                   snippet_params).execute
+
+          render_spam_error! if snippet.spam?
+
+          if snippet.valid?
+            present snippet, with: ::API::V3::Entities::ProjectSnippet
+          else
+            render_validation_error!(snippet)
+          end
+        end
+
+        desc 'Delete a project snippet'
+        params do
+          requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
+        end
+        delete ":id/snippets/:snippet_id" do
+          snippet = snippets_for_current_user.find_by(id: params[:snippet_id])
+          not_found!('Snippet') unless snippet
+
+          authorize! :admin_project_snippet, snippet
+          snippet.destroy
+
+          status(200)
+        end
+
+        desc 'Get a raw project snippet'
+        params do
+          requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
+        end
+        get ":id/snippets/:snippet_id/raw" do
+          snippet = snippets_for_current_user.find_by(id: params[:snippet_id])
+          not_found!('Snippet') unless snippet
+
+          env['api.format'] = :txt
+          content_type 'text/plain'
+          present snippet.content
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb
new file mode 100644
index 00000000000..eb090453b48
--- /dev/null
+++ b/lib/api/v3/projects.rb
@@ -0,0 +1,474 @@
+module API
+  module V3
+    class Projects < Grape::API
+      include PaginationParams
+
+      before { authenticate_non_get! }
+
+      after_validation do
+        set_only_allow_merge_if_pipeline_succeeds!
+      end
+
+      helpers do
+        params :optional_params do
+          optional :description, type: String, desc: 'The description of the project'
+          optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled'
+          optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled'
+          optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled'
+          optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled'
+          optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled'
+          optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project'
+          optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project'
+          optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project'
+          optional :public, type: Boolean, desc: 'Create a public project. The same as visibility_level = 20.'
+          optional :visibility_level, type: Integer, values: [
+            Gitlab::VisibilityLevel::PRIVATE,
+            Gitlab::VisibilityLevel::INTERNAL,
+            Gitlab::VisibilityLevel::PUBLIC
+          ], desc: 'Create a public project. The same as visibility_level = 20.'
+          optional :public_builds, type: Boolean, desc: 'Perform public builds'
+          optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
+          optional :only_allow_merge_if_build_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed'
+          optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed'
+          optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved'
+        end
+
+        def map_public_to_visibility_level(attrs)
+          publik = attrs.delete(:public)
+          if !publik.nil? && !attrs[:visibility_level].present?
+            # Since setting the public attribute to private could mean either
+            # private or internal, use the more conservative option, private.
+            attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE
+          end
+          attrs
+        end
+
+        def set_only_allow_merge_if_pipeline_succeeds!
+          if params.key?(:only_allow_merge_if_build_succeeds)
+            params[:only_allow_merge_if_pipeline_succeeds] = params.delete(:only_allow_merge_if_build_succeeds)
+          end
+        end
+      end
+
+      resource :projects do
+        helpers do
+          params :collection_params do
+            use :sort_params
+            use :filter_params
+            use :pagination
+
+            optional :simple, type: Boolean, default: false,
+                              desc: 'Return only the ID, URL, name, and path of each project'
+          end
+
+          params :sort_params do
+            optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
+                                default: 'created_at', desc: 'Return projects ordered by field'
+            optional :sort, type: String, values: %w[asc desc], default: 'desc',
+                            desc: 'Return projects sorted in ascending and descending order'
+          end
+
+          params :filter_params do
+            optional :archived, type: Boolean, default: nil, desc: 'Limit by archived status'
+            optional :visibility, type: String, values: %w[public internal private],
+                                  desc: 'Limit by visibility'
+            optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
+          end
+
+          params :statistics_params do
+            optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
+          end
+
+          params :create_params do
+            optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
+            optional :import_url, type: String, desc: 'URL from which the project is imported'
+          end
+
+          def present_projects(projects, options = {})
+            options = options.reverse_merge(
+              with: ::API::V3::Entities::Project,
+              current_user: current_user,
+              simple: params[:simple]
+            )
+
+            projects = filter_projects(projects)
+            projects = projects.with_statistics if options[:statistics]
+            options[:with] = ::API::Entities::BasicProjectDetails if options[:simple]
+
+            present paginate(projects), options
+          end
+        end
+
+        desc 'Get a list of visible projects for authenticated user' do
+          success ::API::Entities::BasicProjectDetails
+        end
+        params do
+          use :collection_params
+        end
+        get '/visible' do
+          entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails
+          present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity
+        end
+
+        desc 'Get a projects list for authenticated user' do
+          success ::API::Entities::BasicProjectDetails
+        end
+        params do
+          use :collection_params
+        end
+        get do
+          authenticate!
+
+          present_projects current_user.authorized_projects,
+            with: ::API::V3::Entities::ProjectWithAccess
+        end
+
+        desc 'Get an owned projects list for authenticated user' do
+          success ::API::Entities::BasicProjectDetails
+        end
+        params do
+          use :collection_params
+          use :statistics_params
+        end
+        get '/owned' do
+          authenticate!
+
+          present_projects current_user.owned_projects,
+            with: ::API::V3::Entities::ProjectWithAccess,
+            statistics: params[:statistics]
+        end
+
+        desc 'Gets starred project for the authenticated user' do
+          success ::API::Entities::BasicProjectDetails
+        end
+        params do
+          use :collection_params
+        end
+        get '/starred' do
+          authenticate!
+
+          present_projects ProjectsFinder.new(current_user: current_user, params: { starred: true }).execute
+        end
+
+        desc 'Get all projects for admin user' do
+          success ::API::Entities::BasicProjectDetails
+        end
+        params do
+          use :collection_params
+          use :statistics_params
+        end
+        get '/all' do
+          authenticated_as_admin!
+
+          present_projects Project.all, with: ::API::V3::Entities::ProjectWithAccess, statistics: params[:statistics]
+        end
+
+        desc 'Search for projects the current user has access to' do
+          success ::API::V3::Entities::Project
+        end
+        params do
+          requires :query, type: String, desc: 'The project name to be searched'
+          use :sort_params
+          use :pagination
+        end
+        get "/search/:query", requirements: { query: /[^\/]+/ } do
+          search_service = Search::GlobalService.new(current_user, search: params[:query]).execute
+          projects = search_service.objects('projects', params[:page])
+          projects = projects.reorder(params[:order_by] => params[:sort])
+
+          present paginate(projects), with: ::API::V3::Entities::Project
+        end
+
+        desc 'Create new project' do
+          success ::API::V3::Entities::Project
+        end
+        params do
+          optional :name, type: String, desc: 'The name of the project'
+          optional :path, type: String, desc: 'The path of the repository'
+          at_least_one_of :name, :path
+          use :optional_params
+          use :create_params
+        end
+        post do
+          attrs = map_public_to_visibility_level(declared_params(include_missing: false))
+          project = ::Projects::CreateService.new(current_user, attrs).execute
+
+          if project.saved?
+            present project, with: ::API::V3::Entities::Project,
+                             user_can_admin_project: can?(current_user, :admin_project, project)
+          else
+            if project.errors[:limit_reached].present?
+              error!(project.errors[:limit_reached], 403)
+            end
+            render_validation_error!(project)
+          end
+        end
+
+        desc 'Create new project for a specified user. Only available to admin users.' do
+          success ::API::V3::Entities::Project
+        end
+        params do
+          requires :name, type: String, desc: 'The name of the project'
+          requires :user_id, type: Integer, desc: 'The ID of a user'
+          optional :default_branch, type: String, desc: 'The default branch of the project'
+          use :optional_params
+          use :create_params
+        end
+        post "user/:user_id" do
+          authenticated_as_admin!
+          user = User.find_by(id: params.delete(:user_id))
+          not_found!('User') unless user
+
+          attrs = map_public_to_visibility_level(declared_params(include_missing: false))
+          project = ::Projects::CreateService.new(user, attrs).execute
+
+          if project.saved?
+            present project, with: ::API::V3::Entities::Project,
+                             user_can_admin_project: can?(current_user, :admin_project, project)
+          else
+            render_validation_error!(project)
+          end
+        end
+      end
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get a single project' do
+          success ::API::V3::Entities::ProjectWithAccess
+        end
+        get ":id" do
+          entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails
+          present user_project, with: entity, current_user: current_user,
+                                user_can_admin_project: can?(current_user, :admin_project, user_project)
+        end
+
+        desc 'Get events for a single project' do
+          success ::API::V3::Entities::Event
+        end
+        params do
+          use :pagination
+        end
+        get ":id/events" do
+          present paginate(user_project.events.recent), with: ::API::V3::Entities::Event
+        end
+
+        desc 'Fork new project for the current user or provided namespace.' do
+          success ::API::V3::Entities::Project
+        end
+        params do
+          optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into'
+        end
+        post 'fork/:id' do
+          fork_params = declared_params(include_missing: false)
+          namespace_id = fork_params[:namespace]
+
+          if namespace_id.present?
+            fork_params[:namespace] = if namespace_id =~ /^\d+$/
+                                        Namespace.find_by(id: namespace_id)
+                                      else
+                                        Namespace.find_by_path_or_name(namespace_id)
+                                      end
+
+            unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace])
+              not_found!('Target Namespace')
+            end
+          end
+
+          forked_project = ::Projects::ForkService.new(user_project, current_user, fork_params).execute
+
+          if forked_project.errors.any?
+            conflict!(forked_project.errors.messages)
+          else
+            present forked_project, with: ::API::V3::Entities::Project,
+                                    user_can_admin_project: can?(current_user, :admin_project, forked_project)
+          end
+        end
+
+        desc 'Update an existing project' do
+          success ::API::V3::Entities::Project
+        end
+        params do
+          optional :name, type: String, desc: 'The name of the project'
+          optional :default_branch, type: String, desc: 'The default branch of the project'
+          optional :path, type: String, desc: 'The path of the repository'
+          use :optional_params
+          at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled,
+            :wiki_enabled, :builds_enabled, :snippets_enabled,
+            :shared_runners_enabled, :container_registry_enabled,
+            :lfs_enabled, :public, :visibility_level, :public_builds,
+            :request_access_enabled, :only_allow_merge_if_build_succeeds,
+            :only_allow_merge_if_all_discussions_are_resolved, :path,
+            :default_branch
+        end
+        put ':id' do
+          authorize_admin_project
+          attrs = map_public_to_visibility_level(declared_params(include_missing: false))
+          authorize! :rename_project, user_project if attrs[:name].present?
+          authorize! :change_visibility_level, user_project if attrs[:visibility_level].present?
+
+          result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute
+
+          if result[:status] == :success
+            present user_project, with: ::API::V3::Entities::Project,
+                                  user_can_admin_project: can?(current_user, :admin_project, user_project)
+          else
+            render_validation_error!(user_project)
+          end
+        end
+
+        desc 'Archive a project' do
+          success ::API::V3::Entities::Project
+        end
+        post ':id/archive' do
+          authorize!(:archive_project, user_project)
+
+          user_project.archive!
+
+          present user_project, with: ::API::V3::Entities::Project
+        end
+
+        desc 'Unarchive a project' do
+          success ::API::V3::Entities::Project
+        end
+        post ':id/unarchive' do
+          authorize!(:archive_project, user_project)
+
+          user_project.unarchive!
+
+          present user_project, with: ::API::V3::Entities::Project
+        end
+
+        desc 'Star a project' do
+          success ::API::V3::Entities::Project
+        end
+        post ':id/star' do
+          if current_user.starred?(user_project)
+            not_modified!
+          else
+            current_user.toggle_star(user_project)
+            user_project.reload
+
+            present user_project, with: ::API::V3::Entities::Project
+          end
+        end
+
+        desc 'Unstar a project' do
+          success ::API::V3::Entities::Project
+        end
+        delete ':id/star' do
+          if current_user.starred?(user_project)
+            current_user.toggle_star(user_project)
+            user_project.reload
+
+            present user_project, with: ::API::V3::Entities::Project
+          else
+            not_modified!
+          end
+        end
+
+        desc 'Remove a project'
+        delete ":id" do
+          authorize! :remove_project, user_project
+
+          status(200)
+          ::Projects::DestroyService.new(user_project, current_user, {}).async_execute
+        end
+
+        desc 'Mark this project as forked from another'
+        params do
+          requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from'
+        end
+        post ":id/fork/:forked_from_id" do
+          authenticated_as_admin!
+
+          forked_from_project = find_project!(params[:forked_from_id])
+          not_found!("Source Project") unless forked_from_project
+
+          if user_project.forked_from_project.nil?
+            user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id)
+          else
+            render_api_error!("Project already forked", 409)
+          end
+        end
+
+        desc 'Remove a forked_from relationship'
+        delete ":id/fork" do
+          authorize! :remove_fork_project, user_project
+
+          if user_project.forked?
+            status(200)
+            user_project.forked_project_link.destroy
+          else
+            not_modified!
+          end
+        end
+
+        desc 'Share the project with a group' do
+          success ::API::Entities::ProjectGroupLink
+        end
+        params do
+          requires :group_id, type: Integer, desc: 'The ID of a group'
+          requires :group_access, type: Integer, values: Gitlab::Access.values, desc: 'The group access level'
+          optional :expires_at, type: Date, desc: 'Share expiration date'
+        end
+        post ":id/share" do
+          authorize! :admin_project, user_project
+          group = Group.find_by_id(params[:group_id])
+
+          unless group && can?(current_user, :read_group, group)
+            not_found!('Group')
+          end
+
+          unless user_project.allowed_to_share_with_group?
+            return render_api_error!("The project sharing with group is disabled", 400)
+          end
+
+          link = user_project.project_group_links.new(declared_params(include_missing: false))
+
+          if link.save
+            present link, with: ::API::Entities::ProjectGroupLink
+          else
+            render_api_error!(link.errors.full_messages.first, 409)
+          end
+        end
+
+        params do
+          requires :group_id, type: Integer, desc: 'The ID of the group'
+        end
+        delete ":id/share/:group_id" do
+          authorize! :admin_project, user_project
+
+          link = user_project.project_group_links.find_by(group_id: params[:group_id])
+          not_found!('Group Link') unless link
+
+          link.destroy
+          no_content!
+        end
+
+        desc 'Upload a file'
+        params do
+          requires :file, type: File, desc: 'The file to be uploaded'
+        end
+        post ":id/uploads" do
+          UploadService.new(user_project, params[:file]).execute
+        end
+
+        desc 'Get the users list of a project' do
+          success ::API::Entities::UserBasic
+        end
+        params do
+          optional :search, type: String, desc: 'Return list of users matching the search criteria'
+          use :pagination
+        end
+        get ':id/users' do
+          users = user_project.team.users
+          users = users.search(params[:search]) if params[:search].present?
+
+          present paginate(users), with: ::API::Entities::UserBasic
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb
new file mode 100644
index 00000000000..0eaa0de2eef
--- /dev/null
+++ b/lib/api/v3/repositories.rb
@@ -0,0 +1,109 @@
+require 'mime/types'
+
+module API
+  module V3
+    class Repositories < Grape::API
+      before { authorize! :download_code, user_project }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        helpers do
+          def handle_project_member_errors(errors)
+            if errors[:project_access].any?
+              error!(errors[:project_access], 422)
+            end
+            not_found!
+          end
+        end
+
+        desc 'Get a project repository tree' do
+          success ::API::Entities::RepoTreeObject
+        end
+        params do
+          optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used'
+          optional :path, type: String, desc: 'The path of the tree'
+          optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree'
+        end
+        get ':id/repository/tree' do
+          ref = params[:ref_name] || user_project.try(:default_branch) || 'master'
+          path = params[:path] || nil
+
+          commit = user_project.commit(ref)
+          not_found!('Tree') unless commit
+
+          tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive])
+
+          present tree.sorted_entries, with: ::API::Entities::RepoTreeObject
+        end
+
+        desc 'Get a raw file contents'
+        params do
+          requires :sha, type: String, desc: 'The commit, branch name, or tag name'
+          requires :filepath, type: String, desc: 'The path to the file to display'
+        end
+        get [":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob"] do
+          repo = user_project.repository
+          commit = repo.commit(params[:sha])
+          not_found! "Commit" unless commit
+          blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath])
+          not_found! "File" unless blob
+          send_git_blob repo, blob
+        end
+
+        desc 'Get a raw blob contents by blob sha'
+        params do
+          requires :sha, type: String, desc: 'The commit, branch name, or tag name'
+        end
+        get ':id/repository/raw_blobs/:sha' do
+          repo = user_project.repository
+          begin
+            blob = Gitlab::Git::Blob.raw(repo, params[:sha])
+          rescue
+            not_found! 'Blob'
+          end
+          not_found! 'Blob' unless blob
+          send_git_blob repo, blob
+        end
+
+        desc 'Get an archive of the repository'
+        params do
+          optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded'
+          optional :format, type: String, desc: 'The archive format'
+        end
+        get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do
+          begin
+            send_git_archive user_project.repository, ref: params[:sha], format: params[:format]
+          rescue
+            not_found!('File')
+          end
+        end
+
+        desc 'Compare two branches, tags, or commits' do
+          success ::API::Entities::Compare
+        end
+        params do
+          requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison'
+          requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison'
+        end
+        get ':id/repository/compare' do
+          compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to])
+          present compare, with: ::API::Entities::Compare
+        end
+
+        desc 'Get repository contributors' do
+          success ::API::Entities::Contributor
+        end
+        get ':id/repository/contributors' do
+          begin
+            present user_project.repository.contributors,
+                    with: ::API::Entities::Contributor
+          rescue
+            not_found!
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/runners.rb b/lib/api/v3/runners.rb
new file mode 100644
index 00000000000..faa265f3314
--- /dev/null
+++ b/lib/api/v3/runners.rb
@@ -0,0 +1,65 @@
+module API
+  module V3
+    class Runners < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+
+      resource :runners do
+        desc 'Remove a runner' do
+          success ::API::Entities::Runner
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of the runner'
+        end
+        delete ':id' do
+          runner = Ci::Runner.find(params[:id])
+          not_found!('Runner') unless runner
+
+          authenticate_delete_runner!(runner)
+
+          status(200)
+          runner.destroy
+        end
+      end
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        before { authorize_admin_project }
+
+        desc "Disable project's runner" do
+          success ::API::Entities::Runner
+        end
+        params do
+          requires :runner_id, type: Integer, desc: 'The ID of the runner'
+        end
+        delete ':id/runners/:runner_id' do
+          runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id])
+          not_found!('Runner') unless runner_project
+
+          runner = runner_project.runner
+          forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1
+
+          runner_project.destroy
+
+          present runner, with: ::API::Entities::Runner
+        end
+      end
+
+      helpers do
+        def authenticate_delete_runner!(runner)
+          return if current_user.admin?
+          forbidden!("Runner is shared") if runner.is_shared?
+          forbidden!("Runner associated with more than one project") if runner.projects.count > 1
+          forbidden!("No access granted") unless user_can_access_runner?(runner)
+        end
+
+        def user_can_access_runner?(runner)
+          current_user.ci_authorized_runners.exists?(runner.id)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb
new file mode 100644
index 00000000000..2d13d6fabfd
--- /dev/null
+++ b/lib/api/v3/services.rb
@@ -0,0 +1,650 @@
+module API
+  module V3
+    class Services < Grape::API
+      services = {
+        'asana' => [
+          {
+            required: true,
+            name: :api_key,
+            type: String,
+            desc: 'User API token'
+          },
+          {
+            required: false,
+            name: :restrict_to_branch,
+            type: String,
+            desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches'
+          }
+        ],
+        'assembla' => [
+          {
+            required: true,
+            name: :token,
+            type: String,
+            desc: 'The authentication token'
+          },
+          {
+            required: false,
+            name: :subdomain,
+            type: String,
+            desc: 'Subdomain setting'
+          }
+        ],
+        'bamboo' => [
+          {
+            required: true,
+            name: :bamboo_url,
+            type: String,
+            desc: 'Bamboo root URL like https://bamboo.example.com'
+          },
+          {
+            required: true,
+            name: :build_key,
+            type: String,
+            desc: 'Bamboo build plan key like'
+          },
+          {
+            required: true,
+            name: :username,
+            type: String,
+            desc: 'A user with API access, if applicable'
+          },
+          {
+            required: true,
+            name: :password,
+            type: String,
+            desc: 'Passord of the user'
+          }
+        ],
+        'bugzilla' => [
+          {
+            required: true,
+            name: :new_issue_url,
+            type: String,
+            desc: 'New issue URL'
+          },
+          {
+            required: true,
+            name: :issues_url,
+            type: String,
+            desc: 'Issues URL'
+          },
+          {
+            required: true,
+            name: :project_url,
+            type: String,
+            desc: 'Project URL'
+          },
+          {
+            required: false,
+            name: :description,
+            type: String,
+            desc: 'Description'
+          },
+          {
+            required: false,
+            name: :title,
+            type: String,
+            desc: 'Title'
+          }
+        ],
+        'buildkite' => [
+          {
+            required: true,
+            name: :token,
+            type: String,
+            desc: 'Buildkite project GitLab token'
+          },
+          {
+            required: true,
+            name: :project_url,
+            type: String,
+            desc: 'The buildkite project URL'
+          },
+          {
+            required: false,
+            name: :enable_ssl_verification,
+            type: Boolean,
+            desc: 'Enable SSL verification for communication'
+          }
+        ],
+        'builds-email' => [
+          {
+            required: true,
+            name: :recipients,
+            type: String,
+            desc: 'Comma-separated list of recipient email addresses'
+          },
+          {
+            required: false,
+            name: :add_pusher,
+            type: Boolean,
+            desc: 'Add pusher to recipients list'
+          },
+          {
+            required: false,
+            name: :notify_only_broken_builds,
+            type: Boolean,
+            desc: 'Notify only broken builds'
+          }
+        ],
+        'campfire' => [
+          {
+            required: true,
+            name: :token,
+            type: String,
+            desc: 'Campfire token'
+          },
+          {
+            required: false,
+            name: :subdomain,
+            type: String,
+            desc: 'Campfire subdomain'
+          },
+          {
+            required: false,
+            name: :room,
+            type: String,
+            desc: 'Campfire room'
+          }
+        ],
+        'custom-issue-tracker' => [
+          {
+            required: true,
+            name: :new_issue_url,
+            type: String,
+            desc: 'New issue URL'
+          },
+          {
+            required: true,
+            name: :issues_url,
+            type: String,
+            desc: 'Issues URL'
+          },
+          {
+            required: true,
+            name: :project_url,
+            type: String,
+            desc: 'Project URL'
+          },
+          {
+            required: false,
+            name: :description,
+            type: String,
+            desc: 'Description'
+          },
+          {
+            required: false,
+            name: :title,
+            type: String,
+            desc: 'Title'
+          }
+        ],
+        'drone-ci' => [
+          {
+            required: true,
+            name: :token,
+            type: String,
+            desc: 'Drone CI token'
+          },
+          {
+            required: true,
+            name: :drone_url,
+            type: String,
+            desc: 'Drone CI URL'
+          },
+          {
+            required: false,
+            name: :enable_ssl_verification,
+            type: Boolean,
+            desc: 'Enable SSL verification for communication'
+          }
+        ],
+        'emails-on-push' => [
+          {
+            required: true,
+            name: :recipients,
+            type: String,
+            desc: 'Comma-separated list of recipient email addresses'
+          },
+          {
+            required: false,
+            name: :disable_diffs,
+            type: Boolean,
+            desc: 'Disable code diffs'
+          },
+          {
+            required: false,
+            name: :send_from_committer_email,
+            type: Boolean,
+            desc: 'Send from committer'
+          }
+        ],
+        'external-wiki' => [
+          {
+            required: true,
+            name: :external_wiki_url,
+            type: String,
+            desc: 'The URL of the external Wiki'
+          }
+        ],
+        'flowdock' => [
+          {
+            required: true,
+            name: :token,
+            type: String,
+            desc: 'Flowdock token'
+          }
+        ],
+        'gemnasium' => [
+          {
+            required: true,
+            name: :api_key,
+            type: String,
+            desc: 'Your personal API key on gemnasium.com'
+          },
+          {
+            required: true,
+            name: :token,
+            type: String,
+            desc: "The project's slug on gemnasium.com"
+          }
+        ],
+        'hipchat' => [
+          {
+            required: true,
+            name: :token,
+            type: String,
+            desc: 'The room token'
+          },
+          {
+            required: false,
+            name: :room,
+            type: String,
+            desc: 'The room name or ID'
+          },
+          {
+            required: false,
+            name: :color,
+            type: String,
+            desc: 'The room color'
+          },
+          {
+            required: false,
+            name: :notify,
+            type: Boolean,
+            desc: 'Enable notifications'
+          },
+          {
+            required: false,
+            name: :api_version,
+            type: String,
+            desc: 'Leave blank for default (v2)'
+          },
+          {
+            required: false,
+            name: :server,
+            type: String,
+            desc: 'Leave blank for default. https://hipchat.example.com'
+          }
+        ],
+        'irker' => [
+          {
+            required: true,
+            name: :recipients,
+            type: String,
+            desc: 'Recipients/channels separated by whitespaces'
+          },
+          {
+            required: false,
+            name: :default_irc_uri,
+            type: String,
+            desc: 'Default: irc://irc.network.net:6697'
+          },
+          {
+            required: false,
+            name: :server_host,
+            type: String,
+            desc: 'Server host. Default localhost'
+          },
+          {
+            required: false,
+            name: :server_port,
+            type: Integer,
+            desc: 'Server port. Default 6659'
+          },
+          {
+            required: false,
+            name: :colorize_messages,
+            type: Boolean,
+            desc: 'Colorize messages'
+          }
+        ],
+        'jira' => [
+          {
+            required: true,
+            name: :url,
+            type: String,
+            desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com'
+          },
+          {
+            required: true,
+            name: :project_key,
+            type: String,
+            desc: 'The short identifier for your JIRA project, all uppercase, e.g., PROJ'
+          },
+          {
+            required: false,
+            name: :username,
+            type: String,
+            desc: 'The username of the user created to be used with GitLab/JIRA'
+          },
+          {
+            required: false,
+            name: :password,
+            type: String,
+            desc: 'The password of the user created to be used with GitLab/JIRA'
+          },
+          {
+            required: false,
+            name: :jira_issue_transition_id,
+            type: Integer,
+            desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`'
+          }
+        ],
+
+        'kubernetes' => [
+          {
+            required: true,
+            name: :namespace,
+            type: String,
+            desc: 'The Kubernetes namespace to use'
+          },
+          {
+            required: true,
+            name: :api_url,
+            type: String,
+            desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com'
+          },
+          {
+            required: true,
+            name: :token,
+            type: String,
+            desc: 'The service token to authenticate against the Kubernetes cluster with'
+          },
+          {
+            required: false,
+            name: :ca_pem,
+            type: String,
+            desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)'
+          }
+        ],
+        'mattermost-slash-commands' => [
+          {
+            required: true,
+            name: :token,
+            type: String,
+            desc: 'The Mattermost token'
+          }
+        ],
+        'slack-slash-commands' => [
+          {
+            required: true,
+            name: :token,
+            type: String,
+            desc: 'The Slack token'
+          }
+        ],
+        'pipelines-email' => [
+          {
+            required: true,
+            name: :recipients,
+            type: String,
+            desc: 'Comma-separated list of recipient email addresses'
+          },
+          {
+            required: false,
+            name: :notify_only_broken_builds,
+            type: Boolean,
+            desc: 'Notify only broken builds'
+          }
+        ],
+        'pivotaltracker' => [
+          {
+            required: true,
+            name: :token,
+            type: String,
+            desc: 'The Pivotaltracker token'
+          },
+          {
+            required: false,
+            name: :restrict_to_branch,
+            type: String,
+            desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.'
+          }
+        ],
+        'pushover' => [
+          {
+            required: true,
+            name: :api_key,
+            type: String,
+            desc: 'The application key'
+          },
+          {
+            required: true,
+            name: :user_key,
+            type: String,
+            desc: 'The user key'
+          },
+          {
+            required: true,
+            name: :priority,
+            type: String,
+            desc: 'The priority'
+          },
+          {
+            required: true,
+            name: :device,
+            type: String,
+            desc: 'Leave blank for all active devices'
+          },
+          {
+            required: true,
+            name: :sound,
+            type: String,
+            desc: 'The sound of the notification'
+          }
+        ],
+        'redmine' => [
+          {
+            required: true,
+            name: :new_issue_url,
+            type: String,
+            desc: 'The new issue URL'
+          },
+          {
+            required: true,
+            name: :project_url,
+            type: String,
+            desc: 'The project URL'
+          },
+          {
+            required: true,
+            name: :issues_url,
+            type: String,
+            desc: 'The issues URL'
+          },
+          {
+            required: false,
+            name: :description,
+            type: String,
+            desc: 'The description of the tracker'
+          }
+        ],
+        'slack' => [
+          {
+            required: true,
+            name: :webhook,
+            type: String,
+            desc: 'The Slack webhook. e.g. https://hooks.slack.com/services/...'
+          },
+          {
+            required: false,
+            name: :new_issue_url,
+            type: String,
+            desc: 'The user name'
+          },
+          {
+            required: false,
+            name: :channel,
+            type: String,
+            desc: 'The channel name'
+          }
+        ],
+        'microsoft-teams' => [
+          required: true,
+          name: :webhook,
+          type: String,
+          desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…'
+        ],
+        'mattermost' => [
+          {
+            required: true,
+            name: :webhook,
+            type: String,
+            desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...'
+          }
+        ],
+        'teamcity' => [
+          {
+            required: true,
+            name: :teamcity_url,
+            type: String,
+            desc: 'TeamCity root URL like https://teamcity.example.com'
+          },
+          {
+            required: true,
+            name: :build_type,
+            type: String,
+            desc: 'Build configuration ID'
+          },
+          {
+            required: true,
+            name: :username,
+            type: String,
+            desc: 'A user with permissions to trigger a manual build'
+          },
+          {
+            required: true,
+            name: :password,
+            type: String,
+            desc: 'The password of the user'
+          }
+        ]
+      }
+
+      trigger_services = {
+        'mattermost-slash-commands' => [
+          {
+            name: :token,
+            type: String,
+            desc: 'The Mattermost token'
+          }
+        ],
+        'slack-slash-commands' => [
+          {
+            name: :token,
+            type: String,
+            desc: 'The Slack token'
+          }
+        ]
+      }.freeze
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        before { authenticate! }
+        before { authorize_admin_project }
+
+        helpers do
+          def service_attributes(service)
+            service.fields.inject([]) do |arr, hash|
+              arr << hash[:name].to_sym
+            end
+          end
+        end
+
+        desc "Delete a service for project"
+        params do
+          requires :service_slug, type: String, values: services.keys, desc: 'The name of the service'
+        end
+        delete ":id/services/:service_slug" do
+          service = user_project.find_or_initialize_service(params[:service_slug].underscore)
+
+          attrs = service_attributes(service).inject({}) do |hash, key|
+            hash.merge!(key => nil)
+          end
+
+          if service.update_attributes(attrs.merge(active: false))
+            status(200)
+            true
+          else
+            render_api_error!('400 Bad Request', 400)
+          end
+        end
+
+        desc 'Get the service settings for project' do
+          success Entities::ProjectService
+        end
+        params do
+          requires :service_slug, type: String, values: services.keys, desc: 'The name of the service'
+        end
+        get ":id/services/:service_slug" do
+          service = user_project.find_or_initialize_service(params[:service_slug].underscore)
+          present service, with: Entities::ProjectService, include_passwords: current_user.admin?
+        end
+      end
+
+      trigger_services.each do |service_slug, settings|
+        helpers do
+          def slash_command_service(project, service_slug, params)
+            project.services.active.where(template: false).find do |service|
+              service.try(:token) == params[:token] && service.to_param == service_slug.underscore
+            end
+          end
+        end
+
+        params do
+          requires :id, type: String, desc: 'The ID of a project'
+        end
+        resource :projects, requirements: { id: %r{[^/]+} } do
+          desc "Trigger a slash command for #{service_slug}" do
+            detail 'Added in GitLab 8.13'
+          end
+          params do
+            settings.each do |setting|
+              requires setting[:name], type: setting[:type], desc: setting[:desc]
+            end
+          end
+          post ":id/services/#{service_slug.underscore}/trigger" do
+            project = find_project(params[:id])
+
+            # This is not accurate, but done to prevent leakage of the project names
+            not_found!('Service') unless project
+
+            service = slash_command_service(project, service_slug, params)
+            result = service.try(:trigger, params)
+
+            if result
+              status result[:status] || 200
+              present result
+            else
+              not_found!('Service')
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb
new file mode 100644
index 00000000000..748d6b97d4f
--- /dev/null
+++ b/lib/api/v3/settings.rb
@@ -0,0 +1,137 @@
+module API
+  module V3
+    class Settings < Grape::API
+      before { authenticated_as_admin! }
+
+      helpers do
+        def current_settings
+          @current_setting ||=
+            (ApplicationSetting.current || ApplicationSetting.create_from_defaults)
+        end
+      end
+
+      desc 'Get the current application settings' do
+        success Entities::ApplicationSetting
+      end
+      get "application/settings" do
+        present current_settings, with: Entities::ApplicationSetting
+      end
+
+      desc 'Modify application settings' do
+        success Entities::ApplicationSetting
+      end
+      params do
+        optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master'
+        optional :default_project_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default project visibility'
+        optional :default_snippet_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default snippet visibility'
+        optional :default_group_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default group visibility'
+        optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.'
+        optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project],
+                                  desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com'
+        optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources'
+        optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.'
+        optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled'
+        optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects'
+        optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB'
+        optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.'
+        optional :user_oauth_applications, type: Boolean, desc: 'Allow users to register any application to use GitLab as an OAuth provider'
+        optional :user_default_external, type: Boolean, desc: 'Newly registered users will by default be external'
+        optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled'
+        optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up'
+        optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
+        optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups'
+        given domain_blacklist_enabled: ->(val) { val } do
+          requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com'
+        end
+        optional :after_sign_up_text, type: String, desc: 'Text shown after sign up'
+        optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled'
+        optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication'
+        given require_two_factor_authentication: ->(val) { val } do
+          requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication'
+        end
+        optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page'
+        optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out'
+        optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application'
+        optional :help_page_text, type: String, desc: 'Custom text displayed on the help page'
+        optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects'
+        given shared_runners_enabled: ->(val) { val } do
+          requires :shared_runners_text, type: String, desc: 'Shared runners text '
+        end
+        optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size each build's artifacts can have"
+        optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB'
+        optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)'
+        optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics'
+        given metrics_enabled: ->(val) { val } do
+          requires :metrics_host, type: String, desc: 'The InfluxDB host'
+          requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB'
+          requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open'
+          requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out'
+          requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.'
+          requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds'
+          requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet'
+        end
+        optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling'
+        given sidekiq_throttling_enabled: ->(val) { val } do
+          requires :sidekiq_throttling_queus, type: Array[String], desc: 'Choose which queues you wish to throttle'
+          requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.'
+        end
+        optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts'
+        given recaptcha_enabled: ->(val) { val } do
+          requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha'
+          requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha'
+        end
+        optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues'
+        given akismet_enabled: ->(val) { val } do
+          requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com'
+        end
+        optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.'
+        optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com'
+        given sentry_enabled: ->(val) { val } do
+          requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name'
+        end
+        optional :repository_storage, type: String, desc: 'Storage paths for new projects'
+        optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues."
+        optional :koding_enabled, type: Boolean, desc: 'Enable Koding'
+        given koding_enabled: ->(val) { val } do
+          requires :koding_url, type: String, desc: 'The Koding team URL'
+        end
+        optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML'
+        given plantuml_enabled: ->(val) { val } do
+          requires :plantuml_url, type: String, desc: 'The PlantUML server URL'
+        end
+        optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.'
+        optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.'
+        optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.'
+        optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)'
+        given housekeeping_enabled: ->(val) { val } do
+          requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance."
+          requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run."
+          requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run."
+          requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run."
+        end
+        optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.'
+        at_least_one_of :default_branch_protection, :default_project_visibility, :default_snippet_visibility,
+                        :default_group_visibility, :restricted_visibility_levels, :import_sources,
+                        :enabled_git_access_protocol, :gravatar_enabled, :default_projects_limit,
+                        :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources,
+                        :user_oauth_applications, :user_default_external, :signup_enabled,
+                        :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled,
+                        :after_sign_up_text, :signin_enabled, :require_two_factor_authentication,
+                        :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text,
+                        :shared_runners_enabled, :max_artifacts_size, :max_pages_size, :container_registry_token_expire_delay,
+                        :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled,
+                        :akismet_enabled, :admin_notification_email, :sentry_enabled,
+                        :repository_storage, :repository_checks_enabled, :koding_enabled, :plantuml_enabled,
+                        :version_check_enabled, :email_author_in_body, :html_emails_enabled,
+                        :housekeeping_enabled, :terminal_max_session_time
+      end
+      put "application/settings" do
+        if current_settings.update_attributes(declared_params(include_missing: false))
+          present current_settings, with: Entities::ApplicationSetting
+        else
+          render_validation_error!(current_settings)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb
new file mode 100644
index 00000000000..0762fc02d70
--- /dev/null
+++ b/lib/api/v3/snippets.rb
@@ -0,0 +1,138 @@
+module API
+  module V3
+    class Snippets < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+
+      resource :snippets do
+        helpers do
+          def snippets_for_current_user
+            SnippetsFinder.new(current_user, author: current_user).execute
+          end
+
+          def public_snippets
+            SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute
+          end
+        end
+
+        desc 'Get a snippets list for authenticated user' do
+          detail 'This feature was introduced in GitLab 8.15.'
+          success ::API::Entities::PersonalSnippet
+        end
+        params do
+          use :pagination
+        end
+        get do
+          present paginate(snippets_for_current_user), with: ::API::Entities::PersonalSnippet
+        end
+
+        desc 'List all public snippets current_user has access to' do
+          detail 'This feature was introduced in GitLab 8.15.'
+          success ::API::Entities::PersonalSnippet
+        end
+        params do
+          use :pagination
+        end
+        get 'public' do
+          present paginate(public_snippets), with: ::API::Entities::PersonalSnippet
+        end
+
+        desc 'Get a single snippet' do
+          detail 'This feature was introduced in GitLab 8.15.'
+          success ::API::Entities::PersonalSnippet
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of a snippet'
+        end
+        get ':id' do
+          snippet = snippets_for_current_user.find(params[:id])
+          present snippet, with: ::API::Entities::PersonalSnippet
+        end
+
+        desc 'Create new snippet' do
+          detail 'This feature was introduced in GitLab 8.15.'
+          success ::API::Entities::PersonalSnippet
+        end
+        params do
+          requires :title, type: String, desc: 'The title of a snippet'
+          requires :file_name, type: String, desc: 'The name of a snippet file'
+          requires :content, type: String, desc: 'The content of a snippet'
+          optional :visibility_level, type: Integer,
+                                      values: Gitlab::VisibilityLevel.values,
+                                      default: Gitlab::VisibilityLevel::INTERNAL,
+                                      desc: 'The visibility level of the snippet'
+        end
+        post do
+          attrs = declared_params(include_missing: false).merge(request: request, api: true)
+          snippet = CreateSnippetService.new(nil, current_user, attrs).execute
+
+          if snippet.persisted?
+            present snippet, with: ::API::Entities::PersonalSnippet
+          else
+            render_validation_error!(snippet)
+          end
+        end
+
+        desc 'Update an existing snippet' do
+          detail 'This feature was introduced in GitLab 8.15.'
+          success ::API::Entities::PersonalSnippet
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of a snippet'
+          optional :title, type: String, desc: 'The title of a snippet'
+          optional :file_name, type: String, desc: 'The name of a snippet file'
+          optional :content, type: String, desc: 'The content of a snippet'
+          optional :visibility_level, type: Integer,
+                                      values: Gitlab::VisibilityLevel.values,
+                                      desc: 'The visibility level of the snippet'
+          at_least_one_of :title, :file_name, :content, :visibility_level
+        end
+        put ':id' do
+          snippet = snippets_for_current_user.find_by(id: params.delete(:id))
+          return not_found!('Snippet') unless snippet
+          authorize! :update_personal_snippet, snippet
+
+          attrs = declared_params(include_missing: false)
+
+          UpdateSnippetService.new(nil, current_user, snippet, attrs).execute
+          if snippet.persisted?
+            present snippet, with: ::API::Entities::PersonalSnippet
+          else
+            render_validation_error!(snippet)
+          end
+        end
+
+        desc 'Remove snippet' do
+          detail 'This feature was introduced in GitLab 8.15.'
+          success ::API::Entities::PersonalSnippet
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of a snippet'
+        end
+        delete ':id' do
+          snippet = snippets_for_current_user.find_by(id: params.delete(:id))
+          return not_found!('Snippet') unless snippet
+          authorize! :destroy_personal_snippet, snippet
+          snippet.destroy
+          no_content!
+        end
+
+        desc 'Get a raw snippet' do
+          detail 'This feature was introduced in GitLab 8.15.'
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of a snippet'
+        end
+        get ":id/raw" do
+          snippet = snippets_for_current_user.find_by(id: params.delete(:id))
+          return not_found!('Snippet') unless snippet
+
+          env['api.format'] = :txt
+          content_type 'text/plain'
+          present snippet.content
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/subscriptions.rb b/lib/api/v3/subscriptions.rb
new file mode 100644
index 00000000000..690768db82f
--- /dev/null
+++ b/lib/api/v3/subscriptions.rb
@@ -0,0 +1,53 @@
+module API
+  module V3
+    class Subscriptions < Grape::API
+      before { authenticate! }
+
+      subscribable_types = {
+        'merge_request' => proc { |id| find_merge_request_with_access(id, :update_merge_request) },
+        'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) },
+        'issues' => proc { |id| find_project_issue(id) },
+        'labels' => proc { |id| find_project_label(id) }
+      }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+        requires :subscribable_id, type: String, desc: 'The ID of a resource'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        subscribable_types.each do |type, finder|
+          type_singularized = type.singularize
+          entity_class = ::API::Entities.const_get(type_singularized.camelcase)
+
+          desc 'Subscribe to a resource' do
+            success entity_class
+          end
+          post ":id/#{type}/:subscribable_id/subscription" do
+            resource = instance_exec(params[:subscribable_id], &finder)
+
+            if resource.subscribed?(current_user, user_project)
+              not_modified!
+            else
+              resource.subscribe(current_user, user_project)
+              present resource, with: entity_class, current_user: current_user, project: user_project
+            end
+          end
+
+          desc 'Unsubscribe from a resource' do
+            success entity_class
+          end
+          delete ":id/#{type}/:subscribable_id/subscription" do
+            resource = instance_exec(params[:subscribable_id], &finder)
+
+            if !resource.subscribed?(current_user, user_project)
+              not_modified!
+            else
+              resource.unsubscribe(current_user, user_project)
+              present resource, with: entity_class, current_user: current_user, project: user_project
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/system_hooks.rb b/lib/api/v3/system_hooks.rb
new file mode 100644
index 00000000000..5787c06fc12
--- /dev/null
+++ b/lib/api/v3/system_hooks.rb
@@ -0,0 +1,32 @@
+module API
+  module V3
+    class SystemHooks < Grape::API
+      before do
+        authenticate!
+        authenticated_as_admin!
+      end
+
+      resource :hooks do
+        desc 'Get the list of system hooks' do
+          success ::API::Entities::Hook
+        end
+        get do
+          present SystemHook.all, with: ::API::Entities::Hook
+        end
+
+        desc 'Delete a hook' do
+          success ::API::Entities::Hook
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of the system hook'
+        end
+        delete ":id" do
+          hook = SystemHook.find_by(id: params[:id])
+          not_found!('System hook') unless hook
+
+          present hook.destroy, with: ::API::Entities::Hook
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/tags.rb b/lib/api/v3/tags.rb
new file mode 100644
index 00000000000..7e5875cd030
--- /dev/null
+++ b/lib/api/v3/tags.rb
@@ -0,0 +1,40 @@
+module API
+  module V3
+    class Tags < Grape::API
+      before { authorize! :download_code, user_project }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Get a project repository tags' do
+          success ::API::Entities::RepoTag
+        end
+        get ":id/repository/tags" do
+          tags = user_project.repository.tags.sort_by(&:name).reverse
+          present tags, with: ::API::Entities::RepoTag, project: user_project
+        end
+
+        desc 'Delete a repository tag'
+        params do
+          requires :tag_name, type: String, desc: 'The name of the tag'
+        end
+        delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do
+          authorize_push_project
+
+          result = ::Tags::DestroyService.new(user_project, current_user)
+            .execute(params[:tag_name])
+
+          if result[:status] == :success
+            status(200)
+            {
+              tag_name: params[:tag_name]
+            }
+          else
+            render_api_error!(result[:message], result[:return_code])
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb
new file mode 100644
index 00000000000..4c577a8d2b7
--- /dev/null
+++ b/lib/api/v3/templates.rb
@@ -0,0 +1,122 @@
+module API
+  module V3
+    class Templates < Grape::API
+      GLOBAL_TEMPLATE_TYPES = {
+        gitignores: {
+          klass: Gitlab::Template::GitignoreTemplate,
+          gitlab_version: 8.8
+        },
+        gitlab_ci_ymls: {
+          klass: Gitlab::Template::GitlabCiYmlTemplate,
+          gitlab_version: 8.9
+        },
+        dockerfiles: {
+          klass: Gitlab::Template::DockerfileTemplate,
+          gitlab_version: 8.15
+        }
+      }.freeze
+      PROJECT_TEMPLATE_REGEX =
+        /[\<\{\[]
+          (project|description|
+          one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here
+        [\>\}\]]/xi.freeze
+      YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze
+      FULLNAME_TEMPLATE_REGEX =
+        /[\<\{\[]
+          (fullname|name\sof\s(author|copyright\sowner))
+        [\>\}\]]/xi.freeze
+      DEPRECATION_MESSAGE = ' This endpoint is deprecated and has been removed in V4.'.freeze
+
+      helpers do
+        def parsed_license_template
+          # We create a fresh Licensee::License object since we'll modify its
+          # content in place below.
+          template = Licensee::License.new(params[:name])
+
+          template.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s)
+          template.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present?
+
+          fullname = params[:fullname].presence || current_user.try(:name)
+          template.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname
+          template
+        end
+
+        def render_response(template_type, template)
+          not_found!(template_type.to_s.singularize) unless template
+          present template, with: ::API::Entities::Template
+        end
+      end
+
+      { "licenses" => :deprecated, "templates/licenses" => :ok }.each do |route, status|
+        desc 'Get the list of the available license template' do
+          detailed_desc = 'This feature was introduced in GitLab 8.7.'
+          detailed_desc << DEPRECATION_MESSAGE unless status == :ok
+          detail detailed_desc
+          success ::API::Entities::RepoLicense
+        end
+        params do
+          optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses'
+        end
+        get route do
+          options = {
+            featured: declared(params).popular.present? ? true : nil
+          }
+          present Licensee::License.all(options), with: ::API::Entities::RepoLicense
+        end
+      end
+
+      { "licenses/:name" => :deprecated, "templates/licenses/:name" => :ok }.each do |route, status|
+        desc 'Get the text for a specific license' do
+          detailed_desc = 'This feature was introduced in GitLab 8.7.'
+          detailed_desc << DEPRECATION_MESSAGE unless status == :ok
+          detail detailed_desc
+          success ::API::Entities::RepoLicense
+        end
+        params do
+          requires :name, type: String, desc: 'The name of the template'
+        end
+        get route, requirements: { name: /[\w\.-]+/ } do
+          not_found!('License') unless Licensee::License.find(declared(params).name)
+
+          template = parsed_license_template
+
+          present template, with: ::API::Entities::RepoLicense
+        end
+      end
+
+      GLOBAL_TEMPLATE_TYPES.each do |template_type, properties|
+        klass = properties[:klass]
+        gitlab_version = properties[:gitlab_version]
+
+        { template_type => :deprecated, "templates/#{template_type}" => :ok }.each do |route, status|
+          desc 'Get the list of the available template' do
+            detailed_desc = "This feature was introduced in GitLab #{gitlab_version}."
+            detailed_desc << DEPRECATION_MESSAGE unless status == :ok
+            detail detailed_desc
+            success ::API::Entities::TemplatesList
+          end
+          get route do
+            present klass.all, with: ::API::Entities::TemplatesList
+          end
+        end
+
+        { "#{template_type}/:name" => :deprecated, "templates/#{template_type}/:name" => :ok }.each do |route, status|
+          desc 'Get the text for a specific template present in local filesystem' do
+            detailed_desc = "This feature was introduced in GitLab #{gitlab_version}."
+            detailed_desc << DEPRECATION_MESSAGE unless status == :ok
+            detail detailed_desc
+            success ::API::Entities::Template
+          end
+          params do
+            requires :name, type: String, desc: 'The name of the template'
+          end
+          get route do
+            new_template = klass.find(declared(params).name)
+
+            render_response(template_type, new_template)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/time_tracking_endpoints.rb b/lib/api/v3/time_tracking_endpoints.rb
new file mode 100644
index 00000000000..d5b90e435ba
--- /dev/null
+++ b/lib/api/v3/time_tracking_endpoints.rb
@@ -0,0 +1,116 @@
+module API
+  module V3
+    module TimeTrackingEndpoints
+      extend ActiveSupport::Concern
+
+      included do
+        helpers do
+          def issuable_name
+            declared_params.key?(:issue_id) ? 'issue' : 'merge_request'
+          end
+
+          def issuable_key
+            "#{issuable_name}_id".to_sym
+          end
+
+          def update_issuable_key
+            "update_#{issuable_name}".to_sym
+          end
+
+          def read_issuable_key
+            "read_#{issuable_name}".to_sym
+          end
+
+          def load_issuable
+            @issuable ||= begin
+                            case issuable_name
+                            when 'issue'
+                              find_project_issue(params.delete(issuable_key))
+                            when 'merge_request'
+                              find_project_merge_request(params.delete(issuable_key))
+                            end
+                          end
+          end
+
+          def update_issuable(attrs)
+            custom_params = declared_params(include_missing: false)
+            custom_params.merge!(attrs)
+
+            issuable = update_service.new(user_project, current_user, custom_params).execute(load_issuable)
+            if issuable.valid?
+              present issuable, with: ::API::Entities::IssuableTimeStats
+            else
+              render_validation_error!(issuable)
+            end
+          end
+
+          def update_service
+            issuable_name == 'issue' ? ::Issues::UpdateService : ::MergeRequests::UpdateService
+          end
+        end
+
+        issuable_name            = name.end_with?('Issues') ? 'issue' : 'merge_request'
+        issuable_collection_name = issuable_name.pluralize
+        issuable_key             = "#{issuable_name}_id".to_sym
+
+        desc "Set a time estimate for a project #{issuable_name}"
+        params do
+          requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}"
+          requires :duration, type: String, desc: 'The duration to be parsed'
+        end
+        post ":id/#{issuable_collection_name}/:#{issuable_key}/time_estimate" do
+          authorize! update_issuable_key, load_issuable
+
+          status :ok
+          update_issuable(time_estimate: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)))
+        end
+
+        desc "Reset the time estimate for a project #{issuable_name}"
+        params do
+          requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}"
+        end
+        post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_time_estimate" do
+          authorize! update_issuable_key, load_issuable
+
+          status :ok
+          update_issuable(time_estimate: 0)
+        end
+
+        desc "Add spent time for a project #{issuable_name}"
+        params do
+          requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}"
+          requires :duration, type: String, desc: 'The duration to be parsed'
+        end
+        post ":id/#{issuable_collection_name}/:#{issuable_key}/add_spent_time" do
+          authorize! update_issuable_key, load_issuable
+
+          update_issuable(spend_time: {
+                            duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)),
+                            user: current_user
+                          })
+        end
+
+        desc "Reset spent time for a project #{issuable_name}"
+        params do
+          requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}"
+        end
+        post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_spent_time" do
+          authorize! update_issuable_key, load_issuable
+
+          status :ok
+          update_issuable(spend_time: { duration: :reset, user: current_user })
+        end
+
+        desc "Show time stats for a project #{issuable_name}"
+        params do
+          requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}"
+        end
+        get ":id/#{issuable_collection_name}/:#{issuable_key}/time_stats" do
+          authorize! read_issuable_key, load_issuable
+
+          present load_issuable, with: ::API::Entities::IssuableTimeStats
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/todos.rb b/lib/api/v3/todos.rb
new file mode 100644
index 00000000000..e3b311d61cd
--- /dev/null
+++ b/lib/api/v3/todos.rb
@@ -0,0 +1,30 @@
+module API
+  module V3
+    class Todos < Grape::API
+      before { authenticate! }
+
+      resource :todos do
+        desc 'Mark a todo as done' do
+          success ::API::Entities::Todo
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of the todo being marked as done'
+        end
+        delete ':id' do
+          todo = current_user.todos.find(params[:id])
+          TodoService.new.mark_todos_as_done([todo], current_user)
+
+          present todo.reload, with: ::API::Entities::Todo, current_user: current_user
+        end
+
+        desc 'Mark all todos as done'
+        delete do
+          status(200)
+
+          todos = TodosFinder.new(current_user, params).execute
+          TodoService.new.mark_todos_as_done(todos, current_user).size
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb
new file mode 100644
index 00000000000..a23d6b6b48c
--- /dev/null
+++ b/lib/api/v3/triggers.rb
@@ -0,0 +1,103 @@
+module API
+  module V3
+    class Triggers < Grape::API
+      include PaginationParams
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Trigger a GitLab project build' do
+          success ::API::V3::Entities::TriggerRequest
+        end
+        params do
+          requires :ref, type: String, desc: 'The commit sha or name of a branch or tag'
+          requires :token, type: String, desc: 'The unique token of trigger'
+          optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
+        end
+        post ":id/(ref/:ref/)trigger/builds", requirements: { ref: /.+/ } do
+          project = find_project(params[:id])
+          trigger = Ci::Trigger.find_by_token(params[:token].to_s)
+          not_found! unless project && trigger
+          unauthorized! unless trigger.project == project
+
+          # validate variables
+          variables = params[:variables].to_h
+          unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) }
+            render_api_error!('variables needs to be a map of key-valued strings', 400)
+          end
+
+          # create request and trigger builds
+          trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables)
+          if trigger_request
+            present trigger_request, with: ::API::V3::Entities::TriggerRequest
+          else
+            errors = 'No builds created'
+            render_api_error!(errors, 400)
+          end
+        end
+
+        desc 'Get triggers list' do
+          success ::API::V3::Entities::Trigger
+        end
+        params do
+          use :pagination
+        end
+        get ':id/triggers' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          triggers = user_project.triggers.includes(:trigger_requests)
+
+          present paginate(triggers), with: ::API::V3::Entities::Trigger
+        end
+
+        desc 'Get specific trigger of a project' do
+          success ::API::V3::Entities::Trigger
+        end
+        params do
+          requires :token, type: String, desc: 'The unique token of trigger'
+        end
+        get ':id/triggers/:token' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          trigger = user_project.triggers.find_by(token: params[:token].to_s)
+          return not_found!('Trigger') unless trigger
+
+          present trigger, with: ::API::V3::Entities::Trigger
+        end
+
+        desc 'Create a trigger' do
+          success ::API::V3::Entities::Trigger
+        end
+        post ':id/triggers' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          trigger = user_project.triggers.create
+
+          present trigger, with: ::API::V3::Entities::Trigger
+        end
+
+        desc 'Delete a trigger' do
+          success ::API::V3::Entities::Trigger
+        end
+        params do
+          requires :token, type: String, desc: 'The unique token of trigger'
+        end
+        delete ':id/triggers/:token' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          trigger = user_project.triggers.find_by(token: params[:token].to_s)
+          return not_found!('Trigger') unless trigger
+
+          trigger.destroy
+
+          present trigger, with: ::API::V3::Entities::Trigger
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb
new file mode 100644
index 00000000000..37020019e07
--- /dev/null
+++ b/lib/api/v3/users.rb
@@ -0,0 +1,202 @@
+module API
+  module V3
+    class Users < Grape::API
+      include PaginationParams
+
+      before do
+        allow_access_with_scope :read_user if request.get?
+        authenticate!
+      end
+
+      resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do
+        helpers do
+          params :optional_attributes do
+            optional :skype, type: String, desc: 'The Skype username'
+            optional :linkedin, type: String, desc: 'The LinkedIn username'
+            optional :twitter, type: String, desc: 'The Twitter username'
+            optional :website_url, type: String, desc: 'The website of the user'
+            optional :organization, type: String, desc: 'The organization of the user'
+            optional :projects_limit, type: Integer, desc: 'The number of projects a user can create'
+            optional :extern_uid, type: String, desc: 'The external authentication provider UID'
+            optional :provider, type: String, desc: 'The external provider'
+            optional :bio, type: String, desc: 'The biography of the user'
+            optional :location, type: String, desc: 'The location of the user'
+            optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator'
+            optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups'
+            optional :confirm, type: Boolean, default: true, desc: 'Flag indicating the account needs to be confirmed'
+            optional :external, type: Boolean, desc: 'Flag indicating the user is an external user'
+            all_or_none_of :extern_uid, :provider
+          end
+        end
+
+        desc 'Create a user. Available only for admins.' do
+          success ::API::Entities::UserPublic
+        end
+        params do
+          requires :email, type: String, desc: 'The email of the user'
+          optional :password, type: String, desc: 'The password of the new user'
+          optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token'
+          at_least_one_of :password, :reset_password
+          requires :name, type: String, desc: 'The name of the user'
+          requires :username, type: String, desc: 'The username of the user'
+          use :optional_attributes
+        end
+        post do
+          authenticated_as_admin!
+
+          params = declared_params(include_missing: false)
+          user = ::Users::CreateService.new(current_user, params.merge!(skip_confirmation: !params[:confirm])).execute
+
+          if user.persisted?
+            present user, with: ::API::Entities::UserPublic
+          else
+            conflict!('Email has already been taken') if User
+                .where(email: user.email)
+                .count > 0
+
+            conflict!('Username has already been taken') if User
+                .where(username: user.username)
+                .count > 0
+
+            render_validation_error!(user)
+          end
+        end
+
+        desc 'Get the SSH keys of a specified user. Available only for admins.' do
+          success ::API::Entities::SSHKey
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of the user'
+          use :pagination
+        end
+        get ':id/keys' do
+          authenticated_as_admin!
+
+          user = User.find_by(id: params[:id])
+          not_found!('User') unless user
+
+          present paginate(user.keys), with: ::API::Entities::SSHKey
+        end
+
+        desc 'Get the emails addresses of a specified user. Available only for admins.' do
+          success ::API::Entities::Email
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of the user'
+          use :pagination
+        end
+        get ':id/emails' do
+          authenticated_as_admin!
+          user = User.find_by(id: params[:id])
+          not_found!('User') unless user
+
+          present user.emails, with: ::API::Entities::Email
+        end
+
+        desc 'Block a user. Available only for admins.'
+        params do
+          requires :id, type: Integer, desc: 'The ID of the user'
+        end
+        put ':id/block' do
+          authenticated_as_admin!
+          user = User.find_by(id: params[:id])
+          not_found!('User') unless user
+
+          if !user.ldap_blocked?
+            user.block
+          else
+            forbidden!('LDAP blocked users cannot be modified by the API')
+          end
+        end
+
+        desc 'Unblock a user. Available only for admins.'
+        params do
+          requires :id, type: Integer, desc: 'The ID of the user'
+        end
+        put ':id/unblock' do
+          authenticated_as_admin!
+          user = User.find_by(id: params[:id])
+          not_found!('User') unless user
+
+          if user.ldap_blocked?
+            forbidden!('LDAP blocked users cannot be unblocked by the API')
+          else
+            user.activate
+          end
+        end
+
+        desc 'Get the contribution events of a specified user' do
+          detail 'This feature was introduced in GitLab 8.13.'
+          success ::API::V3::Entities::Event
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of the user'
+          use :pagination
+        end
+        get ':id/events' do
+          user = User.find_by(id: params[:id])
+          not_found!('User') unless user
+
+          events = user.events
+            .merge(ProjectsFinder.new(current_user: current_user).execute)
+            .references(:project)
+            .with_associations
+            .recent
+
+          present paginate(events), with: ::API::V3::Entities::Event
+        end
+
+        desc 'Delete an existing SSH key from a specified user. Available only for admins.' do
+          success ::API::Entities::SSHKey
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of the user'
+          requires :key_id, type: Integer, desc: 'The ID of the SSH key'
+        end
+        delete ':id/keys/:key_id' do
+          authenticated_as_admin!
+
+          user = User.find_by(id: params[:id])
+          not_found!('User') unless user
+
+          key = user.keys.find_by(id: params[:key_id])
+          not_found!('Key') unless key
+
+          present key.destroy, with: ::API::Entities::SSHKey
+        end
+      end
+
+      resource :user do
+        desc "Get the currently authenticated user's SSH keys" do
+          success ::API::Entities::SSHKey
+        end
+        params do
+          use :pagination
+        end
+        get "keys" do
+          present current_user.keys, with: ::API::Entities::SSHKey
+        end
+
+        desc "Get the currently authenticated user's email addresses" do
+          success ::API::Entities::Email
+        end
+        get "emails" do
+          present current_user.emails, with: ::API::Entities::Email
+        end
+
+        desc 'Delete an SSH key from the currently authenticated user' do
+          success ::API::Entities::SSHKey
+        end
+        params do
+          requires :key_id, type: Integer, desc: 'The ID of the SSH key'
+        end
+        delete "keys/:key_id" do
+          key = current_user.keys.find_by(id: params[:key_id])
+          not_found!('Key') unless key
+
+          present key.destroy, with: ::API::Entities::SSHKey
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/v3/variables.rb b/lib/api/v3/variables.rb
new file mode 100644
index 00000000000..83972b1e7ce
--- /dev/null
+++ b/lib/api/v3/variables.rb
@@ -0,0 +1,29 @@
+module API
+  module V3
+    class Variables < Grape::API
+      include PaginationParams
+
+      before { authenticate! }
+      before { authorize! :admin_build, user_project }
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+
+      resource :projects, requirements: { id: %r{[^/]+} } do
+        desc 'Delete an existing variable from a project' do
+          success ::API::Entities::Variable
+        end
+        params do
+          requires :key, type: String, desc: 'The key of the variable'
+        end
+        delete ':id/variables/:key' do
+          variable = user_project.variables.find_by(key: params[:key])
+          not_found!('Variable') unless variable
+
+          present variable.destroy, with: ::API::Entities::Variable
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
index f6495071a11..10374995497 100644
--- a/lib/api/variables.rb
+++ b/lib/api/variables.rb
@@ -1,51 +1,51 @@
 module API
-  # Projects variables API
   class Variables < Grape::API
+    include PaginationParams
+
     before { authenticate! }
     before { authorize! :admin_build, user_project }
 
-    resource :projects do
-      # Get project variables
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   page (optional) - The page number for pagination
-      #   per_page (optional) - The value of items per page to show
-      # Example Request:
-      #   GET /projects/:id/variables
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get project variables' do
+        success Entities::Variable
+      end
+      params do
+        use :pagination
+      end
       get ':id/variables' do
         variables = user_project.variables
         present paginate(variables), with: Entities::Variable
       end
 
-      # Get specific variable of a project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   key (required) - The `key` of variable
-      # Example Request:
-      #   GET /projects/:id/variables/:key
+      desc 'Get a specific variable from a project' do
+        success Entities::Variable
+      end
+      params do
+        requires :key, type: String, desc: 'The key of the variable'
+      end
       get ':id/variables/:key' do
         key = params[:key]
-        variable = user_project.variables.find_by(key: key.to_s)
+        variable = user_project.variables.find_by(key: key)
 
         return not_found!('Variable') unless variable
 
         present variable, with: Entities::Variable
       end
 
-      # Create a new variable in project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   key (required) - The key of variable
-      #   value (required) - The value of variable
-      # Example Request:
-      #   POST /projects/:id/variables
+      desc 'Create a new variable in a project' do
+        success Entities::Variable
+      end
+      params do
+        requires :key, type: String, desc: 'The key of the variable'
+        requires :value, type: String, desc: 'The value of the variable'
+        optional :protected, type: String, desc: 'Whether the variable is protected'
+      end
       post ':id/variables' do
-        required_attributes! [:key, :value]
-
-        variable = user_project.variables.create(key: params[:key], value: params[:value])
+        variable = user_project.variables.create(declared_params(include_missing: false))
 
         if variable.valid?
           present variable, with: Entities::Variable
@@ -54,41 +54,37 @@ module API
         end
       end
 
-      # Update existing variable of a project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   key (optional) - The `key` of variable
-      #   value (optional) - New value for `value` field of variable
-      # Example Request:
-      #   PUT /projects/:id/variables/:key
+      desc 'Update an existing variable from a project' do
+        success Entities::Variable
+      end
+      params do
+        optional :key, type: String, desc: 'The key of the variable'
+        optional :value, type: String, desc: 'The value of the variable'
+        optional :protected, type: String, desc: 'Whether the variable is protected'
+      end
       put ':id/variables/:key' do
-        variable = user_project.variables.find_by(key: params[:key].to_s)
+        variable = user_project.variables.find_by(key: params[:key])
 
         return not_found!('Variable') unless variable
 
-        attrs = attributes_for_keys [:value]
-        if variable.update(attrs)
+        if variable.update(declared_params(include_missing: false).except(:key))
           present variable, with: Entities::Variable
         else
           render_validation_error!(variable)
         end
       end
 
-      # Delete existing variable of a project
-      #
-      # Parameters:
-      #   id (required) - The ID of a project
-      #   key (required) - The ID of a variable
-      # Example Request:
-      #   DELETE /projects/:id/variables/:key
+      desc 'Delete an existing variable from a project' do
+        success Entities::Variable
+      end
+      params do
+        requires :key, type: String, desc: 'The key of the variable'
+      end
       delete ':id/variables/:key' do
-        variable = user_project.variables.find_by(key: params[:key].to_s)
+        variable = user_project.variables.find_by(key: params[:key])
+        not_found!('Variable') unless variable
 
-        return not_found!('Variable') unless variable
         variable.destroy
-
-        present variable, with: Entities::Variable
       end
     end
   end
diff --git a/lib/api/version.rb b/lib/api/version.rb
new file mode 100644
index 00000000000..9ba576bd828
--- /dev/null
+++ b/lib/api/version.rb
@@ -0,0 +1,12 @@
+module API
+  class Version < Grape::API
+    before { authenticate! }
+
+    desc 'Get the version information of the GitLab instance.' do
+      detail 'This feature was introduced in GitLab 8.13.'
+    end
+    get '/version' do
+      { version: Gitlab::VERSION, revision: Gitlab::REVISION }
+    end
+  end
+end
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
index 51fa3867e67..1f4bda6f588 100644
--- a/lib/backup/artifacts.rb
+++ b/lib/backup/artifacts.rb
@@ -3,7 +3,7 @@ require 'backup/files'
 module Backup
   class Artifacts < Files
     def initialize
-      super('artifacts', ArtifactUploader.artifacts_path)
+      super('artifacts', ArtifactUploader.local_artifacts_store)
     end
 
     def create_files_dir
diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index 22319ec6623..d97e5d98229 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -5,7 +5,7 @@ module Backup
     attr_reader :config, :db_file_name
 
     def initialize
-      @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
+      @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
       @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
     end
 
@@ -13,28 +13,32 @@ module Backup
       FileUtils.mkdir_p(File.dirname(db_file_name))
       FileUtils.rm_f(db_file_name)
       compress_rd, compress_wr = IO.pipe
-      compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600])
+      compress_pid = spawn(*%w(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600])
       compress_rd.close
 
-      dump_pid = case config["adapter"]
-      when /^mysql/ then
-        $progress.print "Dumping MySQL database #{config['database']} ... "
-        # Workaround warnings from MySQL 5.6 about passwords on cmd line
-        ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-        spawn('mysqldump', *mysql_args, config['database'], out: compress_wr)
-      when "postgresql" then
-        $progress.print "Dumping PostgreSQL database #{config['database']} ... "
-        pg_env
-        pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump.
-        if Gitlab.config.backup.pg_schema
-          pgsql_args << "-n"
-          pgsql_args << Gitlab.config.backup.pg_schema
+      dump_pid =
+        case config["adapter"]
+        when /^mysql/ then
+          $progress.print "Dumping MySQL database #{config['database']} ... "
+          # Workaround warnings from MySQL 5.6 about passwords on cmd line
+          ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
+          spawn('mysqldump', *mysql_args, config['database'], out: compress_wr)
+        when "postgresql" then
+          $progress.print "Dumping PostgreSQL database #{config['database']} ... "
+          pg_env
+          pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump.
+          if Gitlab.config.backup.pg_schema
+            pgsql_args << "-n"
+            pgsql_args << Gitlab.config.backup.pg_schema
+          end
+          spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr)
         end
-        spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr)
-      end
       compress_wr.close
 
-      success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+      success = [compress_pid, dump_pid].all? do |pid|
+        Process.waitpid(pid)
+        $?.success?
+      end
 
       report_success(success)
       abort 'Backup failed' unless success
@@ -42,23 +46,27 @@ module Backup
 
     def restore
       decompress_rd, decompress_wr = IO.pipe
-      decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
+      decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name)
       decompress_wr.close
 
-      restore_pid = case config["adapter"]
-      when /^mysql/ then
-        $progress.print "Restoring MySQL database #{config['database']} ... "
-        # Workaround warnings from MySQL 5.6 about passwords on cmd line
-        ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-        spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
-      when "postgresql" then
-        $progress.print "Restoring PostgreSQL database #{config['database']} ... "
-        pg_env
-        spawn('psql', config['database'], in: decompress_rd)
-      end
+      restore_pid =
+        case config["adapter"]
+        when /^mysql/ then
+          $progress.print "Restoring MySQL database #{config['database']} ... "
+          # Workaround warnings from MySQL 5.6 about passwords on cmd line
+          ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
+          spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
+        when "postgresql" then
+          $progress.print "Restoring PostgreSQL database #{config['database']} ... "
+          pg_env
+          spawn('psql', config['database'], in: decompress_rd)
+        end
       decompress_rd.close
 
-      success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+      success = [decompress_pid, restore_pid].all? do |pid|
+        Process.waitpid(pid)
+        $?.success?
+      end
 
       report_success(success)
       abort 'Restore failed' unless success
@@ -72,16 +80,32 @@ module Backup
         'port'      => '--port',
         'socket'    => '--socket',
         'username'  => '--user',
-        'encoding'  => '--default-character-set'
+        'encoding'  => '--default-character-set',
+        # SSL
+        'sslkey'    => '--ssl-key',
+        'sslcert'   => '--ssl-cert',
+        'sslca'     => '--ssl-ca',
+        'sslcapath' => '--ssl-capath',
+        'sslcipher' => '--ssl-cipher'
       }
       args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact
     end
 
     def pg_env
-      ENV['PGUSER']     = config["username"] if config["username"]
-      ENV['PGHOST']     = config["host"] if config["host"]
-      ENV['PGPORT']     = config["port"].to_s if config["port"]
-      ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
+      args = {
+        'username'  => 'PGUSER',
+        'host'      => 'PGHOST',
+        'port'      => 'PGPORT',
+        'password'  => 'PGPASSWORD',
+        # SSL
+        'sslmode'         => 'PGSSLMODE',
+        'sslkey'          => 'PGSSLKEY',
+        'sslcert'         => 'PGSSLCERT',
+        'sslrootcert'     => 'PGSSLROOTCERT',
+        'sslcrl'          => 'PGSSLCRL',
+        'sslcompression'  => 'PGSSLCOMPRESSION'
+      }
+      args.each { |opt, arg| ENV[arg] = config[opt].to_s if config[opt] }
     end
 
     def report_success(success)
diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index cedbb289f6a..30a91647b77 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -8,6 +8,7 @@ module Backup
       @name = name
       @app_files_dir = File.realpath(app_files_dir)
       @files_parent_dir = File.realpath(File.join(@app_files_dir, '..'))
+      @backup_files_dir = File.join(Gitlab.config.backup.path, File.basename(@app_files_dir) )
       @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz')
     end
 
@@ -15,14 +16,28 @@ module Backup
     def dump
       FileUtils.mkdir_p(Gitlab.config.backup.path)
       FileUtils.rm_f(backup_tarball)
-      run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600])
+
+      if ENV['STRATEGY'] == 'copy'
+        cmd = %W(cp -a #{app_files_dir} #{Gitlab.config.backup.path})
+        output, status = Gitlab::Popen.popen(cmd)
+
+        unless status.zero?
+          puts output
+          abort 'Backup failed'
+        end
+
+        run_pipeline!([%W(tar -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600])
+        FileUtils.rm_rf(@backup_files_dir)
+      else
+        run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600])
+      end
     end
 
     def restore
       backup_existing_files_dir
       create_files_dir
 
-      run_pipeline!([%W(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball)
+      run_pipeline!([%w(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball)
     end
 
     def backup_existing_files_dir
@@ -32,7 +47,7 @@ module Backup
       end
     end
 
-    def run_pipeline!(cmd_list, options={})
+    def run_pipeline!(cmd_list, options = {})
       status_list = Open3.pipeline(*cmd_list, options)
       abort 'Backup failed' unless status_list.compact.all?(&:success?)
     end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 0dfffaf0bc6..f755c99ea4a 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -1,7 +1,8 @@
 module Backup
   class Manager
-    ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry]
-    FOLDERS_TO_BACKUP = %w[repositories db]
+    ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze
+    FOLDERS_TO_BACKUP = %w[repositories db].freeze
+    FILE_NAME_SUFFIX = '_gitlab_backup.tar'.freeze
 
     def pack
       # Make sure there is a connection
@@ -14,18 +15,17 @@ module Backup
       s[:gitlab_version]     = Gitlab::VERSION
       s[:tar_version]        = tar_version
       s[:skipped]            = ENV["SKIP"]
-      tar_file = "#{s[:backup_created_at].to_i}_gitlab_backup.tar"
+      tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{s[:gitlab_version]}#{FILE_NAME_SUFFIX}"
 
-      Dir.chdir(Gitlab.config.backup.path) do
-        File.open("#{Gitlab.config.backup.path}/backup_information.yml",
-                  "w+") do |file|
-          file << s.to_yaml.gsub(/^---\n/,'')
+      Dir.chdir(backup_path) do
+        File.open("#{backup_path}/backup_information.yml", "w+") do |file|
+          file << s.to_yaml.gsub(/^---\n/, '')
         end
 
         # create archive
         $progress.print "Creating backup archive: #{tar_file} ... "
         # Set file permissions on open to prevent chmod races.
-        tar_system_options = {out: [tar_file, 'w', Gitlab.config.backup.archive_permissions]}
+        tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] }
         if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options)
           $progress.puts "done".color(:green)
         else
@@ -49,8 +49,9 @@ module Backup
       directory = connect_to_remote_directory(connection_settings)
 
       if directory.files.create(key: tar_file, body: File.open(tar_file), public: false,
-          multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
-          encryption: Gitlab.config.backup.upload.encryption)
+                                multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
+                                encryption: Gitlab.config.backup.upload.encryption,
+                                storage_class: Gitlab.config.backup.upload.storage_class)
         $progress.puts "done".color(:green)
       else
         puts "uploading backup to #{remote_directory} failed".color(:red)
@@ -62,9 +63,9 @@ module Backup
       $progress.print "Deleting tmp directories ... "
 
       backup_contents.each do |dir|
-        next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
+        next unless File.exist?(File.join(backup_path, dir))
 
-        if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
+        if FileUtils.rm_rf(File.join(backup_path, dir))
           $progress.puts "done".color(:green)
         else
           puts "deleting tmp directory '#{dir}' failed".color(:red)
@@ -81,13 +82,22 @@ module Backup
       if keep_time > 0
         removed = 0
 
-        Dir.chdir(Gitlab.config.backup.path) do
-          file_list = Dir.glob('*_gitlab_backup.tar')
-          file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ }
-          file_list.sort.each do |timestamp|
+        Dir.chdir(backup_path) do
+          backup_file_list.each do |file|
+            # For backward compatibility, there are 3 names the backups can have:
+            # - 1495527122_gitlab_backup.tar
+            # - 1495527068_2017_05_23_gitlab_backup.tar
+            # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar
+            next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+.*)?)?_gitlab_backup\.tar$/
+
+            timestamp = $1.to_i
+
             if Time.at(timestamp) < (Time.now - keep_time)
-              if Kernel.system(*%W(rm #{timestamp}_gitlab_backup.tar))
+              begin
+                FileUtils.rm(file)
                 removed += 1
+              rescue => e
+                $progress.puts "Deleting #{file} failed: #{e.message}".color(:red)
               end
             end
           end
@@ -100,50 +110,55 @@ module Backup
     end
 
     def unpack
-      Dir.chdir(Gitlab.config.backup.path)
+      Dir.chdir(backup_path)
 
       # check for existing backups in the backup dir
-      file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i }
-      puts "no backups found" if file_list.count == 0
-
-      if file_list.count > 1 && ENV["BACKUP"].nil?
-        puts "Found more than one backup, please specify which one you want to restore:"
-        puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
+      if backup_file_list.empty?
+        $progress.puts "No backups found in #{backup_path}"
+        $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}"
+        exit 1
+      elsif backup_file_list.many? && ENV["BACKUP"].nil?
+        $progress.puts 'Found more than one backup, please specify which one you want to restore:'
+        $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup'
         exit 1
       end
 
-      tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_backup.tar")
+      tar_file = if ENV['BACKUP'].present?
+                   "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}"
+                 else
+                   backup_file_list.first
+                 end
 
       unless File.exist?(tar_file)
-        puts "The specified backup doesn't exist!"
+        $progress.puts "The backup file #{tar_file} does not exist!"
         exit 1
       end
 
-      $progress.print "Unpacking backup ... "
+      $progress.print 'Unpacking backup ... '
 
       unless Kernel.system(*%W(tar -xf #{tar_file}))
-        puts "unpacking backup failed".color(:red)
+        $progress.puts 'unpacking backup failed'.color(:red)
         exit 1
       else
-        $progress.puts "done".color(:green)
+        $progress.puts 'done'.color(:green)
       end
 
       ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
 
       # restoring mismatching backups can lead to unexpected problems
       if settings[:gitlab_version] != Gitlab::VERSION
-        puts "GitLab version mismatch:".color(:red)
-        puts "  Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red)
-        puts "  Please switch to the following version and try again:".color(:red)
-        puts "  version: #{settings[:gitlab_version]}".color(:red)
-        puts
-        puts "Hint: git checkout v#{settings[:gitlab_version]}"
+        $progress.puts 'GitLab version mismatch:'.color(:red)
+        $progress.puts "  Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red)
+        $progress.puts '  Please switch to the following version and try again:'.color(:red)
+        $progress.puts "  version: #{settings[:gitlab_version]}".color(:red)
+        $progress.puts
+        $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}"
         exit 1
       end
     end
 
     def tar_version
-      tar_version, _ = Gitlab::Popen.popen(%W(tar --version))
+      tar_version, _ = Gitlab::Popen.popen(%w(tar --version))
       tar_version.force_encoding('locale').split("\n").first
     end
 
@@ -153,6 +168,14 @@ module Backup
 
     private
 
+    def backup_path
+      Gitlab.config.backup.path
+    end
+
+    def backup_file_list
+      @backup_file_list ||= Dir.glob("*#{FILE_NAME_SUFFIX}")
+    end
+
     def connect_to_remote_directory(connection_settings)
       connection = ::Fog::Storage.new(connection_settings)
 
diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb
new file mode 100644
index 00000000000..215ded93bfe
--- /dev/null
+++ b/lib/backup/pages.rb
@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+  class Pages < Files
+    def initialize
+      super('pages', Gitlab.config.pages.path)
+    end
+
+    def create_files_dir
+      Dir.mkdir(app_files_dir, 0700)
+    end
+  end
+end
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 9fcd9a3f999..a1685c77916 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -2,45 +2,56 @@ require 'yaml'
 
 module Backup
   class Repository
+    # rubocop:disable Metrics/AbcSize
     def dump
       prepare
 
       Project.find_each(batch_size: 1000) do |project|
-        $progress.print " * #{project.path_with_namespace} ... "
+        progress.print " * #{project.path_with_namespace} ... "
+        path_to_project_repo = path_to_repo(project)
+        path_to_project_bundle = path_to_bundle(project)
 
         # Create namespace dir if missing
-        FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace
+        FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
 
-        if project.empty_repo?
-          $progress.puts "[SKIPPED]".color(:cyan)
+        if empty_repo?(project)
+          progress.puts "[SKIPPED]".color(:cyan)
         else
-          cmd = %W(tar -cf #{path_to_bundle(project)} -C #{path_to_repo(project)} .)
+          in_path(path_to_project_repo) do |dir|
+            FileUtils.mkdir_p(path_to_tars(project))
+            cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
+            output, status = Gitlab::Popen.popen(cmd)
+
+            unless status.zero?
+              progress_warn(project, cmd.join(' '), output)
+            end
+          end
+
+          cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
           output, status = Gitlab::Popen.popen(cmd)
+
           if status.zero?
-            $progress.puts "[DONE]".color(:green)
+            progress.puts "[DONE]".color(:green)
           else
-            puts "[FAILED]".color(:red)
-            puts "failed: #{cmd.join(' ')}"
-            puts output
-            abort 'Backup failed'
+            progress_warn(project, cmd.join(' '), output)
           end
         end
 
         wiki = ProjectWiki.new(project)
+        path_to_wiki_repo = path_to_repo(wiki)
+        path_to_wiki_bundle = path_to_bundle(wiki)
 
-        if File.exist?(path_to_repo(wiki))
-          $progress.print " * #{wiki.path_with_namespace} ... "
-          if wiki.repository.empty?
-            $progress.puts " [SKIPPED]".color(:cyan)
+        if File.exist?(path_to_wiki_repo)
+          progress.print " * #{wiki.path_with_namespace} ... "
+          if empty_repo?(wiki)
+            progress.puts " [SKIPPED]".color(:cyan)
           else
-            cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)
+            cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all)
             output, status = Gitlab::Popen.popen(cmd)
             if status.zero?
-              $progress.puts " [DONE]".color(:green)
+              progress.puts " [DONE]".color(:green)
             else
-              puts " [FAILED]".color(:red)
-              puts "failed: #{cmd.join(' ')}"
-              abort 'Backup failed'
+              progress_warn(wiki, cmd.join(' '), output)
             end
           end
         end
@@ -48,7 +59,8 @@ module Backup
     end
 
     def restore
-      Gitlab.config.repositories.storages.each do |name, path|
+      Gitlab.config.repositories.storages.each do |name, repository_storage|
+        path = repository_storage['path']
         next unless File.exist?(path)
 
         # Move repos dir to 'repositories.old' dir
@@ -59,56 +71,69 @@ module Backup
       end
 
       Project.find_each(batch_size: 1000) do |project|
-        $progress.print " * #{project.path_with_namespace} ... "
+        progress.print " * #{project.path_with_namespace} ... "
+        path_to_project_repo = path_to_repo(project)
+        path_to_project_bundle = path_to_bundle(project)
 
         project.ensure_dir_exist
 
-        if File.exist?(path_to_bundle(project))
-          FileUtils.mkdir_p(path_to_repo(project))
-          cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)})
+        cmd = if File.exist?(path_to_project_bundle)
+                %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo})
+              else
+                %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo})
+              end
+
+        output, status = Gitlab::Popen.popen(cmd)
+        if status.zero?
+          progress.puts "[DONE]".color(:green)
         else
-          cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)})
+          progress_warn(project, cmd.join(' '), output)
         end
 
-        if system(*cmd, silent)
-          $progress.puts "[DONE]".color(:green)
-        else
-          puts "[FAILED]".color(:red)
-          puts "failed: #{cmd.join(' ')}"
-          abort 'Restore failed'
+        in_path(path_to_tars(project)) do |dir|
+          cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
+
+          output, status = Gitlab::Popen.popen(cmd)
+          unless status.zero?
+            progress_warn(project, cmd.join(' '), output)
+          end
         end
 
         wiki = ProjectWiki.new(project)
+        path_to_wiki_repo = path_to_repo(wiki)
+        path_to_wiki_bundle = path_to_bundle(wiki)
 
-        if File.exist?(path_to_bundle(wiki))
-          $progress.print " * #{wiki.path_with_namespace} ... "
+        if File.exist?(path_to_wiki_bundle)
+          progress.print " * #{wiki.path_with_namespace} ... "
 
           # If a wiki bundle exists, first remove the empty repo
           # that was initialized with ProjectWiki.new() and then
           # try to restore with 'git clone --bare'.
-          FileUtils.rm_rf(path_to_repo(wiki))
-          cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)})
+          FileUtils.rm_rf(path_to_wiki_repo)
+          cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo})
 
-          if system(*cmd, silent)
-            $progress.puts " [DONE]".color(:green)
+          output, status = Gitlab::Popen.popen(cmd)
+          if status.zero?
+            progress.puts " [DONE]".color(:green)
           else
-            puts " [FAILED]".color(:red)
-            puts "failed: #{cmd.join(' ')}"
-            abort 'Restore failed'
+            progress_warn(project, cmd.join(' '), output)
           end
         end
       end
 
-      $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow)
+      progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow)
       cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args
-      if system(*cmd)
-        $progress.puts " [DONE]".color(:green)
+
+      output, status = Gitlab::Popen.popen(cmd)
+      if status.zero?
+        progress.puts " [DONE]".color(:green)
       else
         puts " [FAILED]".color(:red)
         puts "failed: #{cmd}"
+        puts output
       end
-
     end
+    # rubocop:enable Metrics/AbcSize
 
     protected
 
@@ -117,11 +142,31 @@ module Backup
     end
 
     def path_to_bundle(project)
-      File.join(backup_repos_path, project.path_with_namespace + ".bundle")
+      File.join(backup_repos_path, project.path_with_namespace + '.bundle')
+    end
+
+    def path_to_tars(project, dir = nil)
+      path = File.join(backup_repos_path, project.path_with_namespace)
+
+      if dir
+        File.join(path, "#{dir}.tar")
+      else
+        path
+      end
     end
 
     def backup_repos_path
-      File.join(Gitlab.config.backup.path, "repositories")
+      File.join(Gitlab.config.backup.path, 'repositories')
+    end
+
+    def in_path(path)
+      return unless Dir.exist?(path)
+
+      dir_entries = Dir.entries(path)
+
+      if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar')
+        yield('custom_hooks')
+      end
     end
 
     def prepare
@@ -133,13 +178,30 @@ module Backup
     end
 
     def silent
-      {err: '/dev/null', out: '/dev/null'}
+      { err: '/dev/null', out: '/dev/null' }
     end
 
     private
 
+    def progress_warn(project, cmd, output)
+      progress.puts "[WARNING] Executing #{cmd}".color(:orange)
+      progress.puts "Ignoring error on #{project.path_with_namespace} - #{output}".color(:orange)
+    end
+
+    def empty_repo?(project_or_wiki)
+      project_or_wiki.repository.empty_repo?
+    rescue => e
+      progress.puts "Ignoring repository error and continuing backing up project: #{project_or_wiki.path_with_namespace} - #{e.message}".color(:orange)
+
+      false
+    end
+
     def repository_storage_paths_args
-      Gitlab.config.repositories.storages.values
+      Gitlab.config.repositories.storages.values.map { |rs| rs['path'] }
+    end
+
+    def progress
+      $progress
     end
   end
 end
diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb
index 9261f77f3c9..35118375499 100644
--- a/lib/backup/uploads.rb
+++ b/lib/backup/uploads.rb
@@ -2,7 +2,6 @@ require 'backup/files'
 
 module Backup
   class Uploads < Files
-
     def initialize
       super('uploads', Rails.root.join('public/uploads'))
     end
diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb
index 0257848b6bc..e2b57adf611 100644
--- a/lib/banzai/cross_project_reference.rb
+++ b/lib/banzai/cross_project_reference.rb
@@ -14,7 +14,7 @@ module Banzai
     def project_from_ref(ref)
       return context[:project] unless ref
 
-      Project.find_with_namespace(ref)
+      Project.find_by_full_path(ref)
     end
   end
 end
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index affe34394c2..8bc2dd18bda 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -33,7 +33,12 @@ module Banzai
       # Returns a String replaced with the return of the block.
       def self.references_in(text, pattern = object_class.reference_pattern)
         text.gsub(pattern) do |match|
-          yield match, $~[object_sym].to_i, $~[:project], $~
+          symbol = $~[object_sym]
+          if object_class.reference_valid?(symbol)
+            yield match, symbol.to_i, $~[:project], $~[:namespace], $~
+          else
+            match
+          end
         end
       end
 
@@ -102,10 +107,10 @@ module Banzai
             end
 
           elsif element_node?(node)
-            yield_valid_link(node) do |link, text|
+            yield_valid_link(node) do |link, inner_html|
               if ref_pattern && link =~ /\A#{ref_pattern}\z/
                 replace_link_node_with_href(node, link) do
-                  object_link_filter(link, ref_pattern, link_text: text)
+                  object_link_filter(link, ref_pattern, link_content: inner_html)
                 end
 
                 next
@@ -113,9 +118,9 @@ module Banzai
 
               next unless link_pattern
 
-              if link == text && text =~ /\A#{link_pattern}/
+              if link == inner_html && inner_html =~ /\A#{link_pattern}/
                 replace_link_node_with_text(node, link) do
-                  object_link_filter(text, link_pattern)
+                  object_link_filter(inner_html, link_pattern)
                 end
 
                 next
@@ -123,7 +128,7 @@ module Banzai
 
               if link =~ /\A#{link_pattern}\z/
                 replace_link_node_with_href(node, link) do
-                  object_link_filter(link, link_pattern, link_text: text)
+                  object_link_filter(link, link_pattern, link_content: inner_html)
                 end
 
                 next
@@ -140,40 +145,43 @@ module Banzai
       #
       # text - String text to replace references in.
       # pattern - Reference pattern to match against.
-      # link_text - Original content of the link being replaced.
+      # link_content - Original content of the link being replaced.
       #
       # Returns a String with references replaced with links. All links
       # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling.
-      def object_link_filter(text, pattern, link_text: nil)
-        references_in(text, pattern) do |match, id, project_ref, matches|
-          project = project_from_ref_cached(project_ref)
+      def object_link_filter(text, pattern, link_content: nil)
+        references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches|
+          project_path = full_project_path(namespace_ref, project_ref)
+          project = project_from_ref_cached(project_path)
 
           if project && object = find_object_cached(project, id)
             title = object_link_title(object)
             klass = reference_class(object_sym)
 
-            data = data_attributes_for(link_text || match, project, object)
+            data = data_attributes_for(link_content || match, project, object, link: !!link_content)
 
-            if matches.names.include?("url") && matches[:url]
-              url = matches[:url]
-            else
-              url = url_for_object_cached(object, project)
-            end
+            url =
+              if matches.names.include?("url") && matches[:url]
+                matches[:url]
+              else
+                url_for_object_cached(object, project)
+              end
 
-            text = link_text || object_link_text(object, matches)
+            content = link_content || object_link_text(object, matches)
 
             %(<a href="#{url}" #{data}
                  title="#{escape_once(title)}"
-                 class="#{klass}">#{escape_once(text)}</a>)
+                 class="#{klass}">#{content}</a>)
           else
             match
           end
         end
       end
 
-      def data_attributes_for(text, project, object)
+      def data_attributes_for(text, project, object, link: false)
         data_attribute(
           original:     text,
+          link:         link,
           project:      project.id,
           object_sym => object.id
         )
@@ -208,15 +216,18 @@ module Banzai
         @references_per_project ||= begin
           refs = Hash.new { |hash, key| hash[key] = Set.new }
 
-          regex = Regexp.union(object_class.reference_pattern,
-                               object_class.link_reference_pattern)
+          regex =
+            if uses_reference_pattern?
+              Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern)
+            else
+              object_class.link_reference_pattern
+            end
 
           nodes.each do |node|
             node.to_html.scan(regex) do
-              project = $~[:project] || current_project_path
+              project_path = full_project_path($~[:namespace], $~[:project])
               symbol = $~[object_sym]
-
-              refs[project] << symbol if object_class.reference_valid?(symbol)
+              refs[project_path] << symbol if object_class.reference_valid?(symbol)
             end
           end
 
@@ -228,37 +239,43 @@ module Banzai
       # path.
       def projects_per_reference
         @projects_per_reference ||= begin
-          hash = {}
           refs = Set.new
 
           references_per_project.each do |project_ref, _|
             refs << project_ref
           end
 
-          find_projects_for_paths(refs.to_a).each do |project|
-            hash[project.path_with_namespace] = project
-          end
-
-          hash
+          find_projects_for_paths(refs.to_a).index_by(&:full_path)
         end
       end
 
       def projects_relation_for_paths(paths)
-        Project.where_paths_in(paths).includes(:namespace)
+        Project.where_full_path_in(paths).includes(:namespace)
       end
 
       # Returns projects for the given paths.
       def find_projects_for_paths(paths)
         if RequestStore.active?
-          to_query = paths - project_refs_cache.keys
+          cache = project_refs_cache
+          to_query = paths - cache.keys
 
           unless to_query.empty?
-            projects_relation_for_paths(to_query).each do |project|
-              get_or_set_cache(project_refs_cache, project.path_with_namespace) { project }
+            projects = projects_relation_for_paths(to_query)
+
+            found = []
+            projects.each do |project|
+              ref = project.path_with_namespace
+              get_or_set_cache(cache, ref) { project }
+              found << ref
+            end
+
+            not_found = to_query - found
+            not_found.each do |ref|
+              get_or_set_cache(cache, ref) { nil }
             end
           end
 
-          project_refs_cache.slice(*paths).values
+          cache.slice(*paths).values.compact
         else
           projects_relation_for_paths(paths)
         end
@@ -268,8 +285,19 @@ module Banzai
         @current_project_path ||= project.path_with_namespace
       end
 
+      def current_project_namespace_path
+        @current_project_namespace_path ||= project.namespace.full_path
+      end
+
       private
 
+      def full_project_path(namespace, project_ref)
+        return current_project_path unless project_ref
+
+        namespace_ref = namespace || current_project_namespace_path
+        "#{namespace_ref}/#{project_ref}"
+      end
+
       def project_refs_cache
         RequestStore[:banzai_project_refs] ||= {}
       end
@@ -295,6 +323,14 @@ module Banzai
           value
         end
       end
+
+      # There might be special cases like filters
+      # that should ignore reference pattern
+      # eg: IssueReferenceFilter when using a external issues tracker
+      # In those cases this method should be overridden on the filter subclass
+      def uses_reference_pattern?
+        true
+      end
     end
   end
 end
diff --git a/lib/banzai/filter/ascii_doc_post_processing_filter.rb b/lib/banzai/filter/ascii_doc_post_processing_filter.rb
new file mode 100644
index 00000000000..c9fcf057c5f
--- /dev/null
+++ b/lib/banzai/filter/ascii_doc_post_processing_filter.rb
@@ -0,0 +1,13 @@
+module Banzai
+  module Filter
+    class AsciiDocPostProcessingFilter < HTML::Pipeline::Filter
+      def call
+        doc.search('[data-math-style]').each do |node|
+          node.set_attribute('class', 'code math js-render-math')
+        end
+
+        doc
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb
index 799b83b1069..b8d2673c1a6 100644
--- a/lib/banzai/filter/autolink_filter.rb
+++ b/lib/banzai/filter/autolink_filter.rb
@@ -37,7 +37,7 @@ module Banzai
         and contains(., '://')
         and not(starts-with(., 'http'))
         and not(starts-with(., 'ftp'))
-      ])
+      ]).freeze
 
       def call
         return doc if context[:autolink] == false
@@ -71,6 +71,14 @@ module Banzai
         @doc = parse_html(rinku)
       end
 
+      # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme
+      def contains_unsafe?(scheme)
+        return false unless scheme
+
+        scheme = scheme.strip.downcase
+        Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) }
+      end
+
       # Autolinks any text matching LINK_PATTERN that Rinku didn't already
       # replace
       def text_parse
@@ -89,17 +97,27 @@ module Banzai
         doc
       end
 
-      def autolink_filter(text)
-        text.gsub(LINK_PATTERN) do |match|
-          # Remove any trailing HTML entities and store them for appending
-          # outside the link element. The entity must be marked HTML safe in
-          # order to be output literally rather than escaped.
-          match.gsub!(/((?:&[\w#]+;)+)\z/, '')
-          dropped = ($1 || '').html_safe
-
-          options = link_options.merge(href: match)
-          content_tag(:a, match, options) + dropped
+      def autolink_match(match)
+        # start by stripping out dangerous links
+        begin
+          uri = Addressable::URI.parse(match)
+          return match if contains_unsafe?(uri.scheme)
+        rescue Addressable::URI::InvalidURIError
+          return match
         end
+
+        # Remove any trailing HTML entities and store them for appending
+        # outside the link element. The entity must be marked HTML safe in
+        # order to be output literally rather than escaped.
+        match.gsub!(/((?:&[\w#]+;)+)\z/, '')
+        dropped = ($1 || '').html_safe
+
+        options = link_options.merge(href: match)
+        content_tag(:a, match, options) + dropped
+      end
+
+      def autolink_filter(text)
+        text.gsub(LINK_PATTERN) { |match| autolink_match(match) }
       end
 
       def link_options
diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb
index 4358bf45549..eaacb9591b1 100644
--- a/lib/banzai/filter/commit_range_reference_filter.rb
+++ b/lib/banzai/filter/commit_range_reference_filter.rb
@@ -12,7 +12,7 @@ module Banzai
 
       def self.references_in(text, pattern = CommitRange.reference_pattern)
         text.gsub(pattern) do |match|
-          yield match, $~[:commit_range], $~[:project], $~
+          yield match, $~[:commit_range], $~[:project], $~[:namespace], $~
         end
       end
 
diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb
index a26dd09c25a..69c06117eda 100644
--- a/lib/banzai/filter/commit_reference_filter.rb
+++ b/lib/banzai/filter/commit_reference_filter.rb
@@ -12,7 +12,7 @@ module Banzai
 
       def self.references_in(text, pattern = Commit.reference_pattern)
         text.gsub(pattern) do |match|
-          yield match, $~[:commit], $~[:project], $~
+          yield match, $~[:commit], $~[:project], $~[:namespace], $~
         end
       end
 
diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb
index 2492b5213ac..6255a611dbe 100644
--- a/lib/banzai/filter/emoji_filter.rb
+++ b/lib/banzai/filter/emoji_filter.rb
@@ -1,6 +1,6 @@
 module Banzai
   module Filter
-    # HTML filter that replaces :emoji: with images.
+    # HTML filter that replaces :emoji: and unicode with images.
     #
     # Based on HTML::Pipeline::EmojiFilter
     #
@@ -13,63 +13,65 @@ module Banzai
       def call
         search_text_nodes(doc).each do |node|
           content = node.to_html
-          next unless content.include?(':')
           next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS)
 
-          html = emoji_image_filter(content)
+          next unless content.include?(':') || node.text.match(emoji_unicode_pattern)
+
+          html = emoji_unicode_element_unicode_filter(content)
+          html = emoji_name_element_unicode_filter(html)
 
           next if html == content
 
           node.replace(html)
         end
-
         doc
       end
 
-      # Replace :emoji: with corresponding images.
+      # Replace :emoji: with corresponding gl-emoji unicode.
       #
       # text - String text to replace :emoji: in.
       #
-      # Returns a String with :emoji: replaced with images.
-      def emoji_image_filter(text)
+      # Returns a String with :emoji: replaced with gl-emoji unicode.
+      def emoji_name_element_unicode_filter(text)
         text.gsub(emoji_pattern) do |match|
           name = $1
-          "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{emoji_url(name)}' height='20' width='20' align='absmiddle' />"
+          Gitlab::Emoji.gl_emoji_tag(name)
+        end
+      end
+
+      # Replace unicode emoji with corresponding gl-emoji unicode.
+      #
+      # text - String text to replace unicode emoji in.
+      #
+      # Returns a String with unicode emoji replaced with gl-emoji unicode.
+      def emoji_unicode_element_unicode_filter(text)
+        text.gsub(emoji_unicode_pattern) do |moji|
+          emoji_info = Gitlab::Emoji.emojis_by_moji[moji]
+          Gitlab::Emoji.gl_emoji_tag(emoji_info['name'])
         end
       end
 
       # Build a regexp that matches all valid :emoji: names.
       def self.emoji_pattern
-        @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/
+        @emoji_pattern ||=
+          /(?<=[^[:alnum:]:]|\n|^)
+          :(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):
+          (?=[^[:alnum:]:]|$)/x
       end
 
-      private
-
-      def emoji_url(name)
-        emoji_path = emoji_filename(name)
-
-        if context[:asset_host]
-          # Asset host is specified.
-          url_to_image(emoji_path)
-        elsif context[:asset_root]
-          # Gitlab url is specified
-          File.join(context[:asset_root], url_to_image(emoji_path))
-        else
-          # All other cases
-          url_to_image(emoji_path)
-        end
+      # Build a regexp that matches all valid unicode emojis names.
+      def self.emoji_unicode_pattern
+        @emoji_unicode_pattern ||= /(#{Gitlab::Emoji.emojis_unicodes.map { |moji| Regexp.escape(moji) }.join('|')})/
       end
 
-      def url_to_image(image)
-        ActionController::Base.helpers.url_to_image(image)
-      end
+      private
 
       def emoji_pattern
         self.class.emoji_pattern
       end
 
-      def emoji_filename(name)
-        "#{Gitlab::Emoji.emoji_filename(name)}.png"
+      def emoji_unicode_pattern
+        self.class.emoji_unicode_pattern
       end
     end
   end
diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb
index eaa702952cc..dce4de3ceaf 100644
--- a/lib/banzai/filter/external_issue_reference_filter.rb
+++ b/lib/banzai/filter/external_issue_reference_filter.rb
@@ -8,7 +8,7 @@ module Banzai
 
       # Public: Find `JIRA-123` issue references in text
       #
-      #   ExternalIssueReferenceFilter.references_in(text) do |match, issue|
+      #   ExternalIssueReferenceFilter.references_in(text, pattern) do |match, issue|
       #     "<a href=...>##{issue}</a>"
       #   end
       #
@@ -17,8 +17,8 @@ module Banzai
       # Yields the String match and the String issue reference.
       #
       # Returns a String replaced with the return of the block.
-      def self.references_in(text)
-        text.gsub(ExternalIssue.reference_pattern) do |match|
+      def self.references_in(text, pattern)
+        text.gsub(pattern) do |match|
           yield match, $~[:issue]
         end
       end
@@ -27,7 +27,7 @@ module Banzai
         # Early return if the project isn't using an external tracker
         return doc if project.nil? || default_issues_tracker?
 
-        ref_pattern = ExternalIssue.reference_pattern
+        ref_pattern = issue_reference_pattern
         ref_start_pattern = /\A#{ref_pattern}\z/
 
         each_node do |node|
@@ -37,10 +37,10 @@ module Banzai
             end
 
           elsif element_node?(node)
-            yield_valid_link(node) do |link, text|
+            yield_valid_link(node) do |link, inner_html|
               if link =~ ref_start_pattern
                 replace_link_node_with_href(node, link) do
-                  issue_link_filter(link, link_text: text)
+                  issue_link_filter(link, link_content: inner_html)
                 end
               end
             end
@@ -54,13 +54,14 @@ module Banzai
       # issue's details page.
       #
       # text - String text to replace references in.
+      # link_content - Original content of the link being replaced.
       #
       # Returns a String with `JIRA-123` references replaced with links. All
       # links have `gfm` and `gfm-issue` class names attached for styling.
-      def issue_link_filter(text, link_text: nil)
+      def issue_link_filter(text, link_content: nil)
         project = context[:project]
 
-        self.class.references_in(text) do |match, id|
+        self.class.references_in(text, issue_reference_pattern) do |match, id|
           ExternalIssue.new(id, project)
 
           url = url_for_issue(id, project, only_path: context[:only_path])
@@ -69,11 +70,11 @@ module Banzai
           klass = reference_class(:issue)
           data  = data_attribute(project: project.id, external_issue: id)
 
-          text = link_text || match
+          content = link_content || match
 
           %(<a href="#{url}" #{data}
                title="#{escape_once(title)}"
-               class="#{klass}">#{escape_once(text)}</a>)
+               class="#{klass}">#{content}</a>)
         end
       end
 
@@ -82,18 +83,21 @@ module Banzai
       end
 
       def default_issues_tracker?
-        if RequestStore.active?
-          default_issues_tracker_cache[project.id] ||=
-            project.default_issues_tracker?
-        else
-          project.default_issues_tracker?
-        end
+        external_issues_cached(:default_issues_tracker?)
+      end
+
+      def issue_reference_pattern
+        external_issues_cached(:issue_reference_pattern)
       end
 
       private
 
-      def default_issues_tracker_cache
-        RequestStore[:banzai_default_issues_tracker_cache] ||= {}
+      def external_issues_cached(attribute)
+        return project.public_send(attribute) unless RequestStore.active?
+
+        cached_attributes = RequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} }
+        cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil?
+        cached_attributes[project.id][attribute]
       end
     end
   end
diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb
index 0a29c547a4d..d6327ef31cb 100644
--- a/lib/banzai/filter/external_link_filter.rb
+++ b/lib/banzai/filter/external_link_filter.rb
@@ -2,11 +2,19 @@ module Banzai
   module Filter
     # HTML Filter to modify the attributes of external links
     class ExternalLinkFilter < HTML::Pipeline::Filter
+      SCHEMES = ['http', 'https', nil].freeze
+
       def call
-        # Skip non-HTTP(S) links and internal links
-        doc.xpath("descendant-or-self::a[starts-with(@href, 'http') and not(starts-with(@href, '#{internal_url}'))]").each do |node|
-          node.set_attribute('rel', 'nofollow noreferrer')
-          node.set_attribute('target', '_blank')
+        links.each do |node|
+          uri = uri(node['href'].to_s)
+          next unless uri
+
+          node.set_attribute('href', uri.to_s)
+
+          if SCHEMES.include?(uri.scheme) && external_url?(uri)
+            node.set_attribute('rel', 'nofollow noreferrer noopener')
+            node.set_attribute('target', '_blank')
+          end
         end
 
         doc
@@ -14,8 +22,26 @@ module Banzai
 
       private
 
+      def uri(href)
+        URI.parse(href)
+      rescue URI::Error
+        nil
+      end
+
+      def links
+        query = 'descendant-or-self::a[@href and not(@href = "")]'
+        doc.xpath(query)
+      end
+
+      def external_url?(uri)
+        # Relative URLs miss a hostname
+        return false unless uri.hostname
+
+        uri.hostname != internal_url.hostname
+      end
+
       def internal_url
-        @internal_url ||= Gitlab.config.gitlab.url
+        @internal_url ||= URI.parse(Gitlab.config.gitlab.url)
       end
     end
   end
diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb
index d08267a9d6c..0ea4eeaed5b 100644
--- a/lib/banzai/filter/gollum_tags_filter.rb
+++ b/lib/banzai/filter/gollum_tags_filter.rb
@@ -149,11 +149,12 @@ module Banzai
           name, reference = *parts.compact.map(&:strip)
         end
 
-        if url?(reference)
-          href = reference
-        else
-          href = ::File.join(project_wiki_base_path, reference)
-        end
+        href =
+          if url?(reference)
+            reference
+          else
+            ::File.join(project_wiki_base_path, reference)
+          end
 
         content_tag(:a, name || reference, href: href, class: 'gfm')
       end
diff --git a/lib/banzai/filter/html_entity_filter.rb b/lib/banzai/filter/html_entity_filter.rb
index 4ef8b3b6dcf..f3bd587c28b 100644
--- a/lib/banzai/filter/html_entity_filter.rb
+++ b/lib/banzai/filter/html_entity_filter.rb
@@ -3,9 +3,9 @@ require 'erb'
 module Banzai
   module Filter
     # Text filter that escapes these HTML entities: & " < >
-    class HTMLEntityFilter < HTML::Pipeline::TextFilter
+    class HtmlEntityFilter < HTML::Pipeline::TextFilter
       def call
-        ERB::Util.html_escape(text)
+        ERB::Util.html_escape_once(text)
       end
     end
   end
diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb
index f0fb6084a35..123c92fd250 100644
--- a/lib/banzai/filter/image_link_filter.rb
+++ b/lib/banzai/filter/image_link_filter.rb
@@ -2,29 +2,22 @@ module Banzai
   module Filter
     # HTML filter that wraps links around inline images.
     class ImageLinkFilter < HTML::Pipeline::Filter
-      
       # Find every image that isn't already wrapped in an `a` tag, create
       # a new node (a link to the image source), copy the image as a child
       # of the anchor, and then replace the img with the link-wrapped version.
       def call
         doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |img|
-          div = doc.document.create_element(
-            'div',
-            class: 'image-container'
-          )
-
           link = doc.document.create_element(
             'a',
             class: 'no-attachment-icon',
             href: img['src'],
-            target: '_blank'
+            target: '_blank',
+            rel: 'noopener noreferrer'
           )
 
           link.children = img.clone
 
-          div.children = link
-
-          img.replace(div)
+          img.replace(link)
         end
 
         doc
diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb
new file mode 100644
index 00000000000..327ea9449a1
--- /dev/null
+++ b/lib/banzai/filter/issuable_state_filter.rb
@@ -0,0 +1,37 @@
+module Banzai
+  module Filter
+    # HTML filter that appends state information to issuable links.
+    # Runs as a post-process filter as issuable state might change whilst
+    # Markdown is in the cache.
+    #
+    # This filter supports cross-project references.
+    class IssuableStateFilter < HTML::Pipeline::Filter
+      VISIBLE_STATES = %w(closed merged).freeze
+
+      def call
+        return doc unless context[:issuable_state_filter_enabled]
+
+        extractor = Banzai::IssuableExtractor.new(project, current_user)
+        issuables = extractor.extract([doc])
+
+        issuables.each do |node, issuable|
+          if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project)
+            node.content += " (#{issuable.state})"
+          end
+        end
+
+        doc
+      end
+
+      private
+
+      def current_user
+        context[:current_user]
+      end
+
+      def project
+        context[:project]
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
index 54c5f9a71a4..044d18ff824 100644
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ b/lib/banzai/filter/issue_reference_filter.rb
@@ -4,6 +4,10 @@ module Banzai
     # issues that do not exist are ignored.
     #
     # This filter supports cross-project references.
+    #
+    # When external issues tracker like Jira is activated we should not
+    # use issue reference pattern, but we should still be able
+    # to reference issues from other GitLab projects.
     class IssueReferenceFilter < AbstractReferenceFilter
       self.reference_type = :issue
 
@@ -11,6 +15,10 @@ module Banzai
         Issue
       end
 
+      def uses_reference_pattern?
+        context[:project].default_issues_tracker?
+      end
+
       def find_object(project, iid)
         issues_per_project[project][iid]
       end
@@ -31,11 +39,12 @@ module Banzai
           projects_per_reference.each do |path, project|
             issue_ids = references_per_project[path]
 
-            if project.default_issues_tracker?
-              issues = project.issues.where(iid: issue_ids.to_a)
-            else
-              issues = issue_ids.map { |id| ExternalIssue.new(id, project) }
-            end
+            issues =
+              if project.default_issues_tracker?
+                project.issues.where(iid: issue_ids.to_a)
+              else
+                issue_ids.map { |id| ExternalIssue.new(id, project) }
+              end
 
             issues.each do |issue|
               hash[project][issue.iid.to_i] = issue
@@ -54,7 +63,7 @@ module Banzai
         end
       end
 
-      def data_attributes_for(text, project, object)
+      def data_attributes_for(text, project, object, link: false)
         if object.is_a?(ExternalIssue)
           data_attribute(
             project: project.id,
diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb
index 8f262ef3d8d..a605dea149e 100644
--- a/lib/banzai/filter/label_reference_filter.rb
+++ b/lib/banzai/filter/label_reference_filter.rb
@@ -9,21 +9,23 @@ module Banzai
       end
 
       def find_object(project, id)
-        project.labels.find(id)
+        find_labels(project).find(id)
       end
 
       def self.references_in(text, pattern = Label.reference_pattern)
         unescape_html_entities(text).gsub(pattern) do |match|
-          yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~
+          yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~[:namespace], $~
         end
       end
 
       def references_in(text, pattern = Label.reference_pattern)
         unescape_html_entities(text).gsub(pattern) do |match|
-          label = find_label($~[:project], $~[:label_id], $~[:label_name])
+          namespace, project = $~[:namespace], $~[:project]
+          project_path = full_project_path(namespace, project)
+          label = find_label(project_path, $~[:label_id], $~[:label_name])
 
           if label
-            yield match, label.id, $~[:project], $~
+            yield match, label.id, project, namespace, $~
           else
             match
           end
@@ -35,7 +37,11 @@ module Banzai
         return unless project
 
         label_params = label_params(label_id, label_name)
-        project.labels.find_by(label_params)
+        find_labels(project).find_by(label_params)
+      end
+
+      def find_labels(project)
+        LabelsFinder.new(nil, project_id: project.id).execute(skip_authorization: true)
       end
 
       # Parameters to pass to `Label.find_by` based on the given arguments
@@ -60,11 +66,12 @@ module Banzai
       end
 
       def object_link_text(object, matches)
-        if context[:project] == object.project
-          LabelsHelper.render_colored_label(object)
-        else
-          LabelsHelper.render_colored_cross_project_label(object)
-        end
+        project_path     = full_project_path(matches[:namespace], matches[:project])
+        project_from_ref = project_from_ref_cached(project_path)
+        reference        = project_from_ref.to_human_reference(project)
+        label_suffix     = " <i>in #{reference}</i>" if reference.present?
+
+        LabelsHelper.render_colored_label(object, label_suffix)
       end
 
       def unescape_html_entities(text)
diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb
index ff580ec68f8..ee73fa91589 100644
--- a/lib/banzai/filter/markdown_filter.rb
+++ b/lib/banzai/filter/markdown_filter.rb
@@ -14,7 +14,7 @@ module Banzai
 
       def self.renderer
         @renderer ||= begin
-          renderer = Redcarpet::Render::HTML.new
+          renderer = Banzai::Renderer::HTML.new
           Redcarpet::Markdown.new(renderer, redcarpet_options)
         end
       end
diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb
new file mode 100644
index 00000000000..b6e784c886b
--- /dev/null
+++ b/lib/banzai/filter/math_filter.rb
@@ -0,0 +1,46 @@
+require 'uri'
+
+module Banzai
+  module Filter
+    # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$.
+    #
+    class MathFilter < HTML::Pipeline::Filter
+      # Attribute indicating inline or display math.
+      STYLE_ATTRIBUTE = 'data-math-style'.freeze
+
+      # Class used for tagging elements that should be rendered
+      TAG_CLASS = 'js-render-math'.freeze
+
+      INLINE_CLASSES = "code math #{TAG_CLASS}".freeze
+
+      DOLLAR_SIGN = '$'.freeze
+
+      def call
+        doc.css('code').each do |code|
+          closing = code.next
+          opening = code.previous
+
+          # We need a sibling before and after.
+          # They should end and start with $ respectively.
+          if closing && opening &&
+              closing.text? && opening.text? &&
+              closing.content.first == DOLLAR_SIGN &&
+              opening.content.last == DOLLAR_SIGN
+
+            code[:class] = INLINE_CLASSES
+            code[STYLE_ATTRIBUTE] = 'inline'
+            closing.content = closing.content[1..-1]
+            opening.content = opening.content[0..-2]
+          end
+        end
+
+        doc.css('pre.code.math').each do |el|
+          el[STYLE_ATTRIBUTE] = 'display'
+          el[:class] += " #{TAG_CLASS}"
+        end
+
+        doc
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb
index ac5216d9cfb..3888acf935e 100644
--- a/lib/banzai/filter/merge_request_reference_filter.rb
+++ b/lib/banzai/filter/merge_request_reference_filter.rb
@@ -11,8 +11,8 @@ module Banzai
         MergeRequest
       end
 
-      def find_object(project, id)
-        project.merge_requests.find_by(iid: id)
+      def find_object(project, iid)
+        merge_requests_per_project[project][iid]
       end
 
       def url_for_object(mr, project)
@@ -21,6 +21,31 @@ module Banzai
                                             only_path: context[:only_path])
       end
 
+      def project_from_ref(ref)
+        projects_per_reference[ref || current_project_path]
+      end
+
+      # Returns a Hash containing the merge_requests per Project instance.
+      def merge_requests_per_project
+        @merge_requests_per_project ||= begin
+          hash = Hash.new { |h, k| h[k] = {} }
+
+          projects_per_reference.each do |path, project|
+            merge_request_ids = references_per_project[path]
+
+            merge_requests = project.merge_requests
+              .where(iid: merge_request_ids.to_a)
+              .includes(target_project: :namespace)
+
+            merge_requests.each do |merge_request|
+              hash[project][merge_request.iid.to_i] = merge_request
+            end
+          end
+
+          hash
+        end
+      end
+
       def object_link_text_extras(object, matches)
         extras = super
 
diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb
index 58fff496d00..f12014e191f 100644
--- a/lib/banzai/filter/milestone_reference_filter.rb
+++ b/lib/banzai/filter/milestone_reference_filter.rb
@@ -19,18 +19,20 @@ module Banzai
         return super(text, pattern) if pattern != Milestone.reference_pattern
 
         text.gsub(pattern) do |match|
-          milestone = find_milestone($~[:project], $~[:milestone_iid], $~[:milestone_name])
+          milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name])
 
           if milestone
-            yield match, milestone.iid, $~[:project], $~
+            yield match, milestone.iid, $~[:project], $~[:namespace], $~
           else
             match
           end
         end
       end
 
-      def find_milestone(project_ref, milestone_id, milestone_name)
-        project = project_from_ref(project_ref)
+      def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name)
+        project_path = full_project_path(namespace_ref, project_ref)
+        project = project_from_ref(project_path)
+
         return unless project
 
         milestone_params = milestone_params(milestone_id, milestone_name)
@@ -52,11 +54,13 @@ module Banzai
       end
 
       def object_link_text(object, matches)
-        if context[:project] == object.project
-          super
+        milestone_link = escape_once(super)
+        reference = object.project.to_reference(project)
+
+        if reference.present?
+          "#{milestone_link} <i>in #{reference}</i>".html_safe
         else
-          "#{escape_once(super)} <i>in #{escape_once(object.project.path_with_namespace)}</i>".
-            html_safe
+          milestone_link
         end
       end
 
diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb
new file mode 100644
index 00000000000..5325819d828
--- /dev/null
+++ b/lib/banzai/filter/plantuml_filter.rb
@@ -0,0 +1,39 @@
+require "nokogiri"
+require "asciidoctor-plantuml/plantuml"
+
+module Banzai
+  module Filter
+    # HTML that replaces all `code plantuml` tags with PlantUML img tags.
+    #
+    class PlantumlFilter < HTML::Pipeline::Filter
+      def call
+        return doc unless doc.at('pre > code[lang="plantuml"]') && settings.plantuml_enabled
+
+        plantuml_setup
+
+        doc.css('pre > code[lang="plantuml"]').each do |node|
+          img_tag = Nokogiri::HTML::DocumentFragment.parse(
+            Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {}))
+          node.parent.replace(img_tag)
+        end
+
+        doc
+      end
+
+      private
+
+      def settings
+        ApplicationSetting.current || ApplicationSetting.create_from_defaults
+      end
+
+      def plantuml_setup
+        Asciidoctor::PlantUml.configure do |conf|
+          conf.url = settings.plantuml_url
+          conf.png_enable = settings.plantuml_enabled
+          conf.svg_enable = false
+          conf.txt_enable = false
+        end
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb
index c59a80dd1c7..9f9882b3b40 100644
--- a/lib/banzai/filter/redactor_filter.rb
+++ b/lib/banzai/filter/redactor_filter.rb
@@ -7,7 +7,7 @@ module Banzai
     #
     class RedactorFilter < HTML::Pipeline::Filter
       def call
-        Redactor.new(project, current_user).redact([doc])
+        Redactor.new(project, current_user).redact([doc]) unless context[:skip_redaction]
 
         doc
       end
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 2d221290f7e..6640168bfa2 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -20,10 +20,10 @@ module Banzai
       # Examples:
       #
       #   data_attribute(project: 1, issue: 2)
-      #   # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\""
+      #   # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\""
       #
       #   data_attribute(project: 3, merge_request: 4)
-      #   # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\""
+      #   # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\""
       #
       # Returns a String
       def data_attribute(attributes = {})
@@ -31,7 +31,9 @@ module Banzai
 
         attributes[:reference_type] ||= self.class.reference_type
         attributes.delete(:original) if context[:no_original_data]
-        attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ")
+        attributes.map do |key, value|
+          %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}")
+        end.join(' ')
       end
 
       def escape_once(html)
@@ -51,6 +53,10 @@ module Banzai
         context[:project]
       end
 
+      def skip_project_check?
+        context[:skip_project_check]
+      end
+
       def reference_class(type)
         "gfm gfm-#{type} has-tooltip"
       end
@@ -85,14 +91,14 @@ module Banzai
         @nodes ||= each_node.to_a
       end
 
-      # Yields the link's URL and text whenever the node is a valid <a> tag.
+      # Yields the link's URL and inner HTML whenever the node is a valid <a> tag.
       def yield_valid_link(node)
         link = CGI.unescape(node.attr('href').to_s)
-        text = node.text
+        inner_html = node.inner_html
 
         return unless link.force_encoding('UTF-8').valid_encoding?
 
-        yield link, text
+        yield link, inner_html
       end
 
       def replace_text_when_pattern_matches(node, pattern)
diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb
index 4fa8d05481f..9e23c8f8c55 100644
--- a/lib/banzai/filter/relative_link_filter.rb
+++ b/lib/banzai/filter/relative_link_filter.rb
@@ -46,21 +46,23 @@ module Banzai
       end
 
       def rebuild_relative_uri(uri)
-        file_path = relative_file_path(uri.path)
+        file_path = relative_file_path(uri)
 
         uri.path = [
           relative_url_root,
           context[:project].path_with_namespace,
           uri_type(file_path),
-          ref,
-          file_path
+          Addressable::URI.escape(ref),
+          Addressable::URI.escape(file_path)
         ].compact.join('/').squeeze('/').chomp('/')
 
         uri
       end
 
-      def relative_file_path(path)
-        nested_path = build_relative_path(path, context[:requested_path])
+      def relative_file_path(uri)
+        path = Addressable::URI.unescape(uri.path)
+        request_path = Addressable::URI.unescape(context[:requested_path])
+        nested_path = build_relative_path(path, request_path)
         file_exists?(nested_path) ? nested_path : path
       end
 
@@ -108,11 +110,7 @@ module Banzai
       end
 
       def uri_type(path)
-        @uri_types[path] ||= begin
-          unescaped_path = Addressable::URI.unescape(path)
-
-          current_commit.uri_type(unescaped_path)
-        end
+        @uri_types[path] ||= current_commit.uri_type(path)
       end
 
       def current_commit
diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb
index af1e575fc89..2d6e8ffc90f 100644
--- a/lib/banzai/filter/sanitization_filter.rb
+++ b/lib/banzai/filter/sanitization_filter.rb
@@ -24,10 +24,6 @@ module Banzai
         # Only push these customizations once
         return if customized?(whitelist[:transformers])
 
-        # Allow code highlighting
-        whitelist[:attributes]['pre'] = %w(class v-pre)
-        whitelist[:attributes]['span'] = %w(class)
-
         # Allow table alignment
         whitelist[:attributes]['th'] = %w(style)
         whitelist[:attributes]['td'] = %w(style)
@@ -35,6 +31,14 @@ module Banzai
         # Allow span elements
         whitelist[:elements].push('span')
 
+        # Allow data-math-style attribute in order to support LaTeX formatting
+        whitelist[:attributes]['code'] = %w(data-math-style)
+        whitelist[:attributes]['pre'] = %w(data-math-style)
+
+        # Allow html5 details/summary elements
+        whitelist[:elements].push('details')
+        whitelist[:elements].push('summary')
+
         # Allow abbr elements with title attribute
         whitelist[:elements].push('abbr')
         whitelist[:attributes]['abbr'] = %w(title)
@@ -48,9 +52,6 @@ module Banzai
         # Remove `rel` attribute from `a` elements
         whitelist[:transformers].push(self.class.remove_rel)
 
-        # Remove `class` attribute from non-highlight spans
-        whitelist[:transformers].push(self.class.clean_spans)
-
         whitelist
       end
 
@@ -80,21 +81,6 @@ module Banzai
             end
           end
         end
-
-        def clean_spans
-          lambda do |env|
-            node = env[:node]
-
-            return unless node.name == 'span'
-            return unless node.has_attribute?('class')
-
-            unless node.ancestors.any? { |n| n.name.casecmp('pre').zero? }
-              node.remove_attribute('class')
-            end
-
-            { node_whitelist: [node] }
-          end
-        end
       end
     end
   end
diff --git a/lib/banzai/filter/set_direction_filter.rb b/lib/banzai/filter/set_direction_filter.rb
new file mode 100644
index 00000000000..c2976aeb7c6
--- /dev/null
+++ b/lib/banzai/filter/set_direction_filter.rb
@@ -0,0 +1,15 @@
+module Banzai
+  module Filter
+    # HTML filter that sets dir="auto" for RTL languages support
+    class SetDirectionFilter < HTML::Pipeline::Filter
+      def call
+        # select these elements just on top level of the document
+        doc.xpath('p|h1|h2|h3|h4|h5|h6|ol|ul[not(@class="section-nav")]|blockquote|table').each do |el|
+          el['dir'] = 'auto'
+        end
+
+        doc
+      end
+    end
+  end
+end
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index 026b81ac175..7da565043d1 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -5,8 +5,6 @@ module Banzai
     # HTML Filter to highlight fenced code blocks
     #
     class SyntaxHighlightFilter < HTML::Pipeline::Filter
-      include Rouge::Plugins::Redcarpet
-
       def call
         doc.search('pre > code').each do |node|
           highlight_node(node)
@@ -16,21 +14,23 @@ module Banzai
       end
 
       def highlight_node(node)
-        language = node.attr('class')
+        language = node.attr('lang')
         code = node.text
         css_classes = "code highlight"
         lexer = lexer_for(language)
+        lang = lexer.tag
 
         begin
-          code = format(lex(lexer, code))
+          code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code), tag: lang)
 
-          css_classes << " js-syntax-highlight #{lexer.tag}"
+          css_classes << " js-syntax-highlight #{lang}"
         rescue
+          lang = nil
           # Gracefully handle syntax highlighter bugs/errors to ensure
           # users can still access an issue/comment/etc.
         end
 
-        highlighted = %(<pre class="#{css_classes}" v-pre="true"><code>#{code}</code></pre>)
+        highlighted = %(<pre class="#{css_classes}" lang="#{lang}" v-pre="true"><code>#{code}</code></pre>)
 
         # Extracted to a method to measure it
         replace_parent_pre_element(node, highlighted)
@@ -43,10 +43,6 @@ module Banzai
         lexer.lex(code)
       end
 
-      def format(tokens)
-        rouge_formatter.format(tokens)
-      end
-
       def lexer_for(language)
         (Rouge::Lexer.find(language) || Rouge::Lexers::PlainText).new
       end
@@ -55,11 +51,6 @@ module Banzai
         # Replace the parent `pre` element with the entire highlighted block
         node.parent.replace(highlighted)
       end
-
-      # Override Rouge::Plugins::Redcarpet#rouge_formatter
-      def rouge_formatter(lexer = nil)
-        @rouge_formatter ||= Rouge::Formatters::HTML.new
-      end
     end
   end
 end
diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb
index a4eda6fdf76..8e7084f2543 100644
--- a/lib/banzai/filter/table_of_contents_filter.rb
+++ b/lib/banzai/filter/table_of_contents_filter.rb
@@ -35,9 +35,11 @@ module Banzai
           headers[id] += 1
 
           if header_content = node.children.first
+            # namespace detection will be automatically handled via javascript (see issue #22781)
+            namespace = "user-content-"
             href = "#{id}#{uniq}"
             push_toc(href, text)
-            header_content.add_previous_sibling(anchor_tag(href))
+            header_content.add_previous_sibling(anchor_tag("#{namespace}#{href}", href))
           end
         end
 
@@ -48,8 +50,8 @@ module Banzai
 
       private
 
-      def anchor_tag(href)
-        %Q{<a id="#{href}" class="anchor" href="##{href}" aria-hidden="true"></a>}
+      def anchor_tag(id, href)
+        %Q{<a id="#{id}" class="anchor" href="##{href}" aria-hidden="true"></a>}
       end
 
       def push_toc(href, text)
diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb
index c6302b586d3..a798927823f 100644
--- a/lib/banzai/filter/user_reference_filter.rb
+++ b/lib/banzai/filter/user_reference_filter.rb
@@ -24,7 +24,7 @@ module Banzai
       end
 
       def call
-        return doc if project.nil?
+        return doc if project.nil? && !skip_project_check?
 
         ref_pattern = User.reference_pattern
         ref_pattern_start = /\A#{ref_pattern}\z/
@@ -35,10 +35,10 @@ module Banzai
               user_link_filter(content)
             end
           elsif element_node?(node)
-            yield_valid_link(node) do |link, text|
+            yield_valid_link(node) do |link, inner_html|
               if link =~ ref_pattern_start
                 replace_link_node_with_href(node, link) do
-                  user_link_filter(link, link_text: text)
+                  user_link_filter(link, link_content: inner_html)
                 end
               end
             end
@@ -52,15 +52,16 @@ module Banzai
       # user's profile page.
       #
       # text - String text to replace references in.
+      # link_content - Original content of the link being replaced.
       #
       # Returns a String with `@user` references replaced with links. All links
       # have `gfm` and `gfm-project_member` class names attached for styling.
-      def user_link_filter(text, link_text: nil)
+      def user_link_filter(text, link_content: nil)
         self.class.references_in(text) do |match, username|
-          if username == 'all'
-            link_to_all(link_text: link_text)
-          elsif namespace = namespaces[username]
-            link_to_namespace(namespace, link_text: link_text) || match
+          if username == 'all' && !skip_project_check?
+            link_to_all(link_content: link_content)
+          elsif namespace = namespaces[username.downcase]
+            link_to_namespace(namespace, link_content: link_content) || match
           else
             match
           end
@@ -73,10 +74,7 @@ module Banzai
       # The keys of this Hash are the namespace paths, the values the
       # corresponding Namespace objects.
       def namespaces
-        @namespaces ||=
-          Namespace.where(path: usernames).each_with_object({}) do |row, hash|
-            hash[row.path] = row
-          end
+        @namespaces ||= Namespace.where_full_path_in(usernames).index_by(&:full_path).transform_keys(&:downcase)
       end
 
       # Returns all usernames referenced in the current document.
@@ -102,49 +100,49 @@ module Banzai
         reference_class(:project_member)
       end
 
-      def link_to_all(link_text: nil)
+      def link_to_all(link_content: nil)
         project = context[:project]
         author = context[:author]
 
         if author && !project.team.member?(author)
-          link_text
+          link_content
         else
           url = urls.namespace_project_url(project.namespace, project,
                                            only_path: context[:only_path])
 
           data = data_attribute(project: project.id, author: author.try(:id))
-          text = link_text || User.reference_prefix + 'all'
+          content = link_content || User.reference_prefix + 'all'
 
-          link_tag(url, data, text, 'All Project and Group Members')
+          link_tag(url, data, content, 'All Project and Group Members')
         end
       end
 
-      def link_to_namespace(namespace, link_text: nil)
+      def link_to_namespace(namespace, link_content: nil)
         if namespace.is_a?(Group)
-          link_to_group(namespace.path, namespace, link_text: link_text)
+          link_to_group(namespace.full_path, namespace, link_content: link_content)
         else
-          link_to_user(namespace.path, namespace, link_text: link_text)
+          link_to_user(namespace.path, namespace, link_content: link_content)
         end
       end
 
-      def link_to_group(group, namespace, link_text: nil)
+      def link_to_group(group, namespace, link_content: nil)
         url = urls.group_url(group, only_path: context[:only_path])
         data = data_attribute(group: namespace.id)
-        text = link_text || Group.reference_prefix + group
+        content = link_content || Group.reference_prefix + group
 
-        link_tag(url, data, text, namespace.name)
+        link_tag(url, data, content, namespace.full_name)
       end
 
-      def link_to_user(user, namespace, link_text: nil)
+      def link_to_user(user, namespace, link_content: nil)
         url = urls.user_url(user, only_path: context[:only_path])
         data = data_attribute(user: namespace.owner_id)
-        text = link_text || User.reference_prefix + user
+        content = link_content || User.reference_prefix + user
 
-        link_tag(url, data, text, namespace.owner_name)
+        link_tag(url, data, content, namespace.owner_name)
       end
 
-      def link_tag(url, data, text, title)
-        %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{escape_once(text)}</a>)
+      def link_tag(url, data, link_content, title)
+        %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>)
       end
     end
   end
diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb
index ac7bbcb0d10..35cb10eae5d 100644
--- a/lib/banzai/filter/video_link_filter.rb
+++ b/lib/banzai/filter/video_link_filter.rb
@@ -35,13 +35,15 @@ module Banzai
           src: element['src'],
           width: '400',
           controls: true,
-          'data-setup' => '{}')
+          'data-setup' => '{}',
+          'data-title' => element['title'] || element['alt'])
 
         link = doc.document.create_element(
           'a',
           element['title'] || element['alt'],
           href: element['src'],
           target: '_blank',
+          rel: 'noopener noreferrer',
           title: "Download '#{element['title'] || element['alt']}'")
         download_paragraph = doc.document.create_element('p')
         download_paragraph.children = link
diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb
new file mode 100644
index 00000000000..cbabf9156de
--- /dev/null
+++ b/lib/banzai/issuable_extractor.rb
@@ -0,0 +1,40 @@
+module Banzai
+  # Extract references to issuables from multiple documents
+
+  # This populates RequestStore cache used in Banzai::ReferenceParser::IssueParser
+  # and Banzai::ReferenceParser::MergeRequestParser
+  # Populating the cache should happen before processing documents one-by-one
+  # so we can avoid N+1 queries problem
+
+  class IssuableExtractor
+    QUERY = %q(
+      descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")]
+      [@data-reference-type="issue" or @data-reference-type="merge_request"]
+    ).freeze
+
+    attr_reader :project, :user
+
+    def initialize(project, user)
+      @project = project
+      @user = user
+    end
+
+    # Returns Hash in the form { node => issuable_instance }
+    def extract(documents)
+      nodes = documents.flat_map do |document|
+        document.xpath(QUERY)
+      end
+
+      issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user)
+      merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user)
+
+      issuables_for_nodes = issue_parser.issues_for_nodes(nodes).merge(
+        merge_request_parser.merge_requests_for_nodes(nodes)
+      )
+
+      # The project for the issue/MR might be pending for deletion!
+      # Filter them out because we don't care about them.
+      issuables_for_nodes.select { |node, issuable| issuable.project }
+    end
+  end
+end
diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb
index 9f8eb0931b8..002a3341ccd 100644
--- a/lib/banzai/object_renderer.rb
+++ b/lib/banzai/object_renderer.rb
@@ -31,7 +31,8 @@ module Banzai
     #
     # Returns the same input objects.
     def render(objects, attribute)
-      documents = render_objects(objects, attribute)
+      documents = render_documents(objects, attribute)
+      documents = post_process_documents(documents, objects, attribute)
       redacted = redact_documents(documents)
 
       objects.each_with_index do |object, index|
@@ -41,9 +42,24 @@ module Banzai
       end
     end
 
-    # Renders the attribute of every given object.
-    def render_objects(objects, attribute)
-      render_attributes(objects, attribute)
+    private
+
+    def render_documents(objects, attribute)
+      pipeline = HTML::Pipeline.new([])
+
+      objects.map do |object|
+        pipeline.to_document(Banzai.render_field(object, attribute))
+      end
+    end
+
+    def post_process_documents(documents, objects, attribute)
+      # Called here to populate cache, refer to IssuableExtractor docs
+      IssuableExtractor.new(project, user).extract(documents)
+
+      documents.zip(objects).map do |document, object|
+        context = context_for(object, attribute)
+        Banzai::Pipeline[:post_process].to_document(document, context)
+      end
     end
 
     # Redacts the list of documents.
@@ -57,25 +73,15 @@ module Banzai
 
     # Returns a Banzai context for the given object and attribute.
     def context_for(object, attribute)
-      context = base_context.dup
-      context = context.merge(object.banzai_render_context(attribute))
-      context
-    end
-
-    # Renders the attributes of a set of objects.
-    #
-    # Returns an Array of `Nokogiri::HTML::Document`.
-    def render_attributes(objects, attribute)
-      objects.map do |object|
-        string = Banzai.render_field(object, attribute)
-        context = context_for(object, attribute)
-
-        Banzai::Pipeline[:relative_link].to_document(string, context)
-      end
+      base_context.merge(object.banzai_render_context(attribute))
     end
 
     def base_context
-      @base_context ||= @redaction_context.merge(current_user: user, project: project)
+      @base_context ||= @redaction_context.merge(
+        current_user: user,
+        project: project,
+        skip_redaction: true
+      )
     end
   end
 end
diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb
new file mode 100644
index 00000000000..1048b927cd3
--- /dev/null
+++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb
@@ -0,0 +1,14 @@
+module Banzai
+  module Pipeline
+    class AsciiDocPipeline < BasePipeline
+      def self.filters
+        FilterArray[
+          Filter::SanitizationFilter,
+          Filter::ExternalLinkFilter,
+          Filter::PlantumlFilter,
+          Filter::AsciiDocPostProcessingFilter
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index 8d94b199c66..bd4d1aa9ff8 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -1,11 +1,19 @@
 module Banzai
   module Pipeline
     class GfmPipeline < BasePipeline
+      # These filters convert GitLab Flavored Markdown (GFM) to HTML.
+      # The handlers defined in app/assets/javascripts/copy_as_gfm.js
+      # consequently convert that same HTML to GFM to be copied to the clipboard.
+      # Every filter that generates HTML from GFM should have a handler in
+      # app/assets/javascripts/copy_as_gfm.js, in reverse order.
+      # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb.
       def self.filters
         @filters ||= FilterArray[
-          Filter::SyntaxHighlightFilter,
+          Filter::PlantumlFilter,
           Filter::SanitizationFilter,
+          Filter::SyntaxHighlightFilter,
 
+          Filter::MathFilter,
           Filter::UploadLinkFilter,
           Filter::VideoLinkFilter,
           Filter::ImageLinkFilter,
@@ -25,7 +33,9 @@ module Banzai
           Filter::MilestoneReferenceFilter,
 
           Filter::TaskListFilter,
-          Filter::InlineDiffFilter
+          Filter::InlineDiffFilter,
+
+          Filter::SetDirectionFilter
         ]
       end
 
diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb
new file mode 100644
index 00000000000..c56d908009f
--- /dev/null
+++ b/lib/banzai/pipeline/markup_pipeline.rb
@@ -0,0 +1,13 @@
+module Banzai
+  module Pipeline
+    class MarkupPipeline < BasePipeline
+      def self.filters
+        @filters ||= FilterArray[
+          Filter::SanitizationFilter,
+          Filter::ExternalLinkFilter,
+          Filter::PlantumlFilter
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb
index ecff094b1e5..131ac3b0eec 100644
--- a/lib/banzai/pipeline/post_process_pipeline.rb
+++ b/lib/banzai/pipeline/post_process_pipeline.rb
@@ -4,6 +4,7 @@ module Banzai
       def self.filters
         FilterArray[
           Filter::RelativeLinkFilter,
+          Filter::IssuableStateFilter,
           Filter::RedactorFilter
         ]
       end
diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb
index 30bc035d085..1929099931b 100644
--- a/lib/banzai/pipeline/single_line_pipeline.rb
+++ b/lib/banzai/pipeline/single_line_pipeline.rb
@@ -3,7 +3,7 @@ module Banzai
     class SingleLinePipeline < GfmPipeline
       def self.filters
         @filters ||= FilterArray[
-          Filter::HTMLEntityFilter,
+          Filter::HtmlEntityFilter,
           Filter::SanitizationFilter,
 
           Filter::EmojiFilter,
diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb
index 1e1b51e683e..fb2faae02bc 100644
--- a/lib/banzai/querying.rb
+++ b/lib/banzai/querying.rb
@@ -1,18 +1,64 @@
 module Banzai
   module Querying
+    module_function
+
     # Searches a Nokogiri document using a CSS query, optionally optimizing it
     # whenever possible.
     #
-    # document - A document/element to search.
-    # query    - The CSS query to use.
+    # document          - A document/element to search.
+    # query             - The CSS query to use.
+    # reference_options - A hash with nodes filter options
     #
-    # Returns a Nokogiri::XML::NodeSet.
-    def self.css(document, query)
+    # Returns an array of Nokogiri::XML::Element objects if location is specified
+    # in reference_options. Otherwise it would a Nokogiri::XML::NodeSet.
+    def css(document, query, reference_options = {})
       # When using "a.foo" Nokogiri compiles this to "//a[...]" but
       # "descendant::a[...]" is quite a bit faster and achieves the same result.
       xpath = Nokogiri::CSS.xpath_for(query)[0].gsub(%r{^//}, 'descendant::')
+      xpath = restrict_to_p_nodes_at_root(xpath) if filter_nodes_at_beginning?(reference_options)
+      nodes = document.xpath(xpath)
+
+      filter_nodes(nodes, reference_options)
+    end
+
+    def restrict_to_p_nodes_at_root(xpath)
+      xpath.gsub('descendant::', './p/')
+    end
+
+    def filter_nodes(nodes, reference_options)
+      if filter_nodes_at_beginning?(reference_options)
+        filter_nodes_at_beginning(nodes)
+      else
+        nodes
+      end
+    end
+
+    def filter_nodes_at_beginning?(reference_options)
+      reference_options && reference_options[:location] == :beginning
+    end
+
+    # Selects child nodes if they are present in the beginning among other siblings.
+    #
+    # nodes - A Nokogiri::XML::NodeSet.
+    #
+    # Returns an array of Nokogiri::XML::Element objects.
+    def filter_nodes_at_beginning(nodes)
+      parents_and_nodes = nodes.group_by(&:parent)
+      filtered_nodes = []
+
+      parents_and_nodes.each do |parent, nodes|
+        children = parent.children
+        nodes    = nodes.to_a
+
+        children.each do |child|
+          next if child.text.blank?
+          node = nodes.shift
+          break unless node == child
+          filtered_nodes << node
+        end
+      end
 
-      document.xpath(xpath)
+      filtered_nodes
     end
   end
 end
diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb
index 0df3a72d1c4..de3ebe72720 100644
--- a/lib/banzai/redactor.rb
+++ b/lib/banzai/redactor.rb
@@ -41,10 +41,10 @@ module Banzai
           next if visible.include?(node)
 
           doc_data[:visible_reference_count] -= 1
-          # The reference should be replaced by the original text,
-          # which is not always the same as the rendered text.
-          text = node.attr('data-original') || node.text
-          node.replace(text)
+          # The reference should be replaced by the original link's content,
+          # which is not always the same as the rendered one.
+          content = node.attr('data-original') || node.inner_html
+          node.replace(content)
         end
       end
 
diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb
index b26a41a1f3b..7e6357f8a00 100644
--- a/lib/banzai/reference_extractor.rb
+++ b/lib/banzai/reference_extractor.rb
@@ -10,12 +10,17 @@ module Banzai
     end
 
     def references(type, project, current_user = nil)
-      processor = Banzai::ReferenceParser[type].
-        new(project, current_user)
+      processor = Banzai::ReferenceParser[type]
+        .new(project, current_user)
 
       processor.process(html_documents)
     end
 
+    def reset_memoized_values
+      @html_documents     = nil
+      @texts_and_contexts = []
+    end
+
     private
 
     def html_documents
diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb
index f5d110e987b..279fca8d043 100644
--- a/lib/banzai/reference_parser/base_parser.rb
+++ b/lib/banzai/reference_parser/base_parser.rb
@@ -33,7 +33,7 @@ module Banzai
     # they have access to.
     class BaseParser
       class << self
-        attr_accessor :reference_type
+        attr_accessor :reference_type, :reference_options
       end
 
       # Returns the attribute name containing the value for every object to be
@@ -62,13 +62,7 @@ module Banzai
 
         nodes.select do |node|
           if node.has_attribute?(project_attr)
-            node_id = node.attr(project_attr).to_i
-
-            if project && project.id == node_id
-              true
-            else
-              can?(user, :read_project, projects[node_id])
-            end
+            can_read_reference?(user, projects[node], node)
           else
             true
           end
@@ -117,12 +111,12 @@ module Banzai
         per_project
       end
 
-      # Returns a Hash containing objects for an attribute grouped per their
-      # IDs.
+      # Returns a Hash containing objects for an attribute grouped per the
+      # nodes that reference them.
       #
       # The returned Hash uses the following format:
       #
-      #     { id value => row }
+      #     { node => row }
       #
       # nodes - An Array of HTML nodes to process.
       #
@@ -137,10 +131,14 @@ module Banzai
         return {} if nodes.empty?
 
         ids = unique_attribute_values(nodes, attribute)
-        rows = collection_objects_for_ids(collection, ids)
+        collection_objects = collection_objects_for_ids(collection, ids)
+        objects_by_id = collection_objects.index_by(&:id)
 
-        rows.each_with_object({}) do |row, hash|
-          hash[row.id] = row
+        nodes.each_with_object({}) do |node, hash|
+          if node.has_attribute?(attribute)
+            obj = objects_by_id[node.attr(attribute).to_i]
+            hash[node] = obj if obj
+          end
         end
       end
 
@@ -165,14 +163,15 @@ module Banzai
       # been queried the object is returned from the cache.
       def collection_objects_for_ids(collection, ids)
         if RequestStore.active?
+          ids = ids.map(&:to_i)
           cache = collection_cache[collection_cache_key(collection)]
-          to_query = ids.map(&:to_i) - cache.keys
+          to_query = ids - cache.keys
 
           unless to_query.empty?
             collection.where(id: to_query).each { |row| cache[row.id] = row }
           end
 
-          cache.values
+          cache.values_at(*ids).compact
         else
           collection.where(id: ids)
         end
@@ -187,9 +186,10 @@ module Banzai
       # the references.
       def process(documents)
         type = self.class.reference_type
+        reference_options = self.class.reference_options
 
         nodes = documents.flat_map do |document|
-          Querying.css(document, "a[data-reference-type='#{type}'].gfm").to_a
+          Querying.css(document, "a[data-reference-type='#{type}'].gfm", reference_options).to_a
         end
 
         gather_references(nodes)
@@ -207,14 +207,14 @@ module Banzai
       #
       # The returned Hash uses the following format:
       #
-      #     { project ID => project }
+      #     { node => project }
       #
       def projects_for_nodes(nodes)
         @projects_for_nodes ||=
           grouped_objects_for_nodes(nodes, Project, 'data-project')
       end
 
-      def can?(user, permission, subject)
+      def can?(user, permission, subject = :global)
         Ability.allowed?(user, permission, subject)
       end
 
@@ -226,6 +226,15 @@ module Banzai
 
       attr_reader :current_user, :project
 
+      # When a feature is disabled or visible only for
+      # team members we should not allow team members
+      # see reference comments.
+      # Override this method on subclasses
+      # to check if user can read resource
+      def can_read_reference?(user, ref_project, node)
+        raise NotImplementedError
+      end
+
       def lazy(&block)
         Gitlab::Lazy.new(&block)
       end
diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb
index 0fee9d267de..30dc87248b4 100644
--- a/lib/banzai/reference_parser/commit_parser.rb
+++ b/lib/banzai/reference_parser/commit_parser.rb
@@ -29,6 +29,12 @@ module Banzai
 
         commits
       end
+
+      private
+
+      def can_read_reference?(user, ref_project, node)
+        can?(user, :download_code, ref_project)
+      end
     end
   end
 end
diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb
index 69d01f8db15..a50e6f8ef8f 100644
--- a/lib/banzai/reference_parser/commit_range_parser.rb
+++ b/lib/banzai/reference_parser/commit_range_parser.rb
@@ -33,6 +33,12 @@ module Banzai
 
         range.valid_commits? ? range : nil
       end
+
+      private
+
+      def can_read_reference?(user, ref_project, node)
+        can?(user, :download_code, ref_project)
+      end
     end
   end
 end
diff --git a/lib/banzai/reference_parser/directly_addressed_user_parser.rb b/lib/banzai/reference_parser/directly_addressed_user_parser.rb
new file mode 100644
index 00000000000..77df9bbd024
--- /dev/null
+++ b/lib/banzai/reference_parser/directly_addressed_user_parser.rb
@@ -0,0 +1,8 @@
+module Banzai
+  module ReferenceParser
+    class DirectlyAddressedUserParser < UserParser
+      self.reference_type = :user
+      self.reference_options = { location: :beginning }
+    end
+  end
+end
diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb
index a1264db2111..6307c1b571a 100644
--- a/lib/banzai/reference_parser/external_issue_parser.rb
+++ b/lib/banzai/reference_parser/external_issue_parser.rb
@@ -20,6 +20,12 @@ module Banzai
       def issue_ids_per_project(nodes)
         gather_attributes_per_project(nodes, self.class.data_attribute)
       end
+
+      private
+
+      def can_read_reference?(user, ref_project, node)
+        can?(user, :read_issue, ref_project)
+      end
     end
   end
 end
diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb
index 6c20dec5734..9fd4bd68d43 100644
--- a/lib/banzai/reference_parser/issue_parser.rb
+++ b/lib/banzai/reference_parser/issue_parser.rb
@@ -9,18 +9,18 @@ module Banzai
 
         issues = issues_for_nodes(nodes)
 
-        readable_issues = Ability.
-          issues_readable_by_user(issues.values, user).to_set
+        readable_issues = Ability
+          .issues_readable_by_user(issues.values, user).to_set
 
         nodes.select do |node|
-          readable_issues.include?(issue_for_node(issues, node))
+          readable_issues.include?(issues[node])
         end
       end
 
       def referenced_by(nodes)
         issues = issues_for_nodes(nodes)
 
-        nodes.map { |node| issue_for_node(issues, node) }.uniq
+        nodes.map { |node| issues[node] }.compact.uniq
       end
 
       def issues_for_nodes(nodes)
@@ -28,7 +28,7 @@ module Banzai
           nodes,
           Issue.all.includes(
             :author,
-            :assignee,
+            :assignees,
             {
               # These associations are primarily used for checking permissions.
               # Eager loading these ensures we don't end up running dozens of
@@ -44,12 +44,6 @@ module Banzai
           self.class.data_attribute
         )
       end
-
-      private
-
-      def issue_for_node(issues, node)
-        issues[node.attr(self.class.data_attribute).to_i]
-      end
     end
   end
 end
diff --git a/lib/banzai/reference_parser/label_parser.rb b/lib/banzai/reference_parser/label_parser.rb
index e5d1eb11d7f..30e2a012f09 100644
--- a/lib/banzai/reference_parser/label_parser.rb
+++ b/lib/banzai/reference_parser/label_parser.rb
@@ -6,6 +6,12 @@ module Banzai
       def references_relation
         Label
       end
+
+      private
+
+      def can_read_reference?(user, ref_project, node)
+        can?(user, :read_label, ref_project)
+      end
     end
   end
 end
diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb
index c9a9ca79c09..75cbc7fdac4 100644
--- a/lib/banzai/reference_parser/merge_request_parser.rb
+++ b/lib/banzai/reference_parser/merge_request_parser.rb
@@ -3,8 +3,46 @@ module Banzai
     class MergeRequestParser < BaseParser
       self.reference_type = :merge_request
 
-      def references_relation
-        MergeRequest.includes(:author, :assignee, :target_project)
+      def nodes_visible_to_user(user, nodes)
+        merge_requests = merge_requests_for_nodes(nodes)
+
+        nodes.select do |node|
+          merge_request = merge_requests[node]
+
+          merge_request && can?(user, :read_merge_request, merge_request.project)
+        end
+      end
+
+      def referenced_by(nodes)
+        merge_requests = merge_requests_for_nodes(nodes)
+
+        nodes.map { |node| merge_requests[node] }.compact.uniq
+      end
+
+      def merge_requests_for_nodes(nodes)
+        @merge_requests_for_nodes ||= grouped_objects_for_nodes(
+          nodes,
+          MergeRequest.includes(
+            :author,
+            :assignee,
+            {
+              # These associations are primarily used for checking permissions.
+              # Eager loading these ensures we don't end up running dozens of
+              # queries in this process.
+              target_project: [
+                { namespace: :owner },
+                { group: [:owners, :group_members] },
+                :invited_groups,
+                :project_members,
+                :project_feature
+              ]
+            }),
+          self.class.data_attribute
+        )
+      end
+
+      def can_read_reference?(user, ref_project, node)
+        can?(user, :read_merge_request, ref_project)
       end
     end
   end
diff --git a/lib/banzai/reference_parser/milestone_parser.rb b/lib/banzai/reference_parser/milestone_parser.rb
index a000ac61e5c..68675abe22a 100644
--- a/lib/banzai/reference_parser/milestone_parser.rb
+++ b/lib/banzai/reference_parser/milestone_parser.rb
@@ -6,6 +6,12 @@ module Banzai
       def references_relation
         Milestone
       end
+
+      private
+
+      def can_read_reference?(user, ref_project, node)
+        can?(user, :read_milestone, ref_project)
+      end
     end
   end
 end
diff --git a/lib/banzai/reference_parser/snippet_parser.rb b/lib/banzai/reference_parser/snippet_parser.rb
index fa71b3c952a..3ade168b566 100644
--- a/lib/banzai/reference_parser/snippet_parser.rb
+++ b/lib/banzai/reference_parser/snippet_parser.rb
@@ -6,6 +6,12 @@ module Banzai
       def references_relation
         Snippet
       end
+
+      private
+
+      def can_read_reference?(user, ref_project, node)
+        can?(user, :read_project_snippet, referenced_by([node]).first)
+      end
     end
   end
 end
diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb
index 863f5725d3b..4d336068861 100644
--- a/lib/banzai/reference_parser/user_parser.rb
+++ b/lib/banzai/reference_parser/user_parser.rb
@@ -30,22 +30,36 @@ module Banzai
 
         nodes.each do |node|
           if node.has_attribute?(group_attr)
-            node_group = groups[node.attr(group_attr).to_i]
-
-            if node_group &&
-              can?(user, :read_group, node_group)
-              visible << node
-            end
-          # Remaining nodes will be processed by the parent class'
-          # implementation of this method.
+            next unless can_read_group_reference?(node, user, groups)
+            visible << node
+          elsif can_read_project_reference?(node)
+            visible << node
           else
             remaining << node
           end
         end
 
+        # If project does not belong to a group
+        # and does not have the same project id as the current project
+        # base class will check if user can read the project that contains
+        # the user reference.
         visible + super(current_user, remaining)
       end
 
+      # Check if project belongs to a group which
+      # user can read.
+      def can_read_group_reference?(node, user, groups)
+        node_group = groups[node]
+
+        node_group && can?(user, :read_group, node_group)
+      end
+
+      def can_read_project_reference?(node)
+        node_id = node.attr('data-project').to_i
+
+        project && project.id == node_id
+      end
+
       def nodes_user_can_reference(current_user, nodes)
         project_attr = 'data-project'
         author_attr = 'data-author'
@@ -60,8 +74,8 @@ module Banzai
           if project && project_id && project.id == project_id.to_i
             true
           elsif project_id && user_id
-            project = projects[project_id.to_i]
-            user = users[user_id.to_i]
+            project = projects[node]
+            user = users[node]
 
             project && user ? project.team.member?(user) : false
           else
@@ -85,8 +99,12 @@ module Banzai
       def find_users_for_projects(ids)
         return [] if ids.empty?
 
-        collection_objects_for_ids(Project, ids).
-          flat_map { |p| p.team.members.to_a }
+        collection_objects_for_ids(Project, ids)
+          .flat_map { |p| p.team.members.to_a }
+      end
+
+      def can_read_reference?(user, ref_project, node)
+        can?(user, :read_project, ref_project)
       end
     end
   end
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index 6924a293da8..c7801cb5baf 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -1,7 +1,5 @@
 module Banzai
   module Renderer
-    extend self
-
     # Convert a Markdown String into an HTML-safe String of HTML
     #
     # Note that while the returned HTML will have been sanitized of dangerous
@@ -16,7 +14,7 @@ module Banzai
     # context  - Hash of context options passed to our HTML Pipeline
     #
     # Returns an HTML-safe String
-    def render(text, context = {})
+    def self.render(text, context = {})
       cache_key = context.delete(:cache_key)
       cache_key = full_cache_key(cache_key, context[:pipeline])
 
@@ -35,26 +33,18 @@ module Banzai
     # of HTML. This method is analogous to calling render(object.field), but it
     # can cache the rendered HTML in the object, rather than Redis.
     #
-    # The context to use is learned from the passed-in object by calling
-    # #banzai_render_context(field), and cannot be changed. Use #render, passing
-    # it the field text, if a custom rendering is needed. The generated context
-    # is returned along with the HTML.
-    def render_field(object, field)
-      html_field = object.markdown_cache_field_for(field)
-
-      html = object.__send__(html_field)
-      return html if html.present?
-
-      html = cacheless_render_field(object, field)
-      object.update_column(html_field, html) unless object.new_record? || object.destroyed?
+    # The context to use is managed by the object and cannot be changed.
+    # Use #render, passing it the field text, if a custom rendering is needed.
+    def self.render_field(object, field)
+      object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field)
 
-      html
+      object.cached_html_for(field)
     end
 
     # Same as +render_field+, but without consulting or updating the cache field
-    def cacheless_render_field(object, field)
+    def self.cacheless_render_field(object, field, options = {})
       text = object.__send__(field)
-      context = object.banzai_render_context(field)
+      context = object.banzai_render_context(field).merge(options)
 
       cacheless_render(text, context)
     end
@@ -82,7 +72,7 @@ module Banzai
     #    texts_and_contexts
     #    => [{ text: '### Hello',
     #          context: { cache_key: [note, :note] } }]
-    def cache_collection_render(texts_and_contexts)
+    def self.cache_collection_render(texts_and_contexts)
       items_collection = texts_and_contexts.each_with_index do |item, index|
         context = item[:context]
         cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline])
@@ -111,7 +101,7 @@ module Banzai
       items_collection.map { |item| item[:rendered] }
     end
 
-    def render_result(text, context = {})
+    def self.render_result(text, context = {})
       text = Pipeline[:pre_process].to_html(text, context) if text
 
       Pipeline[context[:pipeline]].call(text, context)
@@ -130,7 +120,7 @@ module Banzai
     #            :user      - User object
     #
     # Returns an HTML-safe String
-    def post_process(html, context)
+    def self.post_process(html, context)
       context = Pipeline[context[:pipeline]].transform_context(context)
 
       pipeline = Pipeline[:post_process]
@@ -141,9 +131,7 @@ module Banzai
       end.html_safe
     end
 
-    private
-
-    def cacheless_render(text, context = {})
+    def self.cacheless_render(text, context = {})
       Gitlab::Metrics.measure(:banzai_cacheless_render) do
         result = render_result(text, context)
 
@@ -156,7 +144,7 @@ module Banzai
       end
     end
 
-    def full_cache_key(cache_key, pipeline_name)
+    def self.full_cache_key(cache_key, pipeline_name)
       return unless cache_key
       ["banzai", *cache_key, pipeline_name || :full]
     end
@@ -164,9 +152,14 @@ module Banzai
     # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key.
     # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key
     # method.
-    def full_cache_multi_key(cache_key, pipeline_name)
+    def self.full_cache_multi_key(cache_key, pipeline_name)
       return unless cache_key
       Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name))
     end
+
+    # GitLab EE needs to disable updates on GET requests in Geo
+    def self.update_object?(object)
+      true
+    end
   end
 end
diff --git a/lib/banzai/renderer/html.rb b/lib/banzai/renderer/html.rb
new file mode 100644
index 00000000000..252caa35947
--- /dev/null
+++ b/lib/banzai/renderer/html.rb
@@ -0,0 +1,13 @@
+module Banzai
+  module Renderer
+    class HTML < Redcarpet::Render::HTML
+      def block_code(code, lang)
+        lang_attr = lang ? %Q{ lang="#{lang}"} : ''
+
+        "\n<pre>" \
+          "<code#{lang_attr}>#{html_escape(code)}</code>" \
+        "</pre>"
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb
new file mode 100644
index 00000000000..f8ee7e0f9ae
--- /dev/null
+++ b/lib/bitbucket/client.rb
@@ -0,0 +1,58 @@
+module Bitbucket
+  class Client
+    attr_reader :connection
+
+    def initialize(options = {})
+      @connection = Connection.new(options)
+    end
+
+    def issues(repo)
+      path = "/repositories/#{repo}/issues"
+      get_collection(path, :issue)
+    end
+
+    def issue_comments(repo, issue_id)
+      path = "/repositories/#{repo}/issues/#{issue_id}/comments"
+      get_collection(path, :comment)
+    end
+
+    def pull_requests(repo)
+      path = "/repositories/#{repo}/pullrequests?state=ALL"
+      get_collection(path, :pull_request)
+    end
+
+    def pull_request_comments(repo, pull_request)
+      path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments"
+      get_collection(path, :pull_request_comment)
+    end
+
+    def pull_request_diff(repo, pull_request)
+      path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff"
+      connection.get(path)
+    end
+
+    def repo(name)
+      parsed_response = connection.get("/repositories/#{name}")
+      Representation::Repo.new(parsed_response)
+    end
+
+    def repos
+      path = "/repositories?role=member"
+      get_collection(path, :repo)
+    end
+
+    def user
+      @user ||= begin
+        parsed_response = connection.get('/user')
+        Representation::User.new(parsed_response)
+      end
+    end
+
+    private
+
+    def get_collection(path, type)
+      paginator = Paginator.new(connection, path, type)
+      Collection.new(paginator)
+    end
+  end
+end
diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb
new file mode 100644
index 00000000000..3a9379ff680
--- /dev/null
+++ b/lib/bitbucket/collection.rb
@@ -0,0 +1,21 @@
+module Bitbucket
+  class Collection < Enumerator
+    def initialize(paginator)
+      super() do |yielder|
+        loop do
+          paginator.items.each { |item| yielder << item }
+        end
+      end
+
+      lazy
+    end
+
+    def method_missing(method, *args)
+      return super unless self.respond_to?(method)
+
+      self.send(method, *args) do |item|
+        block_given? ? yield(item) : item
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb
new file mode 100644
index 00000000000..b9279c33f5b
--- /dev/null
+++ b/lib/bitbucket/connection.rb
@@ -0,0 +1,67 @@
+module Bitbucket
+  class Connection
+    DEFAULT_API_VERSION = '2.0'.freeze
+    DEFAULT_BASE_URI    = 'https://api.bitbucket.org/'.freeze
+    DEFAULT_QUERY       = {}.freeze
+
+    attr_reader :expires_at, :expires_in, :refresh_token, :token
+
+    def initialize(options = {})
+      @api_version   = options.fetch(:api_version, DEFAULT_API_VERSION)
+      @base_uri      = options.fetch(:base_uri, DEFAULT_BASE_URI)
+      @default_query = options.fetch(:query, DEFAULT_QUERY)
+
+      @token         = options[:token]
+      @expires_at    = options[:expires_at]
+      @expires_in    = options[:expires_in]
+      @refresh_token = options[:refresh_token]
+    end
+
+    def get(path, extra_query = {})
+      refresh! if expired?
+
+      response = connection.get(build_url(path), params: @default_query.merge(extra_query))
+      response.parsed
+    end
+
+    delegate :expired?, to: :connection
+
+    def refresh!
+      response = connection.refresh!
+
+      @token         = response.token
+      @expires_at    = response.expires_at
+      @expires_in    = response.expires_in
+      @refresh_token = response.refresh_token
+      @connection    = nil
+    end
+
+    private
+
+    def client
+      @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options)
+    end
+
+    def connection
+      @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in)
+    end
+
+    def build_url(path)
+      return path if path.starts_with?(root_url)
+
+      "#{root_url}#{path}"
+    end
+
+    def root_url
+      @root_url ||= "#{@base_uri}#{@api_version}"
+    end
+
+    def provider
+      Gitlab::OAuth::Provider.config_for('bitbucket')
+    end
+
+    def options
+      OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys
+    end
+  end
+end
diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb
new file mode 100644
index 00000000000..efe10542f19
--- /dev/null
+++ b/lib/bitbucket/error/unauthorized.rb
@@ -0,0 +1,5 @@
+module Bitbucket
+  module Error
+    Unauthorized = Class.new(StandardError)
+  end
+end
diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb
new file mode 100644
index 00000000000..2b0a3fe7b1a
--- /dev/null
+++ b/lib/bitbucket/page.rb
@@ -0,0 +1,34 @@
+module Bitbucket
+  class Page
+    attr_reader :attrs, :items
+
+    def initialize(raw, type)
+      @attrs = parse_attrs(raw)
+      @items = parse_values(raw, representation_class(type))
+    end
+
+    def next?
+      attrs.fetch(:next, false)
+    end
+
+    def next
+      attrs.fetch(:next)
+    end
+
+    private
+
+    def parse_attrs(raw)
+      raw.slice(*%w(size page pagelen next previous)).symbolize_keys
+    end
+
+    def parse_values(raw, bitbucket_rep_class)
+      return [] unless raw['values'] && raw['values'].is_a?(Array)
+
+      bitbucket_rep_class.decorate(raw['values'])
+    end
+
+    def representation_class(type)
+      Bitbucket::Representation.const_get(type.to_s.camelize)
+    end
+  end
+end
diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb
new file mode 100644
index 00000000000..135d0d55674
--- /dev/null
+++ b/lib/bitbucket/paginator.rb
@@ -0,0 +1,36 @@
+module Bitbucket
+  class Paginator
+    PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100.
+
+    def initialize(connection, url, type)
+      @connection = connection
+      @type = type
+      @url = url
+      @page = nil
+    end
+
+    def items
+      raise StopIteration unless has_next_page?
+
+      @page = fetch_next_page
+      @page.items
+    end
+
+    private
+
+    attr_reader :connection, :page, :url, :type
+
+    def has_next_page?
+      page.nil? || page.next?
+    end
+
+    def next_url
+      page.nil? ? url : page.next
+    end
+
+    def fetch_next_page
+      parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on)
+      Page.new(parsed_response, type)
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb
new file mode 100644
index 00000000000..800d5a075c6
--- /dev/null
+++ b/lib/bitbucket/representation/base.rb
@@ -0,0 +1,15 @@
+module Bitbucket
+  module Representation
+    class Base
+      attr_reader :raw
+
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def self.decorate(entries)
+        entries.map { |entry| new(entry)}
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb
new file mode 100644
index 00000000000..4937aa9728f
--- /dev/null
+++ b/lib/bitbucket/representation/comment.rb
@@ -0,0 +1,27 @@
+module Bitbucket
+  module Representation
+    class Comment < Representation::Base
+      def author
+        user['username']
+      end
+
+      def note
+        raw.fetch('content', {}).fetch('raw', nil)
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['updated_on'] || raw['created_on']
+      end
+
+      private
+
+      def user
+        raw.fetch('user', {})
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb
new file mode 100644
index 00000000000..054064395c3
--- /dev/null
+++ b/lib/bitbucket/representation/issue.rb
@@ -0,0 +1,53 @@
+module Bitbucket
+  module Representation
+    class Issue < Representation::Base
+      CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze
+
+      def iid
+        raw['id']
+      end
+
+      def kind
+        raw['kind']
+      end
+
+      def author
+        raw.fetch('reporter', {}).fetch('username', nil)
+      end
+
+      def description
+        raw.fetch('content', {}).fetch('raw', nil)
+      end
+
+      def state
+        closed? ? 'closed' : 'opened'
+      end
+
+      def title
+        raw['title']
+      end
+
+      def milestone
+        raw['milestone']['name'] if raw['milestone'].present?
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['edited_on']
+      end
+
+      def to_s
+        iid
+      end
+
+      private
+
+      def closed?
+        CLOSED_STATUS.include?(raw['state'])
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb
new file mode 100644
index 00000000000..eebf8093380
--- /dev/null
+++ b/lib/bitbucket/representation/pull_request.rb
@@ -0,0 +1,65 @@
+module Bitbucket
+  module Representation
+    class PullRequest < Representation::Base
+      def author
+        raw.fetch('author', {}).fetch('username', nil)
+      end
+
+      def description
+        raw['description']
+      end
+
+      def iid
+        raw['id']
+      end
+
+      def state
+        if raw['state'] == 'MERGED'
+          'merged'
+        elsif raw['state'] == 'DECLINED'
+          'closed'
+        else
+          'opened'
+        end
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['updated_on']
+      end
+
+      def title
+        raw['title']
+      end
+
+      def source_branch_name
+        source_branch.fetch('branch', {}).fetch('name', nil)
+      end
+
+      def source_branch_sha
+        source_branch.fetch('commit', {}).fetch('hash', nil)
+      end
+
+      def target_branch_name
+        target_branch.fetch('branch', {}).fetch('name', nil)
+      end
+
+      def target_branch_sha
+        target_branch.fetch('commit', {}).fetch('hash', nil)
+      end
+
+      private
+
+      def source_branch
+        raw['source']
+      end
+
+      def target_branch
+        raw['destination']
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb
new file mode 100644
index 00000000000..c52acbc3ddc
--- /dev/null
+++ b/lib/bitbucket/representation/pull_request_comment.rb
@@ -0,0 +1,39 @@
+module Bitbucket
+  module Representation
+    class PullRequestComment < Comment
+      def iid
+        raw['id']
+      end
+
+      def file_path
+        inline.fetch('path')
+      end
+
+      def old_pos
+        inline.fetch('from')
+      end
+
+      def new_pos
+        inline.fetch('to')
+      end
+
+      def parent_id
+        raw.fetch('parent', {}).fetch('id', nil)
+      end
+
+      def inline?
+        raw.key?('inline')
+      end
+
+      def has_parent?
+        raw.key?('parent')
+      end
+
+      private
+
+      def inline
+        raw.fetch('inline', {})
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb
new file mode 100644
index 00000000000..59b0fda8e14
--- /dev/null
+++ b/lib/bitbucket/representation/repo.rb
@@ -0,0 +1,71 @@
+module Bitbucket
+  module Representation
+    class Repo < Representation::Base
+      attr_reader :owner, :slug
+
+      def initialize(raw)
+        super(raw)
+      end
+
+      def owner_and_slug
+        @owner_and_slug ||= full_name.split('/', 2)
+      end
+
+      def owner
+        owner_and_slug.first
+      end
+
+      def slug
+        owner_and_slug.last
+      end
+
+      def clone_url(token = nil)
+        url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href')
+
+        if token.present?
+          clone_url = URI.parse(url)
+          clone_url.user = "x-token-auth:#{token}"
+          clone_url.to_s
+        else
+          url
+        end
+      end
+
+      def description
+        raw['description']
+      end
+
+      def full_name
+        raw['full_name']
+      end
+
+      def issues_enabled?
+        raw['has_issues']
+      end
+
+      def name
+        raw['name']
+      end
+
+      def valid?
+        raw['scm'] == 'git'
+      end
+
+      def has_wiki?
+        raw['has_wiki']
+      end
+
+      def visibility_level
+        if raw['is_private']
+          Gitlab::VisibilityLevel::PRIVATE
+        else
+          Gitlab::VisibilityLevel::PUBLIC
+        end
+      end
+
+      def to_s
+        full_name
+      end
+    end
+  end
+end
diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb
new file mode 100644
index 00000000000..ba6b7667b49
--- /dev/null
+++ b/lib/bitbucket/representation/user.rb
@@ -0,0 +1,9 @@
+module Bitbucket
+  module Representation
+    class User < Representation::Base
+      def username
+        raw['username']
+      end
+    end
+  end
+end
diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb
index 229050151d3..55402101e43 100644
--- a/lib/ci/ansi2html.rb
+++ b/lib/ci/ansi2html.rb
@@ -13,15 +13,15 @@ module Ci
       5 => 'magenta',
       6 => 'cyan',
       7 => 'white', # not that this is gray in the dark (aka default) color table
-    }
+    }.freeze
 
     STYLE_SWITCHES = {
       bold:       0x01,
       italic:     0x02,
       underline:  0x04,
       conceal:    0x08,
-      cross:      0x10,
-    }
+      cross:      0x10
+    }.freeze
 
     def self.convert(ansi, state = nil)
       Converter.new.convert(ansi, state)
@@ -29,93 +29,157 @@ module Ci
 
     class Converter
       def on_0(s) reset()                            end
+
       def on_1(s) enable(STYLE_SWITCHES[:bold])      end
+
       def on_3(s) enable(STYLE_SWITCHES[:italic])    end
+
       def on_4(s) enable(STYLE_SWITCHES[:underline]) end
+
       def on_8(s) enable(STYLE_SWITCHES[:conceal])   end
+
       def on_9(s) enable(STYLE_SWITCHES[:cross])     end
 
       def on_21(s) disable(STYLE_SWITCHES[:bold])      end
+
       def on_22(s) disable(STYLE_SWITCHES[:bold])      end
+
       def on_23(s) disable(STYLE_SWITCHES[:italic])    end
+
       def on_24(s) disable(STYLE_SWITCHES[:underline]) end
+
       def on_28(s) disable(STYLE_SWITCHES[:conceal])   end
+
       def on_29(s) disable(STYLE_SWITCHES[:cross])     end
 
       def on_30(s) set_fg_color(0) end
+
       def on_31(s) set_fg_color(1) end
+
       def on_32(s) set_fg_color(2) end
+
       def on_33(s) set_fg_color(3) end
+
       def on_34(s) set_fg_color(4) end
+
       def on_35(s) set_fg_color(5) end
+
       def on_36(s) set_fg_color(6) end
+
       def on_37(s) set_fg_color(7) end
+
       def on_38(s) set_fg_color_256(s) end
+
       def on_39(s) set_fg_color(9) end
 
       def on_40(s) set_bg_color(0) end
+
       def on_41(s) set_bg_color(1) end
+
       def on_42(s) set_bg_color(2) end
+
       def on_43(s) set_bg_color(3) end
+
       def on_44(s) set_bg_color(4) end
+
       def on_45(s) set_bg_color(5) end
+
       def on_46(s) set_bg_color(6) end
+
       def on_47(s) set_bg_color(7) end
+
       def on_48(s) set_bg_color_256(s) end
+
       def on_49(s) set_bg_color(9) end
 
       def on_90(s) set_fg_color(0, 'l') end
+
       def on_91(s) set_fg_color(1, 'l') end
+
       def on_92(s) set_fg_color(2, 'l') end
+
       def on_93(s) set_fg_color(3, 'l') end
+
       def on_94(s) set_fg_color(4, 'l') end
+
       def on_95(s) set_fg_color(5, 'l') end
+
       def on_96(s) set_fg_color(6, 'l') end
+
       def on_97(s) set_fg_color(7, 'l') end
+
       def on_99(s) set_fg_color(9, 'l') end
 
       def on_100(s) set_bg_color(0, 'l') end
+
       def on_101(s) set_bg_color(1, 'l') end
+
       def on_102(s) set_bg_color(2, 'l') end
+
       def on_103(s) set_bg_color(3, 'l') end
+
       def on_104(s) set_bg_color(4, 'l') end
+
       def on_105(s) set_bg_color(5, 'l') end
+
       def on_106(s) set_bg_color(6, 'l') end
+
       def on_107(s) set_bg_color(7, 'l') end
+
       def on_109(s) set_bg_color(9, 'l') end
 
       attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask
 
-      STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask]
+      STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze
 
-      def convert(raw, new_state)
+      def convert(stream, new_state)
         reset_state
-        restore_state(raw, new_state) if new_state.present?
-
-        start = @offset
-        ansi = raw[@offset..-1]
+        restore_state(new_state, stream) if new_state.present?
+
+        append = false
+        truncated = false
+
+        cur_offset = stream.tell
+        if cur_offset > @offset
+          @offset = cur_offset
+          truncated = true
+        else
+          stream.seek(@offset)
+          append = @offset > 0
+        end
+        start_offset = @offset
 
         open_new_tag
 
-        s = StringScanner.new(ansi)
-        until s.eos?
-          if s.scan(/\e([@-_])(.*?)([@-~])/)
-            handle_sequence(s)
-          elsif s.scan(/\e(([@-_])(.*?)?)?$/)
-            break
-          elsif s.scan(/</)
-            @out << '&lt;'
-          elsif s.scan(/\n/)
-            @out << '<br>'
-          else
-            @out << s.scan(/./m)
+        stream.each_line do |line|
+          s = StringScanner.new(line)
+          until s.eos?
+            if s.scan(/\e([@-_])(.*?)([@-~])/)
+              handle_sequence(s)
+            elsif s.scan(/\e(([@-_])(.*?)?)?$/)
+              break
+            elsif s.scan(/</)
+              @out << '&lt;'
+            elsif s.scan(/\r?\n/)
+              @out << '<br>'
+            else
+              @out << s.scan(/./m)
+            end
+            @offset += s.matched_size
           end
-          @offset += s.matched_size
         end
 
         close_open_tags()
 
-        { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 }
+        OpenStruct.new(
+          html: @out.force_encoding(Encoding.default_external),
+          state: state,
+          append: append,
+          truncated: truncated,
+          offset: start_offset,
+          size: stream.tell - start_offset,
+          total: stream.size
+        )
       end
 
       def handle_sequence(s)
@@ -126,7 +190,7 @@ module Ci
         # We are only interested in color and text style changes - triggered by
         # sequences starting with '\e[' and ending with 'm'. Any other control
         # sequence gets stripped (including stuff like "delete last line")
-        return unless indicator == '[' and terminator == 'm'
+        return unless indicator == '[' && terminator == 'm'
 
         close_open_tags()
 
@@ -196,10 +260,10 @@ module Ci
         Base64.urlsafe_encode64(state.to_json)
       end
 
-      def restore_state(raw, new_state)
+      def restore_state(new_state, stream)
         state = Base64.urlsafe_decode64(new_state)
         state = JSON.parse(state, symbolize_names: true)
-        return if state[:offset].to_i > raw.length
+        return if state[:offset].to_i > stream.size
 
         STATE_PARAMS.each do |param|
           send("#{param}=".to_sym, state[param])
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index a6b9beecded..24bb3649a76 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -8,6 +8,16 @@ module Ci
         rack_response({ 'message' => '404 Not found' }.to_json, 404)
       end
 
+      # Retain 405 error rather than a 500 error for Grape 0.15.0+.
+      # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes
+      rescue_from Grape::Exceptions::MethodNotAllowed do |e|
+        error! e.message, e.status, e.headers
+      end
+
+      rescue_from Grape::Exceptions::Base do |e|
+        error! e.message, e.status, e.headers
+      end
+
       rescue_from :all do |exception|
         handle_api_exception(exception)
       end
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index ed87a2603e8..e2e91ce99cd 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -16,17 +16,33 @@ module Ci
           not_found! unless current_runner.active?
           update_runner_info
 
-          build = Ci::RegisterBuildService.new.execute(current_runner)
+          if current_runner.is_runner_queue_value_latest?(params[:last_update])
+            header 'X-GitLab-Last-Update', params[:last_update]
+            Gitlab::Metrics.add_event(:build_not_found_cached)
+            return build_not_found!
+          end
 
-          if build
-            Gitlab::Metrics.add_event(:build_found,
-                                      project: build.project.path_with_namespace)
+          new_update = current_runner.ensure_runner_queue_value
 
-            present build, with: Entities::BuildDetails
-          else
-            Gitlab::Metrics.add_event(:build_not_found)
+          result = Ci::RegisterJobService.new(current_runner).execute
+
+          if result.valid?
+            if result.build
+              Gitlab::Metrics.add_event(:build_found,
+                                        project: result.build.project.path_with_namespace)
+
+              present result.build, with: Entities::BuildDetails
+            else
+              Gitlab::Metrics.add_event(:build_not_found)
 
-            build_not_found!
+              header 'X-GitLab-Last-Update', new_update
+
+              build_not_found!
+            end
+          else
+            # We received build that is invalid due to concurrency conflict
+            Gitlab::Metrics.add_event(:build_invalid)
+            conflict!
           end
         end
 
@@ -41,11 +57,11 @@ module Ci
         put ":id" do
           authenticate_runner!
           build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id])
-          forbidden!('Build has been erased!') if build.erased?
+          validate_build!(build)
 
           update_runner_info
 
-          build.update_attributes(trace: params[:trace]) if params[:trace]
+          build.trace.set(params[:trace]) if params[:trace]
 
           Gitlab::Metrics.add_event(:update_build,
                                     project: build.project.path_with_namespace)
@@ -70,25 +86,20 @@ module Ci
         # Example Request:
         #   PATCH /builds/:id/trace.txt
         patch ":id/trace.txt" do
-          build = Ci::Build.find_by_id(params[:id])
-          not_found! unless build
-          authenticate_build_token!(build)
-          forbidden!('Build has been erased!') if build.erased?
+          build = authenticate_build!
 
-          error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range')
+          error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
           content_range = request.headers['Content-Range']
           content_range = content_range.split('-')
 
-          current_length = build.trace_length
-          unless current_length == content_range[0].to_i
-            return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" })
+          stream_size = build.trace.append(request.body.read, content_range[0].to_i)
+          if stream_size < 0
+            return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" })
           end
 
-          build.append_trace(request.body.read, content_range[0].to_i)
-
           status 202
           header 'Build-Status', build.status
-          header 'Range', "0-#{build.trace_length}"
+          header 'Range', "0-#{stream_size}"
         end
 
         # Authorize artifacts uploading for build - Runners only
@@ -103,9 +114,7 @@ module Ci
           require_gitlab_workhorse!
           Gitlab::Workhorse.verify_api_request!(headers)
           not_allowed! unless Gitlab.config.artifacts.enabled
-          build = Ci::Build.find_by_id(params[:id])
-          not_found! unless build
-          authenticate_build_token!(build)
+          build = authenticate_build!
           forbidden!('build is not running') unless build.running?
 
           if params[:filesize]
@@ -141,11 +150,8 @@ module Ci
         post ":id/artifacts" do
           require_gitlab_workhorse!
           not_allowed! unless Gitlab.config.artifacts.enabled
-          build = Ci::Build.find_by_id(params[:id])
-          not_found! unless build
-          authenticate_build_token!(build)
+          build = authenticate_build!
           forbidden!('Build is not running!') unless build.running?
-          forbidden!('Build has been erased!') if build.erased?
 
           artifacts_upload_path = ArtifactUploader.artifacts_upload_path
           artifacts = uploaded_file(:file, artifacts_upload_path)
@@ -156,7 +162,10 @@ module Ci
 
           build.artifacts_file = artifacts
           build.artifacts_metadata = metadata
-          build.artifacts_expire_in = params['expire_in']
+          build.artifacts_expire_in =
+            params['expire_in'] ||
+            Gitlab::CurrentSettings.current_application_settings
+              .default_artifacts_expire_in
 
           if build.save
             present(build, with: Entities::BuildDetails)
@@ -175,19 +184,17 @@ module Ci
         # Example Request:
         #   GET /builds/:id/artifacts
         get ":id/artifacts" do
-          build = Ci::Build.find_by_id(params[:id])
-          not_found! unless build
-          authenticate_build_token!(build)
+          build = authenticate_build!
           artifacts_file = build.artifacts_file
 
-          unless artifacts_file.file_storage?
-            return redirect_to build.artifacts_file.url
-          end
-
           unless artifacts_file.exists?
             not_found!
           end
 
+          unless artifacts_file.file_storage?
+            return redirect_to build.artifacts_file.url
+          end
+
           present_file!(artifacts_file.path, artifacts_file.filename)
         end
 
@@ -201,10 +208,9 @@ module Ci
         # Example Request:
         #   DELETE /builds/:id/artifacts
         delete ":id/artifacts" do
-          build = Ci::Build.find_by_id(params[:id])
-          not_found! unless build
-          authenticate_build_token!(build)
+          build = authenticate_build!
 
+          status(200)
           build.erase_artifacts!
         end
       end
diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb
index 66c05773b68..6b82b2b4f13 100644
--- a/lib/ci/api/entities.rb
+++ b/lib/ci/api/entities.rb
@@ -32,6 +32,10 @@ module Ci
         expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? }
       end
 
+      class BuildCredentials < Grape::Entity
+        expose :type, :url, :username, :password
+      end
+
       class BuildDetails < Build
         expose :commands
         expose :repo_url
@@ -41,7 +45,21 @@ module Ci
         expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? }
 
         expose :options do |model|
-          model.options
+          # This part ensures that output of old API is still the same after adding support
+          # for extended docker configuration options, used by new API
+          #
+          # I'm leaving this here, not in the model, because it should be removed at the same time
+          # when old API will be removed (planned for August 2017).
+          model.options.dup.tap do |options|
+            options[:image] = options[:image][:name] if options[:image].is_a?(Hash)
+            options[:services].map! do |service|
+              if service.is_a?(Hash)
+                service[:name]
+              else
+                service
+              end
+            end
+          end
         end
 
         expose :timeout do |model|
@@ -50,6 +68,8 @@ module Ci
 
         expose :variables
         expose :depends_on_builds, using: Build
+
+        expose :credentials, using: BuildCredentials
       end
 
       class Runner < Grape::Entity
diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb
index e608f5f6cad..5109dc9670f 100644
--- a/lib/ci/api/helpers.rb
+++ b/lib/ci/api/helpers.rb
@@ -1,7 +1,7 @@
 module Ci
   module API
     module Helpers
-      BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
+      BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN".freeze
       BUILD_TOKEN_PARAM = :token
       UPDATE_RUNNER_EVERY = 10 * 60
 
@@ -13,8 +13,23 @@ module Ci
         forbidden! unless current_runner
       end
 
-      def authenticate_build_token!(build)
-        forbidden! unless build_token_valid?(build)
+      def authenticate_build!
+        build = Ci::Build.find_by_id(params[:id])
+
+        validate_build!(build) do
+          forbidden! unless build_token_valid?(build)
+        end
+
+        build
+      end
+
+      def validate_build!(build)
+        not_found! unless build
+
+        yield if block_given?
+
+        forbidden!('Project has been deleted!') unless build.project
+        forbidden!('Build has been erased!') if build.erased?
       end
 
       def runner_registration_token_valid?
@@ -49,7 +64,7 @@ module Ci
       end
 
       def build_not_found!
-        if headers['User-Agent'].match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /)
+        if headers['User-Agent'].to_s =~ /gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /
           no_content!
         else
           not_found!
@@ -62,7 +77,7 @@ module Ci
 
       def get_runner_version_from_params
         return unless params["info"].present?
-        attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
+        attributes_for_keys(%w(name version revision platform architecture), params["info"])
       end
 
       def max_artifacts_size
diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb
index bcc82969eb3..45aa2adccf5 100644
--- a/lib/ci/api/runners.rb
+++ b/lib/ci/api/runners.rb
@@ -1,44 +1,38 @@
 module Ci
   module API
-    # Runners API
     class Runners < Grape::API
       resource :runners do
-        # Delete runner
-        # Parameters:
-        #   token (required) - The unique token of runner
-        #
-        # Example Request:
-        #   GET /runners/delete
+        desc 'Delete a runner'
+        params do
+          requires :token, type: String, desc: 'The unique token of the runner'
+        end
         delete "delete" do
-          required_attributes! [:token]
           authenticate_runner!
+
+          status(200)
           Ci::Runner.find_by_token(params[:token]).destroy
         end
 
-        # Register a new runner
-        #
-        # Note: This is an "internal" API called when setting up
-        # runners, so it is authenticated differently.
-        #
-        # Parameters:
-        #   token (required) - The unique token of runner
-        #
-        # Example Request:
-        #   POST /runners/register
+        desc 'Register a new runner' do
+          success Entities::Runner
+        end
+        params do
+          requires :token, type: String, desc: 'The unique token of the runner'
+          optional :description, type: String, desc: 'The description of the runner'
+          optional :tag_list, type: Array[String], desc: 'A list of tags the runner should run for'
+          optional :run_untagged, type: Boolean, desc: 'Flag if the runner should execute untagged jobs'
+          optional :locked, type: Boolean, desc: 'Lock this runner for this specific project'
+        end
         post "register" do
-          required_attributes! [:token]
-
-          attributes = attributes_for_keys(
-            [:description, :tag_list, :run_untagged, :locked]
-          )
+          runner_params = declared(params, include_missing: false).except(:token)
 
           runner =
             if runner_registration_token_valid?
               # Create shared runner. Requires admin access
-              Ci::Runner.create(attributes.merge(is_shared: true))
+              Ci::Runner.create(runner_params.merge(is_shared: true))
             elsif project = Project.find_by(runners_token: params[:token])
               # Create a specific runner for project.
-              project.runners.create(attributes)
+              project.runners.create(runner_params)
             end
 
           return forbidden! unless runner
diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb
index 63b42113513..6e622601680 100644
--- a/lib/ci/api/triggers.rb
+++ b/lib/ci/api/triggers.rb
@@ -1,41 +1,30 @@
 module Ci
   module API
-    # Build Trigger API
     class Triggers < Grape::API
       resource :projects do
-        # Trigger a GitLab CI project build
-        #
-        # Parameters:
-        #   id (required) - The ID of a CI project
-        #   ref (required) - The name of project's branch or tag
-        #   token (required) - The uniq token of trigger
-        # Example Request:
-        #   POST /projects/:id/ref/:ref/trigger
+        desc 'Trigger a GitLab CI project build' do
+          success Entities::TriggerRequest
+        end
+        params do
+          requires :id, type: Integer, desc: 'The ID of a CI project'
+          requires :ref, type: String, desc: "The name of project's branch or tag"
+          requires :token, type: String, desc: 'The unique token of the trigger'
+          optional :variables, type: Hash, desc: 'Optional build variables'
+        end
         post ":id/refs/:ref/trigger" do
-          required_attributes! [:token]
-
-          project = Project.find_by(ci_id: params[:id].to_i)
-          trigger = Ci::Trigger.find_by_token(params[:token].to_s)
+          project = Project.find_by(ci_id: params[:id])
+          trigger = Ci::Trigger.find_by_token(params[:token])
           not_found! unless project && trigger
           unauthorized! unless trigger.project == project
 
-          # validate variables
-          variables = params[:variables]
-          if variables
-            unless variables.is_a?(Hash)
-              render_api_error!('variables needs to be a hash', 400)
-            end
-
-            unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) }
-              render_api_error!('variables needs to be a map of key-valued strings', 400)
-            end
-
-            # convert variables from Mash to Hash
-            variables = variables.to_h
+          # Validate variables
+          variables = params[:variables].to_h
+          unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) }
+            render_api_error!('variables needs to be a map of key-valued strings', 400)
           end
 
           # create request and trigger builds
-          trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables)
+          trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref], variables)
           if trigger_request
             present trigger_request, with: Entities::TriggerRequest
           else
diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb
index 3decc3b1a26..872e418c788 100644
--- a/lib/ci/charts.rb
+++ b/lib/ci/charts.rb
@@ -2,10 +2,10 @@ module Ci
   module Charts
     module DailyInterval
       def grouped_count(query)
-        query.
-          group("DATE(#{Ci::Build.table_name}.created_at)").
-          count(:created_at).
-          transform_keys { |date| date.strftime(@format) }
+        query
+          .group("DATE(#{Ci::Pipeline.table_name}.created_at)")
+          .count(:created_at)
+          .transform_keys { |date| date.strftime(@format) }
       end
 
       def interval_step
@@ -16,14 +16,14 @@ module Ci
     module MonthlyInterval
       def grouped_count(query)
         if Gitlab::Database.postgresql?
-          query.
-            group("to_char(#{Ci::Build.table_name}.created_at, '01 Month YYYY')").
-            count(:created_at).
-            transform_keys(&:squish)
+          query
+            .group("to_char(#{Ci::Pipeline.table_name}.created_at, '01 Month YYYY')")
+            .count(:created_at)
+            .transform_keys(&:squish)
         else
-          query.
-            group("DATE_FORMAT(#{Ci::Build.table_name}.created_at, '01 %M %Y')").
-            count(:created_at)
+          query
+            .group("DATE_FORMAT(#{Ci::Pipeline.table_name}.created_at, '01 %M %Y')")
+            .count(:created_at)
         end
       end
 
@@ -33,21 +33,21 @@ module Ci
     end
 
     class Chart
-      attr_reader :labels, :total, :success, :project, :build_times
+      attr_reader :labels, :total, :success, :project, :pipeline_times
 
       def initialize(project)
         @labels = []
         @total = []
         @success = []
-        @build_times = []
+        @pipeline_times = []
         @project = project
 
         collect
       end
 
       def collect
-        query = project.builds.
-          where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", @to, @from)
+        query = project.pipelines
+          .where("? > #{Ci::Pipeline.table_name}.created_at AND #{Ci::Pipeline.table_name}.created_at > ?", @to, @from)
 
         totals_count  = grouped_count(query)
         success_count = grouped_count(query.success)
@@ -101,14 +101,14 @@ module Ci
       end
     end
 
-    class BuildTime < Chart
+    class PipelineTime < Chart
       def collect
         commits = project.pipelines.last(30)
 
         commits.each do |commit|
           @labels << commit.short_sha
           duration = commit.duration || 0
-          @build_times << (duration / 60)
+          @pipeline_times << (duration / 60)
         end
       end
     end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 2fd1fced65c..56ad2c77c7d 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -1,8 +1,8 @@
 module Ci
   class GitlabCiYamlProcessor
-    class ValidationError < StandardError; end
+    ValidationError = Class.new(StandardError)
 
-    include Gitlab::Ci::Config::Node::LegacyValidationHelpers
+    include Gitlab::Ci::Config::Entry::LegacyValidationHelpers
 
     attr_reader :path, :cache, :stages, :jobs
 
@@ -20,26 +20,26 @@ module Ci
       raise ValidationError, e.message
     end
 
-    def jobs_for_ref(ref, tag = false, trigger_request = nil)
+    def jobs_for_ref(ref, tag = false, source = nil)
       @jobs.select do |_, job|
-        process?(job[:only], job[:except], ref, tag, trigger_request)
+        process?(job[:only], job[:except], ref, tag, source)
       end
     end
 
-    def jobs_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil)
-      jobs_for_ref(ref, tag, trigger_request).select do |_, job|
+    def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil)
+      jobs_for_ref(ref, tag, source).select do |_, job|
         job[:stage] == stage
       end
     end
 
-    def builds_for_ref(ref, tag = false, trigger_request = nil)
-      jobs_for_ref(ref, tag, trigger_request).map do |name, _|
+    def builds_for_ref(ref, tag = false, source = nil)
+      jobs_for_ref(ref, tag, source).map do |name, _|
         build_attributes(name)
       end
     end
 
-    def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil)
-      jobs_for_stage_and_ref(stage, ref, tag, trigger_request).map do |name, _|
+    def builds_for_stage_and_ref(stage, ref, tag = false, source = nil)
+      jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _|
         build_attributes(name)
       end
     end
@@ -50,17 +50,29 @@ module Ci
       end
     end
 
+    def stage_seeds(pipeline)
+      seeds = @stages.uniq.map do |stage|
+        builds = builds_for_stage_and_ref(
+          stage, pipeline.ref, pipeline.tag?, pipeline.source)
+
+        Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
+      end
+
+      seeds.compact
+    end
+
     def build_attributes(name)
       job = @jobs[name.to_sym] || {}
-      {
-        stage_idx: @stages.index(job[:stage]),
+
+      { stage_idx: @stages.index(job[:stage]),
         stage: job[:stage],
         commands: job[:commands],
         tag_list: job[:tags] || [],
         name: job[:name].to_s,
-        allow_failure: job[:allow_failure] || false,
+        allow_failure: job[:ignore],
         when: job[:when] || 'on_success',
         environment: job[:environment_name],
+        coverage_regex: job[:coverage],
         yaml_variables: yaml_variables(name),
         options: {
           image: job[:image],
@@ -69,9 +81,8 @@ module Ci
           cache: job[:cache],
           dependencies: job[:dependencies],
           after_script: job[:after_script],
-          environment: job[:environment],
-        }.compact
-      }
+          environment: job[:environment]
+        }.compact }
     end
 
     def self.validation_message(content)
@@ -109,6 +120,7 @@ module Ci
 
         validate_job_stage!(name, job)
         validate_job_dependencies!(name, job)
+        validate_job_environment!(name, job)
       end
     end
 
@@ -117,7 +129,7 @@ module Ci
         .merge(job_variables(name))
 
       variables.map do |key, value|
-        { key: key, value: value, public: true }
+        { key: key.to_s, value: value, public: true }
       end
     end
 
@@ -150,30 +162,64 @@ module Ci
       end
     end
 
-    def process?(only_params, except_params, ref, tag, trigger_request)
+    def validate_job_environment!(name, job)
+      return unless job[:environment]
+      return unless job[:environment].is_a?(Hash)
+
+      environment = job[:environment]
+      validate_on_stop_job!(name, environment, environment[:on_stop])
+    end
+
+    def validate_on_stop_job!(name, environment, on_stop)
+      return unless on_stop
+
+      on_stop_job = @jobs[on_stop.to_sym]
+      unless on_stop_job
+        raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined"
+      end
+
+      unless on_stop_job[:environment]
+        raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined"
+      end
+
+      unless on_stop_job[:environment][:name] == environment[:name]
+        raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name"
+      end
+
+      unless on_stop_job[:environment][:action] == 'stop'
+        raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined"
+      end
+    end
+
+    def process?(only_params, except_params, ref, tag, source)
       if only_params.present?
-        return false unless matching?(only_params, ref, tag, trigger_request)
+        return false unless matching?(only_params, ref, tag, source)
       end
 
       if except_params.present?
-        return false if matching?(except_params, ref, tag, trigger_request)
+        return false if matching?(except_params, ref, tag, source)
       end
 
       true
     end
 
-    def matching?(patterns, ref, tag, trigger_request)
+    def matching?(patterns, ref, tag, source)
       patterns.any? do |pattern|
-        match_ref?(pattern, ref, tag, trigger_request)
+        pattern, path = pattern.split('@', 2)
+        matches_path?(path) && matches_pattern?(pattern, ref, tag, source)
       end
     end
 
-    def match_ref?(pattern, ref, tag, trigger_request)
-      pattern, path = pattern.split('@', 2)
-      return false if path && path != self.path
+    def matches_path?(path)
+      return true unless path
+
+      path == self.path
+    end
+
+    def matches_pattern?(pattern, ref, tag, source)
       return true if tag && pattern == 'tags'
       return true if !tag && pattern == 'branches'
-      return true if trigger_request.present? && pattern == 'triggers'
+      return true if source_to_pattern(source) == pattern
 
       if pattern.first == "/" && pattern.last == "/"
         Regexp.new(pattern[1...-1]) =~ ref
@@ -181,5 +227,13 @@ module Ci
         pattern == ref
       end
     end
+
+    def source_to_pattern(source)
+      if %w[api external web].include?(source)
+        source
+      else
+        source&.pluralize
+      end
+    end
   end
 end
diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb
index ca39b1961ae..6fc1d56d7a0 100644
--- a/lib/constraints/group_url_constrainer.rb
+++ b/lib/constraints/group_url_constrainer.rb
@@ -1,7 +1,9 @@
-require 'constraints/namespace_url_constrainer'
+class GroupUrlConstrainer
+  def matches?(request)
+    full_path = request.params[:group_id] || request.params[:id]
 
-class GroupUrlConstrainer < NamespaceUrlConstrainer
-  def find_resource(id)
-    Group.find_by_path(id)
+    return false unless DynamicPathValidator.valid_group_path?(full_path)
+
+    Group.find_by_full_path(full_path, follow_redirects: request.get?).present?
   end
 end
diff --git a/lib/constraints/namespace_url_constrainer.rb b/lib/constraints/namespace_url_constrainer.rb
deleted file mode 100644
index 23920193743..00000000000
--- a/lib/constraints/namespace_url_constrainer.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-class NamespaceUrlConstrainer
-  def matches?(request)
-    id = request.path.sub(/\A\/+/, '').split('/').first.sub(/.atom\z/, '')
-
-    if id =~ Gitlab::Regex.namespace_regex
-      find_resource(id)
-    end
-  end
-
-  def find_resource(id)
-    Namespace.find_by_path(id)
-  end
-end
diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb
new file mode 100644
index 00000000000..4c0aee6c48f
--- /dev/null
+++ b/lib/constraints/project_url_constrainer.rb
@@ -0,0 +1,11 @@
+class ProjectUrlConstrainer
+  def matches?(request)
+    namespace_path = request.params[:namespace_id]
+    project_path = request.params[:project_id] || request.params[:id]
+    full_path = [namespace_path, project_path].join('/')
+
+    return false unless DynamicPathValidator.valid_project_path?(full_path)
+
+    Project.find_by_full_path(full_path, follow_redirects: request.get?).present?
+  end
+end
diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb
index 504a0f5d93e..d16ae7f3f40 100644
--- a/lib/constraints/user_url_constrainer.rb
+++ b/lib/constraints/user_url_constrainer.rb
@@ -1,7 +1,9 @@
-require 'constraints/namespace_url_constrainer'
+class UserUrlConstrainer
+  def matches?(request)
+    full_path = request.params[:username]
 
-class UserUrlConstrainer < NamespaceUrlConstrainer
-  def find_resource(id)
-    User.find_by('lower(username) = ?', id.downcase)
+    return false unless DynamicPathValidator.valid_user_path?(full_path)
+
+    User.find_by_full_path(full_path, follow_redirects: request.get?).present?
   end
 end
diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb
index eb5a2596177..d5f85f9fcad 100644
--- a/lib/container_registry/blob.rb
+++ b/lib/container_registry/blob.rb
@@ -38,11 +38,11 @@ module ContainerRegistry
     end
 
     def delete
-      client.delete_blob(repository.name, digest)
+      client.delete_blob(repository.path, digest)
     end
 
     def data
-      @data ||= client.blob(repository.name, digest, type)
+      @data ||= client.blob(repository.path, digest, type)
     end
   end
 end
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index 2edddb84fc3..c7263f302ab 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -5,7 +5,7 @@ module ContainerRegistry
   class Client
     attr_accessor :uri
 
-    MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json'
+    MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json'.freeze
 
     # Taken from: FaradayMiddleware::FollowRedirects
     REDIRECT_CODES = Set.new [301, 302, 303, 307]
@@ -75,10 +75,7 @@ module ContainerRegistry
     def redirect_response(location)
       return unless location
 
-      # We explicitly remove authorization token
-      faraday_blob.get(location) do |req|
-        req['Authorization'] = ''
-      end
+      faraday_redirect.get(location)
     end
 
     def faraday
@@ -93,5 +90,14 @@ module ContainerRegistry
         initialize_connection(conn, @options)
       end
     end
+
+    # Create a new request to make sure the Authorization header is not inserted
+    # via the Faraday middleware
+    def faraday_redirect
+      @faraday_redirect ||= Faraday.new(@base_uri) do |conn|
+        conn.request :json
+        conn.adapter :net_http
+      end
+    end
   end
 end
diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb
new file mode 100644
index 00000000000..61849a40383
--- /dev/null
+++ b/lib/container_registry/path.rb
@@ -0,0 +1,76 @@
+module ContainerRegistry
+  ##
+  # Class responsible for extracting project and repository name from
+  # image repository path provided by a containers registry API response.
+  #
+  # Example:
+  #
+  # some/group/my_project/my/image ->
+  #   project: some/group/my_project
+  #   repository: my/image
+  #
+  class Path
+    InvalidRegistryPathError = Class.new(StandardError)
+
+    LEVELS_SUPPORTED = 3
+
+    def initialize(path)
+      @path = path.to_s.downcase
+    end
+
+    def valid?
+      @path =~ Gitlab::Regex.container_repository_name_regex &&
+        components.size > 1 &&
+        components.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED
+    end
+
+    def components
+      @components ||= @path.split('/')
+    end
+
+    def nodes
+      raise InvalidRegistryPathError unless valid?
+
+      @nodes ||= components.size.downto(2).map do |length|
+        components.take(length).join('/')
+      end
+    end
+
+    def has_project?
+      repository_project.present?
+    end
+
+    def has_repository?
+      return false unless has_project?
+
+      repository_project.container_repositories
+        .where(name: repository_name).any?
+    end
+
+    def root_repository?
+      @path == project_path
+    end
+
+    def repository_project
+      @project ||= Project
+        .where_full_path_in(nodes.first(LEVELS_SUPPORTED))
+        .first
+    end
+
+    def repository_name
+      return unless has_project?
+
+      @path.remove(%r(^#{Regexp.escape(project_path)}/?))
+    end
+
+    def project_path
+      return unless has_project?
+
+      repository_project.full_path.downcase
+    end
+
+    def to_s
+      @path
+    end
+  end
+end
diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb
index 0e634f6b6ef..63bce655f57 100644
--- a/lib/container_registry/registry.rb
+++ b/lib/container_registry/registry.rb
@@ -8,10 +8,6 @@ module ContainerRegistry
       @client = ContainerRegistry::Client.new(uri, options)
     end
 
-    def repository(name)
-      ContainerRegistry::Repository.new(self, name)
-    end
-
     private
 
     def default_path
diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb
deleted file mode 100644
index 0e4a7cb3cc9..00000000000
--- a/lib/container_registry/repository.rb
+++ /dev/null
@@ -1,48 +0,0 @@
-module ContainerRegistry
-  class Repository
-    attr_reader :registry, :name
-
-    delegate :client, to: :registry
-
-    def initialize(registry, name)
-      @registry, @name = registry, name
-    end
-
-    def path
-      [registry.path, name].compact.join('/')
-    end
-
-    def tag(tag)
-      ContainerRegistry::Tag.new(self, tag)
-    end
-
-    def manifest
-      return @manifest if defined?(@manifest)
-
-      @manifest = client.repository_tags(name)
-    end
-
-    def valid?
-      manifest.present?
-    end
-
-    def tags
-      return @tags if defined?(@tags)
-      return [] unless manifest && manifest['tags']
-
-      @tags = manifest['tags'].map do |tag|
-        ContainerRegistry::Tag.new(self, tag)
-      end
-    end
-
-    def blob(config)
-      ContainerRegistry::Blob.new(self, config)
-    end
-
-    def delete_tags
-      return unless tags
-
-      tags.all?(&:delete)
-    end
-  end
-end
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
index 59040199920..728deea224f 100644
--- a/lib/container_registry/tag.rb
+++ b/lib/container_registry/tag.rb
@@ -22,15 +22,17 @@ module ContainerRegistry
     end
 
     def manifest
-      return @manifest if defined?(@manifest)
-
-      @manifest = client.repository_manifest(repository.name, name)
+      @manifest ||= client.repository_manifest(repository.path, name)
     end
 
     def path
       "#{repository.path}:#{name}"
     end
 
+    def location
+      "#{repository.location}:#{name}"
+    end
+
     def [](key)
       return unless manifest
 
@@ -38,9 +40,7 @@ module ContainerRegistry
     end
 
     def digest
-      return @digest if defined?(@digest)
-
-      @digest = client.repository_tag_digest(repository.name, name)
+      @digest ||= client.repository_tag_digest(repository.path, name)
     end
 
     def config_blob
@@ -82,7 +82,7 @@ module ContainerRegistry
     def delete
       return unless digest
 
-      client.delete_repository_tag(repository.name, digest)
+      client.delete_repository_tag(repository.path, digest)
     end
   end
 end
diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb
new file mode 100644
index 00000000000..63f9f8d7a5a
--- /dev/null
+++ b/lib/email_template_interceptor.rb
@@ -0,0 +1,13 @@
+# Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails
+class EmailTemplateInterceptor
+  include Gitlab::CurrentSettings
+
+  def self.delivering_email(message)
+    # Remove HTML part if HTML emails are disabled.
+    unless current_application_settings.html_emails_enabled
+      message.parts.delete_if do |part|
+        part.content_type.start_with?('text/html')
+      end
+    end
+  end
+end
diff --git a/lib/event_filter.rb b/lib/event_filter.rb
index 96e70e37e8f..515095af1c2 100644
--- a/lib/event_filter.rb
+++ b/lib/event_filter.rb
@@ -14,6 +14,10 @@ class EventFilter
       'merged'
     end
 
+    def issue
+      'issue'
+    end
+
     def comments
       'comments'
     end
@@ -32,25 +36,20 @@ class EventFilter
   end
 
   def apply_filter(events)
-    return events unless params.present?
-
-    filter = params.dup
-    actions = []
+    return events if params.blank? || params == EventFilter.all
 
-    case filter
+    case params
     when EventFilter.push
-      actions = [Event::PUSHED]
+      events.where(action: Event::PUSHED)
     when EventFilter.merged
-      actions = [Event::MERGED]
+      events.where(action: Event::MERGED)
     when EventFilter.comments
-      actions = [Event::COMMENTED]
+      events.where(action: Event::COMMENTED)
     when EventFilter.team
-      actions = [Event::JOINED, Event::LEFT]
-    when EventFilter.all
-      actions = [Event::PUSHED, Event::MERGED, Event::COMMENTED, Event::JOINED, Event::LEFT]
+      events.where(action: [Event::JOINED, Event::LEFT, Event::EXPIRED])
+    when EventFilter.issue
+      events.where(action: [Event::CREATED, Event::UPDATED, Event::CLOSED, Event::REOPENED])
     end
-
-    events.where(action: actions)
   end
 
   def options(key)
@@ -66,6 +65,10 @@ class EventFilter
   end
 
   def active?(key)
-    params.include? key
+    if params.present?
+      params.include? key
+    else
+      key == EventFilter.all
+    end
   end
 end
diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb
index a4558d157c0..dd864eea3fa 100644
--- a/lib/extracts_path.rb
+++ b/lib/extracts_path.rb
@@ -2,7 +2,7 @@
 # file path string when combined in a request parameter
 module ExtractsPath
   # Raised when given an invalid file path
-  class InvalidPathError < StandardError; end
+  InvalidPathError = Class.new(StandardError)
 
   # Given a string containing both a Git tree-ish, such as a branch or tag, and
   # a filesystem path joined by forward slashes, attempts to separate the two.
@@ -42,7 +42,7 @@ module ExtractsPath
 
     return pair unless @project
 
-    if id.match(/^([[:alnum:]]{40})(.+)/)
+    if id =~ /^(\h{40})(.+)/
       # If the ref appears to be a SHA, we're done, just split the string
       pair = $~.captures
     else
@@ -52,8 +52,7 @@ module ExtractsPath
       # Append a trailing slash if we only get a ref and no file path
       id += '/' unless id.ends_with?('/')
 
-      valid_refs = @project.repository.ref_names
-      valid_refs.select! { |v| id.start_with?("#{v}/") }
+      valid_refs = ref_names.select { |v| id.start_with?("#{v}/") }
 
       if valid_refs.length == 0
         # No exact ref match, so just try our best
@@ -74,6 +73,19 @@ module ExtractsPath
     pair
   end
 
+  # If we have an ID of 'foo.atom', and the controller provides Atom and HTML
+  # formats, then we have to check if the request was for the Atom version of
+  # the ID without the '.atom' suffix, or the HTML version of the ID including
+  # the suffix. We only check this if the version including the suffix doesn't
+  # match, so it is possible to create a branch which has an unroutable Atom
+  # feed.
+  def extract_ref_without_atom(id)
+    id_without_atom = id.sub(/\.atom$/, '')
+    valid_refs = ref_names.select { |v| "#{id_without_atom}/".start_with?("#{v}/") }
+
+    valid_refs.max_by(&:length)
+  end
+
   # Assigns common instance variables for views working with Git tree-ish objects
   #
   # Assignments are:
@@ -86,21 +98,29 @@ module ExtractsPath
   # If the :id parameter appears to be requesting a specific response format,
   # that will be handled as well.
   #
+  # If there is no path and the ref doesn't exist in the repo, try to resolve
+  # the ref without an '.atom' suffix. If _that_ ref is found, set the request's
+  # format to Atom manually.
+  #
   # Automatically renders `not_found!` if a valid tree path could not be
   # resolved (e.g., when a user inserts an invalid path or ref).
   def assign_ref_vars
     # assign allowed options
-    allowed_options = ["filter_ref", "extended_sha1"]
+    allowed_options = ["filter_ref"]
     @options = params.select {|key, value| allowed_options.include?(key) && !value.blank? }
     @options = HashWithIndifferentAccess.new(@options)
 
     @id = get_id
     @ref, @path = extract_ref(@id)
     @repo = @project.repository
-    if @options[:extended_sha1].blank?
+
+    @commit = @repo.commit(@ref)
+
+    if @path.empty? && !@commit && @id.ends_with?('.atom')
+      @id = @ref = extract_ref_without_atom(@id)
       @commit = @repo.commit(@ref)
-    else
-      @commit = @repo.commit(@options[:extended_sha1])
+
+      request.format = :atom if @commit
     end
 
     raise InvalidPathError unless @commit
@@ -125,4 +145,10 @@ module ExtractsPath
     id += "/" + params[:path] unless params[:path].blank?
     id
   end
+
+  def ref_names
+    return [] unless @project
+
+    @ref_names ||= @project.repository.ref_names
+  end
 end
diff --git a/lib/feature.rb b/lib/feature.rb
new file mode 100644
index 00000000000..d3d972564af
--- /dev/null
+++ b/lib/feature.rb
@@ -0,0 +1,51 @@
+require 'flipper/adapters/active_record'
+
+class Feature
+  # Classes to override flipper table names
+  class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature
+    # Using `self.table_name` won't work. ActiveRecord bug?
+    superclass.table_name = 'features'
+  end
+
+  class FlipperGate < Flipper::Adapters::ActiveRecord::Gate
+    superclass.table_name = 'feature_gates'
+  end
+
+  class << self
+    def all
+      flipper.features.to_a
+    end
+
+    def get(key)
+      flipper.feature(key)
+    end
+
+    def persisted?(feature)
+      # Flipper creates on-memory features when asked for a not-yet-created one.
+      # If we want to check if a feature has been actually set, we look for it
+      # on the persisted features list.
+      all.map(&:name).include?(feature.name)
+    end
+
+    def enabled?(key)
+      get(key).enabled?
+    end
+
+    def enable(key)
+      get(key).enable
+    end
+
+    def disable(key)
+      get(key).disable
+    end
+
+    def flipper
+      @flipper ||= begin
+        adapter = Flipper::Adapters::ActiveRecord.new(
+          feature_class: FlipperFeature, gate_class: FlipperGate)
+
+        Flipper.new(adapter)
+      end
+    end
+  end
+end
diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb
index 440dd44ece7..eb19ab45ac3 100644
--- a/lib/file_size_validator.rb
+++ b/lib/file_size_validator.rb
@@ -32,9 +32,9 @@ class FileSizeValidator < ActiveModel::EachValidator
   end
 
   def validate_each(record, attribute, value)
-    raise(ArgumentError, "A CarrierWave::Uploader::Base object was expected") unless value.kind_of? CarrierWave::Uploader::Base
+    raise(ArgumentError, "A CarrierWave::Uploader::Base object was expected") unless value.is_a? CarrierWave::Uploader::Base
 
-    value = (options[:tokenizer] || DEFAULT_TOKENIZER).call(value) if value.kind_of?(String)
+    value = (options[:tokenizer] || DEFAULT_TOKENIZER).call(value) if value.is_a?(String)
 
     CHECKS.each do |key, validity_check|
       next unless check_value = options[key]
diff --git a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb
new file mode 100644
index 00000000000..7cb4bccb23c
--- /dev/null
+++ b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb
@@ -0,0 +1,15 @@
+require 'rails/generators'
+
+module Rails
+  class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase
+    def create_migration_file
+      timestamp = Time.now.strftime('%Y%m%d%H%I%S')
+
+      template "migration.rb", "db/post_migrate/#{timestamp}_#{file_name}.rb"
+    end
+
+    def migration_class_name
+      file_name.camelize
+    end
+  end
+end
diff --git a/lib/github/client.rb b/lib/github/client.rb
new file mode 100644
index 00000000000..e65d908d232
--- /dev/null
+++ b/lib/github/client.rb
@@ -0,0 +1,23 @@
+module Github
+  class Client
+    attr_reader :connection, :rate_limit
+
+    def initialize(options)
+      @connection = Faraday.new(url: options.fetch(:url)) do |faraday|
+        faraday.options.open_timeout = options.fetch(:timeout, 60)
+        faraday.options.timeout = options.fetch(:timeout, 60)
+        faraday.authorization 'token', options.fetch(:token)
+        faraday.adapter :net_http
+      end
+
+      @rate_limit = RateLimit.new(connection)
+    end
+
+    def get(url, query = {})
+      exceed, reset_in = rate_limit.get
+      sleep reset_in if exceed
+
+      Github::Response.new(connection.get(url, query))
+    end
+  end
+end
diff --git a/lib/github/collection.rb b/lib/github/collection.rb
new file mode 100644
index 00000000000..014b2038c4b
--- /dev/null
+++ b/lib/github/collection.rb
@@ -0,0 +1,29 @@
+module Github
+  class Collection
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def fetch(url, query = {})
+      return [] if url.blank?
+
+      Enumerator.new do |yielder|
+        loop do
+          response = client.get(url, query)
+          response.body.each { |item| yielder << item }
+
+          raise StopIteration unless response.rels.key?(:next)
+          url = response.rels[:next]
+        end
+      end.lazy
+    end
+
+    private
+
+    def client
+      @client ||= Github::Client.new(options)
+    end
+  end
+end
diff --git a/lib/github/error.rb b/lib/github/error.rb
new file mode 100644
index 00000000000..66d7afaa787
--- /dev/null
+++ b/lib/github/error.rb
@@ -0,0 +1,3 @@
+module Github
+  RepositoryFetchError = Class.new(StandardError)
+end
diff --git a/lib/github/import.rb b/lib/github/import.rb
new file mode 100644
index 00000000000..ff5d7db2705
--- /dev/null
+++ b/lib/github/import.rb
@@ -0,0 +1,386 @@
+require_relative 'error'
+
+module Github
+  class Import
+    include Gitlab::ShellAdapter
+
+    class MergeRequest < ::MergeRequest
+      self.table_name = 'merge_requests'
+
+      self.reset_callbacks :create
+      self.reset_callbacks :save
+      self.reset_callbacks :commit
+      self.reset_callbacks :update
+      self.reset_callbacks :validate
+    end
+
+    class Issue < ::Issue
+      self.table_name = 'issues'
+
+      self.reset_callbacks :save
+      self.reset_callbacks :create
+      self.reset_callbacks :commit
+      self.reset_callbacks :update
+      self.reset_callbacks :validate
+    end
+
+    class Note < ::Note
+      self.table_name = 'notes'
+
+      self.reset_callbacks :save
+      self.reset_callbacks :commit
+      self.reset_callbacks :update
+      self.reset_callbacks :validate
+    end
+
+    class LegacyDiffNote < ::LegacyDiffNote
+      self.table_name = 'notes'
+
+      self.reset_callbacks :commit
+      self.reset_callbacks :update
+      self.reset_callbacks :validate
+    end
+
+    attr_reader :project, :repository, :repo, :options, :errors, :cached, :verbose
+
+    def initialize(project, options)
+      @project = project
+      @repository = project.repository
+      @repo = project.import_source
+      @options = options
+      @verbose = options.fetch(:verbose, false)
+      @cached  = Hash.new { |hash, key| hash[key] = Hash.new }
+      @errors  = []
+    end
+
+    # rubocop: disable Rails/Output
+    def execute
+      puts 'Fetching repository...'.color(:aqua) if verbose
+      fetch_repository
+      puts 'Fetching labels...'.color(:aqua) if verbose
+      fetch_labels
+      puts 'Fetching milestones...'.color(:aqua) if verbose
+      fetch_milestones
+      puts 'Fetching pull requests...'.color(:aqua) if verbose
+      fetch_pull_requests
+      puts 'Fetching issues...'.color(:aqua) if verbose
+      fetch_issues
+      puts 'Cloning wiki repository...'.color(:aqua) if verbose
+      fetch_wiki_repository
+      puts 'Expiring repository cache...'.color(:aqua) if verbose
+      expire_repository_cache
+
+      true
+    rescue Github::RepositoryFetchError
+      false
+    ensure
+      keep_track_of_errors
+    end
+
+    private
+
+    def fetch_repository
+      begin
+        project.create_repository unless project.repository.exists?
+        project.repository.add_remote('github', "https://#{options.fetch(:token)}@github.com/#{repo}.git")
+        project.repository.set_remote_as_mirror('github')
+        project.repository.fetch_remote('github', forced: true)
+      rescue Gitlab::Shell::Error => e
+        error(:project, "https://github.com/#{repo}.git", e.message)
+        raise Github::RepositoryFetchError
+      end
+    end
+
+    def fetch_wiki_repository
+      wiki_url  = "https://#{options.fetch(:token)}@github.com/#{repo}.wiki.git"
+      wiki_path = "#{project.path_with_namespace}.wiki"
+
+      unless project.wiki.repository_exists?
+        gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url)
+      end
+    rescue Gitlab::Shell::Error => e
+      # GitHub error message when the wiki repo has not been created,
+      # this means that repo has wiki enabled, but have no pages. So,
+      # we can skip the import.
+      if e.message !~ /repository not exported/
+        errors(:wiki, wiki_url, e.message)
+      end
+    end
+
+    def fetch_labels
+      url = "/repos/#{repo}/labels"
+
+      while url
+        response = Github::Client.new(options).get(url)
+
+        response.body.each do |raw|
+          begin
+            representation = Github::Representation::Label.new(raw)
+
+            label = project.labels.find_or_create_by!(title: representation.title) do |label|
+              label.color = representation.color
+            end
+
+            cached[:label_ids][label.title] = label.id
+          rescue => e
+            error(:label, representation.url, e.message)
+          end
+        end
+
+        url = response.rels[:next]
+      end
+    end
+
+    def fetch_milestones
+      url = "/repos/#{repo}/milestones"
+
+      while url
+        response = Github::Client.new(options).get(url, state: :all)
+
+        response.body.each do |raw|
+          begin
+            milestone = Github::Representation::Milestone.new(raw)
+            next if project.milestones.where(iid: milestone.iid).exists?
+
+            project.milestones.create!(
+              iid: milestone.iid,
+              title: milestone.title,
+              description: milestone.description,
+              due_date: milestone.due_date,
+              state: milestone.state,
+              created_at: milestone.created_at,
+              updated_at: milestone.updated_at
+            )
+          rescue => e
+            error(:milestone, milestone.url, e.message)
+          end
+        end
+
+        url = response.rels[:next]
+      end
+    end
+
+    def fetch_pull_requests
+      url = "/repos/#{repo}/pulls"
+
+      while url
+        response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc)
+
+        response.body.each do |raw|
+          pull_request  = Github::Representation::PullRequest.new(raw, options.merge(project: project))
+          merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id)
+          next unless merge_request.new_record? && pull_request.valid?
+
+          begin
+            pull_request.restore_branches!
+
+            author_id   = user_id(pull_request.author, project.creator_id)
+            description = format_description(pull_request.description, pull_request.author)
+
+            merge_request.attributes = {
+              iid: pull_request.iid,
+              title: pull_request.title,
+              description: description,
+              source_project: pull_request.source_project,
+              source_branch: pull_request.source_branch_name,
+              source_branch_sha: pull_request.source_branch_sha,
+              target_project: pull_request.target_project,
+              target_branch: pull_request.target_branch_name,
+              target_branch_sha: pull_request.target_branch_sha,
+              state: pull_request.state,
+              milestone_id: milestone_id(pull_request.milestone),
+              author_id: author_id,
+              assignee_id: user_id(pull_request.assignee),
+              created_at: pull_request.created_at,
+              updated_at: pull_request.updated_at
+            }
+
+            merge_request.save!(validate: false)
+            merge_request.merge_request_diffs.create
+
+            # Fetch review comments
+            review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments"
+            fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote)
+
+            # Fetch comments
+            comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments"
+            fetch_comments(merge_request, :comment, comments_url)
+          rescue => e
+            error(:pull_request, pull_request.url, e.message)
+          ensure
+            pull_request.remove_restored_branches!
+          end
+        end
+
+        url = response.rels[:next]
+      end
+    end
+
+    def fetch_issues
+      url = "/repos/#{repo}/issues"
+
+      while url
+        response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc)
+
+        response.body.each do |raw|
+          representation = Github::Representation::Issue.new(raw, options)
+
+          begin
+            # Every pull request is an issue, but not every issue
+            # is a pull request. For this reason, "shared" actions
+            # for both features, like manipulating assignees, labels
+            # and milestones, are provided within the Issues API.
+            if representation.pull_request?
+              next unless representation.has_labels?
+
+              merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid)
+              merge_request.update_attribute(:label_ids, label_ids(representation.labels))
+            else
+              next if Issue.where(iid: representation.iid, project_id: project.id).exists?
+
+              author_id          = user_id(representation.author, project.creator_id)
+              issue              = Issue.new
+              issue.iid          = representation.iid
+              issue.project_id   = project.id
+              issue.title        = representation.title
+              issue.description  = format_description(representation.description, representation.author)
+              issue.state        = representation.state
+              issue.label_ids    = label_ids(representation.labels)
+              issue.milestone_id = milestone_id(representation.milestone)
+              issue.author_id    = author_id
+              issue.assignee_ids = [user_id(representation.assignee)]
+              issue.created_at   = representation.created_at
+              issue.updated_at   = representation.updated_at
+              issue.save!(validate: false)
+
+              # Fetch comments
+              if representation.has_comments?
+                comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments"
+                fetch_comments(issue, :comment, comments_url)
+              end
+            end
+          rescue => e
+            error(:issue, representation.url, e.message)
+          end
+        end
+
+        url = response.rels[:next]
+      end
+    end
+
+    def fetch_comments(noteable, type, url, klass = Note)
+      while url
+        comments = Github::Client.new(options).get(url)
+
+        ActiveRecord::Base.no_touching do
+          comments.body.each do |raw|
+            begin
+              representation  = Github::Representation::Comment.new(raw, options)
+              author_id       = user_id(representation.author, project.creator_id)
+
+              note            = klass.new
+              note.project_id = project.id
+              note.noteable   = noteable
+              note.note       = format_description(representation.note, representation.author)
+              note.commit_id  = representation.commit_id
+              note.line_code  = representation.line_code
+              note.author_id  = author_id
+              note.created_at = representation.created_at
+              note.updated_at = representation.updated_at
+              note.save!(validate: false)
+            rescue => e
+              error(type, representation.url, e.message)
+            end
+          end
+        end
+
+        url = comments.rels[:next]
+      end
+    end
+
+    def fetch_releases
+      url = "/repos/#{repo}/releases"
+
+      while url
+        response = Github::Client.new(options).get(url)
+
+        response.body.each do |raw|
+          representation = Github::Representation::Release.new(raw)
+          next unless representation.valid?
+
+          release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag)
+          next unless relese.new_record?
+
+          begin
+            release.description = representation.description
+            release.created_at  = representation.created_at
+            release.updated_at  = representation.updated_at
+            release.save!(validate: false)
+          rescue => e
+            error(:release, representation.url, e.message)
+          end
+        end
+
+        url = response.rels[:next]
+      end
+    end
+
+    def label_ids(labels)
+      labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact
+    end
+
+    def milestone_id(milestone)
+      return unless milestone.present?
+
+      project.milestones.select(:id).find_by(iid: milestone.iid)&.id
+    end
+
+    def user_id(user, fallback_id = nil)
+      return unless user.present?
+      return cached[:user_ids][user.id] if cached[:user_ids].key?(user.id)
+
+      gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email)
+
+      cached[:gitlab_user_ids][user.id] = gitlab_user_id.present?
+      cached[:user_ids][user.id] = gitlab_user_id || fallback_id
+    end
+
+    def user_id_by_email(email)
+      return nil unless email
+
+      ::User.find_by_any_email(email)&.id
+    end
+
+    def user_id_by_external_uid(id)
+      return nil unless id
+
+      ::User.select(:id)
+            .joins(:identities)
+            .merge(::Identity.where(provider: :github, extern_uid: id))
+            .first&.id
+    end
+
+    def format_description(body, author)
+      return body if cached[:gitlab_user_ids][author.id]
+
+      "*Created by: #{author.username}*\n\n#{body}"
+    end
+
+    def expire_repository_cache
+      repository.expire_content_cache
+    end
+
+    def keep_track_of_errors
+      return unless errors.any?
+
+      project.update_column(:import_error, {
+        message: 'The remote data could not be fully imported.',
+        errors: errors
+      }.to_json)
+    end
+
+    def error(type, url, message)
+      errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message }
+    end
+  end
+end
diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb
new file mode 100644
index 00000000000..884693d093c
--- /dev/null
+++ b/lib/github/rate_limit.rb
@@ -0,0 +1,27 @@
+module Github
+  class RateLimit
+    SAFE_REMAINING_REQUESTS = 100
+    SAFE_RESET_TIME         = 500
+    RATE_LIMIT_URL          = '/rate_limit'.freeze
+
+    attr_reader :connection
+
+    def initialize(connection)
+      @connection = connection
+    end
+
+    def get
+      response = connection.get(RATE_LIMIT_URL)
+
+      # GitHub Rate Limit API returns 404 when the rate limit is disabled
+      return false unless response.status != 404
+
+      body      = Oj.load(response.body, class_cache: false, mode: :compat)
+      remaining = body.dig('rate', 'remaining').to_i
+      reset_in  = body.dig('rate', 'reset').to_i
+      exceed    = remaining <= SAFE_REMAINING_REQUESTS
+
+      [exceed, reset_in]
+    end
+  end
+end
diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb
new file mode 100644
index 00000000000..c1c9448f305
--- /dev/null
+++ b/lib/github/repositories.rb
@@ -0,0 +1,19 @@
+module Github
+  class Repositories
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def fetch
+      Collection.new(options).fetch(repos_url)
+    end
+
+    private
+
+    def repos_url
+      '/user/repos'
+    end
+  end
+end
diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb
new file mode 100644
index 00000000000..f26bdbdd546
--- /dev/null
+++ b/lib/github/representation/base.rb
@@ -0,0 +1,30 @@
+module Github
+  module Representation
+    class Base
+      def initialize(raw, options = {})
+        @raw     = raw
+        @options = options
+      end
+
+      def id
+        raw['id']
+      end
+
+      def url
+        raw['url']
+      end
+
+      def created_at
+        raw['created_at']
+      end
+
+      def updated_at
+        raw['updated_at']
+      end
+
+      private
+
+      attr_reader :raw, :options
+    end
+  end
+end
diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb
new file mode 100644
index 00000000000..c6fa928d565
--- /dev/null
+++ b/lib/github/representation/branch.rb
@@ -0,0 +1,63 @@
+module Github
+  module Representation
+    class Branch < Representation::Base
+      attr_reader :repository
+
+      def user
+        raw.dig('user', 'login') || 'unknown'
+      end
+
+      def repo
+        return @repo if defined?(@repo)
+
+        @repo = Github::Representation::Repo.new(raw['repo']) if raw['repo'].present?
+      end
+
+      def ref
+        raw['ref']
+      end
+
+      def sha
+        raw['sha']
+      end
+
+      def short_sha
+        Commit.truncate_sha(sha)
+      end
+
+      def exists?
+        @exists ||= branch_exists? && commit_exists?
+      end
+
+      def valid?
+        sha.present? && ref.present?
+      end
+
+      def restore!(name)
+        repository.create_branch(name, sha)
+      rescue Gitlab::Git::Repository::InvalidRef => e
+        Rails.logger.error("#{self.class.name}: Could not restore branch #{name}: #{e}")
+      end
+
+      def remove!(name)
+        repository.delete_branch(name)
+      rescue Rugged::ReferenceError => e
+        Rails.logger.error("#{self.class.name}: Could not remove branch #{name}: #{e}")
+      end
+
+      private
+
+      def branch_exists?
+        repository.branch_exists?(ref)
+      end
+
+      def commit_exists?
+        repository.branch_names_contains(sha).include?(ref)
+      end
+
+      def repository
+        @repository ||= options.fetch(:repository)
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb
new file mode 100644
index 00000000000..1b5be91461b
--- /dev/null
+++ b/lib/github/representation/comment.rb
@@ -0,0 +1,42 @@
+module Github
+  module Representation
+    class Comment < Representation::Base
+      def note
+        raw['body'] || ''
+      end
+
+      def author
+        @author ||= Github::Representation::User.new(raw['user'], options)
+      end
+
+      def commit_id
+        raw['commit_id']
+      end
+
+      def line_code
+        return unless on_diff?
+
+        parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines)
+        generate_line_code(parsed_lines.to_a.last)
+      end
+
+      private
+
+      def generate_line_code(line)
+        Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos)
+      end
+
+      def on_diff?
+        diff_hunk.present?
+      end
+
+      def diff_hunk
+        raw['diff_hunk']
+      end
+
+      def file_path
+        raw['path']
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb
new file mode 100644
index 00000000000..9713b82615d
--- /dev/null
+++ b/lib/github/representation/issuable.rb
@@ -0,0 +1,37 @@
+module Github
+  module Representation
+    class Issuable < Representation::Base
+      def iid
+        raw['number']
+      end
+
+      def title
+        raw['title']
+      end
+
+      def description
+        raw['body'] || ''
+      end
+
+      def milestone
+        return unless raw['milestone'].present?
+
+        @milestone ||= Github::Representation::Milestone.new(raw['milestone'])
+      end
+
+      def author
+        @author ||= Github::Representation::User.new(raw['user'], options)
+      end
+
+      def assignee
+        return unless assigned?
+
+        @assignee ||= Github::Representation::User.new(raw['assignee'], options)
+      end
+
+      def assigned?
+        raw['assignee'].present?
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb
new file mode 100644
index 00000000000..df3540a6e6c
--- /dev/null
+++ b/lib/github/representation/issue.rb
@@ -0,0 +1,25 @@
+module Github
+  module Representation
+    class Issue < Representation::Issuable
+      def labels
+        raw['labels']
+      end
+
+      def state
+        raw['state'] == 'closed' ? 'closed' : 'opened'
+      end
+
+      def has_comments?
+        raw['comments'] > 0
+      end
+
+      def has_labels?
+        labels.count > 0
+      end
+
+      def pull_request?
+        raw['pull_request'].present?
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb
new file mode 100644
index 00000000000..60aa51f9569
--- /dev/null
+++ b/lib/github/representation/label.rb
@@ -0,0 +1,13 @@
+module Github
+  module Representation
+    class Label < Representation::Base
+      def color
+        "##{raw['color']}"
+      end
+
+      def title
+        raw['name']
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb
new file mode 100644
index 00000000000..917e6394ad4
--- /dev/null
+++ b/lib/github/representation/milestone.rb
@@ -0,0 +1,25 @@
+module Github
+  module Representation
+    class Milestone < Representation::Base
+      def iid
+        raw['number']
+      end
+
+      def title
+        raw['title']
+      end
+
+      def description
+        raw['description']
+      end
+
+      def due_date
+        raw['due_on']
+      end
+
+      def state
+        raw['state'] == 'closed' ? 'closed' : 'active'
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb
new file mode 100644
index 00000000000..55461097e8a
--- /dev/null
+++ b/lib/github/representation/pull_request.rb
@@ -0,0 +1,120 @@
+module Github
+  module Representation
+    class PullRequest < Representation::Issuable
+      delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true
+      delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true
+
+      def source_project
+        project
+      end
+
+      def source_branch_name
+        @source_branch_name ||=
+          if cross_project? || !source_branch_exists?
+            source_branch_name_prefixed
+          else
+            source_branch_ref
+          end
+      end
+
+      def source_branch_exists?
+        return @source_branch_exists if defined?(@source_branch_exists)
+
+        @source_branch_exists = !cross_project? && source_branch.exists?
+      end
+
+      def target_project
+        project
+      end
+
+      def target_branch_name
+        @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed
+      end
+
+      def target_branch_exists?
+        @target_branch_exists ||= target_branch.exists?
+      end
+
+      def state
+        return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present?
+        return 'closed' if raw['state'] == 'closed'
+
+        'opened'
+      end
+
+      def opened?
+        state == 'opened'
+      end
+
+      def valid?
+        source_branch.valid? && target_branch.valid?
+      end
+
+      def restore_branches!
+        restore_source_branch!
+        restore_target_branch!
+      end
+
+      def remove_restored_branches!
+        return if opened?
+
+        remove_source_branch!
+        remove_target_branch!
+      end
+
+      private
+
+      def project
+        @project ||= options.fetch(:project)
+      end
+
+      def source_branch
+        @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository)
+      end
+
+      def source_branch_name_prefixed
+        "gh-#{target_branch_short_sha}/#{iid}/#{source_branch_user}/#{source_branch_ref}"
+      end
+
+      def target_branch
+        @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository)
+      end
+
+      def target_branch_name_prefixed
+        "gl-#{target_branch_short_sha}/#{iid}/#{target_branch_user}/#{target_branch_ref}"
+      end
+
+      def cross_project?
+        return true if source_branch_repo.nil?
+
+        source_branch_repo.id != target_branch_repo.id
+      end
+
+      def restore_source_branch!
+        return if source_branch_exists?
+
+        source_branch.restore!(source_branch_name)
+      end
+
+      def restore_target_branch!
+        return if target_branch_exists?
+
+        target_branch.restore!(target_branch_name)
+      end
+
+      def remove_source_branch!
+        # We should remove the source/target branches only if they were
+        # restored. Otherwise, we'll remove branches like 'master' that
+        # target_branch_exists? returns true. In other words, we need
+        # to clean up only the restored branches that (source|target)_branch_exists?
+        # returns false for the first time it has been called, because of
+        # this that is important to memoize these values.
+        source_branch.remove!(source_branch_name) unless source_branch_exists?
+      end
+
+      def remove_target_branch!
+        target_branch.remove!(target_branch_name) unless target_branch_exists?
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/release.rb b/lib/github/representation/release.rb
new file mode 100644
index 00000000000..e7e4b428c1a
--- /dev/null
+++ b/lib/github/representation/release.rb
@@ -0,0 +1,17 @@
+module Github
+  module Representation
+    class Release < Representation::Base
+      def description
+        raw['body']
+      end
+
+      def tag
+        raw['tag_name']
+      end
+
+      def valid?
+        !raw['draft']
+      end
+    end
+  end
+end
diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb
new file mode 100644
index 00000000000..6938aa7db05
--- /dev/null
+++ b/lib/github/representation/repo.rb
@@ -0,0 +1,6 @@
+module Github
+  module Representation
+    class Repo < Representation::Base
+    end
+  end
+end
diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb
new file mode 100644
index 00000000000..18591380e25
--- /dev/null
+++ b/lib/github/representation/user.rb
@@ -0,0 +1,15 @@
+module Github
+  module Representation
+    class User < Representation::Base
+      def email
+        return @email if defined?(@email)
+
+        @email = Github::User.new(username, options).get.fetch('email', nil)
+      end
+
+      def username
+        raw['login']
+      end
+    end
+  end
+end
diff --git a/lib/github/response.rb b/lib/github/response.rb
new file mode 100644
index 00000000000..761c524b553
--- /dev/null
+++ b/lib/github/response.rb
@@ -0,0 +1,25 @@
+module Github
+  class Response
+    attr_reader :raw, :headers, :status
+
+    def initialize(response)
+      @raw     = response
+      @headers = response.headers
+      @status  = response.status
+    end
+
+    def body
+      Oj.load(raw.body, class_cache: false, mode: :compat)
+    end
+
+    def rels
+      links = headers['Link'].to_s.split(', ').map do |link|
+        href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures
+
+        [name.to_sym, href]
+      end
+
+      Hash[*links.flatten]
+    end
+  end
+end
diff --git a/lib/github/user.rb b/lib/github/user.rb
new file mode 100644
index 00000000000..f88a29e590b
--- /dev/null
+++ b/lib/github/user.rb
@@ -0,0 +1,24 @@
+module Github
+  class User
+    attr_reader :username, :options
+
+    def initialize(username, options)
+      @username = username
+      @options  = options
+    end
+
+    def get
+      client.get(user_url).body
+    end
+
+    private
+
+    def client
+      @client ||= Github::Client.new(options)
+    end
+
+    def user_url
+      "/users/#{username}"
+    end
+  end
+end
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index c3064163e07..11f7c8b9510 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -1,9 +1,11 @@
 require_dependency 'gitlab/git'
 
 module Gitlab
+  COM_URL = 'https://gitlab.com'.freeze
+
   def self.com?
     # Check `staging?` as well to keep parity with gitlab.com
-    Gitlab.config.gitlab.url == 'https://gitlab.com' || staging?
+    Gitlab.config.gitlab.url == COM_URL || staging?
   end
 
   def self.staging?
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb
index 9b484a2ecfd..4714ab18cc1 100644
--- a/lib/gitlab/access.rb
+++ b/lib/gitlab/access.rb
@@ -5,7 +5,7 @@
 #
 module Gitlab
   module Access
-    class AccessDeniedError < StandardError; end
+    AccessDeniedError = Class.new(StandardError)
 
     NO_ACCESS = 0
     GUEST     = 10
@@ -21,9 +21,7 @@ module Gitlab
     PROTECTION_DEV_CAN_MERGE = 3
 
     class << self
-      def values
-        options.values
-      end
+      delegate :values, to: :options
 
       def all_values
         options_with_owner.values
@@ -34,7 +32,7 @@ module Gitlab
           "Guest"     => GUEST,
           "Reporter"  => REPORTER,
           "Developer" => DEVELOPER,
-          "Master"    => MASTER,
+          "Master"    => MASTER
         }
       end
 
@@ -49,7 +47,7 @@ module Gitlab
           guest:     GUEST,
           reporter:  REPORTER,
           developer: DEVELOPER,
-          master:    MASTER,
+          master:    MASTER
         }
       end
 
@@ -62,7 +60,7 @@ module Gitlab
           "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE,
           "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch." => PROTECTION_DEV_CAN_MERGE,
           "Partially protected: Developers can push new commits, but cannot force push or delete the branch. Masters can do all of those." => PROTECTION_DEV_CAN_PUSH,
-          "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL,
+          "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL
         }
       end
 
diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb
new file mode 100644
index 00000000000..e4f7cad2b79
--- /dev/null
+++ b/lib/gitlab/allowable.rb
@@ -0,0 +1,7 @@
+module Gitlab
+  module Allowable
+    def can?(user, action, subject = :global)
+      Ability.allowed?(user, action, subject)
+    end
+  end
+end
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb
index 1a22ad9acf5..3d41ac76406 100644
--- a/lib/gitlab/asciidoc.rb
+++ b/lib/gitlab/asciidoc.rb
@@ -1,4 +1,6 @@
 require 'asciidoctor'
+require 'asciidoctor/converter/html5'
+require "asciidoctor-plantuml"
 
 module Gitlab
   # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters
@@ -6,33 +8,58 @@ module Gitlab
   module Asciidoc
     DEFAULT_ADOC_ATTRS = [
       'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab',
-      'env-gitlab', 'source-highlighter=html-pipeline'
+      'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font'
     ].freeze
 
     # Public: Converts the provided Asciidoc markup into HTML.
     #
     # input         - the source text in Asciidoc format
-    # context       - a Hash with the template context:
-    #                 :commit
-    #                 :project
-    #                 :project_wiki
-    #                 :requested_path
-    #                 :ref
-    # asciidoc_opts - a Hash of options to pass to the Asciidoctor converter
     #
-    def self.render(input, context, asciidoc_opts = {})
-      asciidoc_opts.reverse_merge!(
-        safe: :secure,
-        backend: :html5,
-        attributes: []
-      )
-      asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS)
+    def self.render(input, context)
+      asciidoc_opts = { safe: :secure,
+                        backend: :gitlab_html5,
+                        attributes: DEFAULT_ADOC_ATTRS }
 
-      html = ::Asciidoctor.convert(input, asciidoc_opts)
+      context[:pipeline] = :ascii_doc
 
-      html = Banzai.post_process(html, context)
+      plantuml_setup
 
+      html = ::Asciidoctor.convert(input, asciidoc_opts)
+      html = Banzai.render(html, context)
       html.html_safe
     end
+
+    def self.plantuml_setup
+      Asciidoctor::PlantUml.configure do |conf|
+        conf.url = current_application_settings.plantuml_url
+        conf.svg_enable = current_application_settings.plantuml_enabled
+        conf.png_enable = current_application_settings.plantuml_enabled
+        conf.txt_enable = false
+      end
+    end
+
+    class Html5Converter < Asciidoctor::Converter::Html5Converter
+      extend Asciidoctor::Converter::Config
+
+      register_for 'gitlab_html5'
+
+      def stem(node)
+        return super unless node.style.to_sym == :latexmath
+
+        %(<pre#{id_attribute(node)} data-math-style="display"><code>#{node.content}</code></pre>)
+      end
+
+      def inline_quoted(node)
+        return super unless node.type.to_sym == :latexmath
+
+        %(<code#{id_attribute(node)} data-math-style="inline">#{node.text}</code>)
+      end
+
+      private
+
+      def id_attribute(node)
+        node.id ? %( id="#{node.id}") : nil
+      end
+    end
   end
 end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index aca5d0020cf..3933c3b04dd 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -1,37 +1,66 @@
 module Gitlab
   module Auth
-    class MissingPersonalTokenError < StandardError; end
+    MissingPersonalTokenError = Class.new(StandardError)
+
+    REGISTRY_SCOPES = [:read_registry].freeze
+
+    # Scopes used for GitLab API access
+    API_SCOPES = [:api, :read_user].freeze
+
+    # Scopes used for OpenID Connect
+    OPENID_SCOPES = [:openid].freeze
+
+    # Default scopes for OAuth applications that don't define their own
+    DEFAULT_SCOPES = [:api].freeze
+
+    AVAILABLE_SCOPES = (API_SCOPES + REGISTRY_SCOPES).freeze
+
+    # Other available scopes
+    OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze
 
     class << self
       def find_for_git_client(login, password, project:, ip:)
         raise "Must provide an IP for rate limiting" if ip.nil?
 
+        # `user_with_password_for_git` should be the last check
+        # because it's the most expensive, especially when LDAP
+        # is enabled.
         result =
           service_request_check(login, password, project) ||
           build_access_token_check(login, password) ||
-          user_with_password_for_git(login, password) ||
-          oauth_access_token_check(login, password) ||
           lfs_token_check(login, password) ||
-          personal_access_token_check(login, password) ||
+          oauth_access_token_check(login, password) ||
+          personal_access_token_check(password) ||
+          user_with_password_for_git(login, password) ||
           Gitlab::Auth::Result.new
 
         rate_limit!(ip, success: result.success?, login: login)
+        Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor)
 
-        result
+        return result if result.success? || current_application_settings.signin_enabled? || Gitlab::LDAP::Config.enabled?
+
+        # If sign-in is disabled and LDAP is not configured, recommend a
+        # personal access token on failed auth attempts
+        raise Gitlab::Auth::MissingPersonalTokenError
       end
 
       def find_with_user_password(login, password)
-        user = User.by_login(login)
+        # Avoid resource intensive login checks if password is not provided
+        return unless password.present?
 
-        # If no user is found, or it's an LDAP server, try LDAP.
-        #   LDAP users are only authenticated via LDAP
-        if user.nil? || user.ldap_user?
-          # Second chance - try LDAP authentication
-          return nil unless Gitlab::LDAP::Config.enabled?
+        Gitlab::Auth::UniqueIpsLimiter.limit_user! do
+          user = User.by_login(login)
 
-          Gitlab::LDAP::Authentication.login(login, password)
-        else
-          user if user.valid_password?(password)
+          # If no user is found, or it's an LDAP server, try LDAP.
+          #   LDAP users are only authenticated via LDAP
+          if user.nil? || user.ldap_user?
+            # Second chance - try LDAP authentication
+            return unless Gitlab::LDAP::Config.enabled?
+
+            Gitlab::LDAP::Authentication.login(login, password)
+          else
+            user if user.active? && user.valid_password?(password)
+          end
         end
       end
 
@@ -88,21 +117,38 @@ module Gitlab
       def oauth_access_token_check(login, password)
         if login == "oauth2" && password.present?
           token = Doorkeeper::AccessToken.by_token(password)
-          if token && token.accessible?
+
+          if valid_oauth_token?(token)
             user = User.find_by(id: token.resource_owner_id)
-            Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities)
+            Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities)
           end
         end
       end
 
-      def personal_access_token_check(login, password)
-        if login && password
-          user = User.find_by_personal_access_token(password)
-          validation = User.by_login(login)
-          Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation
+      def personal_access_token_check(password)
+        return unless password.present?
+
+        token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password)
+
+        if token && valid_scoped_token?(token, AVAILABLE_SCOPES.map(&:to_s))
+          Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes))
         end
       end
 
+      def valid_oauth_token?(token)
+        token && token.accessible? && valid_scoped_token?(token, ["api"])
+      end
+
+      def valid_scoped_token?(token, scopes)
+        AccessTokenValidationService.new(token).include_any_scope?(scopes)
+      end
+
+      def abilities_for_scope(scopes)
+        scopes.map do |scope|
+          self.public_send(:"#{scope}_scope_authentication_abilities")
+        end.flatten.uniq
+      end
+
       def lfs_token_check(login, password)
         deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/)
 
@@ -124,7 +170,9 @@ module Gitlab
             read_authentication_abilities
           end
 
-        Result.new(actor, nil, token_handler.type, authentication_abilities) if Devise.secure_compare(token_handler.token, password)
+        if Devise.secure_compare(token_handler.token, password)
+          Gitlab::Auth::Result.new(actor, nil, token_handler.type, authentication_abilities)
+        end
       end
 
       def build_access_token_check(login, password)
@@ -169,6 +217,16 @@ module Gitlab
           :create_container_image
         ]
       end
+      alias_method :api_scope_authentication_abilities, :full_authentication_abilities
+
+      def read_registry_scope_authentication_abilities
+        [:read_container_image]
+      end
+
+      # The currently used auth method doesn't allow any actions for this scope
+      def read_user_scope_authentication_abilities
+        []
+      end
     end
   end
 end
diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb
index 6be7f690676..75451cf8aa9 100644
--- a/lib/gitlab/auth/result.rb
+++ b/lib/gitlab/auth/result.rb
@@ -9,13 +9,16 @@ module Gitlab
 
       def lfs_deploy_token?(for_project)
         type == :lfs_deploy_token &&
-          actor &&
-          actor.projects.include?(for_project)
+          actor.try(:has_access_to?, for_project)
       end
 
       def success?
         actor.present? || type == :ci
       end
+
+      def failed?
+        !success?
+      end
     end
   end
 end
diff --git a/lib/gitlab/auth/too_many_ips.rb b/lib/gitlab/auth/too_many_ips.rb
new file mode 100644
index 00000000000..ed862791551
--- /dev/null
+++ b/lib/gitlab/auth/too_many_ips.rb
@@ -0,0 +1,17 @@
+module Gitlab
+  module Auth
+    class TooManyIps < StandardError
+      attr_reader :user_id, :ip, :unique_ips_count
+
+      def initialize(user_id, ip, unique_ips_count)
+        @user_id = user_id
+        @ip = ip
+        @unique_ips_count = unique_ips_count
+      end
+
+      def message
+        "User #{user_id} from IP: #{ip} tried logging from too many ips: #{unique_ips_count}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb
new file mode 100644
index 00000000000..bf2239ca150
--- /dev/null
+++ b/lib/gitlab/auth/unique_ips_limiter.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module Auth
+    class UniqueIpsLimiter
+      USER_UNIQUE_IPS_PREFIX = 'user_unique_ips'.freeze
+
+      class << self
+        def limit_user_id!(user_id)
+          if config.unique_ips_limit_enabled
+            ip = RequestContext.client_ip
+            unique_ips = update_and_return_ips_count(user_id, ip)
+
+            raise TooManyIps.new(user_id, ip, unique_ips) if unique_ips > config.unique_ips_limit_per_user
+          end
+        end
+
+        def limit_user!(user = nil)
+          user ||= yield if block_given?
+          limit_user_id!(user.id) unless user.nil?
+          user
+        end
+
+        def config
+          Gitlab::CurrentSettings.current_application_settings
+        end
+
+        def update_and_return_ips_count(user_id, ip)
+          time = Time.now.utc.to_i
+          key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}"
+
+          Gitlab::Redis.with do |redis|
+            unique_ips_count = nil
+            redis.multi do |r|
+              r.zadd(key, time, ip)
+              r.zremrangebyscore(key, 0, time - config.unique_ips_limit_time_window)
+              unique_ips_count = r.zcard(key)
+            end
+            unique_ips_count.value
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/award_emoji.rb b/lib/gitlab/award_emoji.rb
deleted file mode 100644
index 39b43ab5489..00000000000
--- a/lib/gitlab/award_emoji.rb
+++ /dev/null
@@ -1,83 +0,0 @@
-module Gitlab
-  class AwardEmoji
-    CATEGORIES = {
-      objects: "Objects",
-      travel: "Travel",
-      symbols: "Symbols",
-      nature: "Nature",
-      people: "People",
-      activity: "Activity",
-      flags: "Flags",
-      food: "Food"
-    }.with_indifferent_access
-
-    def self.normalize_emoji_name(name)
-      aliases[name] || name
-    end
-
-    def self.emoji_by_category
-      unless @emoji_by_category
-        @emoji_by_category = Hash.new { |h, key| h[key] = [] }
-
-        emojis.each do |emoji_name, data|
-          data["name"] = emoji_name
-
-          # Skip Fitzpatrick(tone) modifiers
-          next if data["category"] == "modifier"
-
-          category = data["category"]
-
-          @emoji_by_category[category] << data
-        end
-
-        @emoji_by_category = @emoji_by_category.sort.to_h
-      end
-
-      @emoji_by_category
-    end
-
-    def self.emojis
-      @emojis ||=
-        begin
-          json_path = File.join(Rails.root, 'fixtures', 'emojis', 'index.json' )
-          JSON.parse(File.read(json_path))
-        end
-    end
-
-    def self.aliases
-      @aliases ||=
-        begin
-          json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json')
-          JSON.parse(File.read(json_path))
-        end
-    end
-
-    # Returns an Array of Emoji names and their asset URLs.
-    def self.urls
-      @urls ||= begin
-                  path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json')
-                  # Construct the full asset path ourselves because
-                  # ActionView::Helpers::AssetUrlHelper.asset_url is slow for hundreds
-                  # of entries since it has to do a lot of extra work (e.g. regexps).
-                  prefix = Gitlab::Application.config.assets.prefix
-                  digest = Gitlab::Application.config.assets.digest
-                  base =
-                    if defined?(Gitlab::Application.config.relative_url_root) && Gitlab::Application.config.relative_url_root
-                      Gitlab::Application.config.relative_url_root
-                    else
-                      ''
-                    end
-
-                  JSON.parse(File.read(path)).map do |hash|
-                    if digest
-                      fname = "#{hash['unicode']}-#{hash['digest']}"
-                    else
-                      fname = hash['unicode']
-                    end
-
-                    { name: hash['name'], path: File.join(base, prefix, "#{fname}.png") }
-                  end
-                end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb
new file mode 100644
index 00000000000..d95ecd7b291
--- /dev/null
+++ b/lib/gitlab/background_migration.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module BackgroundMigration
+    # Begins stealing jobs from the background migrations queue, blocking the
+    # caller until all jobs have been completed.
+    #
+    # steal_class - The name of the class for which to steal jobs.
+    def self.steal(steal_class)
+      queue = Sidekiq::Queue
+        .new(BackgroundMigrationWorker.sidekiq_options['queue'])
+
+      queue.each do |job|
+        migration_class, migration_args = job.args
+
+        next unless migration_class == steal_class
+
+        perform(migration_class, migration_args)
+
+        job.delete
+      end
+    end
+
+    # class_name - The name of the background migration class as defined in the
+    #              Gitlab::BackgroundMigration namespace.
+    #
+    # arguments - The arguments to pass to the background migration's "perform"
+    #             method.
+    def self.perform(class_name, arguments)
+      const_get(class_name).new.perform(*arguments)
+    end
+  end
+end
diff --git a/lib/tasks/.gitkeep b/lib/gitlab/background_migration/.gitkeep
index e69de29bb2d..e69de29bb2d 100644
--- a/lib/tasks/.gitkeep
+++ b/lib/gitlab/background_migration/.gitkeep
diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb
index 50aa45e5406..b762d85b6e5 100644
--- a/lib/gitlab/badge/build/status.rb
+++ b/lib/gitlab/badge/build/status.rb
@@ -20,8 +20,8 @@ module Gitlab
 
         def status
           @project.pipelines
-            .where(sha: @sha, ref: @ref)
-            .status || 'unknown'
+            .where(sha: @sha)
+            .latest_status(@ref) || 'unknown'
         end
 
         def metadata
diff --git a/lib/gitlab/badge/build/template.rb b/lib/gitlab/badge/build/template.rb
index 2b95ddfcb53..bc0e0cd441d 100644
--- a/lib/gitlab/badge/build/template.rb
+++ b/lib/gitlab/badge/build/template.rb
@@ -15,7 +15,7 @@ module Gitlab
           canceled: '#9f9f9f',
           skipped: '#9f9f9f',
           unknown: '#9f9f9f'
-        }
+        }.freeze
 
         def initialize(badge)
           @entity = badge.entity
diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb
index 06e0d084e9f..fcecb1d9665 100644
--- a/lib/gitlab/badge/coverage/template.rb
+++ b/lib/gitlab/badge/coverage/template.rb
@@ -13,7 +13,7 @@ module Gitlab
           medium: '#dfb317',
           low: '#e05d44',
           unknown: '#9f9f9f'
-        }
+        }.freeze
 
         def initialize(badge)
           @entity = badge.entity
diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb
index 548f85b78bb..4a049ef758d 100644
--- a/lib/gitlab/badge/metadata.rb
+++ b/lib/gitlab/badge/metadata.rb
@@ -20,6 +20,10 @@ module Gitlab
         "[![#{title}](#{image_url})](#{link_url})"
       end
 
+      def to_asciidoc
+        "image:#{image_url}[link=\"#{link_url}\",title=\"#{title}\"]"
+      end
+
       def title
         raise NotImplementedError
       end
diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb
deleted file mode 100644
index 7298152e7e9..00000000000
--- a/lib/gitlab/bitbucket_import.rb
+++ /dev/null
@@ -1,6 +0,0 @@
-module Gitlab
-  module BitbucketImport
-    mattr_accessor :public_key
-    @public_key = nil
-  end
-end
diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb
deleted file mode 100644
index 8d1ad62fae0..00000000000
--- a/lib/gitlab/bitbucket_import/client.rb
+++ /dev/null
@@ -1,142 +0,0 @@
-module Gitlab
-  module BitbucketImport
-    class Client
-      class Unauthorized < StandardError; end
-
-      attr_reader :consumer, :api
-
-      def self.from_project(project)
-        import_data_credentials = project.import_data.credentials if project.import_data
-        if import_data_credentials && import_data_credentials[:bb_session]
-          token = import_data_credentials[:bb_session][:bitbucket_access_token]
-          token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret]
-          new(token, token_secret)
-        else
-          raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}"
-        end
-      end
-
-      def initialize(access_token = nil, access_token_secret = nil)
-        @consumer = ::OAuth::Consumer.new(
-          config.app_id,
-          config.app_secret,
-          bitbucket_options
-        )
-
-        if access_token && access_token_secret
-          @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret)
-        end
-      end
-
-      def request_token(redirect_uri)
-        request_token = consumer.get_request_token(oauth_callback: redirect_uri)
-
-        {
-          oauth_token:              request_token.token,
-          oauth_token_secret:       request_token.secret,
-          oauth_callback_confirmed: request_token.callback_confirmed?.to_s
-        }
-      end
-
-      def authorize_url(request_token, redirect_uri)
-        request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
-
-        if request_token.callback_confirmed?
-          request_token.authorize_url
-        else
-          request_token.authorize_url(oauth_callback: redirect_uri)
-        end
-      end
-
-      def get_token(request_token, oauth_verifier, redirect_uri)
-        request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
-
-        if request_token.callback_confirmed?
-          request_token.get_access_token(oauth_verifier: oauth_verifier)
-        else
-          request_token.get_access_token(oauth_callback: redirect_uri)
-        end
-      end
-
-      def user
-        JSON.parse(get("/api/1.0/user").body)
-      end
-
-      def issues(project_identifier)
-        all_issues = []
-        offset = 0
-        per_page = 50 # Maximum number allowed by Bitbucket
-        index = 0
-
-        begin
-          issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body)
-          # Find out how many total issues are present
-          total = issues["count"] if index == 0
-          all_issues.concat(issues["issues"])
-          offset += issues["issues"].count
-          index += 1
-        end while all_issues.count < total
-
-        all_issues
-      end
-
-      def issue_comments(project_identifier, issue_id)
-        comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
-        comments.sort_by { |comment| comment["utc_created_on"] }
-      end
-
-      def project(project_identifier)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body)
-      end
-
-      def find_deploy_key(project_identifier, key)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key|
-          deploy_key["key"].chomp == key.chomp
-        end
-      end
-
-      def add_deploy_key(project_identifier, key)
-        deploy_key = find_deploy_key(project_identifier, key)
-        return if deploy_key
-
-        JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body)
-      end
-
-      def delete_deploy_key(project_identifier, key)
-        deploy_key = find_deploy_key(project_identifier, key)
-        return unless deploy_key
-
-        api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204"
-      end
-
-      def projects
-        JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" }
-      end
-
-      def incompatible_projects
-        JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" }
-      end
-
-      private
-
-      def get(url)
-        response = api.get(url)
-        raise Unauthorized if (400..499).cover?(response.code.to_i)
-
-        response
-      end
-
-      def issue_api_endpoint(project_identifier, per_page, offset)
-        "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}"
-      end
-
-      def config
-        Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" }
-      end
-
-      def bitbucket_options
-        OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index f4b5097adb1..5a6d9ae99a0 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -1,84 +1,252 @@
 module Gitlab
   module BitbucketImport
     class Importer
-      attr_reader :project, :client
+      include Gitlab::ShellAdapter
+
+      LABELS = [{ title: 'bug', color: '#FF0000' },
+                { title: 'enhancement', color: '#428BCA' },
+                { title: 'proposal', color: '#69D100' },
+                { title: 'task', color: '#7F8C8D' }].freeze
+
+      attr_reader :project, :client, :errors, :users
 
       def initialize(project)
         @project = project
-        @client = Client.from_project(@project)
+        @client = Bitbucket::Client.new(project.import_data.credentials)
         @formatter = Gitlab::ImportFormatter.new
+        @labels = {}
+        @errors = []
+        @users = {}
       end
 
       def execute
-        import_issues if has_issues?
+        import_wiki
+        import_issues
+        import_pull_requests
+        handle_errors
 
         true
-      rescue ActiveRecord::RecordInvalid => e
-        raise Projects::ImportService::Error.new, e.message
-      ensure
-        Gitlab::BitbucketImport::KeyDeleter.new(project).execute
       end
 
       private
 
-      def gitlab_user_id(project, bitbucket_id)
-        if bitbucket_id
-          user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
-          (user && user.id) || project.creator_id
-        else
-          project.creator_id
-        end
+      def handle_errors
+        return unless errors.any?
+
+        project.update_column(:import_error, {
+          message: 'The remote data could not be fully imported.',
+          errors: errors
+        }.to_json)
+      end
+
+      def gitlab_user_id(project, username)
+        find_user_id(username) || project.creator_id
       end
 
-      def identifier
-        project.import_source
+      def find_user_id(username)
+        return nil unless username
+
+        return users[username] if users.key?(username)
+
+        users[username] = User.select(:id)
+                              .joins(:identities)
+                              .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username)
+                              .try(:id)
       end
 
-      def has_issues?
-        client.project(identifier)["has_issues"]
+      def repo
+        @repo ||= client.repo(project.import_source)
       end
 
-      def import_issues
-        issues = client.issues(identifier)
+      def import_wiki
+        return if project.wiki.repository_exists?
 
-        issues.each do |issue|
-          body = ''
-          reporter = nil
-          author = 'Anonymous'
+        path_with_namespace = "#{project.path_with_namespace}.wiki"
+        import_url = project.import_url.sub(/\.git\z/, ".git/wiki")
+        gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url)
+      rescue StandardError => e
+        errors << { type: :wiki, errors: e.message }
+      end
 
-          if issue["reported_by"] && issue["reported_by"]["username"]
-            reporter = issue["reported_by"]["username"]
-            author = reporter
+      def import_issues
+        return unless repo.issues_enabled?
+
+        create_labels
+
+        client.issues(repo).each do |issue|
+          begin
+            description = ''
+            description += @formatter.author_line(issue.author) unless find_user_id(issue.author)
+            description += issue.description
+
+            label_name = issue.kind
+            milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil
+
+            gitlab_issue = project.issues.create!(
+              iid: issue.iid,
+              title: issue.title,
+              description: description,
+              state: issue.state,
+              author_id: gitlab_user_id(project, issue.author),
+              milestone: milestone,
+              created_at: issue.created_at,
+              updated_at: issue.updated_at
+            )
+
+            gitlab_issue.labels << @labels[label_name]
+
+            import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted?
+          rescue StandardError => e
+            errors << { type: :issue, iid: issue.iid, errors: e.message }
           end
+        end
+      end
 
-          body = @formatter.author_line(author)
-          body += issue["content"]
+      def import_issue_comments(issue, gitlab_issue)
+        client.issue_comments(repo, issue.iid).each do |comment|
+          # The note can be blank for issue service messages like "Changed title: ..."
+          # We would like to import those comments as well but there is no any
+          # specific parameter that would allow to process them, it's just an empty comment.
+          # To prevent our importer from just crashing or from creating useless empty comments
+          # we do this check.
+          next unless comment.note.present?
+
+          note = ''
+          note += @formatter.author_line(comment.author) unless find_user_id(comment.author)
+          note += comment.note
+
+          begin
+            gitlab_issue.notes.create!(
+              project: project,
+              note: note,
+              author_id: gitlab_user_id(project, comment.author),
+              created_at: comment.created_at,
+              updated_at: comment.updated_at
+            )
+          rescue StandardError => e
+            errors << { type: :issue_comment, iid: issue.iid, errors: e.message }
+          end
+        end
+      end
 
-          comments = client.issue_comments(identifier, issue["local_id"])
+      def create_labels
+        LABELS.each do |label_params|
+          label = ::Labels::CreateService.new(label_params).execute(project: project)
+          if label.valid?
+            @labels[label_params[:title]] = label
+          else
+            raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.name_with_namespace}\""
+          end
+        end
+      end
 
-          if comments.any?
-            body += @formatter.comments_header
+      def import_pull_requests
+        pull_requests = client.pull_requests(repo)
+
+        pull_requests.each do |pull_request|
+          begin
+            description = ''
+            description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author)
+            description += pull_request.description
+
+            merge_request = project.merge_requests.create!(
+              iid: pull_request.iid,
+              title: pull_request.title,
+              description: description,
+              source_project: project,
+              source_branch: pull_request.source_branch_name,
+              source_branch_sha: pull_request.source_branch_sha,
+              target_project: project,
+              target_branch: pull_request.target_branch_name,
+              target_branch_sha: pull_request.target_branch_sha,
+              state: pull_request.state,
+              author_id: gitlab_user_id(project, pull_request.author),
+              assignee_id: nil,
+              created_at: pull_request.created_at,
+              updated_at: pull_request.updated_at
+            )
+
+            import_pull_request_comments(pull_request, merge_request) if merge_request.persisted?
+          rescue StandardError => e
+            errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, trace: e.backtrace.join("\n"), raw_response: pull_request.raw }
           end
+        end
+      end
+
+      def import_pull_request_comments(pull_request, merge_request)
+        comments = client.pull_request_comments(repo, pull_request.iid)
+
+        inline_comments, pr_comments = comments.partition(&:inline?)
+
+        import_inline_comments(inline_comments, pull_request, merge_request)
+        import_standalone_pr_comments(pr_comments, merge_request)
+      end
 
-          comments.each do |comment|
-            author = 'Anonymous'
+      def import_inline_comments(inline_comments, pull_request, merge_request)
+        line_code_map = {}
 
-            if comment["author_info"] && comment["author_info"]["username"]
-              author = comment["author_info"]["username"]
-            end
+        children, parents = inline_comments.partition(&:has_parent?)
 
-            body += @formatter.comment(author, comment["utc_created_on"], comment["content"])
+        # The Bitbucket API returns threaded replies as parent-child
+        # relationships. We assume that the child can appear in any order in
+        # the JSON.
+        parents.each do |comment|
+          line_code_map[comment.iid] = generate_line_code(comment)
+        end
+
+        children.each do |comment|
+          line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil)
+        end
+
+        inline_comments.each do |comment|
+          begin
+            attributes = pull_request_comment_attributes(comment)
+            attributes.merge!(
+              position: build_position(merge_request, comment),
+              line_code: line_code_map.fetch(comment.iid),
+              type: 'DiffNote')
+
+            merge_request.notes.create!(attributes)
+          rescue StandardError => e
+            errors << { type: :pull_request, iid: comment.iid, errors: e.message }
           end
+        end
+      end
+
+      def build_position(merge_request, pr_comment)
+        params = {
+          diff_refs: merge_request.diff_refs,
+          old_path: pr_comment.file_path,
+          new_path: pr_comment.file_path,
+          old_line: pr_comment.old_pos,
+          new_line: pr_comment.new_pos
+        }
 
-          project.issues.create!(
-            description: body,
-            title: issue["title"],
-            state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened',
-            author_id: gitlab_user_id(project, reporter)
-          )
+        Gitlab::Diff::Position.new(params)
+      end
+
+      def import_standalone_pr_comments(pr_comments, merge_request)
+        pr_comments.each do |comment|
+          begin
+            merge_request.notes.create!(pull_request_comment_attributes(comment))
+          rescue StandardError => e
+            errors << { type: :pull_request, iid: comment.iid, errors: e.message }
+          end
         end
-      rescue ActiveRecord::RecordInvalid => e
-        raise Projects::ImportService::Error, e.message
+      end
+
+      def generate_line_code(pr_comment)
+        Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos)
+      end
+
+      def pull_request_comment_attributes(comment)
+        {
+          project: project,
+          note: comment.note,
+          author_id: gitlab_user_id(project, comment.author),
+          created_at: comment.created_at,
+          updated_at: comment.updated_at
+        }
       end
     end
   end
diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb
deleted file mode 100644
index 0b63f025d0a..00000000000
--- a/lib/gitlab/bitbucket_import/key_adder.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-module Gitlab
-  module BitbucketImport
-    class KeyAdder
-      attr_reader :repo, :current_user, :client
-
-      def initialize(repo, current_user, access_params)
-        @repo, @current_user = repo, current_user
-        @client = Client.new(access_params[:bitbucket_access_token],
-                             access_params[:bitbucket_access_token_secret])
-      end
-
-      def execute
-        return false unless BitbucketImport.public_key.present?
-
-        project_identifier = "#{repo["owner"]}/#{repo["slug"]}"
-        client.add_deploy_key(project_identifier, BitbucketImport.public_key)
-
-        true
-      rescue
-        false
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb
deleted file mode 100644
index e03c3155b3e..00000000000
--- a/lib/gitlab/bitbucket_import/key_deleter.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-module Gitlab
-  module BitbucketImport
-    class KeyDeleter
-      attr_reader :project, :current_user, :client
-
-      def initialize(project)
-        @project = project
-        @current_user = project.creator
-        @client = Client.from_project(@project)
-      end
-
-      def execute
-        return false unless BitbucketImport.public_key.present?
-
-        client.delete_deploy_key(project.import_source, BitbucketImport.public_key)
-
-        true
-      rescue
-        false
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb
index b90ef0b0fba..d94f70fd1fb 100644
--- a/lib/gitlab/bitbucket_import/project_creator.rb
+++ b/lib/gitlab/bitbucket_import/project_creator.rb
@@ -1,10 +1,11 @@
 module Gitlab
   module BitbucketImport
     class ProjectCreator
-      attr_reader :repo, :namespace, :current_user, :session_data
+      attr_reader :repo, :name, :namespace, :current_user, :session_data
 
-      def initialize(repo, namespace, current_user, session_data)
+      def initialize(repo, name, namespace, current_user, session_data)
         @repo = repo
+        @name = name
         @namespace = namespace
         @current_user = current_user
         @session_data = session_data
@@ -13,17 +14,24 @@ module Gitlab
       def execute
         ::Projects::CreateService.new(
           current_user,
-          name: repo["name"],
-          path: repo["slug"],
-          description: repo["description"],
+          name: name,
+          path: name,
+          description: repo.description,
           namespace_id: namespace.id,
-          visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
-          import_type: "bitbucket",
-          import_source: "#{repo["owner"]}/#{repo["slug"]}",
-          import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git",
-          import_data: { credentials: { bb_session: session_data } }
+          visibility_level: repo.visibility_level,
+          import_type: 'bitbucket',
+          import_source: repo.full_name,
+          import_url: repo.clone_url(session_data[:token]),
+          import_data: { credentials: session_data },
+          skip_wiki: skip_wiki
         ).execute
       end
+
+      private
+
+      def skip_wiki
+        repo.has_wiki?
+      end
     end
   end
 end
diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb
index d62bc50ce78..169aac79854 100644
--- a/lib/gitlab/blame.rb
+++ b/lib/gitlab/blame.rb
@@ -40,7 +40,7 @@ module Gitlab
     end
 
     def highlighted_lines
-      @blob.load_all_data!(repository)
+      @blob.load_all_data!
       @highlighted_lines ||=
         Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines
     end
diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb
new file mode 100644
index 00000000000..9c2e09943b0
--- /dev/null
+++ b/lib/gitlab/cache/ci/project_pipeline_status.rb
@@ -0,0 +1,138 @@
+# This class is not backed by a table in the main database.
+# It loads the latest Pipeline for the HEAD of a repository, and caches that
+# in Redis.
+module Gitlab
+  module Cache
+    module Ci
+      class ProjectPipelineStatus
+        attr_accessor :sha, :status, :ref, :project, :loaded
+
+        delegate :commit, to: :project
+
+        def self.load_for_project(project)
+          new(project).tap do |status|
+            status.load_status
+          end
+        end
+
+        def self.load_in_batch_for_projects(projects)
+          cached_results_for_projects(projects).zip(projects).each do |result, project|
+            project.pipeline_status = new(project, result)
+            project.pipeline_status.load_status
+          end
+        end
+
+        def self.cached_results_for_projects(projects)
+          result = Gitlab::Redis.with do |redis|
+            redis.multi do
+              projects.each do |project|
+                cache_key = cache_key_for_project(project)
+                redis.exists(cache_key)
+                redis.hmget(cache_key, :sha, :status, :ref)
+              end
+            end
+          end
+
+          result.each_slice(2).map do |(cache_key_exists, (sha, status, ref))|
+            pipeline_info = { sha: sha, status: status, ref: ref }
+            { loaded_from_cache: cache_key_exists, pipeline_info: pipeline_info }
+          end
+        end
+
+        def self.cache_key_for_project(project)
+          "projects/#{project.id}/pipeline_status"
+        end
+
+        def self.update_for_pipeline(pipeline)
+          pipeline_info = {
+            sha: pipeline.sha,
+            status: pipeline.status,
+            ref: pipeline.ref
+          }
+
+          new(pipeline.project, pipeline_info: pipeline_info)
+            .store_in_cache_if_needed
+        end
+
+        def initialize(project, pipeline_info: {}, loaded_from_cache: nil)
+          @project = project
+          @sha = pipeline_info[:sha]
+          @ref = pipeline_info[:ref]
+          @status = pipeline_info[:status]
+          @loaded = loaded_from_cache
+        end
+
+        def has_status?
+          loaded? && sha.present? && status.present?
+        end
+
+        def load_status
+          return if loaded?
+
+          if has_cache?
+            load_from_cache
+          else
+            load_from_project
+            store_in_cache
+          end
+
+          self.loaded = true
+        end
+
+        def load_from_project
+          return unless commit
+
+          self.sha = commit.sha
+          self.status = commit.status
+          self.ref = project.default_branch
+        end
+
+        # We only cache the status for the HEAD commit of a project
+        # This status is rendered in project lists
+        def store_in_cache_if_needed
+          return delete_from_cache unless commit
+          return unless sha
+          return unless ref
+
+          if commit.sha == sha && project.default_branch == ref
+            store_in_cache
+          end
+        end
+
+        def load_from_cache
+          Gitlab::Redis.with do |redis|
+            self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref)
+          end
+        end
+
+        def store_in_cache
+          Gitlab::Redis.with do |redis|
+            redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref })
+          end
+        end
+
+        def delete_from_cache
+          Gitlab::Redis.with do |redis|
+            redis.del(cache_key)
+          end
+        end
+
+        def has_cache?
+          return self.loaded unless self.loaded.nil?
+
+          Gitlab::Redis.with do |redis|
+            redis.exists(cache_key)
+          end
+        end
+
+        def loaded?
+          self.loaded
+        end
+
+        def cache_key
+          self.class.cache_key_for_project(project)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb
index 95308aca95f..5b32fca00a4 100644
--- a/lib/gitlab/changes_list.rb
+++ b/lib/gitlab/changes_list.rb
@@ -5,7 +5,7 @@ module Gitlab
     attr_reader :raw_changes
 
     def initialize(changes)
-      @raw_changes = changes.kind_of?(String) ? changes.lines : changes
+      @raw_changes = changes.is_a?(String) ? changes.lines : changes
     end
 
     def each(&block)
diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb
new file mode 100644
index 00000000000..1b081aa9b1d
--- /dev/null
+++ b/lib/gitlab/chat_name_token.rb
@@ -0,0 +1,45 @@
+require 'json'
+
+module Gitlab
+  class ChatNameToken
+    attr_reader :token
+
+    TOKEN_LENGTH = 50
+    EXPIRY_TIME = 10.minutes
+
+    def initialize(token = new_token)
+      @token = token
+    end
+
+    def get
+      Gitlab::Redis.with do |redis|
+        data = redis.get(redis_key)
+        JSON.parse(data, symbolize_names: true) if data
+      end
+    end
+
+    def store!(params)
+      Gitlab::Redis.with do |redis|
+        params = params.to_json
+        redis.set(redis_key, params, ex: EXPIRY_TIME)
+        token
+      end
+    end
+
+    def delete
+      Gitlab::Redis.with do |redis|
+        redis.del(redis_key)
+      end
+    end
+
+    private
+
+    def new_token
+      Devise.friendly_token(TOKEN_LENGTH)
+    end
+
+    def redis_key
+      "gitlab:chat_names:#{token}"
+    end
+  end
+end
diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb
index cb1065223d4..b6805230348 100644
--- a/lib/gitlab/checks/change_access.rb
+++ b/lib/gitlab/checks/change_access.rb
@@ -1,76 +1,138 @@
 module Gitlab
   module Checks
     class ChangeAccess
-      attr_reader :user_access, :project
+      ERROR_MESSAGES = {
+        push_code: 'You are not allowed to push code to this project.',
+        delete_default_branch: 'The default branch of a project cannot be deleted.',
+        force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.',
+        non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.',
+        non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.',
+        merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.',
+        push_protected_branch: 'You are not allowed to push code to protected branches on this project.',
+        change_existing_tags: 'You are not allowed to change existing tags on this project.',
+        update_protected_tag: 'Protected tags cannot be updated.',
+        delete_protected_tag: 'Protected tags cannot be deleted.',
+        create_protected_tag: 'You are not allowed to create this tag as it is protected.'
+      }.freeze
 
-      def initialize(change, user_access:, project:)
+      attr_reader :user_access, :project, :skip_authorization, :protocol
+
+      def initialize(
+        change, user_access:, project:, skip_authorization: false,
+        protocol:
+      )
         @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref)
         @branch_name = Gitlab::Git.branch_name(@ref)
+        @tag_name = Gitlab::Git.tag_name(@ref)
         @user_access = user_access
         @project = project
+        @skip_authorization = skip_authorization
+        @protocol = protocol
       end
 
       def exec
-        error = push_checks || tag_checks || protected_branch_checks
+        return true if skip_authorization
 
-        if error
-          GitAccessStatus.new(false, error)
-        else
-          GitAccessStatus.new(true)
-        end
+        push_checks
+        branch_checks
+        tag_checks
+
+        true
       end
 
       protected
 
-      def protected_branch_checks
+      def push_checks
+        if user_access.cannot_do_action?(:push_code)
+          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code]
+        end
+      end
+
+      def branch_checks
         return unless @branch_name
-        return unless project.protected_branch?(@branch_name)
 
-        if forced_push? && user_access.cannot_do_action?(:force_push_code_to_protected_branches)
-          return "You are not allowed to force push code to a protected branch on this project."
-        elsif Gitlab::Git.blank_ref?(@newrev) && user_access.cannot_do_action?(:remove_protected_branches)
-          return "You are not allowed to delete protected branches from this project."
+        if deletion? && @branch_name == project.default_branch
+          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch]
+        end
+
+        protected_branch_checks
+      end
+
+      def protected_branch_checks
+        return unless ProtectedBranch.protected?(project, @branch_name)
+
+        if forced_push?
+          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch]
+        end
+
+        if deletion?
+          protected_branch_deletion_checks
+        else
+          protected_branch_push_checks
+        end
+      end
+
+      def protected_branch_deletion_checks
+        unless user_access.can_delete_branch?(@branch_name)
+          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch]
         end
 
+        unless protocol == 'web'
+          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch]
+        end
+      end
+
+      def protected_branch_push_checks
         if matching_merge_request?
-          if user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name)
-            return
-          else
-            "You are not allowed to merge code into protected branches on this project."
+          unless user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name)
+            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch]
           end
         else
-          if user_access.can_push_to_branch?(@branch_name)
-            return
-          else
-            "You are not allowed to push code to protected branches on this project."
+          unless user_access.can_push_to_branch?(@branch_name)
+            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_protected_branch]
           end
         end
       end
 
       def tag_checks
-        tag_ref = Gitlab::Git.tag_name(@ref)
+        return unless @tag_name
 
-        if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project)
-          "You are not allowed to change existing tags on this project."
+        if tag_exists? && user_access.cannot_do_action?(:admin_project)
+          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags]
         end
+
+        protected_tag_checks
       end
 
-      def push_checks
-        if user_access.cannot_do_action?(:push_code)
-          "You are not allowed to push code to this project."
+      def protected_tag_checks
+        return unless ProtectedTag.protected?(project, @tag_name)
+
+        raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update?
+        raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion?
+
+        unless user_access.can_create_tag?(@tag_name)
+          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag]
         end
       end
 
       private
 
-      def protected_tag?(tag_name)
-        project.repository.tag_exists?(tag_name)
+      def tag_exists?
+        project.repository.tag_exists?(@tag_name)
       end
 
       def forced_push?
         Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev)
       end
 
+      def update?
+        !Gitlab::Git.blank_ref?(@oldrev) && !deletion?
+      end
+
+      def deletion?
+        Gitlab::Git.blank_ref?(@newrev)
+      end
+
       def matching_merge_request?
         Checks::MatchingMergeRequest.new(@newrev, @branch_name, @project).match?
       end
diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb
index 5fe86553bd0..1e73f89158d 100644
--- a/lib/gitlab/checks/force_push.rb
+++ b/lib/gitlab/checks/force_push.rb
@@ -8,8 +8,9 @@ module Gitlab
         if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev)
           false
         else
-          missed_ref, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list --max-count=1 #{oldrev} ^#{newrev}))
-          missed_ref.present?
+          Gitlab::Git::RevList.new(
+            path_to_repo: project.repository.path_to_repo,
+            oldrev: oldrev, newrev: newrev).missed_ref.present?
         end
       end
     end
diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb
index cd2e83b4c27..a375ccbece0 100644
--- a/lib/gitlab/ci/build/artifacts/metadata.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata.rb
@@ -6,7 +6,7 @@ module Gitlab
     module Build
       module Artifacts
         class Metadata
-          class ParserError < StandardError; end
+          ParserError = Class.new(StandardError)
 
           VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/
           INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}
diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
index 7f4c750b6fd..2e073334abc 100644
--- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
@@ -27,6 +27,8 @@ module Gitlab
           end
         end
 
+        delegate :empty?, to: :children
+
         def directory?
           blank_node? || @path.end_with?('/')
         end
@@ -35,6 +37,12 @@ module Gitlab
           !directory?
         end
 
+        def blob
+          return unless file?
+
+          @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil)
+        end
+
         def has_parent?
           nodes > 0
         end
@@ -91,10 +99,6 @@ module Gitlab
           blank_node? || @entries.include?(@path)
         end
 
-        def empty?
-          children.empty?
-        end
-
         def total_size
           descendant_pattern = %r{^#{Regexp.escape(@path)}}
           entries.sum do |path, entry|
diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb
new file mode 100644
index 00000000000..29a7a27c963
--- /dev/null
+++ b/lib/gitlab/ci/build/credentials/base.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module Ci
+    module Build
+      module Credentials
+        class Base
+          def type
+            self.class.name.demodulize.underscore
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb
new file mode 100644
index 00000000000..2423aa8857d
--- /dev/null
+++ b/lib/gitlab/ci/build/credentials/factory.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Ci
+    module Build
+      module Credentials
+        class Factory
+          def initialize(build)
+            @build = build
+          end
+
+          def create!
+            credentials.select(&:valid?)
+          end
+
+          private
+
+          def credentials
+            providers.map { |provider| provider.new(@build) }
+          end
+
+          def providers
+            [Registry]
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb
new file mode 100644
index 00000000000..55eafcaed10
--- /dev/null
+++ b/lib/gitlab/ci/build/credentials/registry.rb
@@ -0,0 +1,24 @@
+module Gitlab
+  module Ci
+    module Build
+      module Credentials
+        class Registry < Base
+          attr_reader :username, :password
+
+          def initialize(build)
+            @username = 'gitlab-ci-token'
+            @password = build.token
+          end
+
+          def url
+            Gitlab.config.registry.host_port
+          end
+
+          def valid?
+            Gitlab.config.registry.enabled
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb
new file mode 100644
index 00000000000..b88b2e36d53
--- /dev/null
+++ b/lib/gitlab/ci/build/image.rb
@@ -0,0 +1,40 @@
+module Gitlab
+  module Ci
+    module Build
+      class Image
+        attr_reader :alias, :command, :entrypoint, :name
+
+        class << self
+          def from_image(job)
+            image = Gitlab::Ci::Build::Image.new(job.options[:image])
+            return unless image.valid?
+            image
+          end
+
+          def from_services(job)
+            services = job.options[:services].to_a.map do |service|
+              Gitlab::Ci::Build::Image.new(service)
+            end
+
+            services.select(&:valid?).compact
+          end
+        end
+
+        def initialize(image)
+          if image.is_a?(String)
+            @name = image
+          elsif image.is_a?(Hash)
+            @alias = image[:alias]
+            @command = image[:command]
+            @entrypoint = image[:entrypoint]
+            @name = image[:name]
+          end
+        end
+
+        def valid?
+          @name.present?
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb
new file mode 100644
index 00000000000..ee034d9cc56
--- /dev/null
+++ b/lib/gitlab/ci/build/step.rb
@@ -0,0 +1,41 @@
+module Gitlab
+  module Ci
+    module Build
+      class Step
+        WHEN_ON_FAILURE = 'on_failure'.freeze
+        WHEN_ON_SUCCESS = 'on_success'.freeze
+        WHEN_ALWAYS = 'always'.freeze
+
+        attr_reader :name
+        attr_accessor :script, :timeout, :when, :allow_failure
+
+        class << self
+          def from_commands(job)
+            self.new(:script).tap do |step|
+              step.script = job.commands.split("\n")
+              step.timeout = job.timeout
+              step.when = WHEN_ON_SUCCESS
+            end
+          end
+
+          def from_after_script(job)
+            after_script = job.options[:after_script]
+            return unless after_script
+
+            self.new(:after_script).tap do |step|
+              step.script = after_script
+              step.timeout = job.timeout
+              step.when = WHEN_ALWAYS
+              step.allow_failure = true
+            end
+          end
+        end
+
+        def initialize(name)
+          @name = name
+          @allow_failure = false
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index bbfa6cf7d05..f7ff7ea212e 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -4,16 +4,10 @@ module Gitlab
     # Base GitLab CI Configuration facade
     #
     class Config
-      ##
-      # Temporary delegations that should be removed after refactoring
-      #
-      delegate :before_script, :image, :services, :after_script, :variables,
-               :stages, :cache, :jobs, to: :@global
-
       def initialize(config)
         @config = Loader.new(config).load!
 
-        @global = Node::Global.new(@config)
+        @global = Entry::Global.new(@config)
         @global.compose!
       end
 
@@ -28,6 +22,41 @@ module Gitlab
       def to_hash
         @config
       end
+
+      ##
+      # Temporary method that should be removed after refactoring
+      #
+      def before_script
+        @global.before_script_value
+      end
+
+      def image
+        @global.image_value
+      end
+
+      def services
+        @global.services_value
+      end
+
+      def after_script
+        @global.after_script_value
+      end
+
+      def variables
+        @global.variables_value
+      end
+
+      def stages
+        @global.stages_value
+      end
+
+      def cache
+        @global.cache_value
+      end
+
+      def jobs
+        @global.jobs_value
+      end
     end
   end
 end
diff --git a/lib/gitlab/ci/config/node/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb
index 844bd2fe998..8275aacee9b 100644
--- a/lib/gitlab/ci/config/node/artifacts.rb
+++ b/lib/gitlab/ci/config/entry/artifacts.rb
@@ -1,15 +1,15 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents a configuration of job artifacts.
         #
-        class Artifacts < Entry
+        class Artifacts < Node
           include Validatable
           include Attributable
 
-          ALLOWED_KEYS = %i[name untracked paths when expire_in]
+          ALLOWED_KEYS = %i[name untracked paths when expire_in].freeze
 
           attributes ALLOWED_KEYS
 
diff --git a/lib/gitlab/ci/config/node/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb
index 221b666f9f6..1c8b55ee4c4 100644
--- a/lib/gitlab/ci/config/node/attributable.rb
+++ b/lib/gitlab/ci/config/entry/attributable.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         module Attributable
           extend ActiveSupport::Concern
 
diff --git a/lib/gitlab/ci/config/node/boolean.rb b/lib/gitlab/ci/config/entry/boolean.rb
index 84b03ee7832..f3357f85b99 100644
--- a/lib/gitlab/ci/config/node/boolean.rb
+++ b/lib/gitlab/ci/config/entry/boolean.rb
@@ -1,11 +1,11 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents a boolean value.
         #
-        class Boolean < Entry
+        class Boolean < Node
           include Validatable
 
           validations do
diff --git a/lib/gitlab/ci/config/node/cache.rb b/lib/gitlab/ci/config/entry/cache.rb
index b4bda2841ac..f074df9c7a1 100644
--- a/lib/gitlab/ci/config/node/cache.rb
+++ b/lib/gitlab/ci/config/entry/cache.rb
@@ -1,27 +1,33 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents a cache configuration
         #
-        class Cache < Entry
+        class Cache < Node
           include Configurable
 
-          ALLOWED_KEYS = %i[key untracked paths]
+          ALLOWED_KEYS = %i[key untracked paths].freeze
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
           end
 
-          node :key, Node::Key,
+          entry :key, Entry::Key,
             description: 'Cache key used to define a cache affinity.'
 
-          node :untracked, Node::Boolean,
+          entry :untracked, Entry::Boolean,
             description: 'Cache all untracked files.'
 
-          node :paths, Node::Paths,
+          entry :paths, Entry::Paths,
             description: 'Specify which paths should be cached across builds.'
+
+          helpers :key
+
+          def value
+            super.merge(key: key_value)
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/node/commands.rb b/lib/gitlab/ci/config/entry/commands.rb
index d7657ae314b..65d19db249c 100644
--- a/lib/gitlab/ci/config/node/commands.rb
+++ b/lib/gitlab/ci/config/entry/commands.rb
@@ -1,11 +1,11 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents a job script.
         #
-        class Commands < Entry
+        class Commands < Node
           include Validatable
 
           validations do
diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb
index 6b7ab2fdaf2..e05aca9881b 100644
--- a/lib/gitlab/ci/config/node/configurable.rb
+++ b/lib/gitlab/ci/config/entry/configurable.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # This mixin is responsible for adding DSL, which purpose is to
         # simplifly process of adding child nodes.
@@ -48,8 +48,8 @@ module Gitlab
 
             private # rubocop:disable Lint/UselessAccessModifier
 
-            def node(key, node, metadata)
-              factory = Node::Factory.new(node)
+            def entry(key, entry, metadata)
+              factory = Entry::Factory.new(entry)
                 .with(description: metadata[:description])
 
               (@nodes ||= {}).merge!(key.to_sym => factory)
@@ -58,7 +58,7 @@ module Gitlab
             def helpers(*nodes)
               nodes.each do |symbol|
                 define_method("#{symbol}_defined?") do
-                  @entries[symbol].specified? if @entries[symbol]
+                  @entries[symbol]&.specified?
                 end
 
                 define_method("#{symbol}_value") do
@@ -66,8 +66,6 @@ module Gitlab
 
                   @entries[symbol].value
                 end
-
-                alias_method symbol.to_sym, "#{symbol}_value".to_sym
               end
             end
           end
diff --git a/lib/gitlab/ci/config/entry/coverage.rb b/lib/gitlab/ci/config/entry/coverage.rb
new file mode 100644
index 00000000000..12a063059cb
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/coverage.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents Coverage settings.
+        #
+        class Coverage < Node
+          include Validatable
+
+          validations do
+            validates :config, regexp: true
+          end
+
+          def value
+            @config[1...-1]
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/environment.rb b/lib/gitlab/ci/config/entry/environment.rb
index d388ab6b879..0c1f9eb7cbf 100644
--- a/lib/gitlab/ci/config/node/environment.rb
+++ b/lib/gitlab/ci/config/entry/environment.rb
@@ -1,14 +1,14 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents an environment.
         #
-        class Environment < Entry
+        class Environment < Node
           include Validatable
 
-          ALLOWED_KEYS = %i[name url]
+          ALLOWED_KEYS = %i[name url action on_stop].freeze
 
           validations do
             validate do
@@ -21,20 +21,27 @@ module Gitlab
             validates :name,
               type: {
                 with: String,
-                message: Gitlab::Regex.environment_name_regex_message }
+                message: Gitlab::Regex.environment_name_regex_message
+              }
 
             validates :name,
               format: {
                 with: Gitlab::Regex.environment_name_regex,
-                message: Gitlab::Regex.environment_name_regex_message }
+                message: Gitlab::Regex.environment_name_regex_message
+              }
 
             with_options if: :hash? do
               validates :config, allowed_keys: ALLOWED_KEYS
 
               validates :url,
                         length: { maximum: 255 },
-                        addressable_url: true,
                         allow_nil: true
+
+              validates :action,
+                        inclusion: { in: %w[start stop], message: 'should be start or stop' },
+                        allow_nil: true
+
+              validates :on_stop, type: String, allow_nil: true
             end
           end
 
@@ -54,9 +61,17 @@ module Gitlab
             value[:url]
           end
 
+          def action
+            value[:action] || 'start'
+          end
+
+          def on_stop
+            value[:on_stop]
+          end
+
           def value
             case @config
-            when String then { name: @config }
+            when String then { name: @config, action: 'start' }
             when Hash then @config
             else {}
             end
diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/entry/factory.rb
index 5387f29ad59..6be8288748f 100644
--- a/lib/gitlab/ci/config/node/factory.rb
+++ b/lib/gitlab/ci/config/entry/factory.rb
@@ -1,15 +1,15 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
-        # Factory class responsible for fabricating node entry objects.
+        # Factory class responsible for fabricating entry objects.
         #
         class Factory
-          class InvalidFactory < StandardError; end
+          InvalidFactory = Class.new(StandardError)
 
-          def initialize(node)
-            @node = node
+          def initialize(entry)
+            @entry = entry
             @metadata = {}
             @attributes = {}
           end
@@ -37,11 +37,11 @@ module Gitlab
             # See issue #18775.
             #
             if @value.nil?
-              Node::Unspecified.new(
+              Entry::Unspecified.new(
                 fabricate_unspecified
               )
             else
-              fabricate(@node, @value)
+              fabricate(@entry, @value)
             end
           end
 
@@ -49,21 +49,21 @@ module Gitlab
 
           def fabricate_unspecified
             ##
-            # If node has a default value we fabricate concrete node
+            # If entry has a default value we fabricate concrete node
             # with default value.
             #
-            if @node.default.nil?
-              fabricate(Node::Undefined)
+            if @entry.default.nil?
+              fabricate(Entry::Undefined)
             else
-              fabricate(@node, @node.default)
+              fabricate(@entry, @entry.default)
             end
           end
 
-          def fabricate(node, value = nil)
-            node.new(value, @metadata).tap do |entry|
-              entry.key = @attributes[:key]
-              entry.parent = @attributes[:parent]
-              entry.description = @attributes[:description]
+          def fabricate(entry, value = nil)
+            entry.new(value, @metadata).tap do |node|
+              node.key = @attributes[:key]
+              node.parent = @attributes[:parent]
+              node.description = @attributes[:description]
             end
           end
         end
diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/entry/global.rb
index 2a2943c9288..a4ec8f0ff2f 100644
--- a/lib/gitlab/ci/config/node/global.rb
+++ b/lib/gitlab/ci/config/entry/global.rb
@@ -1,36 +1,36 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
-        # This class represents a global entry - root node for entire
+        # This class represents a global entry - root Entry for entire
         # GitLab CI Configuration file.
         #
-        class Global < Entry
+        class Global < Node
           include Configurable
 
-          node :before_script, Node::Script,
+          entry :before_script, Entry::Script,
             description: 'Script that will be executed before each job.'
 
-          node :image, Node::Image,
+          entry :image, Entry::Image,
             description: 'Docker image that will be used to execute jobs.'
 
-          node :services, Node::Services,
+          entry :services, Entry::Services,
             description: 'Docker images that will be linked to the container.'
 
-          node :after_script, Node::Script,
+          entry :after_script, Entry::Script,
             description: 'Script that will be executed after each job.'
 
-          node :variables, Node::Variables,
+          entry :variables, Entry::Variables,
             description: 'Environment variables that will be used.'
 
-          node :stages, Node::Stages,
+          entry :stages, Entry::Stages,
             description: 'Configuration of stages for this pipeline.'
 
-          node :types, Node::Stages,
+          entry :types, Entry::Stages,
             description: 'Deprecated: stages for this pipeline.'
 
-          node :cache, Node::Cache,
+          entry :cache, Entry::Cache,
             description: 'Configure caching between build jobs.'
 
           helpers :before_script, :image, :services, :after_script,
@@ -46,7 +46,7 @@ module Gitlab
           private
 
           def compose_jobs!
-            factory = Node::Factory.new(Node::Jobs)
+            factory = Entry::Factory.new(Entry::Jobs)
               .value(@config.except(*self.class.nodes.keys))
               .with(key: :jobs, parent: self,
                     description: 'Jobs definition for this pipeline')
diff --git a/lib/gitlab/ci/config/node/hidden.rb b/lib/gitlab/ci/config/entry/hidden.rb
index fe4ee8a7fc6..6fc3aa385bc 100644
--- a/lib/gitlab/ci/config/node/hidden.rb
+++ b/lib/gitlab/ci/config/entry/hidden.rb
@@ -1,11 +1,11 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
-        # Entry that represents a hidden CI/CD job.
+        # Entry that represents a hidden CI/CD key.
         #
-        class Hidden < Entry
+        class Hidden < Node
           include Validatable
 
           validations do
diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb
new file mode 100644
index 00000000000..897dcff8012
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/image.rb
@@ -0,0 +1,46 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a Docker image.
+        #
+        class Image < Node
+          include Validatable
+
+          ALLOWED_KEYS = %i[name entrypoint].freeze
+
+          validations do
+            validates :config, hash_or_string: true
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            validates :name, type: String, presence: true
+            validates :entrypoint, type: String, allow_nil: true
+          end
+
+          def hash?
+            @config.is_a?(Hash)
+          end
+
+          def string?
+            @config.is_a?(String)
+          end
+
+          def name
+            value[:name]
+          end
+
+          def entrypoint
+            value[:entrypoint]
+          end
+
+          def value
+            return { name: @config } if string?
+            return @config if hash?
+            {}
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 603334d6793..176301bcca1 100644
--- a/lib/gitlab/ci/config/node/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -1,24 +1,22 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents a concrete CI/CD job.
         #
-        class Job < Entry
+        class Job < Node
           include Configurable
           include Attributable
 
           ALLOWED_KEYS = %i[tags script only except type image services allow_failure
                             type stage when artifacts cache dependencies before_script
-                            after_script variables environment]
-
-          attributes :tags, :allow_failure, :when, :dependencies
+                            after_script variables environment coverage].freeze
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
-
             validates :config, presence: true
+            validates :script, presence: true
             validates :name, presence: true
             validates :name, type: Symbol
 
@@ -34,48 +32,53 @@ module Gitlab
             end
           end
 
-          node :before_script, Node::Script,
+          entry :before_script, Entry::Script,
             description: 'Global before script overridden in this job.'
 
-          node :script, Node::Commands,
+          entry :script, Entry::Commands,
             description: 'Commands that will be executed in this job.'
 
-          node :stage, Node::Stage,
+          entry :stage, Entry::Stage,
             description: 'Pipeline stage this job will be executed into.'
 
-          node :type, Node::Stage,
+          entry :type, Entry::Stage,
             description: 'Deprecated: stage this job will be executed into.'
 
-          node :after_script, Node::Script,
+          entry :after_script, Entry::Script,
             description: 'Commands that will be executed when finishing job.'
 
-          node :cache, Node::Cache,
+          entry :cache, Entry::Cache,
             description: 'Cache definition for this job.'
 
-          node :image, Node::Image,
+          entry :image, Entry::Image,
             description: 'Image that will be used to execute this job.'
 
-          node :services, Node::Services,
+          entry :services, Entry::Services,
             description: 'Services that will be used to execute this job.'
 
-          node :only, Node::Trigger,
+          entry :only, Entry::Trigger,
             description: 'Refs policy this job will be executed for.'
 
-          node :except, Node::Trigger,
+          entry :except, Entry::Trigger,
             description: 'Refs policy this job will be executed for.'
 
-          node :variables, Node::Variables,
+          entry :variables, Entry::Variables,
             description: 'Environment variables available for this job.'
 
-          node :artifacts, Node::Artifacts,
+          entry :artifacts, Entry::Artifacts,
             description: 'Artifacts configuration for this job.'
 
-          node :environment, Node::Environment,
+          entry :environment, Entry::Environment,
                description: 'Environment configuration for this job.'
 
+          entry :coverage, Entry::Coverage,
+               description: 'Coverage configuration for this job.'
+
           helpers :before_script, :script, :stage, :type, :after_script,
                   :cache, :image, :services, :only, :except, :variables,
-                  :artifacts, :commands, :environment
+                  :artifacts, :commands, :environment, :coverage
+
+          attributes :script, :tags, :allow_failure, :when, :dependencies
 
           def compose!(deps = nil)
             super do
@@ -101,6 +104,14 @@ module Gitlab
             (before_script_value.to_a + script_value.to_a).join("\n")
           end
 
+          def manual_action?
+            self.when == 'manual'
+          end
+
+          def ignored?
+            allow_failure.nil? ? manual_action? : allow_failure
+          end
+
           private
 
           def inherit!(deps)
@@ -108,7 +119,7 @@ module Gitlab
 
             self.class.nodes.each_key do |key|
               global_entry = deps[key]
-              job_entry = @entries[key]
+              job_entry = self[key]
 
               if global_entry.specified? && !job_entry.specified?
                 @entries[key] = global_entry
@@ -118,20 +129,22 @@ module Gitlab
 
           def to_hash
             { name: name,
-              before_script: before_script,
-              script: script,
+              before_script: before_script_value,
+              script: script_value,
               commands: commands,
-              image: image,
-              services: services,
-              stage: stage,
-              cache: cache,
-              only: only,
-              except: except,
-              variables: variables_defined? ? variables : nil,
-              environment: environment_defined? ? environment : nil,
-              environment_name: environment_defined? ? environment[:name] : nil,
-              artifacts: artifacts,
-              after_script: after_script }
+              image: image_value,
+              services: services_value,
+              stage: stage_value,
+              cache: cache_value,
+              only: only_value,
+              except: except_value,
+              variables: variables_defined? ? variables_value : nil,
+              environment: environment_defined? ? environment_value : nil,
+              environment_name: environment_defined? ? environment_value[:name] : nil,
+              coverage: coverage_defined? ? coverage_value : nil,
+              artifacts: artifacts_value,
+              after_script: after_script_value,
+              ignore: ignored? }
           end
         end
       end
diff --git a/lib/gitlab/ci/config/node/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb
index d10e80d1a7d..5671a09480b 100644
--- a/lib/gitlab/ci/config/node/jobs.rb
+++ b/lib/gitlab/ci/config/entry/jobs.rb
@@ -1,11 +1,11 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents a set of jobs.
         #
-        class Jobs < Entry
+        class Jobs < Node
           include Validatable
 
           validations do
@@ -29,9 +29,9 @@ module Gitlab
           def compose!(deps = nil)
             super do
               @config.each do |name, config|
-                node = hidden?(name) ? Node::Hidden : Node::Job
+                node = hidden?(name) ? Entry::Hidden : Entry::Job
 
-                factory = Node::Factory.new(node)
+                factory = Entry::Factory.new(node)
                   .value(config || {})
                   .metadata(name: name)
                   .with(key: name, parent: self,
diff --git a/lib/gitlab/ci/config/node/key.rb b/lib/gitlab/ci/config/entry/key.rb
index f8b461ca098..f27ad0a7759 100644
--- a/lib/gitlab/ci/config/node/key.rb
+++ b/lib/gitlab/ci/config/entry/key.rb
@@ -1,16 +1,20 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents a key.
         #
-        class Key < Entry
+        class Key < Node
           include Validatable
 
           validations do
             validates :config, key: true
           end
+
+          def self.default
+            'default'
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb
index 0c291efe6a5..a78a85397bd 100644
--- a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb
+++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         module LegacyValidationHelpers
           private
 
@@ -21,24 +21,34 @@ module Gitlab
 
           def validate_variables(variables)
             variables.is_a?(Hash) &&
-              variables.all? { |key, value| validate_string(key) && validate_string(value) }
+              variables.flatten.all? do |value|
+                validate_string(value) || validate_integer(value)
+              end
+          end
+
+          def validate_integer(value)
+            value.is_a?(Integer)
           end
 
           def validate_string(value)
             value.is_a?(String) || value.is_a?(Symbol)
           end
 
+          def validate_regexp(value)
+            !value.nil? && Regexp.new(value.to_s) && true
+          rescue RegexpError, TypeError
+            false
+          end
+
           def validate_string_or_regexp(value)
             return true if value.is_a?(Symbol)
             return false unless value.is_a?(String)
 
             if value.first == '/' && value.last == '/'
-              Regexp.new(value[1...-1])
+              validate_regexp(value[1...-1])
             else
               true
             end
-          rescue RegexpError
-            false
           end
 
           def validate_boolean(value)
diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/entry/node.rb
index 8717eabf81e..a6a914d79c1 100644
--- a/lib/gitlab/ci/config/node/entry.rb
+++ b/lib/gitlab/ci/config/entry/node.rb
@@ -1,12 +1,12 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Base abstract class for each configuration entry node.
         #
-        class Entry
-          class InvalidError < StandardError; end
+        class Node
+          InvalidError = Class.new(StandardError)
 
           attr_reader :config, :metadata
           attr_accessor :key, :parent, :description
@@ -21,7 +21,7 @@ module Gitlab
           end
 
           def [](key)
-            @entries[key] || Node::Undefined.new
+            @entries[key] || Entry::Undefined.new
           end
 
           def compose!(deps = nil)
@@ -70,6 +70,12 @@ module Gitlab
             true
           end
 
+          def inspect
+            val = leaf? ? config : descendants
+            unspecified = specified? ? '' : '(unspecified) '
+            "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>"
+          end
+
           def self.default
           end
 
diff --git a/lib/gitlab/ci/config/node/paths.rb b/lib/gitlab/ci/config/entry/paths.rb
index 3c6d3a52966..68dad161149 100644
--- a/lib/gitlab/ci/config/node/paths.rb
+++ b/lib/gitlab/ci/config/entry/paths.rb
@@ -1,11 +1,11 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents an array of paths.
         #
-        class Paths < Entry
+        class Paths < Node
           include Validatable
 
           validations do
diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/entry/script.rb
index 39328f0fade..29ecd9995ca 100644
--- a/lib/gitlab/ci/config/node/script.rb
+++ b/lib/gitlab/ci/config/entry/script.rb
@@ -1,11 +1,11 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents a script.
         #
-        class Script < Entry
+        class Script < Node
           include Validatable
 
           validations do
diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb
new file mode 100644
index 00000000000..b52faf48b58
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/service.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a configuration of Docker service.
+        #
+        class Service < Image
+          include Validatable
+
+          ALLOWED_KEYS = %i[name entrypoint command alias].freeze
+
+          validations do
+            validates :config, hash_or_string: true
+            validates :config, allowed_keys: ALLOWED_KEYS
+
+            validates :name, type: String, presence: true
+            validates :entrypoint, type: String, allow_nil: true
+            validates :command, type: String, allow_nil: true
+            validates :alias, type: String, allow_nil: true
+          end
+
+          def alias
+            value[:alias]
+          end
+
+          def command
+            value[:command]
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb
new file mode 100644
index 00000000000..0066894e069
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/services.rb
@@ -0,0 +1,41 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a configuration of Docker services.
+        #
+        class Services < Node
+          include Validatable
+
+          validations do
+            validates :config, type: Array
+          end
+
+          def compose!(deps = nil)
+            super do
+              @entries = []
+              @config.each do |config|
+                @entries << Entry::Factory.new(Entry::Service)
+                  .value(config || {})
+                  .create!
+              end
+
+              @entries.each do |entry|
+                entry.compose!(deps)
+              end
+            end
+          end
+
+          def value
+            @entries.map(&:value)
+          end
+
+          def descendants
+            @entries
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/stage.rb b/lib/gitlab/ci/config/entry/stage.rb
index cbc97641f5a..b7afaba1de8 100644
--- a/lib/gitlab/ci/config/node/stage.rb
+++ b/lib/gitlab/ci/config/entry/stage.rb
@@ -1,11 +1,11 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents a stage for a job.
         #
-        class Stage < Entry
+        class Stage < Node
           include Validatable
 
           validations do
diff --git a/lib/gitlab/ci/config/node/stages.rb b/lib/gitlab/ci/config/entry/stages.rb
index b1fe45357ff..ec187bd3732 100644
--- a/lib/gitlab/ci/config/node/stages.rb
+++ b/lib/gitlab/ci/config/entry/stages.rb
@@ -1,11 +1,11 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents a configuration for pipeline stages.
         #
-        class Stages < Entry
+        class Stages < Node
           include Validatable
 
           validations do
diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb
new file mode 100644
index 00000000000..16b234e6c59
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/trigger.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a trigger policy for the job.
+        #
+        class Trigger < Node
+          include Validatable
+
+          validations do
+            validates :config, array_of_strings_or_regexps: true
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/entry/undefined.rb
index 33e78023539..1171ac10f22 100644
--- a/lib/gitlab/ci/config/node/undefined.rb
+++ b/lib/gitlab/ci/config/entry/undefined.rb
@@ -1,13 +1,11 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
-        # This class represents an undefined node.
+        # This class represents an undefined entry.
         #
-        # Implements the Null Object pattern.
-        #
-        class Undefined < Entry
+        class Undefined < Node
           def initialize(*)
             super(nil)
           end
@@ -31,6 +29,10 @@ module Gitlab
           def relevant?
             false
           end
+
+          def inspect
+            "#<#{self.class.name}>"
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/node/unspecified.rb b/lib/gitlab/ci/config/entry/unspecified.rb
index a7d1f6131b8..fbb2551e870 100644
--- a/lib/gitlab/ci/config/node/unspecified.rb
+++ b/lib/gitlab/ci/config/entry/unspecified.rb
@@ -1,9 +1,9 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
-        # This class represents an unspecified entry node.
+        # This class represents an unspecified entry.
         #
         # It decorates original entry adding method that indicates it is
         # unspecified.
diff --git a/lib/gitlab/ci/config/node/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb
index 085e6e988d1..f7f1b111571 100644
--- a/lib/gitlab/ci/config/node/validatable.rb
+++ b/lib/gitlab/ci/config/entry/validatable.rb
@@ -1,13 +1,13 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         module Validatable
           extend ActiveSupport::Concern
 
           class_methods do
             def validator
-              @validator ||= Class.new(Node::Validator).tap do |validator|
+              @validator ||= Class.new(Entry::Validator).tap do |validator|
                 if defined?(@validations)
                   @validations.each { |rules| validator.class_eval(&rules) }
                 end
diff --git a/lib/gitlab/ci/config/node/validator.rb b/lib/gitlab/ci/config/entry/validator.rb
index 43c7e102b50..55343005fe3 100644
--- a/lib/gitlab/ci/config/node/validator.rb
+++ b/lib/gitlab/ci/config/entry/validator.rb
@@ -1,14 +1,14 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         class Validator < SimpleDelegator
           include ActiveModel::Validations
-          include Node::Validators
+          include Entry::Validators
 
-          def initialize(node)
-            super(node)
-            @node = node
+          def initialize(entry)
+            super(entry)
+            @entry = entry
           end
 
           def messages
@@ -30,7 +30,7 @@ module Gitlab
 
           def key_name
             if key.blank?
-              @node.class.name.demodulize.underscore.humanize
+              @entry.class.name.demodulize.underscore.humanize
             else
               key
             end
diff --git a/lib/gitlab/ci/config/node/validators.rb b/lib/gitlab/ci/config/entry/validators.rb
index e20908ad3cb..b2ca3c881e4 100644
--- a/lib/gitlab/ci/config/node/validators.rb
+++ b/lib/gitlab/ci/config/entry/validators.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         module Validators
           class AllowedKeysValidator < ActiveModel::EachValidator
             def validate_each(record, attribute, value)
@@ -44,6 +44,14 @@ module Gitlab
             end
           end
 
+          class HashOrStringValidator < ActiveModel::EachValidator
+            def validate_each(record, attribute, value)
+              unless value.is_a?(Hash) || value.is_a?(String)
+                record.errors.add(attribute, 'should be a hash or a string')
+              end
+            end
+          end
+
           class KeyValidator < ActiveModel::EachValidator
             include LegacyValidationHelpers
 
@@ -54,6 +62,51 @@ module Gitlab
             end
           end
 
+          class RegexpValidator < ActiveModel::EachValidator
+            include LegacyValidationHelpers
+
+            def validate_each(record, attribute, value)
+              unless validate_regexp(value)
+                record.errors.add(attribute, 'must be a regular expression')
+              end
+            end
+
+            private
+
+            def look_like_regexp?(value)
+              value.is_a?(String) && value.start_with?('/') &&
+                value.end_with?('/')
+            end
+
+            def validate_regexp(value)
+              look_like_regexp?(value) &&
+                Regexp.new(value.to_s[1...-1]) &&
+                true
+            rescue RegexpError
+              false
+            end
+          end
+
+          class ArrayOfStringsOrRegexpsValidator < RegexpValidator
+            def validate_each(record, attribute, value)
+              unless validate_array_of_strings_or_regexps(value)
+                record.errors.add(attribute, 'should be an array of strings or regexps')
+              end
+            end
+
+            private
+
+            def validate_array_of_strings_or_regexps(values)
+              values.is_a?(Array) && values.all?(&method(:validate_string_or_regexp))
+            end
+
+            def validate_string_or_regexp(value)
+              return false unless value.is_a?(String)
+              return validate_regexp(value) if look_like_regexp?(value)
+              true
+            end
+          end
+
           class TypeValidator < ActiveModel::EachValidator
             def validate_each(record, attribute, value)
               type = options[:with]
diff --git a/lib/gitlab/ci/config/node/variables.rb b/lib/gitlab/ci/config/entry/variables.rb
index 5f813f81f55..8acab605c91 100644
--- a/lib/gitlab/ci/config/node/variables.rb
+++ b/lib/gitlab/ci/config/entry/variables.rb
@@ -1,11 +1,11 @@
 module Gitlab
   module Ci
     class Config
-      module Node
+      module Entry
         ##
         # Entry that represents environment variables.
         #
-        class Variables < Entry
+        class Variables < Node
           include Validatable
 
           validations do
@@ -15,6 +15,10 @@ module Gitlab
           def self.default
             {}
           end
+
+          def value
+            Hash[@config.map { |key, value| [key.to_s, value.to_s] }]
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/ci/config/loader.rb
index dbf6eb0edbe..e7d9f6a7761 100644
--- a/lib/gitlab/ci/config/loader.rb
+++ b/lib/gitlab/ci/config/loader.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Ci
     class Config
       class Loader
-        class FormatError < StandardError; end
+        FormatError = Class.new(StandardError)
 
         def initialize(config)
           @config = YAML.safe_load(config, [Symbol], [], true)
diff --git a/lib/gitlab/ci/config/node/image.rb b/lib/gitlab/ci/config/node/image.rb
deleted file mode 100644
index 5d3c7c5eab0..00000000000
--- a/lib/gitlab/ci/config/node/image.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Node
-        ##
-        # Entry that represents a Docker image.
-        #
-        class Image < Entry
-          include Validatable
-
-          validations do
-            validates :config, type: String
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/node/services.rb b/lib/gitlab/ci/config/node/services.rb
deleted file mode 100644
index 481e2b66adc..00000000000
--- a/lib/gitlab/ci/config/node/services.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Node
-        ##
-        # Entry that represents a configuration of Docker services.
-        #
-        class Services < Entry
-          include Validatable
-
-          validations do
-            validates :config, array_of_strings: true
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/node/trigger.rb b/lib/gitlab/ci/config/node/trigger.rb
deleted file mode 100644
index d8b31975088..00000000000
--- a/lib/gitlab/ci/config/node/trigger.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-module Gitlab
-  module Ci
-    class Config
-      module Node
-        ##
-        # Entry that represents a trigger policy for the job.
-        #
-        class Trigger < Entry
-          include Validatable
-
-          validations do
-            include LegacyValidationHelpers
-
-            validate :array_of_strings_or_regexps
-
-            def array_of_strings_or_regexps
-              unless validate_array_of_strings_or_regexps(config)
-                errors.add(:config, 'should be an array of strings or regexps')
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb
new file mode 100644
index 00000000000..551483d0aaa
--- /dev/null
+++ b/lib/gitlab/ci/cron_parser.rb
@@ -0,0 +1,49 @@
+module Gitlab
+  module Ci
+    class CronParser
+      VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'.freeze
+      VALID_SYNTAX_SAMPLE_CRON = '* * * * *'.freeze
+
+      def initialize(cron, cron_timezone = 'UTC')
+        @cron = cron
+        @cron_timezone = ActiveSupport::TimeZone.find_tzinfo(cron_timezone).name
+      end
+
+      def next_time_from(time)
+        @cron_line ||= try_parse_cron(@cron, @cron_timezone)
+        @cron_line.next_time(time).utc.in_time_zone(Time.zone) if @cron_line.present?
+      end
+
+      def cron_valid?
+        try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present?
+      end
+
+      def cron_timezone_valid?
+        try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_timezone).present?
+      end
+
+      private
+
+      # NOTE:
+      # cron_timezone can only accept timezones listed in TZInfo::Timezone.
+      # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted,
+      # because Rufus::Scheduler only supports TZInfo::Timezone.
+      #
+      # For example, those codes have the same effect.
+      # Time.zone = 'Pacific Time (US & Canada)' (ActiveSupport::TimeZone)
+      # Time.zone = 'America/Los_Angeles' (TZInfo::Timezone)
+      #
+      # However, try_parse_cron only accepts the latter format.
+      # try_parse_cron('* * * * *', 'Pacific Time (US & Canada)') -> Doesn't work
+      # try_parse_cron('* * * * *', 'America/Los_Angeles') -> Works
+      # If you want to know more, please take a look
+      # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/values/time_zone.rb
+      def try_parse_cron(cron, cron_timezone)
+        cron_line = Rufus::Scheduler.parse("#{cron} #{cron_timezone}")
+        cron_line if cron_line.is_a?(Rufus::Scheduler::CronLine)
+      rescue
+        # noop
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb
index a210e76acaa..3208cc2bef6 100644
--- a/lib/gitlab/ci/pipeline_duration.rb
+++ b/lib/gitlab/ci/pipeline_duration.rb
@@ -87,8 +87,8 @@ module Gitlab
 
       def from_pipeline(pipeline)
         status = %w[success failed running canceled]
-        builds = pipeline.builds.latest.
-          where(status: status).where.not(started_at: nil).order(:started_at)
+        builds = pipeline.builds.latest
+          .where(status: status).where.not(started_at: nil).order(:started_at)
 
         from_builds(builds)
       end
diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb
new file mode 100644
index 00000000000..f81f9347b4d
--- /dev/null
+++ b/lib/gitlab/ci/stage/seed.rb
@@ -0,0 +1,49 @@
+module Gitlab
+  module Ci
+    module Stage
+      class Seed
+        attr_reader :pipeline
+        delegate :project, to: :pipeline
+
+        def initialize(pipeline, stage, jobs)
+          @pipeline = pipeline
+          @stage = { name: stage }
+          @jobs = jobs.to_a.dup
+        end
+
+        def user=(current_user)
+          @jobs.map! do |attributes|
+            attributes.merge(user: current_user)
+          end
+        end
+
+        def stage
+          @stage.merge(project: project)
+        end
+
+        def builds
+          trigger = pipeline.trigger_requests.first
+
+          @jobs.map do |attributes|
+            attributes.merge(project: project,
+                             ref: pipeline.ref,
+                             tag: pipeline.tag,
+                             trigger_request: trigger)
+          end
+        end
+
+        def create!
+          pipeline.stages.create!(stage).tap do |stage|
+            builds_attributes = builds.map do |attributes|
+              attributes.merge(stage_id: stage.id)
+            end
+
+            pipeline.builds.create!(builds_attributes).each do |build|
+              yield build if block_given?
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/action.rb b/lib/gitlab/ci/status/build/action.rb
new file mode 100644
index 00000000000..45fd0d4aa07
--- /dev/null
+++ b/lib/gitlab/ci/status/build/action.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Action < Status::Extended
+          def label
+            if has_action?
+              @status.label
+            else
+              "#{@status.label} (not allowed)"
+            end
+          end
+
+          def self.matches?(build, user)
+            build.action?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb
new file mode 100644
index 00000000000..439ef0ce015
--- /dev/null
+++ b/lib/gitlab/ci/status/build/cancelable.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Cancelable < Status::Extended
+          def has_action?
+            can?(user, :update_build, subject)
+          end
+
+          def action_icon
+            'icon_action_cancel'
+          end
+
+          def action_path
+            cancel_namespace_project_job_path(subject.project.namespace,
+                                                subject.project,
+                                                subject)
+          end
+
+          def action_method
+            :post
+          end
+
+          def action_title
+            'Cancel'
+          end
+
+          def self.matches?(build, user)
+            build.cancelable?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb
new file mode 100644
index 00000000000..b173c23fba4
--- /dev/null
+++ b/lib/gitlab/ci/status/build/common.rb
@@ -0,0 +1,19 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        module Common
+          def has_details?
+            can?(user, :read_build, subject)
+          end
+
+          def details_path
+            namespace_project_job_path(subject.project.namespace,
+                                         subject.project,
+                                         subject)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb
new file mode 100644
index 00000000000..c852d607373
--- /dev/null
+++ b/lib/gitlab/ci/status/build/factory.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Factory < Status::Factory
+          def self.extended_statuses
+            [[Status::Build::Cancelable,
+              Status::Build::Retryable],
+             [Status::Build::FailedAllowed,
+              Status::Build::Play,
+              Status::Build::Stop],
+             [Status::Build::Action]]
+          end
+
+          def self.common_helpers
+            Status::Build::Common
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb
new file mode 100644
index 00000000000..e42d3574357
--- /dev/null
+++ b/lib/gitlab/ci/status/build/failed_allowed.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class FailedAllowed < Status::Extended
+          def label
+            'failed (allowed to fail)'
+          end
+
+          def icon
+            'icon_status_warning'
+          end
+
+          def group
+            'failed_with_warnings'
+          end
+
+          def self.matches?(build, user)
+            build.failed? && build.allow_failure?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb
new file mode 100644
index 00000000000..e80f3263794
--- /dev/null
+++ b/lib/gitlab/ci/status/build/play.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Play < Status::Extended
+          def label
+            'manual play action'
+          end
+
+          def has_action?
+            can?(user, :update_build, subject)
+          end
+
+          def action_icon
+            'icon_action_play'
+          end
+
+          def action_title
+            'Play'
+          end
+
+          def action_path
+            play_namespace_project_job_path(subject.project.namespace,
+                                              subject.project,
+                                              subject)
+          end
+
+          def action_method
+            :post
+          end
+
+          def self.matches?(build, user)
+            build.playable? && !build.stops_environment?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb
new file mode 100644
index 00000000000..56303e4cb17
--- /dev/null
+++ b/lib/gitlab/ci/status/build/retryable.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Retryable < Status::Extended
+          def has_action?
+            can?(user, :update_build, subject)
+          end
+
+          def action_icon
+            'icon_action_retry'
+          end
+
+          def action_title
+            'Retry'
+          end
+
+          def action_path
+            retry_namespace_project_job_path(subject.project.namespace,
+                                               subject.project,
+                                               subject)
+          end
+
+          def action_method
+            :post
+          end
+
+          def self.matches?(build, user)
+            build.retryable?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb
new file mode 100644
index 00000000000..2778d6f3b52
--- /dev/null
+++ b/lib/gitlab/ci/status/build/stop.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Stop < Status::Extended
+          def label
+            'manual stop action'
+          end
+
+          def has_action?
+            can?(user, :update_build, subject)
+          end
+
+          def action_icon
+            'icon_action_stop'
+          end
+
+          def action_title
+            'Stop'
+          end
+
+          def action_path
+            play_namespace_project_job_path(subject.project.namespace,
+                                              subject.project,
+                                              subject)
+          end
+
+          def action_method
+            :post
+          end
+
+          def self.matches?(build, user)
+            build.playable? && build.stops_environment?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb
new file mode 100644
index 00000000000..e5fdc1f8136
--- /dev/null
+++ b/lib/gitlab/ci/status/canceled.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Ci
+    module Status
+      class Canceled < Status::Core
+        def text
+          s_('CiStatusText|canceled')
+        end
+
+        def label
+          s_('CiStatusLabel|canceled')
+        end
+
+        def icon
+          'icon_status_canceled'
+        end
+
+        def favicon
+          'favicon_status_canceled'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb
new file mode 100644
index 00000000000..d4fd83b93f8
--- /dev/null
+++ b/lib/gitlab/ci/status/core.rb
@@ -0,0 +1,63 @@
+module Gitlab
+  module Ci
+    module Status
+      # Base abstract class fore core status
+      #
+      class Core
+        include Gitlab::Routing
+        include Gitlab::Allowable
+
+        attr_reader :subject, :user
+
+        def initialize(subject, user)
+          @subject = subject
+          @user = user
+        end
+
+        def icon
+          raise NotImplementedError
+        end
+
+        def favicon
+          raise NotImplementedError
+        end
+
+        def label
+          raise NotImplementedError
+        end
+
+        def group
+          self.class.name.demodulize.underscore
+        end
+
+        def has_details?
+          false
+        end
+
+        def details_path
+          raise NotImplementedError
+        end
+
+        def has_action?
+          false
+        end
+
+        def action_icon
+          raise NotImplementedError
+        end
+
+        def action_path
+          raise NotImplementedError
+        end
+
+        def action_method
+          raise NotImplementedError
+        end
+
+        def action_title
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb
new file mode 100644
index 00000000000..d188bd286a6
--- /dev/null
+++ b/lib/gitlab/ci/status/created.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Ci
+    module Status
+      class Created < Status::Core
+        def text
+          s_('CiStatusText|created')
+        end
+
+        def label
+          s_('CiStatusLabel|created')
+        end
+
+        def icon
+          'icon_status_created'
+        end
+
+        def favicon
+          'favicon_status_created'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb
new file mode 100644
index 00000000000..1e8101f8949
--- /dev/null
+++ b/lib/gitlab/ci/status/extended.rb
@@ -0,0 +1,15 @@
+module Gitlab
+  module Ci
+    module Status
+      class Extended < SimpleDelegator
+        def initialize(status)
+          super(@status = status)
+        end
+
+        def self.matches?(_subject, _user)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb
new file mode 100644
index 00000000000..9307545b5b1
--- /dev/null
+++ b/lib/gitlab/ci/status/external/common.rb
@@ -0,0 +1,26 @@
+module Gitlab
+  module Ci
+    module Status
+      module External
+        module Common
+          def label
+            subject.description
+          end
+
+          def has_details?
+            subject.target_url.present? &&
+              can?(user, :read_commit_status, subject)
+          end
+
+          def details_path
+            subject.target_url
+          end
+
+          def has_action?
+            false
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/external/factory.rb b/lib/gitlab/ci/status/external/factory.rb
new file mode 100644
index 00000000000..07b15bd8d97
--- /dev/null
+++ b/lib/gitlab/ci/status/external/factory.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module Ci
+    module Status
+      module External
+        class Factory < Status::Factory
+          def self.common_helpers
+            Status::External::Common
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb
new file mode 100644
index 00000000000..15836c699c7
--- /dev/null
+++ b/lib/gitlab/ci/status/factory.rb
@@ -0,0 +1,52 @@
+module Gitlab
+  module Ci
+    module Status
+      class Factory
+        def initialize(subject, user)
+          @subject = subject
+          @user = user
+          @status = subject.status || HasStatus::DEFAULT_STATUS
+        end
+
+        def fabricate!
+          if extended_statuses.none?
+            core_status
+          else
+            compound_extended_status
+          end
+        end
+
+        def core_status
+          Gitlab::Ci::Status
+            .const_get(@status.capitalize)
+            .new(@subject, @user)
+            .extend(self.class.common_helpers)
+        end
+
+        def compound_extended_status
+          extended_statuses.inject(core_status) do |status, extended|
+            extended.new(status)
+          end
+        end
+
+        def extended_statuses
+          return @extended_statuses if defined?(@extended_statuses)
+
+          groups = self.class.extended_statuses.map do |group|
+            Array(group).find { |status| status.matches?(@subject, @user) }
+          end
+
+          @extended_statuses = groups.flatten.compact
+        end
+
+        def self.extended_statuses
+          []
+        end
+
+        def self.common_helpers
+          Module.new
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb
new file mode 100644
index 00000000000..38e45714c22
--- /dev/null
+++ b/lib/gitlab/ci/status/failed.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Ci
+    module Status
+      class Failed < Status::Core
+        def text
+          s_('CiStatusText|failed')
+        end
+
+        def label
+          s_('CiStatusLabel|failed')
+        end
+
+        def icon
+          'icon_status_failed'
+        end
+
+        def favicon
+          'favicon_status_failed'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/group/common.rb b/lib/gitlab/ci/status/group/common.rb
new file mode 100644
index 00000000000..cfd4329a923
--- /dev/null
+++ b/lib/gitlab/ci/status/group/common.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module Ci
+    module Status
+      module Group
+        module Common
+          def has_details?
+            false
+          end
+
+          def details_path
+            nil
+          end
+
+          def has_action?
+            false
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb
new file mode 100644
index 00000000000..d118116cfc3
--- /dev/null
+++ b/lib/gitlab/ci/status/group/factory.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module Ci
+    module Status
+      module Group
+        class Factory < Status::Factory
+          def self.common_helpers
+            Status::Group::Common
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb
new file mode 100644
index 00000000000..a4a7edadac9
--- /dev/null
+++ b/lib/gitlab/ci/status/manual.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Ci
+    module Status
+      class Manual < Status::Core
+        def text
+          s_('CiStatusText|manual')
+        end
+
+        def label
+          s_('CiStatusLabel|manual action')
+        end
+
+        def icon
+          'icon_status_manual'
+        end
+
+        def favicon
+          'favicon_status_manual'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb
new file mode 100644
index 00000000000..5164260b861
--- /dev/null
+++ b/lib/gitlab/ci/status/pending.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Ci
+    module Status
+      class Pending < Status::Core
+        def text
+          s_('CiStatusText|pending')
+        end
+
+        def label
+          s_('CiStatusLabel|pending')
+        end
+
+        def icon
+          'icon_status_pending'
+        end
+
+        def favicon
+          'favicon_status_pending'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb
new file mode 100644
index 00000000000..bf7e484ee9b
--- /dev/null
+++ b/lib/gitlab/ci/status/pipeline/blocked.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module Ci
+    module Status
+      module Pipeline
+        class Blocked < Status::Extended
+          def text
+            s_('CiStatusText|blocked')
+          end
+
+          def label
+            s_('CiStatusLabel|waiting for manual action')
+          end
+
+          def self.matches?(pipeline, user)
+            pipeline.blocked?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb
new file mode 100644
index 00000000000..76bfd18bf40
--- /dev/null
+++ b/lib/gitlab/ci/status/pipeline/common.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Ci
+    module Status
+      module Pipeline
+        module Common
+          def has_details?
+            can?(user, :read_pipeline, subject)
+          end
+
+          def details_path
+            namespace_project_pipeline_path(subject.project.namespace,
+                                            subject.project,
+                                            subject)
+          end
+
+          def has_action?
+            false
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb
new file mode 100644
index 00000000000..17f9a75f436
--- /dev/null
+++ b/lib/gitlab/ci/status/pipeline/factory.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module Ci
+    module Status
+      module Pipeline
+        class Factory < Status::Factory
+          def self.extended_statuses
+            [[Status::SuccessWarning,
+              Status::Pipeline::Blocked]]
+          end
+
+          def self.common_helpers
+            Status::Pipeline::Common
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb
new file mode 100644
index 00000000000..993937e98ca
--- /dev/null
+++ b/lib/gitlab/ci/status/running.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Ci
+    module Status
+      class Running < Status::Core
+        def text
+          s_('CiStatus|running')
+        end
+
+        def label
+          s_('CiStatus|running')
+        end
+
+        def icon
+          'icon_status_running'
+        end
+
+        def favicon
+          'favicon_status_running'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb
new file mode 100644
index 00000000000..0c942920b02
--- /dev/null
+++ b/lib/gitlab/ci/status/skipped.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Ci
+    module Status
+      class Skipped < Status::Core
+        def text
+          s_('CiStatusText|skipped')
+        end
+
+        def label
+          s_('CiStatusLabel|skipped')
+        end
+
+        def icon
+          'icon_status_skipped'
+        end
+
+        def favicon
+          'favicon_status_skipped'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb
new file mode 100644
index 00000000000..7852f492e1d
--- /dev/null
+++ b/lib/gitlab/ci/status/stage/common.rb
@@ -0,0 +1,24 @@
+module Gitlab
+  module Ci
+    module Status
+      module Stage
+        module Common
+          def has_details?
+            can?(user, :read_pipeline, subject.pipeline)
+          end
+
+          def details_path
+            namespace_project_pipeline_path(subject.project.namespace,
+                                            subject.project,
+                                            subject.pipeline,
+                                            anchor: subject.name)
+          end
+
+          def has_action?
+            false
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb
new file mode 100644
index 00000000000..4c37f084d07
--- /dev/null
+++ b/lib/gitlab/ci/status/stage/factory.rb
@@ -0,0 +1,17 @@
+module Gitlab
+  module Ci
+    module Status
+      module Stage
+        class Factory < Status::Factory
+          def self.extended_statuses
+            [Status::SuccessWarning]
+          end
+
+          def self.common_helpers
+            Status::Stage::Common
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb
new file mode 100644
index 00000000000..d7af98857b0
--- /dev/null
+++ b/lib/gitlab/ci/status/success.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module Ci
+    module Status
+      class Success < Status::Core
+        def text
+          s_('CiStatusText|passed')
+        end
+
+        def label
+          s_('CiStatusLabel|passed')
+        end
+
+        def icon
+          'icon_status_success'
+        end
+
+        def favicon
+          'favicon_status_success'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb
new file mode 100644
index 00000000000..4d7d82e04cf
--- /dev/null
+++ b/lib/gitlab/ci/status/success_warning.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module Ci
+    module Status
+      ##
+      # Extended status used when pipeline or stage passed conditionally.
+      # This means that failed jobs that are allowed to fail were present.
+      #
+      class SuccessWarning < Status::Extended
+        def text
+          s_('CiStatusText|passed')
+        end
+
+        def label
+          s_('CiStatusLabel|passed with warnings')
+        end
+
+        def icon
+          'icon_status_warning'
+        end
+
+        def group
+          'success_with_warnings'
+        end
+
+        def self.matches?(subject, user)
+          subject.success? && subject.has_warnings?
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
new file mode 100644
index 00000000000..5b835bb669a
--- /dev/null
+++ b/lib/gitlab/ci/trace.rb
@@ -0,0 +1,136 @@
+module Gitlab
+  module Ci
+    class Trace
+      attr_reader :job
+
+      delegate :old_trace, to: :job
+
+      def initialize(job)
+        @job = job
+      end
+
+      def html(last_lines: nil)
+        read do |stream|
+          stream.html(last_lines: last_lines)
+        end
+      end
+
+      def raw(last_lines: nil)
+        read do |stream|
+          stream.raw(last_lines: last_lines)
+        end
+      end
+
+      def extract_coverage(regex)
+        read do |stream|
+          stream.extract_coverage(regex)
+        end
+      end
+
+      def set(data)
+        write do |stream|
+          data = job.hide_secrets(data)
+          stream.set(data)
+        end
+      end
+
+      def append(data, offset)
+        write do |stream|
+          current_length = stream.size
+          return -current_length unless current_length == offset
+
+          data = job.hide_secrets(data)
+          stream.append(data, offset)
+          stream.size
+        end
+      end
+
+      def exist?
+        current_path.present? || old_trace.present?
+      end
+
+      def read
+        stream = Gitlab::Ci::Trace::Stream.new do
+          if current_path
+            File.open(current_path, "rb")
+          elsif old_trace
+            StringIO.new(old_trace)
+          end
+        end
+
+        yield stream
+      ensure
+        stream&.close
+      end
+
+      def write
+        stream = Gitlab::Ci::Trace::Stream.new do
+          File.open(ensure_path, "a+b")
+        end
+
+        yield(stream).tap do
+          job.touch if job.needs_touch?
+        end
+      ensure
+        stream&.close
+      end
+
+      def erase!
+        paths.each do |trace_path|
+          FileUtils.rm(trace_path, force: true)
+        end
+
+        job.erase_old_trace!
+      end
+
+      private
+
+      def ensure_path
+        return current_path if current_path
+
+        ensure_directory
+        default_path
+      end
+
+      def ensure_directory
+        unless Dir.exist?(default_directory)
+          FileUtils.mkdir_p(default_directory)
+        end
+      end
+
+      def current_path
+        @current_path ||= paths.find do |trace_path|
+          File.exist?(trace_path)
+        end
+      end
+
+      def paths
+        [
+          default_path,
+          deprecated_path
+        ].compact
+      end
+
+      def default_directory
+        File.join(
+          Settings.gitlab_ci.builds_path,
+          job.created_at.utc.strftime("%Y_%m"),
+          job.project_id.to_s
+        )
+      end
+
+      def default_path
+        File.join(default_directory, "#{job.id}.log")
+      end
+
+      def deprecated_path
+        File.join(
+          Settings.gitlab_ci.builds_path,
+          job.created_at.utc.strftime("%Y_%m"),
+          job.project.ci_id.to_s,
+          "#{job.id}.log"
+        ) if job.project&.ci_id
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
new file mode 100644
index 00000000000..c4c0623df6c
--- /dev/null
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -0,0 +1,126 @@
+module Gitlab
+  module Ci
+    class Trace
+      # This was inspired from: http://stackoverflow.com/a/10219411/1520132
+      class Stream
+        BUFFER_SIZE = 4096
+        LIMIT_SIZE = 500.kilobytes
+
+        attr_reader :stream
+
+        delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true
+
+        delegate :valid?, to: :stream, as: :present?, allow_nil: true
+
+        def initialize
+          @stream = yield
+          @stream&.binmode
+        end
+
+        def valid?
+          self.stream.present?
+        end
+
+        def file?
+          self.path.present?
+        end
+
+        def limit(last_bytes = LIMIT_SIZE)
+          if last_bytes < size
+            stream.seek(-last_bytes, IO::SEEK_END)
+            stream.readline
+          end
+        end
+
+        def append(data, offset)
+          stream.truncate(offset)
+          stream.seek(0, IO::SEEK_END)
+          stream.write(data)
+          stream.flush()
+        end
+
+        def set(data)
+          truncate(0)
+          stream.write(data)
+          stream.flush()
+        end
+
+        def raw(last_lines: nil)
+          return unless valid?
+
+          if last_lines.to_i > 0
+            read_last_lines(last_lines)
+          else
+            stream.read
+          end.force_encoding(Encoding.default_external)
+        end
+
+        def html_with_state(state = nil)
+          ::Ci::Ansi2html.convert(stream, state)
+        end
+
+        def html(last_lines: nil)
+          text = raw(last_lines: last_lines)
+          buffer = StringIO.new(text)
+          ::Ci::Ansi2html.convert(buffer).html
+        end
+
+        def extract_coverage(regex)
+          return unless valid?
+          return unless regex
+
+          regex = Regexp.new(regex)
+
+          match = ""
+
+          reverse_line do |line|
+            matches = line.scan(regex)
+            next unless matches.is_a?(Array)
+            next if matches.empty?
+
+            match = matches.flatten.last
+            coverage = match.gsub(/\d+(\.\d+)?/).first
+            return coverage if coverage.present?
+          end
+
+          nil
+        rescue
+          # if bad regex or something goes wrong we dont want to interrupt transition
+          # so we just silently ignore error for now
+        end
+
+        private
+
+        def read_last_lines(limit)
+          to_enum(:reverse_line).first(limit).reverse.join
+        end
+
+        def reverse_line
+          stream.seek(0, IO::SEEK_END)
+          debris = ''
+
+          until (buf = read_backward(BUFFER_SIZE)).empty?
+            buf += debris
+            debris, *lines = buf.each_line.to_a
+            lines.reverse_each do |line|
+              yield(line.force_encoding('UTF-8'))
+            end
+          end
+
+          yield(debris.force_encoding('UTF-8')) unless debris.empty?
+        end
+
+        def read_backward(length)
+          cur_offset = stream.tell
+          start = cur_offset - length
+          start = 0 if start < 0
+
+          stream.seek(start, IO::SEEK_SET)
+          stream.read(cur_offset - start).tap do
+            stream.seek(start, IO::SEEK_SET)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci_access.rb b/lib/gitlab/ci_access.rb
new file mode 100644
index 00000000000..def1373d8cf
--- /dev/null
+++ b/lib/gitlab/ci_access.rb
@@ -0,0 +1,9 @@
+module Gitlab
+  # For backwards compatibility, generic CI (which is a build without a user) is
+  # allowed to :build_download_code without any other checks.
+  class CiAccess
+    def can_do_action?(action)
+      action == :build_download_code
+    end
+  end
+end
diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb
index dff9e29c6a5..75a213ef752 100644
--- a/lib/gitlab/conflict/file.rb
+++ b/lib/gitlab/conflict/file.rb
@@ -4,8 +4,7 @@ module Gitlab
       include Gitlab::Routing.url_helpers
       include IconsHelper
 
-      class MissingResolution < StandardError
-      end
+      MissingResolution = Class.new(ResolutionError)
 
       CONTEXT_LINES = 3
 
@@ -21,12 +20,34 @@ module Gitlab
         @match_line_headers = {}
       end
 
+      def content
+        merge_file_result[:data]
+      end
+
+      def our_blob
+        @our_blob ||= repository.blob_at(merge_request.diff_refs.head_sha, our_path)
+      end
+
+      def type
+        lines unless @type
+
+        @type.inquiry
+      end
+
       # Array of Gitlab::Diff::Line objects
       def lines
-        @lines ||= Gitlab::Conflict::Parser.new.parse(merge_file_result[:data],
+        return @lines if defined?(@lines)
+
+        begin
+          @type = 'text'
+          @lines = Gitlab::Conflict::Parser.new.parse(content,
                                                       our_path: our_path,
                                                       their_path: their_path,
                                                       parent_file: self)
+        rescue Gitlab::Conflict::Parser::ParserError
+          @type = 'text-editor'
+          @lines = nil
+        end
       end
 
       def resolve_lines(resolution)
@@ -53,6 +74,14 @@ module Gitlab
         end.compact
       end
 
+      def resolve_content(resolution)
+        if resolution == content
+          raise MissingResolution, "Resolved content has no changes for file #{our_path}"
+        end
+
+        resolution
+      end
+
       def highlight_lines!
         their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n")
         our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n")
@@ -61,11 +90,12 @@ module Gitlab
         our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines
 
         lines.each do |line|
-          if line.type == 'old'
-            line.rich_text = their_highlight[line.old_line - 1].try(:html_safe)
-          else
-            line.rich_text = our_highlight[line.new_line - 1].try(:html_safe)
-          end
+          line.rich_text =
+            if line.type == 'old'
+              their_highlight[line.old_line - 1].try(:html_safe)
+            else
+              our_highlight[line.new_line - 1].try(:html_safe)
+            end
         end
       end
 
@@ -170,21 +200,39 @@ module Gitlab
         match_line.text = "@@ -#{match_line.old_pos},#{line.old_pos} +#{match_line.new_pos},#{line.new_pos} @@#{header}"
       end
 
-      def as_json(opts = nil)
-        {
+      def as_json(opts = {})
+        json_hash = {
           old_path: their_path,
           new_path: our_path,
           blob_icon: file_type_icon_class('file', our_mode, our_path),
           blob_path: namespace_project_blob_path(merge_request.project.namespace,
                                                  merge_request.project,
-                                                 ::File.join(merge_request.diff_refs.head_sha, our_path)),
-          sections: sections
+                                                 ::File.join(merge_request.diff_refs.head_sha, our_path))
         }
+
+        json_hash.tap do |json_hash|
+          if opts[:full_content]
+            json_hash[:content] = content
+            json_hash[:blob_ace_mode] = our_blob && our_blob.language.try(:ace_mode)
+          else
+            json_hash[:sections] = sections if type.text?
+            json_hash[:type] = type
+            json_hash[:content_path] = content_path
+          end
+        end
+      end
+
+      def content_path
+        conflict_for_path_namespace_project_merge_request_path(merge_request.project.namespace,
+                                                               merge_request.project,
+                                                               merge_request,
+                                                               old_path: their_path,
+                                                               new_path: our_path)
       end
 
       # Don't try to print merge_request or repository.
       def inspect
-        instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode].map do |instance_variable|
+        instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode, :type].map do |instance_variable|
           value = instance_variable_get("@#{instance_variable}")
 
           "#{instance_variable}=\"#{value}\""
diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb
index bbd0427a2c8..1611eba31da 100644
--- a/lib/gitlab/conflict/file_collection.rb
+++ b/lib/gitlab/conflict/file_collection.rb
@@ -1,19 +1,35 @@
 module Gitlab
   module Conflict
     class FileCollection
-      class ConflictSideMissing < StandardError
-      end
+      ConflictSideMissing = Class.new(StandardError)
 
-      attr_reader :merge_request, :our_commit, :their_commit
+      attr_reader :merge_request, :our_commit, :their_commit, :project
 
-      def initialize(merge_request)
-        @merge_request = merge_request
-        @our_commit = merge_request.source_branch_head.raw.raw_commit
-        @their_commit = merge_request.target_branch_head.raw.raw_commit
-      end
+      delegate :repository, to: :project
+
+      class << self
+        # We can only write when getting the merge index from the source
+        # project, because we will write to that project. We don't use this all
+        # the time because this fetches a ref into the source project, which
+        # isn't needed for reading.
+        def for_resolution(merge_request)
+          project = merge_request.source_project
+
+          new(merge_request, project).tap do |file_collection|
+            project
+              .repository
+              .with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do
+
+              yield file_collection
+            end
+          end
+        end
 
-      def repository
-        merge_request.project.repository
+        # We don't need to do `with_repo_branch_commit` here, because the target
+        # project always fetches source refs when creating merge request diffs.
+        def read_only(merge_request)
+          new(merge_request, merge_request.target_project)
+        end
       end
 
       def merge_index
@@ -30,6 +46,10 @@ module Gitlab
         end
       end
 
+      def file_for_path(old_path, new_path)
+        files.find { |file| file.their_path == old_path && file.our_path == new_path }
+      end
+
       def as_json(opts = nil)
         {
           target_branch: merge_request.target_branch,
@@ -52,6 +72,15 @@ Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branc
 #{conflict_filenames.join("\n")}
 EOM
       end
+
+      private
+
+      def initialize(merge_request, project)
+        @merge_request = merge_request
+        @our_commit = merge_request.source_branch_head.raw.raw_commit
+        @their_commit = merge_request.target_branch_head.raw.raw_commit
+        @project = project
+      end
     end
   end
 end
diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb
index 98e842cded3..84f9ecd3d23 100644
--- a/lib/gitlab/conflict/parser.rb
+++ b/lib/gitlab/conflict/parser.rb
@@ -1,30 +1,23 @@
 module Gitlab
   module Conflict
     class Parser
-      class ParserError < StandardError
-      end
-
-      class UnexpectedDelimiter < ParserError
-      end
-
-      class MissingEndDelimiter < ParserError
-      end
+      UnresolvableError = Class.new(StandardError)
+      UnmergeableFile = Class.new(UnresolvableError)
+      UnsupportedEncoding = Class.new(UnresolvableError)
 
-      class UnmergeableFile < ParserError
-      end
-
-      class UnsupportedEncoding < ParserError
-      end
+      # Recoverable errors - the conflict can be resolved in an editor, but not with
+      # sections.
+      ParserError = Class.new(StandardError)
+      UnexpectedDelimiter = Class.new(ParserError)
+      MissingEndDelimiter = Class.new(ParserError)
 
       def parse(text, our_path:, their_path:, parent_file: nil)
         raise UnmergeableFile if text.blank? # Typically a binary file
         raise UnmergeableFile if text.length > 200.kilobytes
 
-        begin
-          text.to_json
-        rescue Encoding::UndefinedConversionError
-          raise UnsupportedEncoding
-        end
+        text.force_encoding('UTF-8')
+
+        raise UnsupportedEncoding unless text.valid_encoding?
 
         line_obj_index = 0
         line_old = 1
diff --git a/lib/gitlab/conflict/resolution_error.rb b/lib/gitlab/conflict/resolution_error.rb
new file mode 100644
index 00000000000..0b61256b35a
--- /dev/null
+++ b/lib/gitlab/conflict/resolution_error.rb
@@ -0,0 +1,5 @@
+module Gitlab
+  module Conflict
+    ResolutionError = Class.new(StandardError)
+  end
+end
diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb
index b164f5a2eea..bf557103cfd 100644
--- a/lib/gitlab/contributions_calendar.rb
+++ b/lib/gitlab/contributions_calendar.rb
@@ -1,45 +1,46 @@
 module Gitlab
   class ContributionsCalendar
-    attr_reader :activity_dates, :projects, :user
+    attr_reader :contributor
+    attr_reader :current_user
+    attr_reader :projects
 
-    def initialize(projects, user)
-      @projects = projects
-      @user = user
+    def initialize(contributor, current_user = nil)
+      @contributor = contributor
+      @current_user = current_user
+      @projects = ContributedProjectsFinder.new(contributor).execute(current_user)
     end
 
     def activity_dates
       return @activity_dates if @activity_dates.present?
 
-      @activity_dates = {}
+      # Can't use Event.contributions here because we need to check 3 different
+      # project_features for the (currently) 3 different contribution types
       date_from = 1.year.ago
+      repo_events = event_counts(date_from, :repository)
+        .having(action: Event::PUSHED)
+      issue_events = event_counts(date_from, :issues)
+        .having(action: [Event::CREATED, Event::CLOSED], target_type: "Issue")
+      mr_events = event_counts(date_from, :merge_requests)
+        .having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest")
+      note_events = event_counts(date_from, :merge_requests)
+        .having(action: [Event::COMMENTED], target_type: "Note")
 
-      events = Event.reorder(nil).contributions.where(author_id: user.id).
-        where("created_at > ?", date_from).where(project_id: projects).
-        group('date(created_at)').
-        select('date(created_at) as date, count(id) as total_amount').
-        map(&:attributes)
+      union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events])
+      events = Event.find_by_sql(union.to_sql).map(&:attributes)
 
-      activity_dates = (1.year.ago.to_date..Date.today).to_a
-
-      activity_dates.each do |date|
-        day_events = events.find { |day_events| day_events["date"] == date }
-
-        if day_events
-          @activity_dates[date] = day_events["total_amount"]
-        end
+      @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities|
+        activities[event["date"]] += event["total_amount"]
       end
-
-      @activity_dates
     end
 
     def events_by_date(date)
-      events = Event.contributions.where(author_id: user.id).
-        where("created_at > ? AND created_at < ?", date.beginning_of_day, date.end_of_day).
-        where(project_id: projects)
+      events = Event.contributions.where(author_id: contributor.id)
+        .where(created_at: date.beginning_of_day..date.end_of_day)
+        .where(project_id: projects)
 
-      events.select do |event|
-        event.push? || event.issue? || event.merge_request?
-      end
+      # Use visible_to_user? instead of the complicated logic in activity_dates
+      # because we're only viewing the events for a single day.
+      events.select { |event| event.visible_to_user?(current_user) }
     end
 
     def starting_year
@@ -49,5 +50,30 @@ module Gitlab
     def starting_month
       Date.today.month
     end
+
+    private
+
+    def event_counts(date_from, feature)
+      t = Event.arel_table
+
+      # re-running the contributed projects query in each union is expensive, so
+      # use IN(project_ids...) instead. It's the intersection of two users so
+      # the list will be (relatively) short
+      @contributed_project_ids ||= projects.uniq.pluck(:id)
+      authed_projects = Project.where(id: @contributed_project_ids)
+        .with_feature_available_for_user(feature, current_user)
+        .reorder(nil)
+        .select(:id)
+
+      conditions = t[:created_at].gteq(date_from.beginning_of_day)
+        .and(t[:created_at].lteq(Date.today.end_of_day))
+        .and(t[:author_id].eq(contributor.id))
+
+      Event.reorder(nil)
+        .select(t[:project_id], t[:target_type], t[:action], 'date(created_at) AS date', 'count(id) as total_amount')
+        .group(t[:project_id], t[:target_type], t[:action], 'date(created_at)')
+        .where(conditions)
+        .having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql)))
+    end
   end
 end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index ef9160d6437..818b3d9c46b 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -8,52 +8,60 @@ module Gitlab
       end
     end
 
+    delegate :sidekiq_throttling_enabled?, to: :current_application_settings
+
+    def fake_application_settings(defaults = ::ApplicationSetting.defaults)
+      FakeApplicationSettings.new(defaults)
+    end
+
+    private
+
     def ensure_application_settings!
-      if connect_to_db?
-        begin
-          settings = ::ApplicationSetting.current
+      return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true'
+
+      cached_application_settings || uncached_application_settings
+    end
+
+    def cached_application_settings
+      begin
+        ::ApplicationSetting.cached
+      rescue ::Redis::BaseError, ::Errno::ENOENT
         # In case Redis isn't running or the Redis UNIX socket file is not available
-        rescue ::Redis::BaseError, ::Errno::ENOENT
-          settings = ::ApplicationSetting.last
-        end
+      end
+    end
+
+    def uncached_application_settings
+      return fake_application_settings unless connect_to_db?
 
-        settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration?
+      # This loads from the database into the cache, so handle Redis errors
+      begin
+        db_settings = ::ApplicationSetting.current
+      rescue ::Redis::BaseError, ::Errno::ENOENT
+        # In case Redis isn't running or the Redis UNIX socket file is not available
       end
 
-      settings || fake_application_settings
-    end
+      # If there are pending migrations, it's possible there are columns that
+      # need to be added to the application settings. To prevent Rake tasks
+      # and other callers from failing, use any loaded settings and return
+      # defaults for missing columns.
+      if ActiveRecord::Migrator.needs_migration?
+        defaults = ::ApplicationSetting.defaults
+        defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present?
+        return fake_application_settings(defaults)
+      end
+
+      return db_settings if db_settings.present?
 
-    def fake_application_settings
-      OpenStruct.new(
-        default_projects_limit: Settings.gitlab['default_projects_limit'],
-        default_branch_protection: Settings.gitlab['default_branch_protection'],
-        signup_enabled: Settings.gitlab['signup_enabled'],
-        signin_enabled: Settings.gitlab['signin_enabled'],
-        gravatar_enabled: Settings.gravatar['enabled'],
-        koding_enabled: false,
-        sign_in_text: nil,
-        after_sign_up_text: nil,
-        help_page_text: nil,
-        shared_runners_text: nil,
-        restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'],
-        max_attachment_size: Settings.gitlab['max_attachment_size'],
-        session_expire_delay: Settings.gitlab['session_expire_delay'],
-        default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
-        default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
-        domain_whitelist: Settings.gitlab['domain_whitelist'],
-        import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project],
-        shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
-        max_artifacts_size: Settings.artifacts['max_size'],
-        require_two_factor_authentication: false,
-        two_factor_grace_period: 48,
-        akismet_enabled: false,
-        repository_checks_enabled: true,
-        container_registry_token_expire_delay: 5,
-        user_default_external: false,
-      )
+      ::ApplicationSetting.create_from_defaults || in_memory_application_settings
     end
 
-    private
+    def in_memory_application_settings
+      @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults)
+    rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError
+      # In case migrations the application_settings table is not created yet,
+      # we fallback to a simple OpenStruct
+      fake_application_settings
+    end
 
     def connect_to_db?
       # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised
diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb
new file mode 100644
index 00000000000..ab115afcaa5
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb
@@ -0,0 +1,67 @@
+module Gitlab
+  module CycleAnalytics
+    class BaseEventFetcher
+      include BaseQuery
+
+      attr_reader :projections, :query, :stage, :order
+
+      MAX_EVENTS = 50
+
+      def initialize(project:, stage:, options:)
+        @project = project
+        @stage = stage
+        @options = options
+      end
+
+      def fetch
+        update_author!
+
+        event_result.map do |event|
+          serialize(event) if has_permission?(event['id'])
+        end.compact
+      end
+
+      def order
+        @order || default_order
+      end
+
+      private
+
+      def update_author!
+        return unless event_result.any? && event_result.first['author_id']
+
+        Updater.update!(event_result, from: 'author_id', to: 'author', klass: User)
+      end
+
+      def event_result
+        @event_result ||= ActiveRecord::Base.connection.exec_query(events_query.to_sql).to_a
+      end
+
+      def events_query
+        diff_fn = subtract_datetimes_diff(base_query, @options[:start_time_attrs], @options[:end_time_attrs])
+
+        base_query.project(extract_diff_epoch(diff_fn).as('total_time'), *projections).order(order.desc).take(MAX_EVENTS)
+      end
+
+      def default_order
+        [@options[:start_time_attrs]].flatten.first
+      end
+
+      def serialize(_event)
+        raise NotImplementedError.new("Expected #{self.name} to implement serialize(event)")
+      end
+
+      def has_permission?(id)
+        allowed_ids.nil? || allowed_ids.include?(id.to_i)
+      end
+
+      def allowed_ids
+        nil
+      end
+
+      def event_result_ids
+        event_result.map { |event| event['id'] }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb
new file mode 100644
index 00000000000..58729d3ced8
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/base_query.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module CycleAnalytics
+    module BaseQuery
+      include MetricsTables
+      include Gitlab::Database::Median
+      include Gitlab::Database::DateTime
+
+      private
+
+      def base_query
+        @base_query ||= stage_query
+      end
+
+      def stage_query
+        query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id]))
+          .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id]))
+          .where(issue_table[:project_id].eq(@project.id))
+          .where(issue_table[:deleted_at].eq(nil))
+          .where(issue_table[:created_at].gteq(@options[:from]))
+
+        # Load merge_requests
+        query = query.join(mr_table, Arel::Nodes::OuterJoin)
+          .on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id]))
+          .join(mr_metrics_table)
+          .on(mr_table[:id].eq(mr_metrics_table[:merge_request_id]))
+
+        query
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb
new file mode 100644
index 00000000000..cac31ea8cff
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/base_stage.rb
@@ -0,0 +1,54 @@
+module Gitlab
+  module CycleAnalytics
+    class BaseStage
+      include BaseQuery
+
+      def initialize(project:, options:)
+        @project = project
+        @options = options
+      end
+
+      def events
+        event_fetcher.fetch
+      end
+
+      def as_json
+        AnalyticsStageSerializer.new.represent(self)
+      end
+
+      def title
+        raise NotImplementedError.new("Expected #{self.name} to implement title")
+      end
+
+      def median
+        cte_table = Arel::Table.new("cte_table_for_#{name}")
+
+        # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time).
+        # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time).
+        # We compute the (end_time - start_time) interval, and give it an alias based on the current
+        # cycle analytics stage.
+        interval_query = Arel::Nodes::As.new(
+          cte_table,
+          subtract_datetimes(base_query.dup, start_time_attrs, end_time_attrs, name.to_s))
+
+        median_datetime(cte_table, interval_query, name)
+      end
+
+      def name
+        raise NotImplementedError.new("Expected #{self.name} to implement name")
+      end
+
+      private
+
+      def event_fetcher
+        @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(project: @project,
+                                                                          stage: name,
+                                                                          options: event_options)
+      end
+
+      def event_options
+        @options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb
new file mode 100644
index 00000000000..d5bf6149749
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module CycleAnalytics
+    class CodeEventFetcher < BaseEventFetcher
+      include MergeRequestAllowed
+
+      def initialize(*args)
+        @projections = [mr_table[:title],
+                        mr_table[:iid],
+                        mr_table[:id],
+                        mr_table[:created_at],
+                        mr_table[:state],
+                        mr_table[:author_id]]
+        @order = mr_table[:created_at]
+
+        super(*args)
+      end
+
+      private
+
+      def serialize(event)
+        AnalyticsMergeRequestSerializer.new(project: @project).represent(event)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb
new file mode 100644
index 00000000000..5f9dc9a4303
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/code_stage.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module CycleAnalytics
+    class CodeStage < BaseStage
+      def start_time_attrs
+        @start_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at]
+      end
+
+      def end_time_attrs
+        @end_time_attrs ||= mr_table[:created_at]
+      end
+
+      def name
+        :code
+      end
+
+      def title
+        s_('CycleAnalyticsStage|Code')
+      end
+
+      def legend
+        _("Related Merge Requests")
+      end
+
+      def description
+        _("Time until first merge request")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb
new file mode 100644
index 00000000000..50e126cf00b
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/event_fetcher.rb
@@ -0,0 +1,9 @@
+module Gitlab
+  module CycleAnalytics
+    module EventFetcher
+      def self.[](stage_name)
+        CycleAnalytics.const_get("#{stage_name.to_s.camelize}EventFetcher")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/issue_allowed.rb b/lib/gitlab/cycle_analytics/issue_allowed.rb
new file mode 100644
index 00000000000..a7652a70641
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/issue_allowed.rb
@@ -0,0 +1,9 @@
+module Gitlab
+  module CycleAnalytics
+    module IssueAllowed
+      def allowed_ids
+        @allowed_ids ||= IssuesFinder.new(@options[:current_user], project_id: @project.id).execute.where(id: event_result_ids).pluck(:id)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb
new file mode 100644
index 00000000000..3df9cbdcfce
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module CycleAnalytics
+    class IssueEventFetcher < BaseEventFetcher
+      include IssueAllowed
+
+      def initialize(*args)
+        @projections = [issue_table[:title],
+                        issue_table[:iid],
+                        issue_table[:id],
+                        issue_table[:created_at],
+                        issue_table[:author_id]]
+
+        super(*args)
+      end
+
+      private
+
+      def serialize(event)
+        AnalyticsIssueSerializer.new(project: @project).represent(event)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb
new file mode 100644
index 00000000000..7b03811efb2
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/issue_stage.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module CycleAnalytics
+    class IssueStage < BaseStage
+      def start_time_attrs
+        @start_time_attrs ||= issue_table[:created_at]
+      end
+
+      def end_time_attrs
+        @end_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at],
+                             issue_metrics_table[:first_added_to_board_at]]
+      end
+
+      def name
+        :issue
+      end
+
+      def title
+        s_('CycleAnalyticsStage|Issue')
+      end
+
+      def legend
+        _("Related Issues")
+      end
+
+      def description
+        _("Time before an issue gets scheduled")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/merge_request_allowed.rb b/lib/gitlab/cycle_analytics/merge_request_allowed.rb
new file mode 100644
index 00000000000..28f6db44759
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/merge_request_allowed.rb
@@ -0,0 +1,9 @@
+module Gitlab
+  module CycleAnalytics
+    module MergeRequestAllowed
+      def allowed_ids
+        @allowed_ids ||= MergeRequestsFinder.new(@options[:current_user], project_id: @project.id).execute.where(id: event_result_ids).pluck(:id)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb
new file mode 100644
index 00000000000..9d25ef078e8
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/metrics_tables.rb
@@ -0,0 +1,37 @@
+module Gitlab
+  module CycleAnalytics
+    module MetricsTables
+      def mr_metrics_table
+        MergeRequest::Metrics.arel_table
+      end
+
+      def mr_table
+        MergeRequest.arel_table
+      end
+
+      def mr_diff_table
+        MergeRequestDiff.arel_table
+      end
+
+      def mr_closing_issues_table
+        MergeRequestsClosingIssues.arel_table
+      end
+
+      def issue_table
+        Issue.arel_table
+      end
+
+      def issue_metrics_table
+        Issue::Metrics.arel_table
+      end
+
+      def user_table
+        User.arel_table
+      end
+
+      def build_table
+        ::CommitStatus.arel_table
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb
new file mode 100644
index 00000000000..1e11e84a9cb
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/permissions.rb
@@ -0,0 +1,44 @@
+module Gitlab
+  module CycleAnalytics
+    class Permissions
+      STAGE_PERMISSIONS = {
+        issue: :read_issue,
+        code: :read_merge_request,
+        test: :read_build,
+        review: :read_merge_request,
+        staging: :read_build,
+        production: :read_issue
+      }.freeze
+
+      def self.get(*args)
+        new(*args).get
+      end
+
+      def initialize(user:, project:)
+        @user = user
+        @project = project
+        @stage_permission_hash = {}
+      end
+
+      def get
+        ::CycleAnalytics::STAGES.each do |stage|
+          @stage_permission_hash[stage] = authorized_stage?(stage)
+        end
+
+        @stage_permission_hash
+      end
+
+      private
+
+      def authorized_stage?(stage)
+        return false unless authorize_project(:read_cycle_analytics)
+
+        STAGE_PERMISSIONS[stage] ? authorize_project(STAGE_PERMISSIONS[stage]) : true
+      end
+
+      def authorize_project(permission)
+        Ability.allowed?(@user, permission, @project)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
new file mode 100644
index 00000000000..7d342a2d2cb
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb
@@ -0,0 +1,44 @@
+module Gitlab
+  module CycleAnalytics
+    class PlanEventFetcher < BaseEventFetcher
+      def initialize(*args)
+        @projections = [mr_diff_table[:st_commits].as('commits'),
+                        issue_metrics_table[:first_mentioned_in_commit_at]]
+
+        super(*args)
+      end
+
+      def events_query
+        base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id]))
+
+        super
+      end
+
+      private
+
+      def serialize(event)
+        st_commit = first_time_reference_commit(event.delete('commits'), event)
+
+        return unless st_commit
+
+        serialize_commit(event, st_commit, query)
+      end
+
+      def first_time_reference_commit(commits, event)
+        return nil if commits.blank?
+
+        YAML.load(commits).find do |commit|
+          next unless commit[:committed_date] && event['first_mentioned_in_commit_at']
+
+          commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i
+        end
+      end
+
+      def serialize_commit(event, st_commit, query)
+        commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project)
+
+        AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb
new file mode 100644
index 00000000000..1a0afb56b4f
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/plan_stage.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module CycleAnalytics
+    class PlanStage < BaseStage
+      def start_time_attrs
+        @start_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at],
+                               issue_metrics_table[:first_added_to_board_at]]
+      end
+
+      def end_time_attrs
+        @end_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at]
+      end
+
+      def name
+        :plan
+      end
+
+      def title
+        s_('CycleAnalyticsStage|Plan')
+      end
+
+      def legend
+        _("Related Commits")
+      end
+
+      def description
+        _("Time before an issue starts implementation")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb
new file mode 100644
index 00000000000..0fa2e87f673
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb
@@ -0,0 +1,6 @@
+module Gitlab
+  module CycleAnalytics
+    class ProductionEventFetcher < IssueEventFetcher
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb
new file mode 100644
index 00000000000..d693443bfa4
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/production_helper.rb
@@ -0,0 +1,9 @@
+module Gitlab
+  module CycleAnalytics
+    module ProductionHelper
+      def stage_query
+        super.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@options[:from]))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb
new file mode 100644
index 00000000000..0fa8a65cb99
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/production_stage.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module CycleAnalytics
+    class ProductionStage < BaseStage
+      include ProductionHelper
+
+      def start_time_attrs
+        @start_time_attrs ||= issue_table[:created_at]
+      end
+
+      def end_time_attrs
+        @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at]
+      end
+
+      def name
+        :production
+      end
+
+      def title
+        s_('CycleAnalyticsStage|Production')
+      end
+
+      def legend
+        _("Related Issues")
+      end
+
+      def description
+        _("From issue creation until deploy to production")
+      end
+
+      def query
+        # Limit to merge requests that have been deployed to production after `@from`
+        query.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@from))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb
new file mode 100644
index 00000000000..4c7b3f4467f
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module CycleAnalytics
+    class ReviewEventFetcher < BaseEventFetcher
+      include MergeRequestAllowed
+
+      def initialize(*args)
+        @projections = [mr_table[:title],
+                        mr_table[:iid],
+                        mr_table[:id],
+                        mr_table[:created_at],
+                        mr_table[:state],
+                        mr_table[:author_id]]
+
+        super(*args)
+      end
+
+      def serialize(event)
+        AnalyticsMergeRequestSerializer.new(project: @project).represent(event)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb
new file mode 100644
index 00000000000..cfbbdc43fd9
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/review_stage.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module CycleAnalytics
+    class ReviewStage < BaseStage
+      def start_time_attrs
+        @start_time_attrs ||= mr_table[:created_at]
+      end
+
+      def end_time_attrs
+        @end_time_attrs ||= mr_metrics_table[:merged_at]
+      end
+
+      def name
+        :review
+      end
+
+      def title
+        s_('CycleAnalyticsStage|Review')
+      end
+
+      def legend
+        _("Related Merged Requests")
+      end
+
+      def description
+        _("Time between merge request creation and merge/close")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb
new file mode 100644
index 00000000000..28e0455df59
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage.rb
@@ -0,0 +1,9 @@
+module Gitlab
+  module CycleAnalytics
+    module Stage
+      def self.[](stage_name)
+        CycleAnalytics.const_get("#{stage_name.to_s.camelize}Stage")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb
new file mode 100644
index 00000000000..fc77bd86097
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/stage_summary.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module CycleAnalytics
+    class StageSummary
+      def initialize(project, from:, current_user:)
+        @project = project
+        @from = from
+        @current_user = current_user
+      end
+
+      def data
+        [serialize(Summary::Issue.new(project: @project, from: @from, current_user: @current_user)),
+         serialize(Summary::Commit.new(project: @project, from: @from)),
+         serialize(Summary::Deploy.new(project: @project, from: @from))]
+      end
+
+      private
+
+      def serialize(summary_object)
+        AnalyticsSummarySerializer.new.represent(summary_object)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb
new file mode 100644
index 00000000000..36c0260dbfe
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module CycleAnalytics
+    class StagingEventFetcher < BaseEventFetcher
+      def initialize(*args)
+        @projections = [build_table[:id]]
+        @order = build_table[:created_at]
+
+        super(*args)
+      end
+
+      def fetch
+        Updater.update!(event_result, from: 'id', to: 'build', klass: ::Ci::Build)
+
+        super
+      end
+
+      def events_query
+        base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id]))
+
+        super
+      end
+
+      private
+
+      def serialize(event)
+        AnalyticsBuildSerializer.new.represent(event['build'])
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb
new file mode 100644
index 00000000000..d5684bb9201
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/staging_stage.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module CycleAnalytics
+    class StagingStage < BaseStage
+      include ProductionHelper
+      def start_time_attrs
+        @start_time_attrs ||= mr_metrics_table[:merged_at]
+      end
+
+      def end_time_attrs
+        @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at]
+      end
+
+      def name
+        :staging
+      end
+
+      def title
+        s_('CycleAnalyticsStage|Staging')
+      end
+
+      def legend
+        _("Related Deployed Jobs")
+      end
+
+      def description
+        _("From merge request merge until deploy to production")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb
new file mode 100644
index 00000000000..a917ddccac7
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/summary/base.rb
@@ -0,0 +1,20 @@
+module Gitlab
+  module CycleAnalytics
+    module Summary
+      class Base
+        def initialize(project:, from:)
+          @project = project
+          @from = from
+        end
+
+        def title
+          raise NotImplementedError.new("Expected #{self.name} to implement title")
+        end
+
+        def value
+          raise NotImplementedError.new("Expected #{self.name} to implement value")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb
new file mode 100644
index 00000000000..bea78862757
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/summary/commit.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module CycleAnalytics
+    module Summary
+      class Commit < Base
+        def title
+          n_('Commit', 'Commits', value)
+        end
+
+        def value
+          @value ||= count_commits
+        end
+
+        private
+
+        # Don't use the `Gitlab::Git::Repository#log` method, because it enforces
+        # a limit. Since we need a commit count, we _can't_ enforce a limit, so
+        # the easiest way forward is to replicate the relevant portions of the
+        # `log` function here.
+        def count_commits
+          return unless ref
+
+          repository = @project.repository.raw_repository
+          sha = @project.repository.commit(ref).sha
+
+          cmd = %W(git --git-dir=#{repository.path} log)
+          cmd << '--format=%H'
+          cmd << "--after=#{@from.iso8601}"
+          cmd << sha
+
+          output, status = Gitlab::Popen.popen(cmd)
+
+          raise IOError, output unless status.zero?
+
+          output.lines.count
+        end
+
+        def ref
+          @ref ||= @project.default_branch.presence
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb
new file mode 100644
index 00000000000..099d798aac6
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/summary/deploy.rb
@@ -0,0 +1,15 @@
+module Gitlab
+  module CycleAnalytics
+    module Summary
+      class Deploy < Base
+        def title
+          n_('Deploy', 'Deploys', value)
+        end
+
+        def value
+          @value ||= @project.deployments.where("created_at > ?", @from).count
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb
new file mode 100644
index 00000000000..9bbf7a2685f
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/summary/issue.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module CycleAnalytics
+    module Summary
+      class Issue < Base
+        def initialize(project:, from:, current_user:)
+          @project = project
+          @from = from
+          @current_user = current_user
+        end
+
+        def title
+          n_('New Issue', 'New Issues', value)
+        end
+
+        def value
+          @value ||= IssuesFinder.new(@current_user, project_id: @project.id).execute.created_after(@from).count
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb
new file mode 100644
index 00000000000..a2589c6601a
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/test_event_fetcher.rb
@@ -0,0 +1,6 @@
+module Gitlab
+  module CycleAnalytics
+    class TestEventFetcher < StagingEventFetcher
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb
new file mode 100644
index 00000000000..2b5f72bef89
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/test_stage.rb
@@ -0,0 +1,37 @@
+module Gitlab
+  module CycleAnalytics
+    class TestStage < BaseStage
+      def start_time_attrs
+        @start_time_attrs ||= mr_metrics_table[:latest_build_started_at]
+      end
+
+      def end_time_attrs
+        @end_time_attrs ||= mr_metrics_table[:latest_build_finished_at]
+      end
+
+      def name
+        :test
+      end
+
+      def title
+        s_('CycleAnalyticsStage|Test')
+      end
+
+      def legend
+        _("Related Jobs")
+      end
+
+      def description
+        _("Total test time for all commits/merges")
+      end
+
+      def stage_query
+        if @options[:branch]
+          super.where(build_table[:ref].eq(@options[:branch]))
+        else
+          super
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cycle_analytics/updater.rb b/lib/gitlab/cycle_analytics/updater.rb
new file mode 100644
index 00000000000..953268ebd46
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/updater.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module CycleAnalytics
+    class Updater
+      def self.update!(*args)
+        new(*args).update!
+      end
+
+      def initialize(event_result, from:, to:, klass:)
+        @event_result = event_result
+        @klass = klass
+        @from = from
+        @to = to
+      end
+
+      def update!
+        @event_result.each do |event|
+          event[@to] = items[event.delete(@from).to_i].first
+        end
+      end
+
+      def result_ids
+        @event_result.map { |event| event[@from] }
+      end
+
+      def items
+        @items ||= @klass.find(result_ids).group_by { |item| item['id'] }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb
index 6548e6475c6..8e74e18a311 100644
--- a/lib/gitlab/data_builder/build.rb
+++ b/lib/gitlab/data_builder/build.rb
@@ -8,6 +8,8 @@ module Gitlab
         commit = build.pipeline
         user = build.user
 
+        author_url = build_author_url(build.commit, commit)
+
         data = {
           object_kind: 'build',
 
@@ -34,7 +36,7 @@ module Gitlab
           user: {
             id: user.try(:id),
             name: user.try(:name),
-            email: user.try(:email),
+            email: user.try(:email)
           },
 
           commit: {
@@ -43,10 +45,11 @@ module Gitlab
             message: commit.git_commit_message,
             author_name: commit.git_author_name,
             author_email: commit.git_author_email,
+            author_url: author_url,
             status: commit.status,
             duration: commit.duration,
             started_at: commit.started_at,
-            finished_at: commit.finished_at,
+            finished_at: commit.finished_at
           },
 
           repository: {
@@ -57,11 +60,18 @@ module Gitlab
             git_http_url: project.http_url_to_repo,
             git_ssh_url: project.ssh_url_to_repo,
             visibility_level: project.visibility_level
-          },
+          }
         }
 
         data
       end
+
+      private
+
+      def build_author_url(commit, pipeline)
+        author = commit.try(:author)
+        author ? Gitlab::Routing.url_helpers.user_url(author) : "mailto:#{pipeline.git_author_email}"
+      end
     end
   end
 end
diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb
index 06a783ebc1c..e47fb85b5ee 100644
--- a/lib/gitlab/data_builder/pipeline.rb
+++ b/lib/gitlab/data_builder/pipeline.rb
@@ -22,7 +22,7 @@ module Gitlab
           sha: pipeline.sha,
           before_sha: pipeline.before_sha,
           status: pipeline.status,
-          stages: pipeline.stages,
+          stages: pipeline.stages_names,
           created_at: pipeline.created_at,
           finished_at: pipeline.finished_at,
           duration: pipeline.duration
@@ -39,7 +39,7 @@ module Gitlab
           started_at: build.started_at,
           finished_at: build.finished_at,
           when: build.when,
-          manual: build.manual?,
+          manual: build.action?,
           user: build.user.try(:hook_attrs),
           runner: build.runner && runner_hook_attrs(build.runner),
           artifacts_file: {
diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb
index 4f81863da35..e81d19a7a2e 100644
--- a/lib/gitlab/data_builder/push.rb
+++ b/lib/gitlab/data_builder/push.rb
@@ -11,6 +11,7 @@ module Gitlab
       #   ref: String,
       #   user_id: String,
       #   user_name: String,
+      #   user_username: String,
       #   user_email: String
       #   project_id: String,
       #   repository: {
@@ -41,7 +42,7 @@ module Gitlab
         type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push'
 
         # Hash to be passed as post_receive_data
-        data = {
+        {
           object_kind: type,
           event_name: type,
           before: oldrev,
@@ -51,6 +52,7 @@ module Gitlab
           message: message,
           user_id: user.id,
           user_name: user.name,
+          user_username: user.username,
           user_email: user.email,
           user_avatar: user.avatar_url,
           project_id: project.id,
@@ -61,16 +63,15 @@ module Gitlab
           repository: project.hook_attrs.slice(:name, :url, :description, :homepage,
                                                :git_http_url, :git_ssh_url, :visibility_level)
         }
-
-        data
       end
 
       # This method provide a sample data generated with
       # existing project and commits to test webhooks
       def build_sample(project, user)
-        commits = project.repository.commits(project.default_branch, limit: 3)
         ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}"
-        build(project, user, commits.last.id, commits.first.id, ref, commits)
+        commits = project.repository.commits(project.default_branch.to_s, limit: 3) rescue []
+
+        build(project, user, commits.last&.id, commits.first&.id, ref, commits)
       end
 
       def checkout_sha(repository, newrev, ref)
@@ -83,7 +84,7 @@ module Gitlab
           tag = repository.find_tag(tag_name)
 
           if tag
-            commit = repository.commit(tag.target)
+            commit = repository.commit(tag.dereferenced_target)
             commit.try(:sha)
           end
         else
diff --git a/lib/gitlab/data_builder/repository.rb b/lib/gitlab/data_builder/repository.rb
new file mode 100644
index 00000000000..b42dc052949
--- /dev/null
+++ b/lib/gitlab/data_builder/repository.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module DataBuilder
+    module Repository
+      extend self
+
+      # Produce a hash of post-receive data
+      def update(project, user, changes, refs)
+        {
+          event_name: 'repository_update',
+
+          user_id: user.id,
+          user_name: user.name,
+          user_email: user.email,
+          user_avatar: user.avatar_url,
+
+          project_id: project.id,
+          project: project.hook_attrs,
+
+          changes: changes,
+
+          refs: refs
+        }
+      end
+
+      # Produce a hash of partial data for a single change
+      def single_change(oldrev, newrev, ref)
+        {
+          before: oldrev,
+          after: newrev,
+          ref: ref
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 55b8f888d53..d7dab584a44 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -5,8 +5,12 @@ module Gitlab
     # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html
     MAX_INT_VALUE = 2147483647
 
+    def self.config
+      ActiveRecord::Base.configurations[Rails.env]
+    end
+
     def self.adapter_name
-      connection.adapter_name
+      config['adapter']
     end
 
     def self.mysql?
@@ -24,7 +28,7 @@ module Gitlab
     def self.nulls_last_order(field, direction = 'ASC')
       order = "#{field} #{direction}"
 
-      if Gitlab::Database.postgresql?
+      if postgresql?
         order << ' NULLS LAST'
       else
         # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL
@@ -35,26 +39,86 @@ module Gitlab
       order
     end
 
+    def self.nulls_first_order(field, direction = 'ASC')
+      order = "#{field} #{direction}"
+
+      if postgresql?
+        order << ' NULLS FIRST'
+      else
+        # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL
+        # columns. In the (default) ascending order, `0` comes first.
+        order.prepend("#{field} IS NULL, ") if direction == 'DESC'
+      end
+
+      order
+    end
+
     def self.random
-      Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()"
+      postgresql? ? "RANDOM()" : "RAND()"
     end
 
-    def true_value
-      if Gitlab::Database.postgresql?
+    def self.true_value
+      if postgresql?
         "'t'"
       else
         1
       end
     end
 
-    def false_value
-      if Gitlab::Database.postgresql?
+    def self.false_value
+      if postgresql?
         "'f'"
       else
         0
       end
     end
 
+    def self.with_connection_pool(pool_size)
+      pool = create_connection_pool(pool_size)
+
+      begin
+        yield(pool)
+      ensure
+        pool.disconnect!
+      end
+    end
+
+    def self.bulk_insert(table, rows)
+      return if rows.empty?
+
+      keys = rows.first.keys
+      columns = keys.map { |key| connection.quote_column_name(key) }
+
+      tuples = rows.map do |row|
+        row.values_at(*keys).map { |value| connection.quote(value) }
+      end
+
+      connection.execute <<-EOF
+        INSERT INTO #{table} (#{columns.join(', ')})
+        VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')}
+      EOF
+    end
+
+    # pool_size - The size of the DB pool.
+    # host - An optional host name to use instead of the default one.
+    def self.create_connection_pool(pool_size, host = nil)
+      # See activerecord-4.2.7.1/lib/active_record/connection_adapters/connection_specification.rb
+      env = Rails.env
+      original_config = ActiveRecord::Base.configurations
+
+      env_config = original_config[env].merge('pool' => pool_size)
+      env_config['host'] = host if host
+
+      config = original_config.merge(env => env_config)
+
+      spec =
+        ActiveRecord::
+          ConnectionAdapters::
+          ConnectionSpecification::Resolver.new(config).spec(env.to_sym)
+
+      ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec)
+    end
+
     def self.connection
       ActiveRecord::Base.connection
     end
diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb
index b6a89f715fd..25e56998038 100644
--- a/lib/gitlab/database/date_time.rb
+++ b/lib/gitlab/database/date_time.rb
@@ -7,21 +7,25 @@ module Gitlab
       #
       # Note: For MySQL, the interval is returned in seconds.
       #       For PostgreSQL, the interval is returned as an INTERVAL type.
-      def subtract_datetimes(query_so_far, end_time_attrs, start_time_attrs, as)
-        diff_fn = if Gitlab::Database.postgresql?
-                    Arel::Nodes::Subtraction.new(
-                      Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)),
-                      Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)))
-                  elsif Gitlab::Database.mysql?
-                    Arel::Nodes::NamedFunction.new(
-                      "TIMESTAMPDIFF",
-                      [Arel.sql('second'),
-                       Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)),
-                       Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))])
-                  end
+      def subtract_datetimes(query_so_far, start_time_attrs, end_time_attrs, as)
+        diff_fn = subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs)
 
         query_so_far.project(diff_fn.as(as))
       end
+
+      def subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs)
+        if Gitlab::Database.postgresql?
+          Arel::Nodes::Subtraction.new(
+            Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)),
+            Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)))
+        elsif Gitlab::Database.mysql?
+          Arel::Nodes::NamedFunction.new(
+            "TIMESTAMPDIFF",
+            [Arel.sql('second'),
+             Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)),
+             Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))])
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb
index 1444d25ebc7..059054ac9ff 100644
--- a/lib/gitlab/database/median.rb
+++ b/lib/gitlab/database/median.rb
@@ -29,10 +29,10 @@ module Gitlab
       end
 
       def mysql_median_datetime_sql(arel_table, query_so_far, column_sym)
-        query = arel_table.
-                from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)).
-                project(average([arel_table[column_sym]], 'median')).
-                where(
+        query = arel_table
+                .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name))
+                .project(average([arel_table[column_sym]], 'median'))
+                .where(
                   Arel::Nodes::Between.new(
                     Arel.sql("(select @row_id := @row_id + 1)"),
                     Arel::Nodes::And.new(
@@ -67,8 +67,8 @@ module Gitlab
         cte_table = Arel::Table.new("ordered_records")
         cte = Arel::Nodes::As.new(
           cte_table,
-          arel_table.
-            project(
+          arel_table
+            .project(
               arel_table[column_sym].as(column_sym.to_s),
               Arel::Nodes::Over.new(Arel::Nodes::NamedFunction.new("row_number", []),
                                     Arel::Nodes::Window.new.order(arel_table[column_sym])).as('row_id'),
@@ -79,8 +79,8 @@ module Gitlab
         # From the CTE, select either the middle row or the middle two rows (this is accomplished
         # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the
         # selected rows, and this is the median value.
-        cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")).
-          where(
+        cte_table.project(average([extract_epoch(cte_table[column_sym])], "median"))
+          .where(
             Arel::Nodes::Between.new(
               cte_table[:row_id],
               Arel::Nodes::And.new(
@@ -88,9 +88,9 @@ module Gitlab
                  (cte_table[:ct] / Arel.sql('2.0') + 1)]
               )
             )
-          ).
-          with(query_so_far, cte).
-          to_sql
+          )
+          .with(query_so_far, cte)
+          .to_sql
       end
 
       private
@@ -103,6 +103,12 @@ module Gitlab
         Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")})
       end
 
+      def extract_diff_epoch(diff)
+        return diff unless Gitlab::Database.postgresql?
+
+        Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))})
+      end
+
       # Need to cast '0' to an INTERVAL before we can check if the interval is positive
       def zero_interval
         Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")])
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 0bd6e148ba8..0643c56db9b 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -1,6 +1,39 @@
 module Gitlab
   module Database
     module MigrationHelpers
+      # Adds `created_at` and `updated_at` columns with timezone information.
+      #
+      # This method is an improved version of Rails' built-in method `add_timestamps`.
+      #
+      # Available options are:
+      # default - The default value for the column.
+      # null - When set to `true` the column will allow NULL values.
+      #        The default is to not allow NULL values.
+      def add_timestamps_with_timezone(table_name, options = {})
+        options[:null] = false if options[:null].nil?
+
+        [:created_at, :updated_at].each do |column_name|
+          if options[:default] && transaction_open?
+            raise '`add_timestamps_with_timezone` with default value cannot be run inside a transaction. ' \
+              'You can disable transactions by calling `disable_ddl_transaction!` ' \
+              'in the body of your migration class'
+          end
+
+          # If default value is presented, use `add_column_with_default` method instead.
+          if options[:default]
+            add_column_with_default(
+              table_name,
+              column_name,
+              :datetime_with_timezone,
+              default: options[:default],
+              allow_null: options[:null]
+            )
+          else
+            add_column(table_name, column_name, :datetime_with_timezone, options)
+          end
+        end
+      end
+
       # Creates a new index, concurrently when supported
       #
       # On PostgreSQL this method creates an index concurrently, on MySQL this
@@ -26,11 +59,134 @@ module Gitlab
         add_index(table_name, column_name, options)
       end
 
+      # Removes an existed index, concurrently when supported
+      #
+      # On PostgreSQL this method removes an index concurrently.
+      #
+      # Example:
+      #
+      #     remove_concurrent_index :users, :some_column
+      #
+      # See Rails' `remove_index` for more info on the available arguments.
+      def remove_concurrent_index(table_name, column_name, options = {})
+        if transaction_open?
+          raise 'remove_concurrent_index can not be run inside a transaction, ' \
+            'you can disable transactions by calling disable_ddl_transaction! ' \
+            'in the body of your migration class'
+        end
+
+        if supports_drop_index_concurrently?
+          options = options.merge({ algorithm: :concurrently })
+          disable_statement_timeout
+        end
+
+        remove_index(table_name, options.merge({ column: column_name }))
+      end
+
+      # Removes an existing index, concurrently when supported
+      #
+      # On PostgreSQL this method removes an index concurrently.
+      #
+      # Example:
+      #
+      #     remove_concurrent_index :users, "index_X_by_Y"
+      #
+      # See Rails' `remove_index` for more info on the available arguments.
+      def remove_concurrent_index_by_name(table_name, index_name, options = {})
+        if transaction_open?
+          raise 'remove_concurrent_index_by_name can not be run inside a transaction, ' \
+            'you can disable transactions by calling disable_ddl_transaction! ' \
+            'in the body of your migration class'
+        end
+
+        if supports_drop_index_concurrently?
+          options = options.merge({ algorithm: :concurrently })
+          disable_statement_timeout
+        end
+
+        remove_index(table_name, options.merge({ name: index_name }))
+      end
+
+      # Only available on Postgresql >= 9.2
+      def supports_drop_index_concurrently?
+        return false unless Database.postgresql?
+
+        version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
+
+        version >= 90200
+      end
+
+      # Adds a foreign key with only minimal locking on the tables involved.
+      #
+      # This method only requires minimal locking when using PostgreSQL. When
+      # using MySQL this method will use Rails' default `add_foreign_key`.
+      #
+      # source - The source table containing the foreign key.
+      # target - The target table the key points to.
+      # column - The name of the column to create the foreign key on.
+      # on_delete - The action to perform when associated data is removed,
+      #             defaults to "CASCADE".
+      def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade)
+        # Transactions would result in ALTER TABLE locks being held for the
+        # duration of the transaction, defeating the purpose of this method.
+        if transaction_open?
+          raise 'add_concurrent_foreign_key can not be run inside a transaction'
+        end
+
+        # While MySQL does allow disabling of foreign keys it has no equivalent
+        # of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall
+        # back to the normal foreign key procedure.
+        if Database.mysql?
+          return add_foreign_key(source, target,
+                                 column: column,
+                                 on_delete: on_delete)
+        end
+
+        disable_statement_timeout
+
+        key_name = concurrent_foreign_key_name(source, column)
+
+        # Using NOT VALID allows us to create a key without immediately
+        # validating it. This means we keep the ALTER TABLE lock only for a
+        # short period of time. The key _is_ enforced for any newly created
+        # data.
+        execute <<-EOF.strip_heredoc
+        ALTER TABLE #{source}
+        ADD CONSTRAINT #{key_name}
+        FOREIGN KEY (#{column})
+        REFERENCES #{target} (id)
+        #{on_delete ? "ON DELETE #{on_delete}" : ''}
+        NOT VALID;
+        EOF
+
+        # Validate the existing constraint. This can potentially take a very
+        # long time to complete, but fortunately does not lock the source table
+        # while running.
+        execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};")
+      end
+
+      # Returns the name for a concurrent foreign key.
+      #
+      # PostgreSQL constraint names have a limit of 63 bytes. The logic used
+      # here is based on Rails' foreign_key_name() method, which unfortunately
+      # is private so we can't rely on it directly.
+      def concurrent_foreign_key_name(table, column)
+        "fk_#{Digest::SHA256.hexdigest("#{table}_#{column}_fk").first(10)}"
+      end
+
       # Long-running migrations may take more than the timeout allowed by
       # the database. Disable the session's statement timeout to ensure
       # migrations don't get killed prematurely. (PostgreSQL only)
       def disable_statement_timeout
-        ActiveRecord::Base.connection.execute('SET statement_timeout TO 0') if Database.postgresql?
+        execute('SET statement_timeout TO 0') if Database.postgresql?
+      end
+
+      def true_value
+        Database.true_value
+      end
+
+      def false_value
+        Database.false_value
       end
 
       # Updates the value of a column in batches.
@@ -66,6 +222,12 @@ module Gitlab
       #
       # rubocop: disable Metrics/AbcSize
       def update_column_in_batches(table, column, value)
+        if transaction_open?
+          raise 'update_column_in_batches can not be run inside a transaction, ' \
+            'you can disable transactions by calling disable_ddl_transaction! ' \
+            'in the body of your migration class'
+        end
+
         table = Arel::Table.new(table)
 
         count_arel = table.project(Arel.star.count.as('count'))
@@ -77,25 +239,31 @@ module Gitlab
 
         # Update in batches of 5% until we run out of any rows to update.
         batch_size = ((total / 100.0) * 5.0).ceil
+        max_size = 1000
+
+        # The upper limit is 1000 to ensure we don't lock too many rows. For
+        # example, for "merge_requests" even 1% of the table is around 35 000
+        # rows for GitLab.com.
+        batch_size = max_size if batch_size > max_size
 
         start_arel = table.project(table[:id]).order(table[:id].asc).take(1)
         start_arel = yield table, start_arel if block_given?
         start_id = exec_query(start_arel.to_sql).to_hash.first['id'].to_i
 
         loop do
-          stop_arel = table.project(table[:id]).
-            where(table[:id].gteq(start_id)).
-            order(table[:id].asc).
-            take(1).
-            skip(batch_size)
+          stop_arel = table.project(table[:id])
+            .where(table[:id].gteq(start_id))
+            .order(table[:id].asc)
+            .take(1)
+            .skip(batch_size)
 
           stop_arel = yield table, stop_arel if block_given?
           stop_row = exec_query(stop_arel.to_sql).to_hash.first
 
-          update_arel = Arel::UpdateManager.new(ActiveRecord::Base).
-            table(table).
-            set([[table[column], value]]).
-            where(table[:id].gteq(start_id))
+          update_arel = Arel::UpdateManager.new(ActiveRecord::Base)
+            .table(table)
+            .set([[table[column], value]])
+            .where(table[:id].gteq(start_id))
 
           if stop_row
             stop_id = stop_row['id'].to_i
@@ -169,6 +337,273 @@ module Gitlab
           raise error
         end
       end
+
+      # Renames a column without requiring downtime.
+      #
+      # Concurrent renames work by using database triggers to ensure both the
+      # old and new column are in sync. However, this method will _not_ remove
+      # the triggers or the old column automatically; this needs to be done
+      # manually in a post-deployment migration. This can be done using the
+      # method `cleanup_concurrent_column_rename`.
+      #
+      # table - The name of the database table containing the column.
+      # old - The old column name.
+      # new - The new column name.
+      # type - The type of the new column. If no type is given the old column's
+      #        type is used.
+      def rename_column_concurrently(table, old, new, type: nil)
+        if transaction_open?
+          raise 'rename_column_concurrently can not be run inside a transaction'
+        end
+
+        old_col = column_for(table, old)
+        new_type = type || old_col.type
+
+        add_column(table, new, new_type,
+                   limit: old_col.limit,
+                   precision: old_col.precision,
+                   scale: old_col.scale)
+
+        # We set the default value _after_ adding the column so we don't end up
+        # updating any existing data with the default value. This isn't
+        # necessary since we copy over old values further down.
+        change_column_default(table, new, old_col.default) if old_col.default
+
+        trigger_name = rename_trigger_name(table, old, new)
+        quoted_table = quote_table_name(table)
+        quoted_old = quote_column_name(old)
+        quoted_new = quote_column_name(new)
+
+        if Database.postgresql?
+          install_rename_triggers_for_postgresql(trigger_name, quoted_table,
+                                                 quoted_old, quoted_new)
+        else
+          install_rename_triggers_for_mysql(trigger_name, quoted_table,
+                                            quoted_old, quoted_new)
+        end
+
+        update_column_in_batches(table, new, Arel::Table.new(table)[old])
+
+        change_column_null(table, new, false) unless old_col.null
+
+        copy_indexes(table, old, new)
+        copy_foreign_keys(table, old, new)
+      end
+
+      # Changes the type of a column concurrently.
+      #
+      # table - The table containing the column.
+      # column - The name of the column to change.
+      # new_type - The new column type.
+      def change_column_type_concurrently(table, column, new_type)
+        temp_column = "#{column}_for_type_change"
+
+        rename_column_concurrently(table, column, temp_column, type: new_type)
+      end
+
+      # Performs cleanup of a concurrent type change.
+      #
+      # table - The table containing the column.
+      # column - The name of the column to change.
+      # new_type - The new column type.
+      def cleanup_concurrent_column_type_change(table, column)
+        temp_column = "#{column}_for_type_change"
+
+        transaction do
+          # This has to be performed in a transaction as otherwise we might have
+          # inconsistent data.
+          cleanup_concurrent_column_rename(table, column, temp_column)
+          rename_column(table, temp_column, column)
+        end
+      end
+
+      # Cleans up a concurrent column name.
+      #
+      # This method takes care of removing previously installed triggers as well
+      # as removing the old column.
+      #
+      # table - The name of the database table.
+      # old - The name of the old column.
+      # new - The name of the new column.
+      def cleanup_concurrent_column_rename(table, old, new)
+        trigger_name = rename_trigger_name(table, old, new)
+
+        if Database.postgresql?
+          remove_rename_triggers_for_postgresql(table, trigger_name)
+        else
+          remove_rename_triggers_for_mysql(trigger_name)
+        end
+
+        remove_column(table, old)
+      end
+
+      # Performs a concurrent column rename when using PostgreSQL.
+      def install_rename_triggers_for_postgresql(trigger, table, old, new)
+        execute <<-EOF.strip_heredoc
+        CREATE OR REPLACE FUNCTION #{trigger}()
+        RETURNS trigger AS
+        $BODY$
+        BEGIN
+          NEW.#{new} := NEW.#{old};
+          RETURN NEW;
+        END;
+        $BODY$
+        LANGUAGE 'plpgsql'
+        VOLATILE
+        EOF
+
+        execute <<-EOF.strip_heredoc
+        CREATE TRIGGER #{trigger}
+        BEFORE INSERT OR UPDATE
+        ON #{table}
+        FOR EACH ROW
+        EXECUTE PROCEDURE #{trigger}()
+        EOF
+      end
+
+      # Installs the triggers necessary to perform a concurrent column rename on
+      # MySQL.
+      def install_rename_triggers_for_mysql(trigger, table, old, new)
+        execute <<-EOF.strip_heredoc
+        CREATE TRIGGER #{trigger}_insert
+        BEFORE INSERT
+        ON #{table}
+        FOR EACH ROW
+        SET NEW.#{new} = NEW.#{old}
+        EOF
+
+        execute <<-EOF.strip_heredoc
+        CREATE TRIGGER #{trigger}_update
+        BEFORE UPDATE
+        ON #{table}
+        FOR EACH ROW
+        SET NEW.#{new} = NEW.#{old}
+        EOF
+      end
+
+      # Removes the triggers used for renaming a PostgreSQL column concurrently.
+      def remove_rename_triggers_for_postgresql(table, trigger)
+        execute("DROP TRIGGER #{trigger} ON #{table}")
+        execute("DROP FUNCTION #{trigger}()")
+      end
+
+      # Removes the triggers used for renaming a MySQL column concurrently.
+      def remove_rename_triggers_for_mysql(trigger)
+        execute("DROP TRIGGER #{trigger}_insert")
+        execute("DROP TRIGGER #{trigger}_update")
+      end
+
+      # Returns the (base) name to use for triggers when renaming columns.
+      def rename_trigger_name(table, old, new)
+        'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12)
+      end
+
+      # Returns an Array containing the indexes for the given column
+      def indexes_for(table, column)
+        column = column.to_s
+
+        indexes(table).select { |index| index.columns.include?(column) }
+      end
+
+      # Returns an Array containing the foreign keys for the given column.
+      def foreign_keys_for(table, column)
+        column = column.to_s
+
+        foreign_keys(table).select { |fk| fk.column == column }
+      end
+
+      # Copies all indexes for the old column to a new column.
+      #
+      # table - The table containing the columns and indexes.
+      # old - The old column.
+      # new - The new column.
+      def copy_indexes(table, old, new)
+        old = old.to_s
+        new = new.to_s
+
+        indexes_for(table, old).each do |index|
+          new_columns = index.columns.map do |column|
+            column == old ? new : column
+          end
+
+          # This is necessary as we can't properly rename indexes such as
+          # "ci_taggings_idx".
+          unless index.name.include?(old)
+            raise "The index #{index.name} can not be copied as it does not "\
+              "mention the old column. You have to rename this index manually first."
+          end
+
+          name = index.name.gsub(old, new)
+
+          options = {
+            unique: index.unique,
+            name: name,
+            length: index.lengths,
+            order: index.orders
+          }
+
+          # These options are not supported by MySQL, so we only add them if
+          # they were previously set.
+          options[:using] = index.using if index.using
+          options[:where] = index.where if index.where
+
+          unless index.opclasses.blank?
+            opclasses = index.opclasses.dup
+
+            # Copy the operator classes for the old column (if any) to the new
+            # column.
+            opclasses[new] = opclasses.delete(old) if opclasses[old]
+
+            options[:opclasses] = opclasses
+          end
+
+          add_concurrent_index(table, new_columns, options)
+        end
+      end
+
+      # Copies all foreign keys for the old column to the new column.
+      #
+      # table - The table containing the columns and indexes.
+      # old - The old column.
+      # new - The new column.
+      def copy_foreign_keys(table, old, new)
+        foreign_keys_for(table, old).each do |fk|
+          add_concurrent_foreign_key(fk.from_table,
+                                     fk.to_table,
+                                     column: new,
+                                     on_delete: fk.on_delete)
+        end
+      end
+
+      # Returns the column for the given table and column name.
+      def column_for(table, name)
+        name = name.to_s
+
+        columns(table).find { |column| column.name == name }
+      end
+
+      # This will replace the first occurance of a string in a column with
+      # the replacement
+      # On postgresql we can use `regexp_replace` for that.
+      # On mysql we find the location of the pattern, and overwrite it
+      # with the replacement
+      def replace_sql(column, pattern, replacement)
+        quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s)
+        quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s)
+
+        if Database.mysql?
+          locate = Arel::Nodes::NamedFunction
+            .new('locate', [quoted_pattern, column])
+          insert_in_place = Arel::Nodes::NamedFunction
+            .new('insert', [column, locate, pattern.size, quoted_replacement])
+
+          Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql)
+        else
+          replace = Arel::Nodes::NamedFunction
+            .new("regexp_replace", [column, quoted_pattern, quoted_replacement])
+          Arel::Nodes::SqlLiteral.new(replace.to_sql)
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/database/multi_threaded_migration.rb b/lib/gitlab/database/multi_threaded_migration.rb
new file mode 100644
index 00000000000..7ae5a4c17c8
--- /dev/null
+++ b/lib/gitlab/database/multi_threaded_migration.rb
@@ -0,0 +1,52 @@
+module Gitlab
+  module Database
+    module MultiThreadedMigration
+      MULTI_THREAD_AR_CONNECTION = :thread_local_ar_connection
+
+      # This overwrites the default connection method so that every thread can
+      # use a thread-local connection, while still supporting all of Rails'
+      # migration methods.
+      def connection
+        Thread.current[MULTI_THREAD_AR_CONNECTION] ||
+          ActiveRecord::Base.connection
+      end
+
+      # Starts a thread-pool for N threads, along with N threads each using a
+      # single connection. The provided block is yielded from inside each
+      # thread.
+      #
+      # Example:
+      #
+      #     with_multiple_threads(4) do
+      #       execute('SELECT ...')
+      #     end
+      #
+      # thread_count - The number of threads to start.
+      #
+      # join - When set to true this method will join the threads, blocking the
+      #        caller until all threads have finished running.
+      #
+      # Returns an Array containing the started threads.
+      def with_multiple_threads(thread_count, join: true)
+        pool = Gitlab::Database.create_connection_pool(thread_count)
+
+        threads = Array.new(thread_count) do
+          Thread.new do
+            pool.with_connection do |connection|
+              begin
+                Thread.current[MULTI_THREAD_AR_CONNECTION] = connection
+                yield
+              ensure
+                Thread.current[MULTI_THREAD_AR_CONNECTION] = nil
+              end
+            end
+          end
+        end
+
+        threads.each(&:join) if join
+
+        threads
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
new file mode 100644
index 00000000000..89530082cd2
--- /dev/null
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb
@@ -0,0 +1,35 @@
+# This module can be included in migrations to make it easier to rename paths
+# of `Namespace` & `Project` models certain paths would become `reserved`.
+#
+# If the way things are stored on the filesystem related to namespaces and
+# projects ever changes. Don't update this module, or anything nested in `V1`,
+# since it needs to keep functioning for all migrations using it using the state
+# that the data is in at the time. Instead, create a `V2` module that implements
+# the new way of reserving paths.
+module Gitlab
+  module Database
+    module RenameReservedPathsMigration
+      module V1
+        def self.included(kls)
+          kls.include(MigrationHelpers)
+        end
+
+        def rename_wildcard_paths(one_or_more_paths)
+          rename_child_paths(one_or_more_paths)
+          paths = Array(one_or_more_paths)
+          RenameProjects.new(paths, self).rename_projects
+        end
+
+        def rename_child_paths(one_or_more_paths)
+          paths = Array(one_or_more_paths)
+          RenameNamespaces.new(paths, self).rename_namespaces(type: :child)
+        end
+
+        def rename_root_paths(paths)
+          paths = Array(paths)
+          RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb
new file mode 100644
index 00000000000..5481024db8e
--- /dev/null
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb
@@ -0,0 +1,84 @@
+module Gitlab
+  module Database
+    module RenameReservedPathsMigration
+      module V1
+        module MigrationClasses
+          module Routable
+            def full_path
+              if route && route.path.present?
+                @full_path ||= route.path
+              else
+                update_route if persisted?
+
+                build_full_path
+              end
+            end
+
+            def build_full_path
+              if parent && path
+                parent.full_path + '/' + path
+              else
+                path
+              end
+            end
+
+            def update_route
+              prepare_route
+              route.save
+            end
+
+            def prepare_route
+              route || build_route(source: self)
+              route.path = build_full_path
+              @full_path = nil
+            end
+          end
+
+          class Namespace < ActiveRecord::Base
+            include MigrationClasses::Routable
+            self.table_name = 'namespaces'
+            belongs_to :parent,
+                       class_name: "#{MigrationClasses.name}::Namespace"
+            has_one :route, as: :source
+            has_many :children,
+                     class_name: "#{MigrationClasses.name}::Namespace",
+                     foreign_key: :parent_id
+
+            # Overridden to have the correct `source_type` for the `route` relation
+            def self.name
+              'Namespace'
+            end
+
+            def kind
+              type == 'Group' ? 'group' : 'user'
+            end
+          end
+
+          class User < ActiveRecord::Base
+            self.table_name = 'users'
+          end
+
+          class Route < ActiveRecord::Base
+            self.table_name = 'routes'
+            belongs_to :source, polymorphic: true
+          end
+
+          class Project < ActiveRecord::Base
+            include MigrationClasses::Routable
+            has_one :route, as: :source
+            self.table_name = 'projects'
+
+            def repository_storage_path
+              Gitlab.config.repositories.storages[repository_storage]['path']
+            end
+
+            # Overridden to have the correct `source_type` for the `route` relation
+            def self.name
+              'Project'
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
new file mode 100644
index 00000000000..d8163d7da11
--- /dev/null
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
@@ -0,0 +1,132 @@
+module Gitlab
+  module Database
+    module RenameReservedPathsMigration
+      module V1
+        class RenameBase
+          attr_reader :paths, :migration
+
+          delegate :update_column_in_batches,
+                   :replace_sql,
+                   to: :migration
+
+          def initialize(paths, migration)
+            @paths = paths
+            @migration = migration
+          end
+
+          def path_patterns
+            @path_patterns ||= paths.flat_map { |path| ["%/#{path}", path] }
+          end
+
+          def rename_path_for_routable(routable)
+            old_path = routable.path
+            old_full_path = routable.full_path
+            # Only remove the last occurrence of the path name to get the parent namespace path
+            namespace_path = remove_last_occurrence(old_full_path, old_path)
+            new_path = rename_path(namespace_path, old_path)
+            new_full_path = join_routable_path(namespace_path, new_path)
+
+            # skips callbacks & validations
+            routable.class.where(id: routable)
+              .update_all(path: new_path)
+
+            rename_routes(old_full_path, new_full_path)
+
+            [old_full_path, new_full_path]
+          end
+
+          def rename_routes(old_full_path, new_full_path)
+            replace_statement = replace_sql(Route.arel_table[:path],
+                                            old_full_path,
+                                            new_full_path)
+
+            update_column_in_batches(:routes, :path, replace_statement)  do |table, query|
+              path_or_children = table[:path].matches_any([old_full_path, "#{old_full_path}/%"])
+              query.where(path_or_children)
+            end
+          end
+
+          def rename_path(namespace_path, path_was)
+            counter = 0
+            path = "#{path_was}#{counter}"
+
+            while route_exists?(join_routable_path(namespace_path, path))
+              counter += 1
+              path = "#{path_was}#{counter}"
+            end
+
+            path
+          end
+
+          def remove_last_occurrence(string, pattern)
+            string.reverse.sub(pattern.reverse, "").reverse
+          end
+
+          def join_routable_path(namespace_path, top_level)
+            if namespace_path.present?
+              File.join(namespace_path, top_level)
+            else
+              top_level
+            end
+          end
+
+          def route_exists?(full_path)
+            MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any?
+          end
+
+          def move_pages(old_path, new_path)
+            move_folders(pages_dir, old_path, new_path)
+          end
+
+          def move_uploads(old_path, new_path)
+            return unless file_storage?
+
+            move_folders(uploads_dir, old_path, new_path)
+          end
+
+          def move_folders(directory, old_relative_path, new_relative_path)
+            old_path = File.join(directory, old_relative_path)
+            return unless File.directory?(old_path)
+
+            new_path = File.join(directory, new_relative_path)
+            FileUtils.mv(old_path, new_path)
+          end
+
+          def remove_cached_html_for_projects(project_ids)
+            update_column_in_batches(:projects, :description_html, nil) do |table, query|
+              query.where(table[:id].in(project_ids))
+            end
+
+            update_column_in_batches(:issues, :description_html, nil) do |table, query|
+              query.where(table[:project_id].in(project_ids))
+            end
+
+            update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
+              query.where(table[:target_project_id].in(project_ids))
+            end
+
+            update_column_in_batches(:notes, :note_html, nil) do |table, query|
+              query.where(table[:project_id].in(project_ids))
+            end
+
+            update_column_in_batches(:milestones, :description_html, nil) do |table, query|
+              query.where(table[:project_id].in(project_ids))
+            end
+          end
+
+          def file_storage?
+            CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
+          end
+
+          def uploads_dir
+            File.join(CarrierWave.root, "uploads")
+          end
+
+          def pages_dir
+            Settings.pages.path
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
new file mode 100644
index 00000000000..da7e2cb2e85
--- /dev/null
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
@@ -0,0 +1,78 @@
+module Gitlab
+  module Database
+    module RenameReservedPathsMigration
+      module V1
+        class RenameNamespaces < RenameBase
+          include Gitlab::ShellAdapter
+
+          def rename_namespaces(type:)
+            namespaces_for_paths(type: type).each do |namespace|
+              rename_namespace(namespace)
+            end
+          end
+
+          def namespaces_for_paths(type:)
+            namespaces = case type
+                         when :child
+                           MigrationClasses::Namespace.where.not(parent_id: nil)
+                         when :top_level
+                           MigrationClasses::Namespace.where(parent_id: nil)
+                         end
+            with_paths = MigrationClasses::Route.arel_table[:path]
+                           .matches_any(path_patterns)
+            namespaces.joins(:route).where(with_paths)
+          end
+
+          def rename_namespace(namespace)
+            old_full_path, new_full_path = rename_path_for_routable(namespace)
+
+            move_repositories(namespace, old_full_path, new_full_path)
+            move_uploads(old_full_path, new_full_path)
+            move_pages(old_full_path, new_full_path)
+            rename_user(old_full_path, new_full_path) if namespace.kind == 'user'
+            remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
+          end
+
+          def rename_user(old_username, new_username)
+            MigrationClasses::User.where(username: old_username)
+              .update_all(username: new_username)
+          end
+
+          def move_repositories(namespace, old_full_path, new_full_path)
+            repo_paths_for_namespace(namespace).each do |repository_storage_path|
+              # Ensure old directory exists before moving it
+              gitlab_shell.add_namespace(repository_storage_path, old_full_path)
+
+              unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
+                message = "Exception moving path #{repository_storage_path} \
+                           from #{old_full_path} to #{new_full_path}"
+                Rails.logger.error message
+              end
+            end
+          end
+
+          def repo_paths_for_namespace(namespace)
+            projects_for_namespace(namespace).distinct.select(:repository_storage)
+              .map(&:repository_storage_path)
+          end
+
+          def projects_for_namespace(namespace)
+            namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
+            namespace_or_children = MigrationClasses::Project
+                                      .arel_table[:namespace_id]
+                                      .in(namespace_ids)
+            MigrationClasses::Project.where(namespace_or_children)
+          end
+
+          def child_ids_for_parent(namespace, ids: [])
+            namespace.children.each do |child|
+              ids << child.id
+              child_ids_for_parent(child, ids: ids) if child.children.any?
+            end
+            ids
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
new file mode 100644
index 00000000000..448717eb744
--- /dev/null
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
@@ -0,0 +1,45 @@
+module Gitlab
+  module Database
+    module RenameReservedPathsMigration
+      module V1
+        class RenameProjects < RenameBase
+          include Gitlab::ShellAdapter
+
+          def rename_projects
+            projects_for_paths.each do |project|
+              rename_project(project)
+            end
+
+            remove_cached_html_for_projects(projects_for_paths.map(&:id))
+          end
+
+          def rename_project(project)
+            old_full_path, new_full_path = rename_path_for_routable(project)
+
+            move_repository(project, old_full_path, new_full_path)
+            move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
+            move_uploads(old_full_path, new_full_path)
+            move_pages(old_full_path, new_full_path)
+          end
+
+          def move_repository(project, old_path, new_path)
+            unless gitlab_shell.mv_repository(project.repository_storage_path,
+                                              old_path,
+                                              new_path)
+              Rails.logger.error "Error moving #{old_path} to #{new_path}"
+            end
+          end
+
+          def projects_for_paths
+            return @projects_for_paths if @projects_for_paths
+
+            with_paths = MigrationClasses::Route.arel_table[:path]
+                           .matches_any(path_patterns)
+
+            @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb
new file mode 100644
index 00000000000..3192bf6f667
--- /dev/null
+++ b/lib/gitlab/dependency_linker.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module DependencyLinker
+    LINKERS = [
+      GemfileLinker,
+      GemspecLinker,
+      PackageJsonLinker,
+      ComposerJsonLinker,
+      PodfileLinker,
+      PodspecLinker,
+      PodspecJsonLinker,
+      CartfileLinker,
+      GodepsJsonLinker,
+      RequirementsTxtLinker
+    ].freeze
+
+    def self.linker(blob_name)
+      LINKERS.find { |linker| linker.support?(blob_name) }
+    end
+
+    def self.link(blob_name, plain_text, highlighted_text)
+      linker = linker(blob_name)
+      return highlighted_text unless linker
+
+      linker.link(plain_text, highlighted_text)
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb
new file mode 100644
index 00000000000..7bbd154eb03
--- /dev/null
+++ b/lib/gitlab/dependency_linker/base_linker.rb
@@ -0,0 +1,86 @@
+module Gitlab
+  module DependencyLinker
+    class BaseLinker
+      URL_REGEX = %r{https?://[^'" ]+}.freeze
+      REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze
+
+      class_attribute :file_type
+
+      def self.support?(blob_name)
+        Gitlab::FileDetector.type_of(blob_name) == file_type
+      end
+
+      def self.link(*args)
+        new(*args).link
+      end
+
+      attr_accessor :plain_text, :highlighted_text
+
+      def initialize(plain_text, highlighted_text)
+        @plain_text = plain_text
+        @highlighted_text = highlighted_text
+      end
+
+      def link
+        link_dependencies
+
+        highlighted_lines.join.html_safe
+      end
+
+      private
+
+      def link_dependencies
+        raise NotImplementedError
+      end
+
+      def license_url(name)
+        Licensee::License.find(name)&.url
+      end
+
+      def github_url(name)
+        "https://github.com/#{name}"
+      end
+
+      def link_tag(name, url)
+        %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}
+      end
+
+      # Links package names based on regex.
+      #
+      # Example:
+      #   link_regex(/(github:|:github =>)\s*['"](?<name>[^'"]+)['"]/)
+      #   # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"`
+      def link_regex(regex, &url_proc)
+        highlighted_lines.map!.with_index do |rich_line, i|
+          marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe)
+
+          marker.mark(regex, group: :name) do |text, left:, right:|
+            url = yield(text)
+            url ? link_tag(text, url) : text
+          end
+        end
+      end
+
+      def plain_lines
+        @plain_lines ||= plain_text.lines
+      end
+
+      def highlighted_lines
+        @highlighted_lines ||= highlighted_text.lines
+      end
+
+      def regexp_for_value(value, default: /[^'" ]+/)
+        case value
+        when Array
+          Regexp.union(value.map { |v| regexp_for_value(v, default: default) })
+        when String
+          Regexp.escape(value)
+        when Regexp
+          value
+        else
+          default
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/cartfile_linker.rb b/lib/gitlab/dependency_linker/cartfile_linker.rb
new file mode 100644
index 00000000000..4f69f2c4ab2
--- /dev/null
+++ b/lib/gitlab/dependency_linker/cartfile_linker.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module DependencyLinker
+    class CartfileLinker < MethodLinker
+      self.file_type = :cartfile
+
+      private
+
+      def link_dependencies
+        link_method_call('github', REPO_REGEX, &method(:github_url))
+        link_method_call(%w[github git binary], URL_REGEX, &:itself)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/cocoapods.rb b/lib/gitlab/dependency_linker/cocoapods.rb
new file mode 100644
index 00000000000..2fbde7da1b4
--- /dev/null
+++ b/lib/gitlab/dependency_linker/cocoapods.rb
@@ -0,0 +1,10 @@
+module Gitlab
+  module DependencyLinker
+    module Cocoapods
+      def package_url(name)
+        package = name.split("/", 2).first
+        "https://cocoapods.org/pods/#{package}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb
new file mode 100644
index 00000000000..0245bf4077a
--- /dev/null
+++ b/lib/gitlab/dependency_linker/composer_json_linker.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module DependencyLinker
+    class ComposerJsonLinker < PackageJsonLinker
+      self.file_type = :composer_json
+
+      private
+
+      def link_packages
+        link_packages_at_key("require", &method(:package_url))
+        link_packages_at_key("require-dev", &method(:package_url))
+      end
+
+      def package_url(name)
+        "https://packagist.org/packages/#{name}" if name =~ %r{\A#{REPO_REGEX}\z}
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb
new file mode 100644
index 00000000000..d034ea67387
--- /dev/null
+++ b/lib/gitlab/dependency_linker/gemfile_linker.rb
@@ -0,0 +1,32 @@
+module Gitlab
+  module DependencyLinker
+    class GemfileLinker < MethodLinker
+      self.file_type = :gemfile
+
+      private
+
+      def link_dependencies
+        link_urls
+        link_packages
+      end
+
+      def link_urls
+        # Link `github: "user/repo"` to https://github.com/user/repo
+        link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url))
+
+        # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo
+        link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself)
+
+        # Link `source "https://rubygems.org"` to https://rubygems.org
+        link_method_call('source', URL_REGEX, &:itself)
+      end
+
+      def link_packages
+        # Link `gem "package_name"` to https://rubygems.org/gems/package_name
+        link_method_call('gem') do |name|
+          "https://rubygems.org/gems/#{name}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/gemspec_linker.rb b/lib/gitlab/dependency_linker/gemspec_linker.rb
new file mode 100644
index 00000000000..f1783ee2ab4
--- /dev/null
+++ b/lib/gitlab/dependency_linker/gemspec_linker.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module DependencyLinker
+    class GemspecLinker < MethodLinker
+      self.file_type = :gemspec
+
+      private
+
+      def link_dependencies
+        link_method_call('homepage', URL_REGEX, &:itself)
+        link_method_call('license', &method(:license_url))
+
+        link_method_call(%w[name add_dependency add_runtime_dependency add_development_dependency]) do |name|
+          "https://rubygems.org/gems/#{name}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/godeps_json_linker.rb b/lib/gitlab/dependency_linker/godeps_json_linker.rb
new file mode 100644
index 00000000000..fe091baee6d
--- /dev/null
+++ b/lib/gitlab/dependency_linker/godeps_json_linker.rb
@@ -0,0 +1,26 @@
+module Gitlab
+  module DependencyLinker
+    class GodepsJsonLinker < JsonLinker
+      NESTED_REPO_REGEX = %r{([^/]+/)+[^/]+?}.freeze
+
+      self.file_type = :godeps_json
+
+      private
+
+      def link_dependencies
+        link_json('ImportPath') do |path|
+          case path
+          when %r{\A(?<repo>gitlab\.com/#{NESTED_REPO_REGEX})\.git/(?<path>.+)\z},
+              %r{\A(?<repo>git(lab|hub)\.com/#{REPO_REGEX})/(?<path>.+)\z}
+
+            "https://#{$~[:repo]}/tree/master/#{$~[:path]}"
+          when /\Agolang\.org/
+            "https://godoc.org/#{path}"
+          else
+            "https://#{path}"
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/json_linker.rb b/lib/gitlab/dependency_linker/json_linker.rb
new file mode 100644
index 00000000000..a8ef25233d8
--- /dev/null
+++ b/lib/gitlab/dependency_linker/json_linker.rb
@@ -0,0 +1,44 @@
+module Gitlab
+  module DependencyLinker
+    class JsonLinker < BaseLinker
+      def link
+        return highlighted_text unless json
+
+        super
+      end
+
+      private
+
+      # Links package names in a JSON key or values.
+      #
+      # Example:
+      #   link_json('name')
+      #   # Will link `package` in `"name": "package"`
+      #
+      #   link_json('name', 'specific_package')
+      #   # Will link `specific_package` in `"name": "specific_package"`
+      #
+      #   link_json('name', /[^\/]+\/[^\/]+/)
+      #   # Will link `user/repo` in `"name": "user/repo"`, but not `"name": "package"`
+      #
+      #   link_json('specific_package', '1.0.1', link: :key)
+      #   # Will link `specific_package` in `"specific_package": "1.0.1"`
+      def link_json(key, value = nil, link: :value, &url_proc)
+        key = regexp_for_value(key, default: /[^" ]+/)
+        value = regexp_for_value(value, default: /[^" ]+/)
+
+        if link == :value
+          value = /(?<name>#{value})/
+        else
+          key = /(?<name>#{key})/
+        end
+
+        link_regex(/"#{key}":\s*"#{value}"/, &url_proc)
+      end
+
+      def json
+        @json ||= JSON.parse(plain_text) rescue nil
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/method_linker.rb b/lib/gitlab/dependency_linker/method_linker.rb
new file mode 100644
index 00000000000..0ffa2a83c93
--- /dev/null
+++ b/lib/gitlab/dependency_linker/method_linker.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module DependencyLinker
+    class MethodLinker < BaseLinker
+      private
+
+      # Links package names in a method call or assignment string argument.
+      #
+      # Example:
+      #   link_method_call('gem')
+      #   # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"`
+      #
+      #   link_method_call('gem', 'specific_package')
+      #   # Will link `specific_package` in `gem "specific_package"`
+      #
+      #   link_method_call('github', /[^\/"]+\/[^\/"]+/)
+      #   # Will link `user/repo` in `github "user/repo"`, but not `github "package"`
+      #
+      #   link_method_call(%w[add_dependency add_development_dependency])
+      #   # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"`
+      #
+      #   link_method_call('name')
+      #   # Will link `package` in `self.name = "package"`
+      def link_method_call(method_name, value = nil, &url_proc)
+        method_name = regexp_for_value(method_name)
+        value = regexp_for_value(value)
+
+        regex = %r{
+          #{method_name}            # Method name
+          \s*                       # Whitespace
+          [(=]?                     # Opening brace or equals sign
+          \s*                       # Whitespace
+          ['"](?<name>#{value})['"] # Package name in quotes
+        }x
+
+        link_regex(regex, &url_proc)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/package_json_linker.rb b/lib/gitlab/dependency_linker/package_json_linker.rb
new file mode 100644
index 00000000000..330c95f0880
--- /dev/null
+++ b/lib/gitlab/dependency_linker/package_json_linker.rb
@@ -0,0 +1,44 @@
+module Gitlab
+  module DependencyLinker
+    class PackageJsonLinker < JsonLinker
+      self.file_type = :package_json
+
+      private
+
+      def link_dependencies
+        link_json('name', json["name"], &method(:package_url))
+        link_json('license', &method(:license_url))
+        link_json(%w[homepage url], URL_REGEX, &:itself)
+
+        link_packages
+      end
+
+      def link_packages
+        link_packages_at_key("dependencies", &method(:package_url))
+        link_packages_at_key("devDependencies", &method(:package_url))
+      end
+
+      def link_packages_at_key(key, &url_proc)
+        dependencies = json[key]
+        return unless dependencies
+
+        dependencies.each do |name, version|
+          link_json(name, version, link: :key, &url_proc)
+
+          link_json(name) do |value|
+            case value
+            when /\A#{URL_REGEX}\z/
+              value
+            when /\A#{REPO_REGEX}\z/
+              github_url(value)
+            end
+          end
+        end
+      end
+
+      def package_url(name)
+        "https://npmjs.com/package/#{name}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/podfile_linker.rb b/lib/gitlab/dependency_linker/podfile_linker.rb
new file mode 100644
index 00000000000..60ad166ea17
--- /dev/null
+++ b/lib/gitlab/dependency_linker/podfile_linker.rb
@@ -0,0 +1,15 @@
+module Gitlab
+  module DependencyLinker
+    class PodfileLinker < GemfileLinker
+      include Cocoapods
+
+      self.file_type = :podfile
+
+      private
+
+      def link_packages
+        link_method_call('pod', &method(:package_url))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/podspec_json_linker.rb b/lib/gitlab/dependency_linker/podspec_json_linker.rb
new file mode 100644
index 00000000000..d82237ed3f1
--- /dev/null
+++ b/lib/gitlab/dependency_linker/podspec_json_linker.rb
@@ -0,0 +1,32 @@
+module Gitlab
+  module DependencyLinker
+    class PodspecJsonLinker < JsonLinker
+      include Cocoapods
+
+      self.file_type = :podspec_json
+
+      private
+
+      def link_dependencies
+        link_json('name', json["name"], &method(:package_url))
+        link_json('license', &method(:license_url))
+        link_json(%w[homepage git], URL_REGEX, &:itself)
+
+        link_packages_at_key("dependencies", &method(:package_url))
+
+        json["subspecs"]&.each do |subspec|
+          link_packages_at_key("dependencies", subspec, &method(:package_url))
+        end
+      end
+
+      def link_packages_at_key(key, root = json, &url_proc)
+        dependencies = root[key]
+        return unless dependencies
+
+        dependencies.each do |name, _|
+          link_regex(/"(?<name>#{Regexp.escape(name)})":\s*\[/, &url_proc)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb
new file mode 100644
index 00000000000..a52c7a02439
--- /dev/null
+++ b/lib/gitlab/dependency_linker/podspec_linker.rb
@@ -0,0 +1,24 @@
+module Gitlab
+  module DependencyLinker
+    class PodspecLinker < MethodLinker
+      include Cocoapods
+
+      STRING_REGEX = /['"](?<name>[^'"]+)['"]/.freeze
+
+      self.file_type = :podspec
+
+      private
+
+      def link_dependencies
+        link_method_call('homepage', URL_REGEX, &:itself)
+
+        link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself)
+
+        link_method_call('license', &method(:license_url))
+        link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url))
+
+        link_method_call(%w[name dependency], &method(:package_url))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb
new file mode 100644
index 00000000000..9c9620bc36a
--- /dev/null
+++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb
@@ -0,0 +1,17 @@
+module Gitlab
+  module DependencyLinker
+    class RequirementsTxtLinker < BaseLinker
+      self.file_type = :requirements_txt
+
+      private
+
+      def link_dependencies
+        link_regex(/^(?<name>(?![a-z+]+:)[^#.-][^ ><=~!;\[]+)/) do |name|
+          "https://pypi.python.org/pypi/#{name}"
+        end
+
+        link_regex(%r{^(?<name>https?://[^ ]+)}, &:itself)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb
index 8406ca4269c..371cbe04b9b 100644
--- a/lib/gitlab/diff/diff_refs.rb
+++ b/lib/gitlab/diff/diff_refs.rb
@@ -18,6 +18,12 @@ module Gitlab
           head_sha == other.head_sha
       end
 
+      alias_method :eql?, :==
+
+      def hash
+        [base_sha, start_sha, head_sha].hash
+      end
+
       # There is only one case in which we will have `start_sha` and `head_sha`,
       # but not `base_sha`, which is when a diff is generated between an
       # orphaned branch and another branch, which means there _is_ no base, but
@@ -31,6 +37,16 @@ module Gitlab
       def complete?
         start_sha && head_sha
       end
+
+      def compare_in(project)
+        # We're at the initial commit, so just get that as we can't compare to anything.
+        if Gitlab::Git.blank_ref?(start_sha)
+          project.commit(head_sha)
+        else
+          straight = start_sha == base_sha
+          CompareService.new(project, head_sha).execute(project, start_sha, straight: straight)
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index e47df508ca2..d2863a4da71 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -1,16 +1,30 @@
 module Gitlab
   module Diff
     class File
-      attr_reader :diff, :repository, :diff_refs
+      attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs
 
-      delegate :new_file, :deleted_file, :renamed_file,
-        :old_path, :new_path, :a_mode, :b_mode,
-        :submodule?, :too_large?, :collapsed?, to: :diff, prefix: false
+      delegate :new_file?, :deleted_file?, :renamed_file?,
+        :old_path, :new_path, :a_mode, :b_mode, :mode_changed?,
+        :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, to: :diff, prefix: false
 
-      def initialize(diff, repository:, diff_refs: nil)
+      # Finding a viewer for a diff file happens based only on extension and whether the
+      # diff file blobs are binary or text, which means 1 diff file should only be matched by 1 viewer,
+      # and the order of these viewers doesn't really matter.
+      #
+      # However, when the diff file blobs are LFS pointers, we cannot know for sure whether the
+      # file being pointed to is binary or text. In this case, we match only on
+      # extension, preferring binary viewers over text ones if both exist, since the
+      # large files referred to in "Large File Storage" are much more likely to be
+      # binary than text.
+      RICH_VIEWERS = [
+        DiffViewer::Image
+      ].sort_by { |v| v.binary? ? 0 : 1 }.freeze
+
+      def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil)
         @diff = diff
         @repository = repository
         @diff_refs = diff_refs
+        @fallback_diff_refs = fallback_diff_refs
       end
 
       def position(line)
@@ -49,18 +63,72 @@ module Gitlab
         line_code(line) if line
       end
 
-      def content_commit
-        return unless diff_refs
+      def old_sha
+        diff_refs&.base_sha
+      end
 
-        repository.commit(deleted_file ? old_ref : new_ref)
+      def new_sha
+        diff_refs&.head_sha
       end
 
-      def old_ref
-        diff_refs.try(:base_sha)
+      def new_content_sha
+        return if deleted_file?
+        return @new_content_sha if defined?(@new_content_sha)
+
+        refs = diff_refs || fallback_diff_refs
+        @new_content_sha = refs&.head_sha
       end
 
-      def new_ref
-        diff_refs.try(:head_sha)
+      def new_content_commit
+        return @new_content_commit if defined?(@new_content_commit)
+
+        sha = new_content_commit
+        @new_content_commit = repository.commit(sha) if sha
+      end
+
+      def old_content_sha
+        return if new_file?
+        return @old_content_sha if defined?(@old_content_sha)
+
+        refs = diff_refs || fallback_diff_refs
+        @old_content_sha = refs&.base_sha
+      end
+
+      def old_content_commit
+        return @old_content_commit if defined?(@old_content_commit)
+
+        sha = old_content_sha
+        @old_content_commit = repository.commit(sha) if sha
+      end
+
+      def new_blob
+        return @new_blob if defined?(@new_blob)
+
+        sha = new_content_sha
+        return @new_blob = nil unless sha
+
+        @new_blob = repository.blob_at(sha, file_path)
+      end
+
+      def old_blob
+        return @old_blob if defined?(@old_blob)
+
+        sha = old_content_sha
+        return @old_blob = nil unless sha
+
+        @old_blob = repository.blob_at(sha, old_path)
+      end
+
+      def content_sha
+        new_content_sha || old_content_sha
+      end
+
+      def content_commit
+        new_content_commit || old_content_commit
+      end
+
+      def blob
+        new_blob || old_blob
       end
 
       attr_writer :highlighted_diff_lines
@@ -79,10 +147,6 @@ module Gitlab
         @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize
       end
 
-      def mode_changed?
-        a_mode && b_mode && a_mode != b_mode
-      end
-
       def raw_diff
         diff.diff.to_s
       end
@@ -111,19 +175,114 @@ module Gitlab
         diff_lines.count(&:removed?)
       end
 
-      def old_blob(commit = content_commit)
-        return unless commit
+      def file_identifier
+        "#{file_path}-#{new_file?}-#{deleted_file?}-#{renamed_file?}"
+      end
+
+      def diffable?
+        repository.attributes(file_path).fetch('diff') { true }
+      end
 
-        parent_id = commit.parent_id
-        return unless parent_id
+      def binary?
+        old_blob&.binary? || new_blob&.binary?
+      end
 
-        repository.blob_at(parent_id, old_path)
+      def text?
+        !binary?
       end
 
-      def blob(commit = content_commit)
-        return unless commit
+      def external_storage_error?
+        old_blob&.external_storage_error? || new_blob&.external_storage_error?
+      end
+
+      def stored_externally?
+        old_blob&.stored_externally? || new_blob&.stored_externally?
+      end
+
+      def external_storage
+        old_blob&.external_storage || new_blob&.external_storage
+      end
+
+      def content_changed?
+        old_blob && new_blob && old_blob.id != new_blob.id
+      end
+
+      def different_type?
+        old_blob && new_blob && old_blob.binary? != new_blob.binary?
+      end
+
+      def size
+        [old_blob&.size, new_blob&.size].compact.sum
+      end
+
+      def raw_size
+        [old_blob&.raw_size, new_blob&.raw_size].compact.sum
+      end
+
+      def raw_binary?
+        old_blob&.raw_binary? || new_blob&.raw_binary?
+      end
+
+      def raw_text?
+        !raw_binary? && !different_type?
+      end
+
+      def simple_viewer
+        @simple_viewer ||= simple_viewer_class.new(self)
+      end
+
+      def rich_viewer
+        return @rich_viewer if defined?(@rich_viewer)
+
+        @rich_viewer = rich_viewer_class&.new(self)
+      end
+
+      def rendered_as_text?(ignore_errors: true)
+        simple_viewer.is_a?(DiffViewer::Text) && (ignore_errors || simple_viewer.render_error.nil?)
+      end
+
+      private
+
+      def simple_viewer_class
+        return DiffViewer::NotDiffable unless diffable?
+
+        if content_changed?
+          if raw_text?
+            DiffViewer::Text
+          else
+            DiffViewer::NoPreview
+          end
+        elsif new_file?
+          if raw_text?
+            DiffViewer::Text
+          else
+            DiffViewer::Added
+          end
+        elsif deleted_file?
+          if raw_text?
+            DiffViewer::Text
+          else
+            DiffViewer::Deleted
+          end
+        elsif renamed_file?
+          DiffViewer::Renamed
+        elsif mode_changed?
+          DiffViewer::ModeChanged
+        end
+      end
+
+      def rich_viewer_class
+        viewer_class_from(RICH_VIEWERS)
+      end
+
+      def viewer_class_from(classes)
+        return unless diffable?
+        return if different_type? || external_storage_error?
+        return unless new_file? || deleted_file? || content_changed?
+
+        verify_binary = !stored_externally?
 
-        repository.blob_at(commit.id, file_path)
+        classes.find { |viewer_class| viewer_class.can_render?(self, verify_binary: verify_binary) }
       end
     end
   end
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
index 2b9fc65b985..a6007ebf531 100644
--- a/lib/gitlab/diff/file_collection/base.rb
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -2,32 +2,41 @@ module Gitlab
   module Diff
     module FileCollection
       class Base
-        attr_reader :project, :diff_options, :diff_view, :diff_refs
+        attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs
 
         delegate :count, :size, :real_size, to: :diff_files
 
         def self.default_options
-          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false)
+          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false)
         end
 
-        def initialize(diffable, project:, diff_options: nil, diff_refs: nil)
+        def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil)
           diff_options = self.class.default_options.merge(diff_options || {})
 
-          @diffable     = diffable
-          @diffs        = diffable.raw_diffs(diff_options)
-          @project      = project
+          @diffable = diffable
+          @diffs = diffable.raw_diffs(diff_options)
+          @project = project
           @diff_options = diff_options
-          @diff_refs    = diff_refs
+          @diff_refs = diff_refs
+          @fallback_diff_refs = fallback_diff_refs
         end
 
         def diff_files
           @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) }
         end
 
+        def diff_file_with_old_path(old_path)
+          diff_files.find { |diff_file| diff_file.old_path == old_path }
+        end
+
+        def diff_file_with_new_path(new_path)
+          diff_files.find { |diff_file| diff_file.new_path == new_path }
+        end
+
         private
 
         def decorate_diff!(diff)
-          Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs)
+          Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs)
         end
       end
     end
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb
index 36348b33943..fcda1fe2233 100644
--- a/lib/gitlab/diff/file_collection/merge_request_diff.rb
+++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb
@@ -8,19 +8,24 @@ module Gitlab
           super(merge_request_diff,
             project: merge_request_diff.project,
             diff_options: diff_options,
-            diff_refs: merge_request_diff.diff_refs)
+            diff_refs: merge_request_diff.diff_refs,
+            fallback_diff_refs: merge_request_diff.fallback_diff_refs)
         end
 
         def diff_files
           super.tap { |_| store_highlight_cache }
         end
 
+        def real_size
+          @merge_request_diff.real_size
+        end
+
         private
 
         # Extracted method to highlight in the same iteration to the diff_collection.
         def decorate_diff!(diff)
           diff_file = super
-          cache_highlight!(diff_file) if cacheable?
+          cache_highlight!(diff_file) if cacheable?(diff_file)
           diff_file
         end
 
@@ -35,16 +40,16 @@ module Gitlab
         # for the highlighted ones, so we just skip their execution.
         # If the highlighted diff files lines are not cached we calculate and cache them.
         #
-        # The content of the cache is a Hash where the key correspond to the file_path and the values are Arrays of
+        # The content of the cache is a Hash where the key identifies the file and the values are Arrays of
         # hashes that represent serialized diff lines.
         #
         def cache_highlight!(diff_file)
-          file_path = diff_file.file_path
+          item_key = diff_file.file_identifier
 
-          if highlight_cache[file_path]
-            highlight_diff_file_from_cache!(diff_file, highlight_cache[file_path])
+          if highlight_cache[item_key]
+            highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key])
           else
-            highlight_cache[file_path] = diff_file.highlighted_diff_lines.map(&:to_hash)
+            highlight_cache[item_key] = diff_file.highlighted_diff_lines.map(&:to_hash)
           end
         end
 
@@ -60,8 +65,8 @@ module Gitlab
           Rails.cache.write(cache_key, highlight_cache) if @highlight_cache_was_empty
         end
 
-        def cacheable?
-          @merge_request_diff.present?
+        def cacheable?(diff_file)
+          @merge_request_diff.present? && diff_file.text? && diff_file.diffable?
         end
 
         def cache_key
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index 9ea976e18fa..b669ee5b799 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -3,7 +3,7 @@ module Gitlab
     class Highlight
       attr_reader :diff_file, :diff_lines, :raw_lines, :repository
 
-      delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff
+      delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff
 
       def initialize(diff_lines, repository: nil)
         @repository = repository
@@ -42,15 +42,15 @@ module Gitlab
 
         rich_line =
           if diff_line.unchanged? || diff_line.added?
-            new_lines[diff_line.new_pos - 1]
+            new_lines[diff_line.new_pos - 1]&.html_safe
           elsif diff_line.removed?
-            old_lines[diff_line.old_pos - 1]
+            old_lines[diff_line.old_pos - 1]&.html_safe
           end
 
         # Only update text if line is found. This will prevent
         # issues with submodules given the line only exists in diff content.
         if rich_line
-          line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' '
+          line_prefix = diff_line.text =~ /\A(.)/ ? $1 : ' '
           "#{line_prefix}#{rich_line}".html_safe
         end
       end
@@ -60,13 +60,18 @@ module Gitlab
       end
 
       def old_lines
-        return unless diff_file
-        @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_ref, diff_old_path)
+        @old_lines ||= highlighted_blob_lines(diff_file.old_blob)
       end
 
       def new_lines
-        return unless diff_file
-        @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_ref, diff_new_path)
+        @new_lines ||= highlighted_blob_lines(diff_file.new_blob)
+      end
+
+      def highlighted_blob_lines(blob)
+        return [] unless blob
+
+        blob.load_all_data!
+        Gitlab::Highlight.highlight(blob.path, blob.data, repository: repository).lines
       end
     end
   end
diff --git a/lib/gitlab/diff/inline_diff_markdown_marker.rb b/lib/gitlab/diff/inline_diff_markdown_marker.rb
new file mode 100644
index 00000000000..c2a2eb15931
--- /dev/null
+++ b/lib/gitlab/diff/inline_diff_markdown_marker.rb
@@ -0,0 +1,17 @@
+module Gitlab
+  module Diff
+    class InlineDiffMarkdownMarker < Gitlab::StringRangeMarker
+      MARKDOWN_SYMBOLS = {
+        addition: "+",
+        deletion: "-"
+      }.freeze
+
+      def mark(line_inline_diffs, mode: nil)
+        super(line_inline_diffs) do |text, left:, right:|
+          symbol = MARKDOWN_SYMBOLS[mode]
+          "{#{symbol}#{text}#{symbol}}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb
index 87a9b1e23ac..919965100ae 100644
--- a/lib/gitlab/diff/inline_diff_marker.rb
+++ b/lib/gitlab/diff/inline_diff_marker.rb
@@ -1,137 +1,21 @@
 module Gitlab
   module Diff
-    class InlineDiffMarker
-      MARKDOWN_SYMBOLS = {
-        addition: "+",
-        deletion: "-"
-      }
-
-      attr_accessor :raw_line, :rich_line
-
-      def initialize(raw_line, rich_line = raw_line)
-        @raw_line = raw_line
-        @rich_line = ERB::Util.html_escape(rich_line)
-      end
-
-      def mark(line_inline_diffs, mode: nil, markdown: false)
-        return rich_line unless line_inline_diffs
-
-        marker_ranges = []
-        line_inline_diffs.each do |inline_diff_range|
-          # Map the inline-diff range based on the raw line to character positions in the rich line
-          inline_diff_positions = position_mapping[inline_diff_range].flatten
-          # Turn the array of character positions into ranges
-          marker_ranges.concat(collapse_ranges(inline_diff_positions))
-        end
-
-        offset = 0
-
-        # Mark each range
-        marker_ranges.each_with_index do |range, index|
-          before_content =
-            if markdown
-              "{#{MARKDOWN_SYMBOLS[mode]}"
-            else
-              "<span class='#{html_class_names(marker_ranges, mode, index)}'>"
-            end
-          after_content =
-            if markdown
-              "#{MARKDOWN_SYMBOLS[mode]}}"
-            else
-              "</span>"
-            end
-          offset = insert_around_range(rich_line, range, before_content, after_content, offset)
+    class InlineDiffMarker < Gitlab::StringRangeMarker
+      def mark(line_inline_diffs, mode: nil)
+        super(line_inline_diffs) do |text, left:, right:|
+          %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>}
         end
-
-        rich_line.html_safe
       end
 
       private
 
-      def html_class_names(marker_ranges, mode, index)
+      def html_class_names(left, right, mode)
         class_names = ["idiff"]
-        class_names << "left"  if index == 0
-        class_names << "right" if index == marker_ranges.length - 1
+        class_names << "left"  if left
+        class_names << "right" if right
         class_names << mode if mode
         class_names.join(" ")
       end
-
-      # Mapping of character positions in the raw line, to the rich (highlighted) line
-      def position_mapping
-        @position_mapping ||= begin
-          mapping = []
-          rich_pos = 0
-          (0..raw_line.length).each do |raw_pos|
-            rich_char = rich_line[rich_pos]
-
-            # The raw and rich lines are the same except for HTML tags,
-            # so skip over any `<...>` segment
-            while rich_char == '<'
-              until rich_char == '>'
-                rich_pos += 1
-                rich_char = rich_line[rich_pos]
-              end
-
-              rich_pos += 1
-              rich_char = rich_line[rich_pos]
-            end
-
-            # multi-char HTML entities in the rich line correspond to a single character in the raw line
-            if rich_char == '&'
-              multichar_mapping = [rich_pos]
-              until rich_char == ';'
-                rich_pos += 1
-                multichar_mapping << rich_pos
-                rich_char = rich_line[rich_pos]
-              end
-
-              mapping[raw_pos] = multichar_mapping
-            else
-              mapping[raw_pos] = rich_pos
-            end
-
-            rich_pos += 1
-          end
-
-          mapping
-        end
-      end
-
-      # Takes an array of integers, and returns an array of ranges covering the same integers
-      def collapse_ranges(positions)
-        return [] if positions.empty?
-        ranges = []
-
-        start = prev = positions[0]
-        range = start..prev
-        positions[1..-1].each do |pos|
-          if pos == prev + 1
-            range = start..pos
-            prev = pos
-          else
-            ranges << range
-            start = prev = pos
-            range = start..prev
-          end
-        end
-        ranges << range
-
-        ranges
-      end
-
-      # Inserts tags around the characters identified by the given range
-      def insert_around_range(text, range, before, after, offset = 0)
-        # Just to be sure
-        return offset if offset + range.end + 1 > text.length
-
-        text.insert(offset + range.begin, before)
-        offset += before.length
-
-        text.insert(offset + range.end + 1, after)
-        offset += after.length
-
-        offset
-      end
     end
   end
 end
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 80a146b4a5a..2d89ccfc354 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -33,16 +33,28 @@ module Gitlab
         new_pos unless removed? || meta?
       end
 
+      def line
+        new_line || old_line
+      end
+
       def unchanged?
         type.nil?
       end
 
       def added?
-        type == 'new'
+        %w[new new-nonewline].include?(type)
       end
 
       def removed?
-        type == 'old'
+        %w[old old-nonewline].include?(type)
+      end
+
+      def meta?
+        %w[match new-nonewline old-nonewline].include?(type)
+      end
+
+      def discussable?
+        !meta?
       end
 
       def rich_text
@@ -51,10 +63,6 @@ module Gitlab
         @rich_text
       end
 
-      def meta?
-        type == 'match' || type == 'nonewline'
-      end
-
       def as_json(opts = nil)
         {
           type: type,
diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb
index 481536a380b..0cb26fa45c8 100644
--- a/lib/gitlab/diff/parallel_diff.rb
+++ b/lib/gitlab/diff/parallel_diff.rb
@@ -14,16 +14,7 @@ module Gitlab
         lines = []
         highlighted_diff_lines = diff_file.highlighted_diff_lines
         highlighted_diff_lines.each do |line|
-          if line.meta? || line.unchanged?
-            # line in the right panel is the same as in the left one
-            lines << {
-              left: line,
-              right: line
-            }
-
-            free_right_index = nil
-            i += 1
-          elsif line.removed?
+          if line.removed?
             lines << {
               left: line,
               right: nil
@@ -51,6 +42,15 @@ module Gitlab
               free_right_index = nil
               i += 1
             end
+          elsif line.meta? || line.unchanged?
+            # line in the right panel is the same as in the left one
+            lines << {
+              left: line,
+              right: line
+            }
+
+            free_right_index = nil
+            i += 1
           end
         end
 
diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb
index 59a2367b65d..742f989c50b 100644
--- a/lib/gitlab/diff/parser.rb
+++ b/lib/gitlab/diff/parser.rb
@@ -11,6 +11,7 @@ module Gitlab
         line_old = 1
         line_new = 1
         type = nil
+        context = nil
 
         # By returning an Enumerator we make it possible to search for a single line (with #find)
         # without having to instantiate all the others that come after it.
@@ -20,7 +21,7 @@ module Gitlab
 
             full_line = line.delete("\n")
 
-            if line.match(/^@@ -/)
+            if line =~ /^@@ -/
               type = "match"
 
               line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0
@@ -31,7 +32,8 @@ module Gitlab
               line_obj_index += 1
               next
             elsif line[0] == '\\'
-              type = 'nonewline'
+              type = "#{context}-nonewline"
+
               yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new)
               line_obj_index += 1
             else
@@ -43,9 +45,11 @@ module Gitlab
             case line[0]
             when "+"
               line_new += 1
+              context = :new
             when "-"
               line_old += 1
-            when "\\"
+              context = :old
+            when "\\" # rubocop:disable Lint/EmptyWhen
               # No increment
             else
               line_new += 1
diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb
index ecf62dead35..f80afb20f0c 100644
--- a/lib/gitlab/diff/position.rb
+++ b/lib/gitlab/diff/position.rb
@@ -12,20 +12,26 @@ module Gitlab
       attr_reader :head_sha
 
       def initialize(attrs = {})
+        if diff_file = attrs[:diff_file]
+          attrs[:diff_refs] = diff_file.diff_refs
+          attrs[:old_path] = diff_file.old_path
+          attrs[:new_path] = diff_file.new_path
+        end
+
+        if diff_refs = attrs[:diff_refs]
+          attrs[:base_sha]  = diff_refs.base_sha
+          attrs[:start_sha] = diff_refs.start_sha
+          attrs[:head_sha]  = diff_refs.head_sha
+        end
+
         @old_path = attrs[:old_path]
         @new_path = attrs[:new_path]
+        @base_sha  = attrs[:base_sha]
+        @start_sha = attrs[:start_sha]
+        @head_sha  = attrs[:head_sha]
+
         @old_line = attrs[:old_line]
         @new_line = attrs[:new_line]
-
-        if attrs[:diff_refs]
-          @base_sha  = attrs[:diff_refs].base_sha
-          @start_sha = attrs[:diff_refs].start_sha
-          @head_sha  = attrs[:diff_refs].head_sha
-        else
-          @base_sha  = attrs[:base_sha]
-          @start_sha = attrs[:start_sha]
-          @head_sha  = attrs[:head_sha]
-        end
       end
 
       # `Gitlab::Diff::Position` objects are stored as serialized attributes in
@@ -129,32 +135,19 @@ module Gitlab
       end
 
       def diff_line(repository)
-        @diff_line ||= diff_file(repository).line_for_position(self)
+        @diff_line ||= diff_file(repository)&.line_for_position(self)
       end
 
       def line_code(repository)
-        @line_code ||= diff_file(repository).line_code_for_position(self)
+        @line_code ||= diff_file(repository)&.line_code_for_position(self)
       end
 
       private
 
       def find_diff_file(repository)
-        # We're at the initial commit, so just get that as we can't compare to anything.
-        if Gitlab::Git.blank_ref?(start_sha)
-          compare = Gitlab::Git::Commit.find(repository.raw_repository, head_sha)
-        else
-          compare = Gitlab::Git::Compare.new(
-            repository.raw_repository,
-            start_sha,
-            head_sha
-          )
-        end
-
-        diff = compare.diffs(paths: paths).first
-
-        return unless diff
+        return unless diff_refs.complete?
 
-        Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs)
+        diff_refs.compare_in(repository.project).diffs(paths: paths, expanded: true).diff_files.first
       end
     end
   end
diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb
index 4d04f867268..b68a1636814 100644
--- a/lib/gitlab/diff/position_tracer.rb
+++ b/lib/gitlab/diff/position_tracer.rb
@@ -3,21 +3,21 @@
 module Gitlab
   module Diff
     class PositionTracer
-      attr_accessor :repository
+      attr_accessor :project
       attr_accessor :old_diff_refs
       attr_accessor :new_diff_refs
       attr_accessor :paths
 
-      def initialize(repository:, old_diff_refs:, new_diff_refs:, paths: nil)
-        @repository = repository
+      def initialize(project:, old_diff_refs:, new_diff_refs:, paths: nil)
+        @project = project
         @old_diff_refs = old_diff_refs
         @new_diff_refs = new_diff_refs
         @paths = paths
       end
 
-      def trace(old_position)
-        return unless old_diff_refs.complete? && new_diff_refs.complete?
-        return unless old_position.diff_refs == old_diff_refs
+      def trace(ab_position)
+        return unless old_diff_refs&.complete? && new_diff_refs&.complete?
+        return unless ab_position.diff_refs == old_diff_refs
 
         # Suppose we have an MR with source branch `feature` and target branch `master`.
         # When the MR was created, the head of `master` was commit A, and the
@@ -44,14 +44,16 @@ module Gitlab
         #
         # For diff notes for diff A->B, the position looks like this:
         # Position
-        #   base_sha - ID of commit A
+        #   start_sha - ID of commit A
         #   head_sha - ID of commit B
+        #   base_sha - ID of base commit of A and B
         #   old_path - path as of A (nil if file was newly created)
         #   new_path - path as of B (nil if file was deleted)
         #   old_line - line number as of A (nil if file was newly created)
         #   new_line - line number as of B (nil if file was deleted)
         #
-        # We can easily update `base_sha` and `head_sha` to hold the IDs of commits C and D,
+        # We can easily update `start_sha` and `head_sha` to hold the IDs of
+        # commits C and D, and can trivially determine `base_sha` based on those,
         # but need to find the paths and line numbers as of C and D.
         #
         # If the file was unchanged or newly created in A->B, the path as of D can be found
@@ -68,100 +70,161 @@ module Gitlab
         # by generating diff A->C ("base to base"), finding the diff file with
         # `diff_file.old_path == position.old_path`, and taking `diff_file.new_path`.
         # The path as of D can be found by taking diff C->D, finding the diff file
-        # with that same `old_path` and taking `diff_file.new_path`.
+        # with `old_path` set to that `diff_file.new_path` and taking `diff_file.new_path`.
         # The line number as of C can be found by using the LineMapper on diff A->C
         # and providing the line number as of A.
         # The line number as of D can be found by using the LineMapper on diff C->D
         # and providing the line number as of C.
 
-        results = nil
-        results ||= trace_added_line(old_position)   if old_position.added?   || old_position.unchanged?
-        results ||= trace_removed_line(old_position) if old_position.removed? || old_position.unchanged?
-
-        return unless results
-
-        file_diff, old_line, new_line = results
-
-        Position.new(
-          old_path: file_diff.old_path,
-          new_path: file_diff.new_path,
-          head_sha: new_diff_refs.head_sha,
-          start_sha: new_diff_refs.start_sha,
-          base_sha: new_diff_refs.base_sha,
-          old_line: old_line,
-          new_line: new_line
-        )
+        if ab_position.added?
+          trace_added_line(ab_position)
+        elsif ab_position.removed?
+          trace_removed_line(ab_position)
+        else # unchanged
+          trace_unchanged_line(ab_position)
+        end
       end
 
       private
 
-      def trace_added_line(old_position)
-        file_path = old_position.new_path
-
-        return unless diff_head_to_head
-
-        file_head_to_head = diff_head_to_head.find { |diff_file| diff_file.old_path == file_path }
-
-        file_path = file_head_to_head.new_path if file_head_to_head
-
-        new_line = LineMapper.new(file_head_to_head).old_to_new(old_position.new_line)
-
-        return unless new_line
-
-        file_diff = new_diffs.find { |diff_file| diff_file.new_path == file_path }
-        return unless file_diff
-
-        old_line = LineMapper.new(file_diff).new_to_old(new_line)
-
-        [file_diff, old_line, new_line]
+      def trace_added_line(ab_position)
+        b_path = ab_position.new_path
+        b_line = ab_position.new_line
+
+        bd_diff = bd_diffs.diff_file_with_old_path(b_path)
+
+        d_path = bd_diff&.new_path || b_path
+        d_line = LineMapper.new(bd_diff).old_to_new(b_line)
+
+        if d_line
+          cd_diff = cd_diffs.diff_file_with_new_path(d_path)
+
+          c_path = cd_diff&.old_path || d_path
+          c_line = LineMapper.new(cd_diff).new_to_old(d_line)
+
+          if c_line
+            # If the line is still in D but also in C, it has turned from an
+            # added line into an unchanged one.
+            new_position = position(cd_diff, c_line, d_line)
+            if valid_position?(new_position)
+              # If the line is still in the MR, we don't treat this as outdated.
+              { position: new_position, outdated: false }
+            else
+              # If the line is no longer in the MR, we unfortunately cannot show
+              # the current state on the CD diff, so we treat it as outdated.
+              ac_diff = ac_diffs.diff_file_with_new_path(c_path)
+
+              { position: position(ac_diff, nil, c_line), outdated: true }
+            end
+          else
+            # If the line is still in D and not in C, it is still added.
+            { position: position(cd_diff, nil, d_line), outdated: false }
+          end
+        else
+          # If the line is no longer in D, it has been removed from the MR.
+          { position: position(bd_diff, b_line, nil), outdated: true }
+        end
       end
 
-      def trace_removed_line(old_position)
-        file_path = old_position.old_path
-
-        return unless diff_base_to_base
+      def trace_removed_line(ab_position)
+        a_path = ab_position.old_path
+        a_line = ab_position.old_line
 
-        file_base_to_base = diff_base_to_base.find { |diff_file| diff_file.old_path == file_path }
+        ac_diff = ac_diffs.diff_file_with_old_path(a_path)
 
-        file_path = file_base_to_base.old_path if file_base_to_base
+        c_path = ac_diff&.new_path || a_path
+        c_line = LineMapper.new(ac_diff).old_to_new(a_line)
 
-        old_line = LineMapper.new(file_base_to_base).old_to_new(old_position.old_line)
+        if c_line
+          cd_diff = cd_diffs.diff_file_with_old_path(c_path)
 
-        return unless old_line
+          d_path = cd_diff&.new_path || c_path
+          d_line = LineMapper.new(cd_diff).old_to_new(c_line)
 
-        file_diff = new_diffs.find { |diff_file| diff_file.old_path == file_path }
-        return unless file_diff
+          if d_line
+            # If the line is still in C but also in D, it has turned from a
+            # removed line into an unchanged one.
+            bd_diff = bd_diffs.diff_file_with_new_path(d_path)
 
-        new_line = LineMapper.new(file_diff).old_to_new(old_line)
+            { position: position(bd_diff, nil, d_line), outdated: true }
+          else
+            # If the line is still in C and not in D, it is still removed.
+            { position: position(cd_diff, c_line, nil), outdated: false }
+          end
+        else
+          # If the line is no longer in C, it has been removed outside of the MR.
+          { position: position(ac_diff, a_line, nil), outdated: true }
+        end
+      end
 
-        [file_diff, old_line, new_line]
+      def trace_unchanged_line(ab_position)
+        a_path = ab_position.old_path
+        a_line = ab_position.old_line
+        b_path = ab_position.new_path
+        b_line = ab_position.new_line
+
+        ac_diff = ac_diffs.diff_file_with_old_path(a_path)
+
+        c_path = ac_diff&.new_path || a_path
+        c_line = LineMapper.new(ac_diff).old_to_new(a_line)
+
+        bd_diff = bd_diffs.diff_file_with_old_path(b_path)
+
+        d_line = LineMapper.new(bd_diff).old_to_new(b_line)
+
+        cd_diff = cd_diffs.diff_file_with_old_path(c_path)
+
+        if c_line && d_line
+          # If the line is still in C and D, it is still unchanged.
+          new_position = position(cd_diff, c_line, d_line)
+          if valid_position?(new_position)
+            # If the line is still in the MR, we don't treat this as outdated.
+            { position: new_position, outdated: false }
+          else
+            # If the line is no longer in the MR, we unfortunately cannot show
+            # the current state on the CD diff or any change on the BD diff,
+            # so we treat it as outdated.
+            { position: nil, outdated: true }
+          end
+        elsif d_line # && !c_line
+          # If the line is still in D but no longer in C, it has turned from
+          # an unchanged line into an added one.
+          # We don't treat this as outdated since the line is still in the MR.
+          { position: position(cd_diff, nil, d_line), outdated: false }
+        else # !d_line && (c_line || !c_line)
+          # If the line is no longer in D, it has turned from an unchanged line
+          # into a removed one.
+          { position: position(bd_diff, b_line, nil), outdated: true }
+        end
       end
 
-      def diff_base_to_base
-        @diff_base_to_base ||= diff_files(old_diff_refs.base_sha || old_diff_refs.start_sha, new_diff_refs.base_sha || new_diff_refs.start_sha)
+      def ac_diffs
+        @ac_diffs ||= compare(
+          old_diff_refs.base_sha || old_diff_refs.start_sha,
+          new_diff_refs.base_sha || new_diff_refs.start_sha,
+          straight: true
+        )
       end
 
-      def diff_head_to_head
-        @diff_head_to_head ||= diff_files(old_diff_refs.head_sha, new_diff_refs.head_sha)
+      def bd_diffs
+        @bd_diffs ||= compare(old_diff_refs.head_sha, new_diff_refs.head_sha, straight: true)
       end
 
-      def new_diffs
-        @new_diffs ||= diff_files(new_diff_refs.start_sha, new_diff_refs.head_sha, use_base: true)
+      def cd_diffs
+        @cd_diffs ||= compare(new_diff_refs.start_sha, new_diff_refs.head_sha)
       end
 
-      def diff_files(start_sha, head_sha, use_base: false)
-        base_sha = self.repository.merge_base(start_sha, head_sha) || start_sha
+      def compare(start_sha, head_sha, straight: false)
+        compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight)
+        compare.diffs(paths: paths, expanded: true)
+      end
 
-        diffs = self.repository.raw_repository.diff(
-          use_base ? base_sha : start_sha,
-          head_sha,
-          {},
-          *paths
-        )
+      def position(diff_file, old_line, new_line)
+        Position.new(diff_file: diff_file, old_line: old_line, new_line: new_line)
+      end
 
-        diffs.decorate! do |diff|
-          Gitlab::Diff::File.new(diff, repository: self.repository)
-        end
+      def valid_position?(position)
+        !!position.diff_line(project.repository)
       end
     end
   end
diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb
index ab9537ed7d7..941244694e2 100644
--- a/lib/gitlab/downtime_check.rb
+++ b/lib/gitlab/downtime_check.rb
@@ -50,8 +50,8 @@ module Gitlab
 
     # Returns the class for the given migration file path.
     def class_for_migration_file(path)
-      File.basename(path, File.extname(path)).split('_', 2).last.camelize.
-        constantize
+      File.basename(path, File.extname(path)).split('_', 2).last.camelize
+        .constantize
     end
 
     # Returns true if the given migration can be performed without downtime.
diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb
index 40a4815a9a0..543e62794c5 100644
--- a/lib/gitlab/downtime_check/message.rb
+++ b/lib/gitlab/downtime_check/message.rb
@@ -3,8 +3,8 @@ module Gitlab
     class Message
       attr_reader :path, :offline
 
-      OFFLINE = "\e[31moffline\e[0m"
-      ONLINE = "\e[32monline\e[0m"
+      OFFLINE = "\e[31moffline\e[0m".freeze
+      ONLINE = "\e[32monline\e[0m".freeze
 
       # path - The file path of the migration.
       # offline - When set to `true` the migration will require downtime.
diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
new file mode 100644
index 00000000000..1a5887dab7e
--- /dev/null
+++ b/lib/gitlab/ee_compat_check.rb
@@ -0,0 +1,378 @@
+# rubocop: disable Rails/Output
+module Gitlab
+  # Checks if a set of migrations requires downtime or not.
+  class EeCompatCheck
+    CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze
+    EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
+    CHECK_DIR = Rails.root.join('ee_compat_check')
+    IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze
+    PLEASE_READ_THIS_BANNER = %Q{
+      ============================================================
+      ===================== PLEASE READ THIS =====================
+      ============================================================
+    }.freeze
+    THANKS_FOR_READING_BANNER = %Q{
+      ============================================================
+      ==================== THANKS FOR READING ====================
+      ============================================================\n
+    }.freeze
+
+    attr_reader :ee_repo_dir, :patches_dir, :ce_repo, :ce_branch, :ee_branch_found
+    attr_reader :failed_files
+
+    def initialize(branch:, ce_repo: CE_REPO)
+      @ee_repo_dir = CHECK_DIR.join('ee-repo')
+      @patches_dir = CHECK_DIR.join('patches')
+      @ce_branch = branch
+      @ce_repo = ce_repo
+    end
+
+    def check
+      ensure_patches_dir
+      generate_patch(ce_branch, ce_patch_full_path)
+
+      ensure_ee_repo
+      Dir.chdir(ee_repo_dir) do
+        step("In the #{ee_repo_dir} directory")
+
+        status = catch(:halt_check) do
+          ce_branch_compat_check!
+          delete_ee_branches_locally!
+          ee_branch_presence_check!
+          ee_branch_compat_check!
+        end
+
+        delete_ee_branches_locally!
+
+        if status.nil?
+          true
+        else
+          false
+        end
+      end
+    end
+
+    private
+
+    def ensure_ee_repo
+      if Dir.exist?(ee_repo_dir)
+        step("#{ee_repo_dir} already exists")
+      else
+        step(
+          "Cloning #{EE_REPO} into #{ee_repo_dir}",
+          %W[git clone --branch master --single-branch --depth=200 #{EE_REPO} #{ee_repo_dir}]
+        )
+      end
+    end
+
+    def ensure_patches_dir
+      FileUtils.mkdir_p(patches_dir)
+    end
+
+    def generate_patch(branch, patch_path)
+      FileUtils.rm(patch_path, force: true)
+
+      find_merge_base_with_master(branch: branch)
+
+      step(
+        "Generating the patch against origin/master in #{patch_path}",
+        %w[git diff --binary origin/master...HEAD]
+      ) do |output, status|
+        throw(:halt_check, :ko) unless status.zero?
+
+        File.write(patch_path, output)
+
+        throw(:halt_check, :ko) unless File.exist?(patch_path)
+      end
+    end
+
+    def ce_branch_compat_check!
+      if check_patch(ce_patch_full_path).zero?
+        puts applies_cleanly_msg(ce_branch)
+        throw(:halt_check)
+      end
+    end
+
+    def ee_branch_presence_check!
+      _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch origin #{ee_branch_prefix}])
+
+      if status.zero?
+        @ee_branch_found = ee_branch_prefix
+      else
+        _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}])
+      end
+
+      if status.zero?
+        @ee_branch_found = ee_branch_suffix
+      else
+        puts
+        puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
+
+        throw(:halt_check, :ko)
+      end
+    end
+
+    def ee_branch_compat_check!
+      step("Checking out origin/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} FETCH_HEAD])
+
+      generate_patch(ee_branch_found, ee_patch_full_path)
+
+      unless check_patch(ee_patch_full_path).zero?
+        puts
+        puts ee_branch_doesnt_apply_cleanly_msg
+
+        throw(:halt_check, :ko)
+      end
+
+      puts
+      puts applies_cleanly_msg(ee_branch_found)
+    end
+
+    def check_patch(patch_path)
+      step("Checking out master", %w[git checkout master])
+      step("Resetting to latest master", %w[git reset --hard origin/master])
+      step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}])
+      step(
+        "Checking if #{patch_path} applies cleanly to EE/master",
+        # Don't use --check here because it can result in a 0-exit status even
+        # though the patch doesn't apply cleanly, e.g.:
+        #   > git apply --check --3way foo.patch
+        #   error: patch failed: lib/gitlab/ee_compat_check.rb:74
+        #   Falling back to three-way merge...
+        #   Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts.
+        #   > echo $?
+        #   0
+        %W[git apply --3way #{patch_path}]
+      ) do |output, status|
+        puts output
+        unless status.zero?
+          @failed_files = output.lines.reduce([]) do |memo, line|
+            if line.start_with?('error: patch failed:')
+              file = line.sub(/\Aerror: patch failed: /, '')
+              memo << file unless file =~ IGNORED_FILES_REGEX
+            end
+            memo
+          end
+
+          status = 0 if failed_files.empty?
+        end
+
+        command(%w[git reset --hard])
+        status
+      end
+    end
+
+    def delete_ee_branches_locally!
+      command(%w[git checkout master])
+      command(%W[git branch --delete --force #{ee_branch_prefix}])
+      command(%W[git branch --delete --force #{ee_branch_suffix}])
+    end
+
+    def merge_base_found?
+      step(
+        "Finding merge base with master",
+        %w[git merge-base origin/master HEAD]
+      ) do |output, status|
+        if status.zero?
+          puts "Merge base was found: #{output}"
+          true
+        end
+      end
+    end
+
+    def find_merge_base_with_master(branch:)
+      return if merge_base_found?
+
+      # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403)
+      # In total we go (20 + 54 + 148 + 403 = 625) commits deeper
+      depth = 20
+      success =
+        (3..6).any? do |factor|
+          depth += Math.exp(factor).to_i
+          # Repository is initially cloned with a depth of 20 so we need to fetch
+          # deeper in the case the branch has more than 20 commits on top of master
+          fetch(branch: branch, depth: depth)
+          fetch(branch: 'master', depth: depth)
+
+          merge_base_found?
+        end
+
+      raise "\n#{branch} is too far behind master, please rebase it!\n" unless success
+    end
+
+    def fetch(branch:, depth:)
+      step(
+        "Fetching deeper...",
+        %W[git fetch --depth=#{depth} --prune origin +refs/heads/#{branch}:refs/remotes/origin/#{branch}]
+      ) do |output, status|
+        raise "Fetch failed: #{output}" unless status.zero?
+      end
+    end
+
+    def ce_patch_name
+      @ce_patch_name ||= patch_name_from_branch(ce_branch)
+    end
+
+    def ce_patch_full_path
+      @ce_patch_full_path ||= patches_dir.join(ce_patch_name)
+    end
+
+    def ee_branch_suffix
+      @ee_branch_suffix ||= "#{ce_branch}-ee"
+    end
+
+    def ee_branch_prefix
+      @ee_branch_prefix ||= "ee-#{ce_branch}"
+    end
+
+    def ee_patch_name
+      @ee_patch_name ||= patch_name_from_branch(ee_branch_found)
+    end
+
+    def ee_patch_full_path
+      @ee_patch_full_path ||= patches_dir.join(ee_patch_name)
+    end
+
+    def patch_name_from_branch(branch_name)
+      branch_name.parameterize << '.patch'
+    end
+
+    def step(desc, cmd = nil)
+      puts "\n=> #{desc}\n"
+
+      if cmd
+        start = Time.now
+        puts "\n$ #{cmd.join(' ')}"
+
+        output, status = command(cmd)
+        puts "\n==> Finished in #{Time.now - start} seconds"
+
+        if block_given?
+          yield(output, status)
+        else
+          [output, status]
+        end
+      end
+    end
+
+    def command(cmd)
+      Gitlab::Popen.popen(cmd)
+    end
+
+    def applies_cleanly_msg(branch)
+      %Q{
+        #{PLEASE_READ_THIS_BANNER}
+        🎉 Congratulations!! 🎉
+
+        The `#{branch}` branch applies cleanly to EE/master!
+
+        Much ❤️! For more information, see
+        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
+        #{THANKS_FOR_READING_BANNER}
+      }
+    end
+
+    def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
+      %Q{
+        #{PLEASE_READ_THIS_BANNER}
+        💥 Oh no! 💥
+
+        The `#{ce_branch}` branch does not apply cleanly to the current
+        EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch
+        was found in the EE repository.
+
+        #{conflicting_files_msg}
+
+        We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}`
+        branch that includes changes from `#{ce_branch}` but also specific changes
+        than can be applied cleanly to EE/master. In some cases, the conflicts
+        are trivial and you can ignore the warning from this job. As always,
+        use your best judgment!
+
+        There are different ways to create such branch:
+
+        1. Create a new branch from master and cherry-pick your CE commits
+
+          # In the EE repo
+          $ git fetch origin
+          $ git checkout -b #{ee_branch_prefix} origin/master
+          $ git fetch #{ce_repo} #{ce_branch}
+          $ git cherry-pick SHA # Repeat for all the commits you want to pick
+
+          You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit.
+
+        2. Apply your branch's patch to EE
+
+          # In the CE repo
+          $ git fetch origin master
+          $ git diff --binary origin/master...HEAD -- > #{ce_branch}.patch
+
+          # In the EE repo
+          $ git fetch origin master
+          $ git checkout -b #{ee_branch_prefix} origin/master
+          $ git apply --3way path/to/#{ce_branch}.patch
+
+          At this point you might have conflicts such as:
+
+            error: patch failed: lib/gitlab/ee_compat_check.rb:5
+            Falling back to three-way merge...
+            Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts.
+            U lib/gitlab/ee_compat_check.rb
+
+          Resolve them, stage the changes and commit them.
+
+          If the patch couldn't be applied cleanly, use the following command:
+
+          # In the EE repo
+          $ git apply --reject path/to/#{ce_branch}.patch
+
+          This option makes git apply the parts of the patch that are applicable,
+          and leave the rejected hunks in corresponding `.rej` files.
+          You can then resolve the conflicts highlighted in `.rej` by
+          manually applying the correct diff from the `.rej` file to the file with conflicts.
+          When finished, you can delete the `.rej` files and commit your changes.
+
+        ⚠️ Don't forget to push your branch to gitlab-ee:
+
+          # In the EE repo
+          $ git push origin #{ee_branch_prefix}
+
+        ⚠️ Also, don't forget to create a new merge request on gitlab-ce and
+        cross-link it with the CE merge request.
+
+        Once this is done, you can retry this failed build, and it should pass.
+
+        Stay 💪 ! For more information, see
+        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
+        #{THANKS_FOR_READING_BANNER}
+      }
+    end
+
+    def ee_branch_doesnt_apply_cleanly_msg
+      %Q{
+        #{PLEASE_READ_THIS_BANNER}
+        💥 Oh no! 💥
+
+        The `#{ce_branch}` does not apply cleanly to the current EE/master, and
+        even though a `#{ee_branch_found}` branch
+        exists in the EE repository, it does not apply cleanly either to
+        EE/master!
+
+        #{conflicting_files_msg}
+
+        Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and
+        retry this build.
+
+        Stay 💪 ! For more information, see
+        https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests
+        #{THANKS_FOR_READING_BANNER}
+      }
+    end
+
+    def conflicting_files_msg
+      failed_files.reduce("The conflicts detected were as follows:\n") do |memo, file|
+        memo << "\n        - #{file}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb
index 32cece8316b..83440ae227d 100644
--- a/lib/gitlab/email/attachment_uploader.rb
+++ b/lib/gitlab/email/attachment_uploader.rb
@@ -21,7 +21,7 @@ module Gitlab
               content_type: attachment.content_type
             }
 
-            link = ::Projects::UploadService.new(project, file).execute
+            link = UploadService.new(project, file).execute
             attachments << link if link
           ensure
             tmp.close!
diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb
index 5cf9d5ebe28..b07c68d1498 100644
--- a/lib/gitlab/email/handler.rb
+++ b/lib/gitlab/email/handler.rb
@@ -1,11 +1,15 @@
 require 'gitlab/email/handler/create_note_handler'
 require 'gitlab/email/handler/create_issue_handler'
+require 'gitlab/email/handler/unsubscribe_handler'
 
 module Gitlab
   module Email
     module Handler
-      # The `CreateIssueHandler` feature is disabled for the time being.
-      HANDLERS = [CreateNoteHandler]
+      HANDLERS = [
+        UnsubscribeHandler,
+        CreateNoteHandler,
+        CreateIssueHandler
+      ].freeze
 
       def self.for(mail, mail_key)
         HANDLERS.find do |klass|
diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb
index 7cccf465334..0bba433d04b 100644
--- a/lib/gitlab/email/handler/base_handler.rb
+++ b/lib/gitlab/email/handler/base_handler.rb
@@ -9,51 +9,16 @@ module Gitlab
           @mail_key = mail_key
         end
 
-        def message
-          @message ||= process_message
-        end
-
-        def author
+        def can_execute?
           raise NotImplementedError
         end
 
-        def project
+        def execute
           raise NotImplementedError
         end
 
-        private
-
-        def validate_permission!(permission)
-          raise UserNotFoundError unless author
-          raise UserBlockedError if author.blocked?
-          raise ProjectNotFound unless author.can?(:read_project, project)
-          raise UserNotAuthorizedError unless author.can?(permission, project)
-        end
-
-        def process_message
-          message = ReplyParser.new(mail).execute.strip
-          add_attachments(message)
-        end
-
-        def add_attachments(reply)
-          attachments = Email::AttachmentUploader.new(mail).execute(project)
-
-          reply + attachments.map do |link|
-            "\n\n#{link[:markdown]}"
-          end.join
-        end
-
-        def verify_record!(record:, invalid_exception:, record_name:)
-          return if record.persisted?
-          return if record.errors.key?(:commands_only)
-
-          error_title = "The #{record_name} could not be created for the following reasons:"
-
-          msg = error_title + record.errors.full_messages.map do |error|
-            "\n\n- #{error}"
-          end.join
-
-          raise invalid_exception, msg
+        def metrics_params
+          { handler: self.class.name }
         end
       end
     end
diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb
index 4e6566af8ab..a616a80e8f5 100644
--- a/lib/gitlab/email/handler/create_issue_handler.rb
+++ b/lib/gitlab/email/handler/create_issue_handler.rb
@@ -1,20 +1,20 @@
-
 require 'gitlab/email/handler/base_handler'
 
 module Gitlab
   module Email
     module Handler
       class CreateIssueHandler < BaseHandler
-        attr_reader :project_path, :authentication_token
+        include ReplyProcessing
+        attr_reader :project_path, :incoming_email_token
 
         def initialize(mail, mail_key)
           super(mail, mail_key)
-          @project_path, @authentication_token =
+          @project_path, @incoming_email_token =
             mail_key && mail_key.split('+', 2)
         end
 
         def can_handle?
-          !authentication_token.nil?
+          !incoming_email_token.nil?
         end
 
         def execute
@@ -29,11 +29,15 @@ module Gitlab
         end
 
         def author
-          @author ||= User.find_by(authentication_token: authentication_token)
+          @author ||= User.find_by(incoming_email_token: incoming_email_token)
         end
 
         def project
-          @project ||= Project.find_with_namespace(project_path)
+          @project ||= Project.find_by_full_path(project_path)
+        end
+
+        def metrics_params
+          super.merge(project: project&.full_path)
         end
 
         private
diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb
index 06dae31cc27..31579e94a87 100644
--- a/lib/gitlab/email/handler/create_note_handler.rb
+++ b/lib/gitlab/email/handler/create_note_handler.rb
@@ -1,10 +1,14 @@
-
 require 'gitlab/email/handler/base_handler'
+require 'gitlab/email/handler/reply_processing'
 
 module Gitlab
   module Email
     module Handler
       class CreateNoteHandler < BaseHandler
+        include ReplyProcessing
+
+        delegate :project, to: :sent_notification, allow_nil: true
+
         def can_handle?
           mail_key =~ /\A\w+\z/
         end
@@ -24,30 +28,22 @@ module Gitlab
             record_name: 'comment')
         end
 
-        def author
-          sent_notification.recipient
+        def metrics_params
+          super.merge(project: project&.full_path)
         end
 
-        def project
-          sent_notification.project
+        private
+
+        def author
+          sent_notification.recipient
         end
 
         def sent_notification
           @sent_notification ||= SentNotification.for(mail_key)
         end
 
-        private
-
         def create_note
-          Notes::CreateService.new(
-            project,
-            author,
-            note:           message,
-            noteable_type:  sent_notification.noteable_type,
-            noteable_id:    sent_notification.noteable_id,
-            commit_id:      sent_notification.commit_id,
-            line_code:      sent_notification.line_code
-          ).execute
+          sent_notification.create_reply(message)
         end
       end
     end
diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb
new file mode 100644
index 00000000000..32c5caf93e8
--- /dev/null
+++ b/lib/gitlab/email/handler/reply_processing.rb
@@ -0,0 +1,54 @@
+module Gitlab
+  module Email
+    module Handler
+      module ReplyProcessing
+        private
+
+        def author
+          raise NotImplementedError
+        end
+
+        def project
+          raise NotImplementedError
+        end
+
+        def message
+          @message ||= process_message
+        end
+
+        def process_message
+          message = ReplyParser.new(mail).execute.strip
+          add_attachments(message)
+        end
+
+        def add_attachments(reply)
+          attachments = Email::AttachmentUploader.new(mail).execute(project)
+
+          reply + attachments.map do |link|
+            "\n\n#{link[:markdown]}"
+          end.join
+        end
+
+        def validate_permission!(permission)
+          raise UserNotFoundError unless author
+          raise UserBlockedError if author.blocked?
+          raise ProjectNotFound unless author.can?(:read_project, project)
+          raise UserNotAuthorizedError unless author.can?(permission, project)
+        end
+
+        def verify_record!(record:, invalid_exception:, record_name:)
+          return if record.persisted?
+          return if record.errors.key?(:commands_only)
+
+          error_title = "The #{record_name} could not be created for the following reasons:"
+
+          msg = error_title + record.errors.full_messages.map do |error|
+            "\n\n- #{error}"
+          end.join
+
+          raise invalid_exception, msg
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb
new file mode 100644
index 00000000000..5894384da5d
--- /dev/null
+++ b/lib/gitlab/email/handler/unsubscribe_handler.rb
@@ -0,0 +1,38 @@
+require 'gitlab/email/handler/base_handler'
+
+module Gitlab
+  module Email
+    module Handler
+      class UnsubscribeHandler < BaseHandler
+        delegate :project, to: :sent_notification, allow_nil: true
+
+        def can_handle?
+          mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/
+        end
+
+        def execute
+          raise SentNotificationNotFoundError unless sent_notification
+          return unless sent_notification.unsubscribable?
+
+          noteable = sent_notification.noteable
+          raise NoteableNotFoundError unless noteable
+          noteable.unsubscribe(sent_notification.recipient)
+        end
+
+        def metrics_params
+          super.merge(project: project&.full_path)
+        end
+
+        private
+
+        def sent_notification
+          @sent_notification ||= SentNotification.for(reply_key)
+        end
+
+        def reply_key
+          mail_key.sub(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX, '')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb
new file mode 100644
index 00000000000..50559a48973
--- /dev/null
+++ b/lib/gitlab/email/html_parser.rb
@@ -0,0 +1,41 @@
+module Gitlab
+  module Email
+    class HTMLParser
+      def self.parse_reply(raw_body)
+        new(raw_body).filtered_text
+      end
+
+      attr_reader :raw_body
+      def initialize(raw_body)
+        @raw_body = raw_body
+      end
+
+      def document
+        @document ||= Nokogiri::HTML.parse(raw_body)
+      end
+
+      def filter_replies!
+        document.xpath('//blockquote').each(&:remove)
+        document.xpath('//table').each(&:remove)
+
+        # bogus links with no href are sometimes added by outlook,
+        # and can result in Html2Text adding extra square brackets
+        # to the text, so we unwrap them here.
+        document.xpath('//a[not(@href)]').each do |link|
+          link.replace(link.children)
+        end
+      end
+
+      def filtered_html
+        @filtered_html ||= begin
+          filter_replies!
+          document.inner_html
+        end
+      end
+
+      def filtered_text
+        @filtered_text ||= Html2Text.convert(filtered_html)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index 0e3b65fceb4..ea035e33eff 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -42,11 +42,11 @@ module Gitlab
           return unless compare
 
           # This diff is more moderated in number of files and lines
-          @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, no_collapse: true).diff_files
+          @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, expanded: true).diff_files
         end
 
         def diffs_count
-          diffs.size if diffs
+          diffs&.size
         end
 
         def compare
@@ -58,7 +58,7 @@ module Gitlab
         end
 
         def compare_timeout
-          diffs.overflow? if diffs
+          diffs&.overflow?
         end
 
         def reverse_compare?
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index a40c44eb1bc..0d6b08b5d29 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -1,22 +1,21 @@
-
 require_dependency 'gitlab/email/handler'
 
 # Inspired in great part by Discourse's Email::Receiver
 module Gitlab
   module Email
-    class ProcessingError < StandardError; end
-    class EmailUnparsableError < ProcessingError; end
-    class SentNotificationNotFoundError < ProcessingError; end
-    class ProjectNotFound < ProcessingError; end
-    class EmptyEmailError < ProcessingError; end
-    class AutoGeneratedEmailError < ProcessingError; end
-    class UserNotFoundError < ProcessingError; end
-    class UserBlockedError < ProcessingError; end
-    class UserNotAuthorizedError < ProcessingError; end
-    class NoteableNotFoundError < ProcessingError; end
-    class InvalidNoteError < ProcessingError; end
-    class InvalidIssueError < ProcessingError; end
-    class UnknownIncomingEmail < ProcessingError; end
+    ProcessingError = Class.new(StandardError)
+    EmailUnparsableError = Class.new(ProcessingError)
+    SentNotificationNotFoundError = Class.new(ProcessingError)
+    ProjectNotFound = Class.new(ProcessingError)
+    EmptyEmailError = Class.new(ProcessingError)
+    AutoGeneratedEmailError = Class.new(ProcessingError)
+    UserNotFoundError = Class.new(ProcessingError)
+    UserBlockedError = Class.new(ProcessingError)
+    UserNotAuthorizedError = Class.new(ProcessingError)
+    NoteableNotFoundError = Class.new(ProcessingError)
+    InvalidNoteError = Class.new(ProcessingError)
+    InvalidIssueError = Class.new(ProcessingError)
+    UnknownIncomingEmail = Class.new(ProcessingError)
 
     class Receiver
       def initialize(raw)
@@ -32,9 +31,13 @@ module Gitlab
 
         raise UnknownIncomingEmail unless handler
 
+        Gitlab::Metrics.add_event(:receive_email, handler.metrics_params)
+
         handler.execute
       end
 
+      private
+
       def build_mail
         Mail::Message.new(@raw)
       rescue Encoding::UndefinedConversionError,
@@ -54,11 +57,36 @@ module Gitlab
       end
 
       def key_from_additional_headers(mail)
-        Array(mail.references).find do |mail_id|
+        find_key_from_references(mail) ||
+          find_key_from_delivered_to_header(mail)
+      end
+
+      def ensure_references_array(references)
+        case references
+        when Array
+          references
+        when String
+          # Handle emails from clients which append with commas,
+          # example clients are Microsoft exchange and iOS app
+          Gitlab::IncomingEmail.scan_fallback_references(references)
+        when nil
+          []
+        end
+      end
+
+      def find_key_from_references(mail)
+        ensure_references_array(mail.references).find do |mail_id|
           key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id)
           break key if key
         end
       end
+
+      def find_key_from_delivered_to_header(mail)
+        Array(mail[:delivered_to]).find do |header|
+          key = Gitlab::IncomingEmail.key_from_address(header.value)
+          break key if key
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb
index 3411eb1d9ce..558df87f36d 100644
--- a/lib/gitlab/email/reply_parser.rb
+++ b/lib/gitlab/email/reply_parser.rb
@@ -13,9 +13,17 @@ module Gitlab
 
         encoding = body.encoding
 
-        body = discourse_email_trimmer(body)
+        body = EmailReplyTrimmer.trim(body)
 
-        body = EmailReplyParser.parse_reply(body)
+        return '' unless body
+
+        # not using /\s+$/ here because that deletes empty lines
+        body = body.gsub(/[ \t]$/, '')
+
+        # NOTE: We currently don't support empty quotes.
+        # EmailReplyTrimmer allows this as a special case,
+        # so we detect it manually here.
+        return "" if body.lines.all? { |l| l.strip.empty? || l.start_with?('>') }
 
         body.force_encoding(encoding).encode("UTF-8")
       end
@@ -23,19 +31,27 @@ module Gitlab
       private
 
       def select_body(message)
-        text    = message.text_part if message.multipart?
-        text  ||= message           if message.content_type !~ /text\/html/
+        part =
+          if message.multipart?
+            message.text_part || message.html_part || message
+          else
+            message
+          end
 
-        return "" unless text
+        decoded = fix_charset(part)
 
-        text = fix_charset(text)
+        return "" unless decoded
 
         # Certain trigger phrases that means we didn't parse correctly
-        if text =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/
+        if decoded =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/
           return ""
         end
 
-        text
+        if (part.content_type || '').include? 'text/html'
+          HTMLParser.parse_reply(decoded)
+        else
+          decoded
+        end
       end
 
       # Force encoding to UTF-8 on a Mail::Message or Mail::Part
@@ -50,30 +66,6 @@ module Gitlab
       rescue
         nil
       end
-
-      REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date)
-      REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" })
-
-      def discourse_email_trimmer(body)
-        lines = body.scrub.lines.to_a
-        range_end = 0
-
-        lines.each_with_index do |l, idx|
-          # This one might be controversial but so many reply lines have years, times and end with a colon.
-          # Let's try it and see how well it works.
-          break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
-                   (l =~ /On \w+ \d+,? \d+,?.*wrote:/)
-
-          # Headers on subsequent lines
-          break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX }
-          # Headers on the same line
-          break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3
-
-          range_end = idx
-        end
-
-        lines[0..range_end].join.strip
-      end
     end
   end
 end
diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb
index b63213ae208..e3e36b35ce9 100644
--- a/lib/gitlab/emoji.rb
+++ b/lib/gitlab/emoji.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Emoji
     extend self
-    
+
     def emojis
       Gemojione.index.instance_variable_get(:@emoji_by_name)
     end
@@ -10,12 +10,51 @@ module Gitlab
       Gemojione.index.instance_variable_get(:@emoji_by_moji)
     end
 
+    def emojis_unicodes
+      emojis_by_moji.keys
+    end
+
     def emojis_names
-      emojis.keys.sort
+      emojis.keys
+    end
+
+    def emojis_aliases
+      @emoji_aliases ||= JSON.parse(File.read(Rails.root.join('fixtures', 'emojis', 'aliases.json')))
     end
 
     def emoji_filename(name)
       emojis[name]["unicode"]
     end
+
+    def emoji_unicode_filename(moji)
+      emojis_by_moji[moji]["unicode"]
+    end
+
+    def emoji_unicode_version(name)
+      @emoji_unicode_versions_by_name ||= JSON.parse(File.read(Rails.root.join('fixtures', 'emojis', 'emoji-unicode-version-map.json')))
+      @emoji_unicode_versions_by_name[name]
+    end
+
+    def normalize_emoji_name(name)
+      emojis_aliases[name] || name
+    end
+
+    def emoji_image_tag(name, src)
+      "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{src}' height='20' width='20' align='absmiddle' />"
+    end
+
+    # CSS sprite fallback takes precedence over image fallback
+    def gl_emoji_tag(name)
+      emoji_name = emojis_aliases[name] || name
+      emoji_info = emojis[emoji_name]
+      return unless emoji_info
+
+      data = {
+        name: emoji_name,
+        unicode_version: emoji_unicode_version(emoji_name)
+      }
+
+      ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], title: emoji_info['description'], data: data)
+    end
   end
 end
diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
new file mode 100644
index 00000000000..781f9c56a42
--- /dev/null
+++ b/lib/gitlab/encoding_helper.rb
@@ -0,0 +1,62 @@
+module Gitlab
+  module EncodingHelper
+    extend self
+
+    # This threshold is carefully tweaked to prevent usage of encodings detected
+    # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
+    # we're better off sticking with utf8 encoding.
+    # Reason: git diff can return strings with invalid utf8 byte sequences if it
+    # truncates a diff in the middle of a multibyte character. In this case
+    # CharlockHolmes will try to guess the encoding and will likely suggest an
+    # obscure encoding with low confidence.
+    # There is a lot more info with this merge request:
+    # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
+    ENCODING_CONFIDENCE_THRESHOLD = 40
+
+    def encode!(message)
+      return nil unless message.respond_to? :force_encoding
+
+      # if message is utf-8 encoding, just return it
+      message.force_encoding("UTF-8")
+      return message if message.valid_encoding?
+
+      # return message if message type is binary
+      detect = CharlockHolmes::EncodingDetector.detect(message)
+      return message.force_encoding("BINARY") if detect && detect[:type] == :binary
+
+      # force detected encoding if we have sufficient confidence.
+      if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
+        message.force_encoding(detect[:encoding])
+      end
+
+      # encode and clean the bad chars
+      message.replace clean(message)
+    rescue
+      encoding = detect ? detect[:encoding] : "unknown"
+      "--broken encoding: #{encoding}"
+    end
+
+    def encode_utf8(message)
+      detect = CharlockHolmes::EncodingDetector.detect(message)
+      if detect && detect[:encoding]
+        begin
+          CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
+        rescue ArgumentError => e
+          Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}")
+
+          ''
+        end
+      else
+        clean(message)
+      end
+    end
+
+    private
+
+    def clean(message)
+      message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
+        .encode("UTF-8")
+        .gsub("\0".encode("UTF-8"), "")
+    end
+  end
+end
diff --git a/lib/gitlab/production_logger.rb b/lib/gitlab/environment_logger.rb
index 89ce7144b1b..407cc572656 100644
--- a/lib/gitlab/production_logger.rb
+++ b/lib/gitlab/environment_logger.rb
@@ -1,7 +1,7 @@
 module Gitlab
-  class ProductionLogger < Gitlab::Logger
+  class EnvironmentLogger < Gitlab::Logger
     def self.file_name_noext
-      'production'
+      Rails.env
     end
   end
 end
diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb
new file mode 100644
index 00000000000..1d6f5bb5e1c
--- /dev/null
+++ b/lib/gitlab/etag_caching/middleware.rb
@@ -0,0 +1,71 @@
+module Gitlab
+  module EtagCaching
+    class Middleware
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        request = Rack::Request.new(env)
+        route = Gitlab::EtagCaching::Router.match(request.path_info)
+        return @app.call(env) unless route
+
+        track_event(:etag_caching_middleware_used, route)
+
+        etag, cached_value_present = get_etag(request)
+        if_none_match = env['HTTP_IF_NONE_MATCH']
+
+        if if_none_match == etag
+          handle_cache_hit(etag, route)
+        else
+          track_cache_miss(if_none_match, cached_value_present, route)
+
+          status, headers, body = @app.call(env)
+          headers['ETag'] = etag
+          [status, headers, body]
+        end
+      end
+
+      private
+
+      def get_etag(request)
+        cache_key = request.path
+        store = Gitlab::EtagCaching::Store.new
+        current_value = store.get(cache_key)
+        cached_value_present = current_value.present?
+
+        unless cached_value_present
+          current_value = store.touch(cache_key, only_if_missing: true)
+        end
+
+        [weak_etag_format(current_value), cached_value_present]
+      end
+
+      def weak_etag_format(value)
+        %Q{W/"#{value}"}
+      end
+
+      def handle_cache_hit(etag, route)
+        track_event(:etag_caching_cache_hit, route)
+
+        status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429
+
+        [status_code, { 'ETag' => etag }, []]
+      end
+
+      def track_cache_miss(if_none_match, cached_value_present, route)
+        if if_none_match.blank?
+          track_event(:etag_caching_header_missing, route)
+        elsif !cached_value_present
+          track_event(:etag_caching_key_not_found, route)
+        else
+          track_event(:etag_caching_resource_changed, route)
+        end
+      end
+
+      def track_event(name, route)
+        Gitlab::Metrics.add_event(name, endpoint: route.name)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb
new file mode 100644
index 00000000000..75167a6b088
--- /dev/null
+++ b/lib/gitlab/etag_caching/router.rb
@@ -0,0 +1,61 @@
+module Gitlab
+  module EtagCaching
+    class Router
+      Route = Struct.new(:regexp, :name)
+      # We enable an ETag for every request matching the regex.
+      # To match a regex the path needs to match the following:
+      #   - Don't contain a reserved word (expect for the words used in the
+      #     regex itself)
+      #   - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route
+      #   - Ending in `issues/id`/realtime_changes` for the `issue_title` route
+      USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes
+                          commit pipelines merge_requests builds
+                          new environments].freeze
+      RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES
+      RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape)))
+
+      ROUTES = [
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z),
+          'issue_notes'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/issues/\d+/realtime_changes\z),
+          'issue_title'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/commit/\S+/pipelines\.json\z),
+          'commit_pipelines'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/new\.json\z),
+          'new_merge_request_pipelines'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/\d+/pipelines\.json\z),
+          'merge_request_pipelines'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines\.json\z),
+          'project_pipelines'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines/\d+\.json\z),
+          'project_pipeline'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/builds/\d+\.json\z),
+          'project_build'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z),
+          'environments'
+        )
+      ].freeze
+
+      def self.match(path)
+        ROUTES.find { |route| route.regexp.match(path) }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb
new file mode 100644
index 00000000000..072fcfc65e6
--- /dev/null
+++ b/lib/gitlab/etag_caching/store.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module EtagCaching
+    class Store
+      EXPIRY_TIME = 20.minutes
+      REDIS_NAMESPACE = 'etag:'.freeze
+
+      def get(key)
+        Gitlab::Redis.with { |redis| redis.get(redis_key(key)) }
+      end
+
+      def touch(key, only_if_missing: false)
+        etag = generate_etag
+
+        Gitlab::Redis.with do |redis|
+          redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing)
+        end
+
+        etag
+      end
+
+      private
+
+      def generate_etag
+        SecureRandom.hex
+      end
+
+      def redis_key(key)
+        raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key)
+
+        "#{REDIS_NAMESPACE}#{key}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index ffe49364379..a0f46594eb1 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -1,59 +1,69 @@
+require 'securerandom'
+
 module Gitlab
   # This class implements an 'exclusive lease'. We call it a 'lease'
   # because it has a set expiry time. We call it 'exclusive' because only
   # one caller may obtain a lease for a given key at a time. The
   # implementation is intended to work across GitLab processes and across
-  # servers. It is a 'cheap' alternative to using SQL queries and updates:
+  # servers. It is a cheap alternative to using SQL queries and updates:
   # you do not need to change the SQL schema to start using
   # ExclusiveLease.
   #
-  # It is important to choose the timeout wisely. If the timeout is very
-  # high (1 hour) then the throughput of your operation gets very low (at
-  # most once an hour). If the timeout is lower than how long your
-  # operation may take then you cannot count on exclusivity. For example,
-  # if the timeout is 10 seconds and you do an operation which may take 20
-  # seconds then two overlapping operations may hold a lease for the same
-  # key at the same time.
-  #
-  # This class has no 'cancel' method. I originally decided against adding
-  # it because it would add complexity and a false sense of security. The
-  # complexity: instead of setting '1' we would have to set a UUID, and to
-  # delete it we would have to execute Lua on the Redis server to only
-  # delete the key if the value was our own UUID. Otherwise there is a
-  # chance that when you intend to cancel your lease you actually delete
-  # someone else's. The false sense of security: you cannot design your
-  # system to rely too much on the lease being cancelled after use because
-  # the calling (Ruby) process may crash or be killed. You _cannot_ count
-  # on begin/ensure blocks to cancel a lease, because the 'ensure' does
-  # not always run. Think of 'kill -9' from the Unicorn master for
-  # instance.
-  # 
-  # If you find that leases are getting in your way, ask yourself: would
-  # it be enough to lower the lease timeout? Another thing that might be
-  # appropriate is to only use a lease for bulk/automated operations, and
-  # to ignore the lease when you get a single 'manual' user request (a
-  # button click).
-  #
   class ExclusiveLease
+    LUA_CANCEL_SCRIPT = <<~EOS.freeze
+      local key, uuid = KEYS[1], ARGV[1]
+      if redis.call("get", key) == uuid then
+        redis.call("del", key)
+      end
+    EOS
+
+    LUA_RENEW_SCRIPT = <<~EOS.freeze
+      local key, uuid, ttl = KEYS[1], ARGV[1], ARGV[2]
+      if redis.call("get", key) == uuid then
+        redis.call("expire", key, ttl)
+        return uuid
+      end
+    EOS
+
+    def self.cancel(key, uuid)
+      Gitlab::Redis.with do |redis|
+        redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid])
+      end
+    end
+
+    def self.redis_key(key)
+      "gitlab:exclusive_lease:#{key}"
+    end
+
     def initialize(key, timeout:)
-      @key, @timeout = key, timeout
+      @redis_key = self.class.redis_key(key)
+      @timeout = timeout
+      @uuid = SecureRandom.uuid
     end
 
-    # Try to obtain the lease. Return true on success,
+    # Try to obtain the lease. Return lease UUID on success,
     # false if the lease is already taken.
     def try_obtain
       # Performing a single SET is atomic
       Gitlab::Redis.with do |redis|
-        !!redis.set(redis_key, '1', nx: true, ex: @timeout)
+        redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid
       end
     end
 
-    # No #cancel method. See comments above!
-
-    private
+    # Try to renew an existing lease. Return lease UUID on success,
+    # false if the lease is taken by a different UUID or inexistent.
+    def renew
+      Gitlab::Redis.with do |redis|
+        result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_key], argv: [@uuid, @timeout])
+        result == @uuid
+      end
+    end
 
-    def redis_key
-      "gitlab:exclusive_lease:#{@key}"
+    # Returns true if the key for this lease is set.
+    def exists?
+      Gitlab::Redis.with do |redis|
+        redis.exists(@redis_key)
+      end
     end
   end
 end
diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb
new file mode 100644
index 00000000000..bb14a8cd9e7
--- /dev/null
+++ b/lib/gitlab/fake_application_settings.rb
@@ -0,0 +1,27 @@
+# This class extends an OpenStruct object by adding predicate methods to mimic
+# ActiveRecord access. We rely on the initial values being true or false to
+# determine whether to define a predicate method because for a newly-added
+# column that has not been migrated yet, there is no way to determine the
+# column type without parsing db/schema.rb.
+module Gitlab
+  class FakeApplicationSettings < OpenStruct
+    def initialize(options = {})
+      super
+
+      FakeApplicationSettings.define_predicate_methods(options)
+    end
+
+    # Mimic ActiveRecord predicate methods for boolean values
+    def self.define_predicate_methods(options)
+      options.each do |key, value|
+        next if key.to_s.end_with?('?')
+        next unless [true, false].include?(value)
+
+        define_method "#{key}?" do
+          actual_key = key.to_s.chomp('?')
+          self[actual_key]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb
new file mode 100644
index 00000000000..a8cb7fc3fe7
--- /dev/null
+++ b/lib/gitlab/file_detector.rb
@@ -0,0 +1,81 @@
+require 'set'
+
+module Gitlab
+  # Module that can be used to detect if a path points to a special file such as
+  # a README or a CONTRIBUTING file.
+  module FileDetector
+    PATTERNS = {
+      # Project files
+      readme: /\Areadme/i,
+      changelog: /\A(changelog|history|changes|news)/i,
+      license: /\A(licen[sc]e|copying)(\..+|\z)/i,
+      contributing: /\Acontributing/i,
+      version: 'version',
+      avatar: /\Alogo\.(png|jpg|gif)\z/,
+
+      # Configuration files
+      gitignore: '.gitignore',
+      koding: '.koding.yml',
+      gitlab_ci: '.gitlab-ci.yml',
+      route_map: 'route-map.yml',
+
+      # Dependency files
+      cartfile: /\ACartfile/,
+      composer_json: 'composer.json',
+      gemfile: /\A(Gemfile|gems\.rb)\z/,
+      gemfile_lock: 'Gemfile.lock',
+      gemspec: /\.gemspec\z/,
+      godeps_json: 'Godeps.json',
+      package_json: 'package.json',
+      podfile: 'Podfile',
+      podspec_json: /\.podspec\.json\z/,
+      podspec: /\.podspec\z/,
+      requirements_txt: /requirements\.txt\z/,
+      yarn_lock: 'yarn.lock'
+    }.freeze
+
+    # Returns an Array of file types based on the given paths.
+    #
+    # This method can be used to check if a list of file paths (e.g. of changed
+    # files) involve any special files such as a README or a LICENSE file.
+    #
+    # Example:
+    #
+    #     types_in_paths(%w{README.md foo/bar.txt}) # => [:readme]
+    def self.types_in_paths(paths)
+      types = Set.new
+
+      paths.each do |path|
+        type = type_of(path)
+
+        types << type if type
+      end
+
+      types.to_a
+    end
+
+    # Returns the type of a file path, or nil if none could be detected.
+    #
+    # Returned types are Symbols such as `:readme`, `:version`, etc.
+    #
+    # Example:
+    #
+    #     type_of('README.md') # => :readme
+    #     type_of('VERSION') # => :version
+    def self.type_of(path)
+      name = File.basename(path)
+
+      PATTERNS.each do |type, search|
+        did_match = if search.is_a?(Regexp)
+                      name =~ search
+                    else
+                      name.casecmp(search) == 0
+                    end
+
+        return type if did_match
+      end
+
+      nil
+    end
+  end
+end
diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb
new file mode 100644
index 00000000000..093d9ed8092
--- /dev/null
+++ b/lib/gitlab/file_finder.rb
@@ -0,0 +1,32 @@
+# This class finds files in a repository by name and content
+# the result is joined and sorted by file name
+module Gitlab
+  class FileFinder
+    BATCH_SIZE = 100
+
+    attr_reader :project, :ref
+
+    def initialize(project, ref)
+      @project = project
+      @ref = ref
+    end
+
+    def find(query)
+      blobs = project.repository.search_files_by_content(query, ref).first(BATCH_SIZE)
+      found_file_names = Set.new
+
+      results = blobs.map do |blob|
+        blob = Gitlab::ProjectSearchResults.parse_search_result(blob)
+        found_file_names << blob.filename
+
+        [blob.filename, blob]
+      end
+
+      project.repository.search_files_by_name(query, ref).first(BATCH_SIZE).each do |filename|
+        results << [filename, OpenStruct.new(ref: ref)] unless found_file_names.include?(filename)
+      end
+
+      results.sort_by(&:first)
+    end
+  end
+end
diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb
index 501d5a95547..3dcee681c72 100644
--- a/lib/gitlab/fogbugz_import/importer.rb
+++ b/lib/gitlab/fogbugz_import/importer.rb
@@ -74,8 +74,8 @@ module Gitlab
       end
 
       def create_label(name)
-        color = nice_label_color(name)
-        Label.create!(project_id: project.id, title: name, color: color)
+        params = { title: name, color: nice_label_color(name) }
+        ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true)
       end
 
       def user_info(person_id)
@@ -122,25 +122,21 @@ module Gitlab
           author_id = user_info(bug['ixPersonOpenedBy'])[:gitlab_id] || project.creator_id
 
           issue = Issue.create!(
+            iid:          bug['ixBug'],
             project_id:   project.id,
             title:        bug['sTitle'],
             description:  body,
             author_id:    author_id,
-            assignee_id:  assignee_id,
-            state:        bug['fOpen'] == 'true' ? 'opened' : 'closed'
+            assignee_ids: [assignee_id],
+            state:        bug['fOpen'] == 'true' ? 'opened' : 'closed',
+            created_at:   date,
+            updated_at:   DateTime.parse(bug['dtLastUpdated'])
           )
-          issue.add_labels_by_names(labels)
 
-          if issue.iid != bug['ixBug']
-            issue.update_attribute(:iid, bug['ixBug'])
-          end
+          issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true)
+          issue.update_attribute(:label_ids, issue_labels.pluck(:id))
 
           import_issue_comments(issue, comments)
-
-          issue.update_attribute(:created_at, date)
-
-          last_update = DateTime.parse(bug['dtLastUpdated'])
-          issue.update_attribute(:updated_at, last_update)
         end
       end
 
diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb
index 78d7a4f27cf..b984492d369 100644
--- a/lib/gitlab/gfm/reference_rewriter.rb
+++ b/lib/gitlab/gfm/reference_rewriter.rb
@@ -58,7 +58,7 @@ module Gitlab
         referable = find_referable(reference)
         return reference unless referable
 
-        cross_reference = referable.to_reference(target_project)
+        cross_reference = build_cross_reference(referable, target_project)
         return reference if reference == cross_reference
 
         new_text = before + cross_reference + after
@@ -72,6 +72,14 @@ module Gitlab
         extractor.all.first
       end
 
+      def build_cross_reference(referable, target_project)
+        if referable.respond_to?(:project)
+          referable.to_reference(target_project)
+        else
+          referable.to_reference(@source_project, target_project: target_project)
+        end
+      end
+
       def substitution_valid?(substituted)
         @original_html == markdown(substituted)
       end
diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb
index abc8c8c55e6..8fab5489616 100644
--- a/lib/gitlab/gfm/uploads_rewriter.rb
+++ b/lib/gitlab/gfm/uploads_rewriter.rb
@@ -1,3 +1,5 @@
+require 'fileutils'
+
 module Gitlab
   module Gfm
     ##
@@ -22,7 +24,9 @@ module Gitlab
           return markdown unless file.try(:exists?)
 
           new_uploader = FileUploader.new(target_project)
-          new_uploader.store!(file)
+          with_link_in_tmp_dir(file.file) do |open_tmp_file|
+            new_uploader.store!(open_tmp_file)
+          end
           new_uploader.to_markdown
         end
       end
@@ -46,6 +50,19 @@ module Gitlab
         uploader.retrieve_from_store!(file)
         uploader.file
       end
+
+      # Because the uploaders use 'move_to_store' we must have a temporary
+      # file that is allowed to be (re)moved.
+      def with_link_in_tmp_dir(file)
+        dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file))
+        # The filename matters to Carrierwave so we make sure to preserve it
+        tmp_file = File.join(dir, File.basename(file))
+        File.link(file, tmp_file)
+        # Open the file to placate Carrierwave
+        File.open(tmp_file) { |open_file| yield open_file }
+      ensure
+        FileUtils.rm_rf(dir)
+      end
     end
   end
 end
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index 3cd515e4a3a..936606152e9 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -4,9 +4,11 @@ module Gitlab
     TAG_REF_PREFIX = "refs/tags/".freeze
     BRANCH_REF_PREFIX = "refs/heads/".freeze
 
+    CommandError = Class.new(StandardError)
+
     class << self
       def ref_name(ref)
-        ref.gsub(/\Arefs\/(tags|heads)\//, '')
+        ref.sub(/\Arefs\/(tags|heads)\//, '')
       end
 
       def branch_name(ref)
diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb
new file mode 100644
index 00000000000..42140ecc993
--- /dev/null
+++ b/lib/gitlab/git/attributes.rb
@@ -0,0 +1,131 @@
+module Gitlab
+  module Git
+    # Class for parsing Git attribute files and extracting the attributes for
+    # file patterns.
+    #
+    # Unlike Rugged this parser only needs a single IO call (a call to `open`),
+    # vastly reducing the time spent in extracting attributes.
+    #
+    # This class _only_ supports parsing the attributes file located at
+    # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files
+    # (`.gitattributes` is copied to this particular path).
+    #
+    # Basic usage:
+    #
+    #     attributes = Gitlab::Git::Attributes.new(some_repo.path)
+    #
+    #     attributes.attributes('README.md') # => { "eol" => "lf }
+    class Attributes
+      # path - The path to the Git repository.
+      def initialize(path)
+        @path = File.expand_path(path)
+        @patterns = nil
+      end
+
+      # Returns all the Git attributes for the given path.
+      #
+      # path - A path to a file for which to get the attributes.
+      #
+      # Returns a Hash.
+      def attributes(path)
+        full_path = File.join(@path, path)
+
+        patterns.each do |pattern, attrs|
+          return attrs if File.fnmatch?(pattern, full_path)
+        end
+
+        {}
+      end
+
+      # Returns a Hash containing the file patterns and their attributes.
+      def patterns
+        @patterns ||= parse_file
+      end
+
+      # Parses an attribute string.
+      #
+      # These strings can be in the following formats:
+      #
+      #     text      # => { "text" => true }
+      #     -text     # => { "text" => false }
+      #     key=value # => { "key" => "value" }
+      #
+      # string - The string to parse.
+      #
+      # Returns a Hash containing the attributes and their values.
+      def parse_attributes(string)
+        values = {}
+        dash = '-'
+        equal = '='
+        binary = 'binary'
+
+        string.split(/\s+/).each do |chunk|
+          # Data such as "foo = bar" should be treated as "foo" and "bar" being
+          # separate boolean attributes.
+          next if chunk == equal
+
+          key = chunk
+
+          # Input: "-foo"
+          if chunk.start_with?(dash)
+            key = chunk.byteslice(1, chunk.length - 1)
+            value = false
+
+          # Input: "foo=bar"
+          elsif chunk.include?(equal)
+            key, value = chunk.split(equal, 2)
+
+          # Input: "foo"
+          else
+            value = true
+          end
+
+          values[key] = value
+
+          # When the "binary" option is set the "diff" option should be set to
+          # the inverse. If "diff" is later set it should overwrite the
+          # automatically set value.
+          values['diff'] = false if key == binary && value
+        end
+
+        values
+      end
+
+      # Iterates over every line in the attributes file.
+      def each_line
+        full_path = File.join(@path, 'info/attributes')
+
+        return unless File.exist?(full_path)
+
+        File.open(full_path, 'r') do |handle|
+          handle.each_line do |line|
+            break unless line.valid_encoding?
+
+            yield line.strip
+          end
+        end
+      end
+
+      private
+
+      # Parses the Git attributes file.
+      def parse_file
+        pairs = []
+        comment = '#'
+
+        each_line do |line|
+          next if line.start_with?(comment) || line.empty?
+
+          pattern, attrs = line.split(/\s+/, 2)
+
+          parsed = attrs ? parse_attributes(attrs) : {}
+
+          pairs << [File.join(@path, pattern), parsed]
+        end
+
+        # Newer entries take precedence over older entries.
+        pairs.reverse.to_h
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
new file mode 100644
index 00000000000..66829a03c2e
--- /dev/null
+++ b/lib/gitlab/git/blame.rb
@@ -0,0 +1,75 @@
+module Gitlab
+  module Git
+    class Blame
+      include Gitlab::EncodingHelper
+
+      attr_reader :lines, :blames
+
+      def initialize(repository, sha, path)
+        @repo = repository
+        @sha = sha
+        @path = path
+        @lines = []
+        @blames = load_blame
+      end
+
+      def each
+        @blames.each do |blame|
+          yield(
+            Gitlab::Git::Commit.new(blame.commit),
+            blame.line
+          )
+        end
+      end
+
+      private
+
+      def load_blame
+        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path})
+        # Read in binary mode to ensure ASCII-8BIT
+        raw_output = IO.popen(cmd, 'rb') {|io| io.read }
+        output = encode_utf8(raw_output)
+        process_raw_blame output
+      end
+
+      def process_raw_blame(output)
+        lines, final = [], []
+        info, commits = {}, {}
+
+        # process the output
+        output.split("\n").each do |line|
+          if line[0, 1] == "\t"
+            lines << line[1, line.size]
+          elsif m = /^(\w{40}) (\d+) (\d+)/.match(line)
+            commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i
+            commits[commit_id] = nil unless commits.key?(commit_id)
+            info[lineno] = [commit_id, old_lineno]
+          end
+        end
+
+        # load all commits in single call
+        commits.keys.each do |key|
+          commits[key] = @repo.lookup(key)
+        end
+
+        # get it together
+        info.sort.each do |lineno, (commit_id, old_lineno)|
+          commit = commits[commit_id]
+          final << BlameLine.new(lineno, old_lineno, commit, lines[lineno - 1])
+        end
+
+        @lines = final
+      end
+    end
+
+    class BlameLine
+      attr_accessor :lineno, :oldlineno, :commit, :line
+      def initialize(lineno, oldlineno, commit, line)
+        @lineno = lineno
+        @oldlineno = oldlineno
+        @commit = commit
+        @line = line
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
new file mode 100644
index 00000000000..a7aceab4c14
--- /dev/null
+++ b/lib/gitlab/git/blob.rb
@@ -0,0 +1,224 @@
+module Gitlab
+  module Git
+    class Blob
+      include Linguist::BlobHelper
+      include Gitlab::EncodingHelper
+
+      # This number is the maximum amount of data that we want to display to
+      # the user. We load as much as we can for encoding detection
+      # (Linguist) and LFS pointer parsing. All other cases where we need full
+      # blob data should use load_all_data!.
+      MAX_DATA_DISPLAY_SIZE = 10.megabytes
+
+      attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary
+
+      class << self
+        def find(repository, sha, path)
+          Gitlab::GitalyClient.migrate(:project_raw_show) do |is_enabled|
+            if is_enabled
+              find_by_gitaly(repository, sha, path)
+            else
+              find_by_rugged(repository, sha, path)
+            end
+          end
+        end
+
+        def find_by_gitaly(repository, sha, path)
+          path = path.sub(/\A\/*/, '')
+          path = '/' if path.empty?
+          name = File.basename(path)
+          entry = Gitlab::GitalyClient::Commit.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE)
+          return unless entry
+
+          case entry.type
+          when :COMMIT
+            new(
+              id: entry.oid,
+              name: name,
+              size: 0,
+              data: '',
+              path: path,
+              commit_id: sha
+            )
+          when :BLOB
+            # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks
+            # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15),
+            # which is what we use below to keep a consistent behavior.
+            detect = CharlockHolmes::EncodingDetector.new(8000).detect(entry.data)
+            new(
+              id: entry.oid,
+              name: name,
+              size: entry.size,
+              data: entry.data.dup,
+              mode: entry.mode.to_s(8),
+              path: path,
+              commit_id: sha,
+              binary: detect && detect[:type] == :binary
+            )
+          end
+        end
+
+        def find_by_rugged(repository, sha, path)
+          commit = repository.lookup(sha)
+          root_tree = commit.tree
+
+          blob_entry = find_entry_by_path(repository, root_tree.oid, path)
+
+          return nil unless blob_entry
+
+          if blob_entry[:type] == :commit
+            submodule_blob(blob_entry, path, sha)
+          else
+            blob = repository.lookup(blob_entry[:oid])
+
+            if blob
+              new(
+                id: blob.oid,
+                name: blob_entry[:name],
+                size: blob.size,
+                data: blob.content(MAX_DATA_DISPLAY_SIZE),
+                mode: blob_entry[:filemode].to_s(8),
+                path: path,
+                commit_id: sha,
+                binary: blob.binary?
+              )
+            end
+          end
+        end
+
+        def raw(repository, sha)
+          blob = repository.lookup(sha)
+
+          new(
+            id: blob.oid,
+            size: blob.size,
+            data: blob.content(MAX_DATA_DISPLAY_SIZE),
+            binary: blob.binary?
+          )
+        end
+
+        # Recursive search of blob id by path
+        #
+        # Ex.
+        #   blog/            # oid: 1a
+        #     app/           # oid: 2a
+        #       models/      # oid: 3a
+        #       file.rb      # oid: 4a
+        #
+        #
+        # Blob.find_entry_by_path(repo, '1a', 'app/file.rb') # => '4a'
+        #
+        def find_entry_by_path(repository, root_id, path)
+          root_tree = repository.lookup(root_id)
+          # Strip leading slashes
+          path[/^\/*/] = ''
+          path_arr = path.split('/')
+
+          entry = root_tree.find do |entry|
+            entry[:name] == path_arr[0]
+          end
+
+          return nil unless entry
+
+          if path_arr.size > 1
+            return nil unless entry[:type] == :tree
+            path_arr.shift
+            find_entry_by_path(repository, entry[:oid], path_arr.join('/'))
+          else
+            [:blob, :commit].include?(entry[:type]) ? entry : nil
+          end
+        end
+
+        def submodule_blob(blob_entry, path, sha)
+          new(
+            id: blob_entry[:oid],
+            name: blob_entry[:name],
+            size: 0,
+            data: '',
+            path: path,
+            commit_id: sha
+          )
+        end
+      end
+
+      def initialize(options)
+        %w(id name path size data mode commit_id binary).each do |key|
+          self.send("#{key}=", options[key.to_sym])
+        end
+
+        @loaded_all_data = false
+        # Retain the actual size before it is encoded
+        @loaded_size = @data.bytesize if @data
+      end
+
+      def binary?
+        @binary.nil? ? super : @binary == true
+      end
+
+      def data
+        encode! @data
+      end
+
+      # Load all blob data (not just the first MAX_DATA_DISPLAY_SIZE bytes) into
+      # memory as a Ruby string.
+      def load_all_data!(repository)
+        return if @data == '' # don't mess with submodule blobs
+        return @data if @loaded_all_data
+
+        @loaded_all_data = true
+        @data = repository.lookup(id).content
+        @loaded_size = @data.bytesize
+        @binary = nil
+      end
+
+      def name
+        encode! @name
+      end
+
+      def truncated?
+        size && (size > loaded_size)
+      end
+
+      # Valid LFS object pointer is a text file consisting of
+      # version
+      # oid
+      # size
+      # see https://github.com/github/git-lfs/blob/v1.1.0/docs/spec.md#the-pointer
+      def lfs_pointer?
+        has_lfs_version_key? && lfs_oid.present? && lfs_size.present?
+      end
+
+      def lfs_oid
+        if has_lfs_version_key?
+          oid = data.match(/(?<=sha256:)([0-9a-f]{64})/)
+          return oid[1] if oid
+        end
+
+        nil
+      end
+
+      def lfs_size
+        if has_lfs_version_key?
+          size = data.match(/(?<=size )([0-9]+)/)
+          return size[1].to_i if size
+        end
+
+        nil
+      end
+
+      def external_storage
+        return unless lfs_pointer?
+
+        :lfs
+      end
+
+      alias_method :external_size, :lfs_size
+
+      private
+
+      def has_lfs_version_key?
+        !empty? && text? && data.start_with?("version https://git-lfs.github.com/spec")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb
new file mode 100644
index 00000000000..d7975f88aaa
--- /dev/null
+++ b/lib/gitlab/git/blob_snippet.rb
@@ -0,0 +1,32 @@
+module Gitlab
+  module Git
+    class BlobSnippet
+      include Linguist::BlobHelper
+
+      attr_accessor :ref
+      attr_accessor :lines
+      attr_accessor :filename
+      attr_accessor :startline
+
+      def initialize(ref, lines, startline, filename)
+        @ref, @lines, @startline, @filename = ref, lines, startline, filename
+      end
+
+      def data
+        lines&.join("\n")
+      end
+
+      def name
+        filename
+      end
+
+      def size
+        data.length
+      end
+
+      def mode
+        nil
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb
new file mode 100644
index 00000000000..124526e4b59
--- /dev/null
+++ b/lib/gitlab/git/branch.rb
@@ -0,0 +1,40 @@
+module Gitlab
+  module Git
+    class Branch < Ref
+      def initialize(repository, name, target)
+        if target.is_a?(Gitaly::FindLocalBranchResponse)
+          target = target_from_gitaly_local_branches_response(target)
+        end
+
+        super(repository, name, target)
+      end
+
+      def target_from_gitaly_local_branches_response(response)
+        # Git messages have no encoding enforcements. However, in the UI we only
+        # handle UTF-8, so basically we cross our fingers that the message force
+        # encoded to UTF-8 is readable.
+        message = response.commit_subject.dup.force_encoding('UTF-8')
+
+        # NOTE: For ease of parsing in Gitaly, we have only the subject of
+        # the commit and not the full message. This is ok, since all the
+        # code that uses `local_branches` only cares at most about the
+        # commit message.
+        # TODO: Once gitaly "takes over" Rugged consider separating the
+        # subject from the message to make it clearer when there's one
+        # available but not the other.
+        hash = {
+          id: response.commit_id,
+          message: message,
+          authored_date: Time.at(response.commit_author.date.seconds),
+          author_name: response.commit_author.name,
+          author_email: response.commit_author.email,
+          committed_date: Time.at(response.commit_committer.date.seconds),
+          committer_name: response.commit_committer.name,
+          committer_email: response.commit_committer.email
+        }
+
+        Gitlab::Git::Commit.decorate(hash)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
new file mode 100644
index 00000000000..9c0606d780a
--- /dev/null
+++ b/lib/gitlab/git/commit.rb
@@ -0,0 +1,375 @@
+# Gitlab::Git::Commit is a wrapper around native Rugged::Commit object
+module Gitlab
+  module Git
+    class Commit
+      include Gitlab::EncodingHelper
+
+      attr_accessor :raw_commit, :head
+
+      SERIALIZE_KEYS = [
+        :id, :message, :parent_ids,
+        :authored_date, :author_name, :author_email,
+        :committed_date, :committer_name, :committer_email
+      ].freeze
+
+      attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator
+
+      delegate :tree, to: :raw_commit
+
+      def ==(other)
+        return false unless other.is_a?(Gitlab::Git::Commit)
+
+        id && id == other.id
+      end
+
+      class << self
+        # Get commits collection
+        #
+        # Ex.
+        #   Commit.where(
+        #     repo: repo,
+        #     ref: 'master',
+        #     path: 'app/models',
+        #     limit: 10,
+        #     offset: 5,
+        #   )
+        #
+        def where(options)
+          repo = options.delete(:repo)
+          raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log)
+
+          repo.log(options).map { |c| decorate(c) }
+        end
+
+        # Get single commit
+        #
+        # Ex.
+        #   Commit.find(repo, '29eda46b')
+        #
+        #   Commit.find(repo, 'master')
+        #
+        def find(repo, commit_id = "HEAD")
+          return commit_id if commit_id.is_a?(Gitlab::Git::Commit)
+          return decorate(commit_id) if commit_id.is_a?(Rugged::Commit)
+
+          obj = if commit_id.is_a?(String)
+                  repo.rev_parse_target(commit_id)
+                else
+                  Gitlab::Git::Ref.dereference_object(commit_id)
+                end
+
+          return nil unless obj.is_a?(Rugged::Commit)
+
+          decorate(obj)
+        rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, Gitlab::Git::Repository::NoRepository
+          nil
+        end
+
+        # Get last commit for HEAD
+        #
+        # Ex.
+        #   Commit.last(repo)
+        #
+        def last(repo)
+          find(repo)
+        end
+
+        # Get last commit for specified path and ref
+        #
+        # Ex.
+        #   Commit.last_for_path(repo, '29eda46b', 'app/models')
+        #
+        #   Commit.last_for_path(repo, 'master', 'Gemfile')
+        #
+        def last_for_path(repo, ref, path = nil)
+          where(
+            repo: repo,
+            ref: ref,
+            path: path,
+            limit: 1
+          ).first
+        end
+
+        # Get commits between two revspecs
+        # See also #repository.commits_between
+        #
+        # Ex.
+        #   Commit.between(repo, '29eda46b', 'master')
+        #
+        def between(repo, base, head)
+          repo.commits_between(base, head).map do |commit|
+            decorate(commit)
+          end
+        rescue Rugged::ReferenceError
+          []
+        end
+
+        # Returns commits collection
+        #
+        # Ex.
+        #   Commit.find_all(
+        #     repo,
+        #     ref: 'master',
+        #     max_count: 10,
+        #     skip: 5,
+        #     order: :date
+        #   )
+        #
+        #   +options+ is a Hash of optional arguments to git
+        #     :ref is the ref from which to begin (SHA1 or name)
+        #     :max_count is the maximum number of commits to fetch
+        #     :skip is the number of commits to skip
+        #     :order is the commits order and allowed value is :none (default), :date,
+        #        :topo, or any combination of them (in an array). Commit ordering types
+        #        are documented here:
+        #        http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant)
+        #
+        def find_all(repo, options = {})
+          actual_options = options.dup
+
+          allowed_options = [:ref, :max_count, :skip, :order]
+
+          actual_options.keep_if do |key|
+            allowed_options.include?(key)
+          end
+
+          default_options = { skip: 0 }
+          actual_options = default_options.merge(actual_options)
+
+          rugged = repo.rugged
+          walker = Rugged::Walker.new(rugged)
+
+          if actual_options[:ref]
+            walker.push(rugged.rev_parse_oid(actual_options[:ref]))
+          else
+            rugged.references.each("refs/heads/*") do |ref|
+              walker.push(ref.target_id)
+            end
+          end
+
+          walker.sorting(rugged_sort_type(actual_options[:order]))
+
+          commits = []
+          offset = actual_options[:skip]
+          limit = actual_options[:max_count]
+          walker.each(offset: offset, limit: limit) do |commit|
+            commits.push(decorate(commit))
+          end
+
+          walker.reset
+
+          commits
+        rescue Rugged::OdbError
+          []
+        end
+
+        def decorate(commit, ref = nil)
+          Gitlab::Git::Commit.new(commit, ref)
+        end
+
+        # Returns a diff object for the changes introduced by +rugged_commit+.
+        # If +rugged_commit+ doesn't have a parent, then the diff is between
+        # this commit and an empty repo.  See Repository#diff for the keys
+        # allowed in the +options+ hash.
+        def diff_from_parent(rugged_commit, options = {})
+          options ||= {}
+          break_rewrites = options[:break_rewrites]
+          actual_options = Gitlab::Git::Diff.filter_diff_options(options)
+
+          diff = if rugged_commit.parents.empty?
+                   rugged_commit.diff(actual_options.merge(reverse: true))
+                 else
+                   rugged_commit.parents[0].diff(rugged_commit, actual_options)
+                 end
+
+          diff.find_similar!(break_rewrites: break_rewrites)
+          diff
+        end
+
+        # Returns the `Rugged` sorting type constant for one or more given
+        # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array
+        # containing more than one of them. `:date` uses a combination of date and
+        # topological sorting to closer mimic git's native ordering.
+        def rugged_sort_type(sort_type)
+          @rugged_sort_types ||= {
+            none: Rugged::SORT_NONE,
+            topo: Rugged::SORT_TOPO,
+            date: Rugged::SORT_DATE | Rugged::SORT_TOPO
+          }
+
+          @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE)
+        end
+      end
+
+      def initialize(raw_commit, head = nil)
+        raise "Nil as raw commit passed" unless raw_commit
+
+        if raw_commit.is_a?(Hash)
+          init_from_hash(raw_commit)
+        elsif raw_commit.is_a?(Rugged::Commit)
+          init_from_rugged(raw_commit)
+        else
+          raise "Invalid raw commit type: #{raw_commit.class}"
+        end
+
+        @head = head
+      end
+
+      def sha
+        id
+      end
+
+      def short_id(length = 10)
+        id.to_s[0..length]
+      end
+
+      def safe_message
+        @safe_message ||= message
+      end
+
+      def created_at
+        committed_date
+      end
+
+      # Was this commit committed by a different person than the original author?
+      def different_committer?
+        author_name != committer_name || author_email != committer_email
+      end
+
+      def parent_id
+        parent_ids.first
+      end
+
+      # Shows the diff between the commit's parent and the commit.
+      #
+      # Cuts out the header and stats from #to_patch and returns only the diff.
+      def to_diff
+        diff_from_parent.patch
+      end
+
+      # Returns a diff object for the changes from this commit's first parent.
+      # If there is no parent, then the diff is between this commit and an
+      # empty repo.  See Repository#diff for keys allowed in the +options+
+      # hash.
+      def diff_from_parent(options = {})
+        Commit.diff_from_parent(raw_commit, options)
+      end
+
+      def deltas
+        @deltas ||= diff_from_parent.each_delta.map { |d| Gitlab::Git::Diff.new(d) }
+      end
+
+      def has_zero_stats?
+        stats.total.zero?
+      rescue
+        true
+      end
+
+      def no_commit_message
+        "--no commit message"
+      end
+
+      def to_hash
+        serialize_keys.map.with_object({}) do |key, hash|
+          hash[key] = send(key)
+        end
+      end
+
+      def date
+        committed_date
+      end
+
+      def diffs(options = {})
+        Gitlab::Git::DiffCollection.new(diff_from_parent(options), options)
+      end
+
+      def parents
+        raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) }
+      end
+
+      def stats
+        Gitlab::Git::CommitStats.new(self)
+      end
+
+      def to_patch(options = {})
+        begin
+          raw_commit.to_mbox(options)
+        rescue Rugged::InvalidError => ex
+          if ex.message =~ /Commit \w+ is a merge commit/
+            'Patch format is not currently supported for merge commits.'
+          end
+        end
+      end
+
+      # Get a collection of Rugged::Reference objects for this commit.
+      #
+      # Ex.
+      #   commit.ref(repo)
+      #
+      def refs(repo)
+        repo.refs_hash[id]
+      end
+
+      # Get ref names collection
+      #
+      # Ex.
+      #   commit.ref_names(repo)
+      #
+      def ref_names(repo)
+        refs(repo).map do |ref|
+          ref.name.sub(%r{^refs/(heads|remotes|tags)/}, "")
+        end
+      end
+
+      def message
+        encode! @message
+      end
+
+      def author_name
+        encode! @author_name
+      end
+
+      def author_email
+        encode! @author_email
+      end
+
+      def committer_name
+        encode! @committer_name
+      end
+
+      def committer_email
+        encode! @committer_email
+      end
+
+      private
+
+      def init_from_hash(hash)
+        raw_commit = hash.symbolize_keys
+
+        serialize_keys.each do |key|
+          send("#{key}=", raw_commit[key])
+        end
+      end
+
+      def init_from_rugged(commit)
+        author = commit.author
+        committer = commit.committer
+
+        @raw_commit = commit
+        @id = commit.oid
+        @message = commit.message
+        @authored_date = author[:time]
+        @committed_date = committer[:time]
+        @author_name = author[:name]
+        @author_email = author[:email]
+        @committer_name = committer[:name]
+        @committer_email = committer[:email]
+        @parent_ids = commit.parents.map(&:oid)
+      end
+
+      def serialize_keys
+        SERIALIZE_KEYS
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb
new file mode 100644
index 00000000000..e9118bbed0e
--- /dev/null
+++ b/lib/gitlab/git/commit_stats.rb
@@ -0,0 +1,26 @@
+# Gitlab::Git::CommitStats counts the additions, deletions, and total changes
+# in a commit.
+module Gitlab
+  module Git
+    class CommitStats
+      attr_reader :id, :additions, :deletions, :total
+
+      # Instantiate a CommitStats object
+      def initialize(commit)
+        @id = commit.id
+        @additions = 0
+        @deletions = 0
+        @total = 0
+
+        diff = commit.diff_from_parent
+
+        diff.each_patch do |p|
+          # TODO: Use the new Rugged convenience methods when they're released
+          @additions += p.stat[0]
+          @deletions += p.stat[1]
+          @total += p.changes
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb
new file mode 100644
index 00000000000..78e440395a5
--- /dev/null
+++ b/lib/gitlab/git/compare.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module Git
+    class Compare
+      attr_reader :head, :base, :straight
+
+      def initialize(repository, base, head, straight: false)
+        @repository = repository
+        @straight = straight
+
+        unless base && head
+          @commits = []
+          return
+        end
+
+        @base = Gitlab::Git::Commit.find(repository, base.try(:strip))
+        @head = Gitlab::Git::Commit.find(repository, head.try(:strip))
+
+        @commits = [] unless @base && @head
+        @commits = [] if same
+      end
+
+      def same
+        @base && @head && @base.id == @head.id
+      end
+
+      def commits
+        return @commits if defined?(@commits)
+
+        @commits = Gitlab::Git::Commit.between(@repository, @base.id, @head.id)
+      end
+
+      def diffs(options = {})
+        unless @head && @base
+          return Gitlab::Git::DiffCollection.new([])
+        end
+
+        paths = options.delete(:paths) || []
+        options[:straight] = @straight
+        Gitlab::Git::Diff.between(@repository, @head.id, @base.id, options, *paths)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
new file mode 100644
index 00000000000..cf7829a583b
--- /dev/null
+++ b/lib/gitlab/git/diff.rb
@@ -0,0 +1,381 @@
+# Gitlab::Git::Diff is a wrapper around native Rugged::Diff object
+module Gitlab
+  module Git
+    class Diff
+      TimeoutError = Class.new(StandardError)
+      include Gitlab::EncodingHelper
+
+      # Diff properties
+      attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff
+
+      # Stats properties
+      attr_accessor :new_file, :renamed_file, :deleted_file
+
+      alias_method :new_file?, :new_file
+      alias_method :deleted_file?, :deleted_file
+      alias_method :renamed_file?, :renamed_file
+
+      attr_accessor :expanded
+      attr_writer :too_large
+
+      alias_method :expanded?, :expanded
+
+      SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze
+
+      class << self
+        # The maximum size of a diff to display.
+        def size_limit
+          if RequestStore.active?
+            RequestStore['gitlab_git_diff_size_limit'] ||= find_size_limit
+          else
+            find_size_limit
+          end
+        end
+
+        # The maximum size before a diff is collapsed.
+        def collapse_limit
+          if RequestStore.active?
+            RequestStore['gitlab_git_diff_collapse_limit'] ||= find_collapse_limit
+          else
+            find_collapse_limit
+          end
+        end
+
+        def find_size_limit
+          if Feature.enabled?('gitlab_git_diff_size_limit_increase')
+            200.kilobytes
+          else
+            100.kilobytes
+          end
+        end
+
+        def find_collapse_limit
+          if Feature.enabled?('gitlab_git_diff_size_limit_increase')
+            100.kilobytes
+          else
+            10.kilobytes
+          end
+        end
+
+        def between(repo, head, base, options = {}, *paths)
+          straight = options.delete(:straight) || false
+
+          common_commit = if straight
+                            base
+                          else
+                            # Only show what is new in the source branch
+                            # compared to the target branch, not the other way
+                            # around. The linex below with merge_base is
+                            # equivalent to diff with three dots (git diff
+                            # branch1...branch2) From the git documentation:
+                            # "git diff A...B" is equivalent to "git diff
+                            # $(git-merge-base A B) B"
+                            repo.merge_base_commit(head, base)
+                          end
+
+          options ||= {}
+          actual_options = filter_diff_options(options)
+          repo.diff(common_commit, head, actual_options, *paths)
+        end
+
+        # Return a copy of the +options+ hash containing only keys that can be
+        # passed to Rugged.  Allowed options are:
+        #
+        #  :max_size ::
+        #    An integer specifying the maximum byte size of a file before a it
+        #    will be treated as binary. The default value is 512MB.
+        #
+        #  :context_lines ::
+        #    The number of unchanged lines that define the boundary of a hunk
+        #    (and to display before and after the actual changes). The default is
+        #    3.
+        #
+        #  :interhunk_lines ::
+        #    The maximum number of unchanged lines between hunk boundaries before
+        #    the hunks will be merged into a one. The default is 0.
+        #
+        #  :old_prefix ::
+        #    The virtual "directory" to prefix to old filenames in hunk headers.
+        #    The default is "a".
+        #
+        #  :new_prefix ::
+        #    The virtual "directory" to prefix to new filenames in hunk headers.
+        #    The default is "b".
+        #
+        #  :reverse ::
+        #    If true, the sides of the diff will be reversed.
+        #
+        #  :force_text ::
+        #    If true, all files will be treated as text, disabling binary
+        #    attributes & detection.
+        #
+        #  :ignore_whitespace ::
+        #    If true, all whitespace will be ignored.
+        #
+        #  :ignore_whitespace_change ::
+        #    If true, changes in amount of whitespace will be ignored.
+        #
+        #  :ignore_whitespace_eol ::
+        #    If true, whitespace at end of line will be ignored.
+        #
+        #  :ignore_submodules ::
+        #    if true, submodules will be excluded from the diff completely.
+        #
+        #  :patience ::
+        #    If true, the "patience diff" algorithm will be used (currenlty
+        #    unimplemented).
+        #
+        #  :include_ignored ::
+        #    If true, ignored files will be included in the diff.
+        #
+        #  :include_untracked ::
+        #   If true, untracked files will be included in the diff.
+        #
+        #  :include_unmodified ::
+        #    If true, unmodified files will be included in the diff.
+        #
+        #  :recurse_untracked_dirs ::
+        #    Even if +:include_untracked+ is true, untracked directories will
+        #    only be marked with a single entry in the diff. If this flag is set
+        #    to true, all files under ignored directories will be included in the
+        #    diff, too.
+        #
+        #  :disable_pathspec_match ::
+        #    If true, the given +*paths+ will be applied as exact matches,
+        #    instead of as fnmatch patterns.
+        #
+        #  :deltas_are_icase ::
+        #    If true, filename comparisons will be made with case-insensitivity.
+        #
+        #  :include_untracked_content ::
+        #    if true, untracked content will be contained in the the diff patch
+        #    text.
+        #
+        #  :skip_binary_check ::
+        #    If true, diff deltas will be generated without spending time on
+        #    binary detection. This is useful to improve performance in cases
+        #    where the actual file content difference is not needed.
+        #
+        #  :include_typechange ::
+        #    If true, type changes for files will not be interpreted as deletion
+        #    of the "old file" and addition of the "new file", but will generate
+        #    typechange records.
+        #
+        #  :include_typechange_trees ::
+        #    Even if +:include_typechange+ is true, blob -> tree changes will
+        #    still usually be handled as a deletion of the blob. If this flag is
+        #    set to true, blob -> tree changes will be marked as typechanges.
+        #
+        #  :ignore_filemode ::
+        #    If true, file mode changes will be ignored.
+        #
+        #  :recurse_ignored_dirs ::
+        #    Even if +:include_ignored+ is true, ignored directories will only be
+        #    marked with a single entry in the diff. If this flag is set to true,
+        #    all files under ignored directories will be included in the diff,
+        #    too.
+        def filter_diff_options(options, default_options = {})
+          allowed_options = [:max_size, :context_lines, :interhunk_lines,
+                             :old_prefix, :new_prefix, :reverse, :force_text,
+                             :ignore_whitespace, :ignore_whitespace_change,
+                             :ignore_whitespace_eol, :ignore_submodules,
+                             :patience, :include_ignored, :include_untracked,
+                             :include_unmodified, :recurse_untracked_dirs,
+                             :disable_pathspec_match, :deltas_are_icase,
+                             :include_untracked_content, :skip_binary_check,
+                             :include_typechange, :include_typechange_trees,
+                             :ignore_filemode, :recurse_ignored_dirs, :paths,
+                             :max_files, :max_lines, :limits, :expanded]
+
+          if default_options
+            actual_defaults = default_options.dup
+            actual_defaults.keep_if do |key|
+              allowed_options.include?(key)
+            end
+          else
+            actual_defaults = {}
+          end
+
+          if options
+            filtered_opts = options.dup
+            filtered_opts.keep_if do |key|
+              allowed_options.include?(key)
+            end
+            filtered_opts = actual_defaults.merge(filtered_opts)
+          else
+            filtered_opts = actual_defaults
+          end
+
+          filtered_opts
+        end
+      end
+
+      def initialize(raw_diff, expanded: true)
+        @expanded = expanded
+
+        case raw_diff
+        when Hash
+          init_from_hash(raw_diff)
+          prune_diff_if_eligible
+        when Rugged::Patch, Rugged::Diff::Delta
+          init_from_rugged(raw_diff)
+        when Gitlab::GitalyClient::Diff
+          init_from_gitaly(raw_diff)
+          prune_diff_if_eligible
+        when Gitaly::CommitDelta
+          init_from_gitaly(raw_diff)
+        when nil
+          raise "Nil as raw diff passed"
+        else
+          raise "Invalid raw diff type: #{raw_diff.class}"
+        end
+      end
+
+      def to_hash
+        hash = {}
+
+        SERIALIZE_KEYS.each do |key|
+          hash[key] = send(key)
+        end
+
+        hash
+      end
+
+      def mode_changed?
+        a_mode && b_mode && a_mode != b_mode
+      end
+
+      def submodule?
+        a_mode == '160000' || b_mode == '160000'
+      end
+
+      def line_count
+        @line_count ||= Util.count_lines(@diff)
+      end
+
+      def too_large?
+        if @too_large.nil?
+          @too_large = @diff.bytesize >= self.class.size_limit
+        else
+          @too_large
+        end
+      end
+
+      # This is used by `to_hash` and `init_from_hash`.
+      alias_method :too_large, :too_large?
+
+      def too_large!
+        @diff = ''
+        @line_count = 0
+        @too_large = true
+      end
+
+      def collapsed?
+        return @collapsed if defined?(@collapsed)
+
+        @collapsed = !expanded && @diff.bytesize >= self.class.collapse_limit
+      end
+
+      def collapse!
+        @diff = ''
+        @line_count = 0
+        @collapsed = true
+      end
+
+      private
+
+      def init_from_rugged(rugged)
+        if rugged.is_a?(Rugged::Patch)
+          init_from_rugged_patch(rugged)
+          d = rugged.delta
+        else
+          d = rugged
+        end
+
+        @new_path = encode!(d.new_file[:path])
+        @old_path = encode!(d.old_file[:path])
+        @a_mode = d.old_file[:mode].to_s(8)
+        @b_mode = d.new_file[:mode].to_s(8)
+        @new_file = d.added?
+        @renamed_file = d.renamed?
+        @deleted_file = d.deleted?
+      end
+
+      def init_from_rugged_patch(patch)
+        # Don't bother initializing diffs that are too large. If a diff is
+        # binary we're not going to display anything so we skip the size check.
+        return if !patch.delta.binary? && prune_large_patch(patch)
+
+        @diff = encode!(strip_diff_headers(patch.to_s))
+      end
+
+      def init_from_hash(hash)
+        raw_diff = hash.symbolize_keys
+
+        SERIALIZE_KEYS.each do |key|
+          send(:"#{key}=", raw_diff[key.to_sym])
+        end
+      end
+
+      def init_from_gitaly(diff)
+        @diff = encode!(diff.patch) if diff.respond_to?(:patch)
+        @new_path = encode!(diff.to_path.dup)
+        @old_path = encode!(diff.from_path.dup)
+        @a_mode = diff.old_mode.to_s(8)
+        @b_mode = diff.new_mode.to_s(8)
+        @new_file = diff.from_id == BLANK_SHA
+        @renamed_file = diff.from_path != diff.to_path
+        @deleted_file = diff.to_id == BLANK_SHA
+      end
+
+      def prune_diff_if_eligible
+        if too_large?
+          too_large!
+        elsif collapsed?
+          collapse!
+        end
+      end
+
+      # If the patch surpasses any of the diff limits it calls the appropiate
+      # prune method and returns true. Otherwise returns false.
+      def prune_large_patch(patch)
+        size = 0
+
+        patch.each_hunk do |hunk|
+          hunk.each_line do |line|
+            size += line.content.bytesize
+
+            if size >= self.class.size_limit
+              too_large!
+              return true
+            end
+          end
+        end
+
+        if !expanded && size >= self.class.collapse_limit
+          collapse!
+          return true
+        end
+
+        false
+      end
+
+      # Strip out the information at the beginning of the patch's text to match
+      # Grit's output
+      def strip_diff_headers(diff_text)
+        # Delete everything up to the first line that starts with '---' or
+        # 'Binary'
+        diff_text.sub!(/\A.*?^(---|Binary)/m, '\1')
+
+        if diff_text.start_with?('---', 'Binary')
+          diff_text
+        else
+          # If the diff_text did not contain a line starting with '---' or
+          # 'Binary', return the empty string. No idea why; we are just
+          # preserving behavior from before the refactor.
+          ''
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
new file mode 100644
index 00000000000..555894907cc
--- /dev/null
+++ b/lib/gitlab/git/diff_collection.rb
@@ -0,0 +1,125 @@
+module Gitlab
+  module Git
+    class DiffCollection
+      include Enumerable
+
+      DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze
+
+      def initialize(iterator, options = {})
+        @iterator = iterator
+        @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files])
+        @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines])
+        @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file
+        @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min
+        @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min
+        @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file
+        @enforce_limits = !!options.fetch(:limits, true)
+        @expanded = !!options.fetch(:expanded, true)
+
+        @line_count = 0
+        @byte_count = 0
+        @overflow = false
+        @empty = true
+        @array = Array.new
+      end
+
+      def each(&block)
+        Gitlab::GitalyClient.migrate(:commit_raw_diffs) do
+          each_patch(&block)
+        end
+      end
+
+      def empty?
+        any? # Make sure the iterator has been exercised
+        @empty
+      end
+
+      def overflow?
+        populate!
+        !!@overflow
+      end
+
+      def size
+        @size ||= count # forces a loop using each method
+      end
+
+      def real_size
+        populate!
+
+        if @overflow
+          "#{size}+"
+        else
+          size.to_s
+        end
+      end
+
+      def decorate!
+        collection = each_with_index do |element, i|
+          @array[i] = yield(element)
+        end
+        collection
+      end
+
+      alias_method :to_ary, :to_a
+
+      private
+
+      def populate!
+        return if @populated
+
+        each { nil } # force a loop through all diffs
+        nil
+      end
+
+      def over_safe_limits?(files)
+        files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes
+      end
+
+      def each_patch
+        i = 0
+        @array.each do |diff|
+          yield diff
+          i += 1
+        end
+
+        return if @overflow
+        return if @iterator.nil?
+
+        @iterator.each do |raw|
+          @empty = false
+
+          if @enforce_limits && i >= @max_files
+            @overflow = true
+            break
+          end
+
+          expanded = !@enforce_limits || @expanded
+
+          diff = Gitlab::Git::Diff.new(raw, expanded: expanded)
+
+          if !expanded && over_safe_limits?(i) && diff.line_count > 0
+            diff.collapse!
+          end
+
+          @line_count += diff.line_count
+          @byte_count += diff.diff.bytesize
+
+          if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes)
+            # This last Diff instance pushes us over the lines limit. We stop and
+            # discard it.
+            @overflow = true
+            break
+          end
+
+          yield @array[i] = diff
+          i += 1
+        end
+
+        @populated = true
+
+        # Allow iterator to be garbage-collected. It cannot be reused anyway.
+        @iterator = nil
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb
new file mode 100644
index 00000000000..0fdc57ec954
--- /dev/null
+++ b/lib/gitlab/git/env.rb
@@ -0,0 +1,38 @@
+module Gitlab
+  module Git
+    # Ephemeral (per request) storage for environment variables that some Git
+    # commands may need.
+    #
+    # For example, in pre-receive hooks, new objects are put in a temporary
+    # $GIT_OBJECT_DIRECTORY. Without it set, the new objects cannot be retrieved
+    # (this would break push rules for instance).
+    #
+    # This class is thread-safe via RequestStore.
+    class Env
+      WHITELISTED_GIT_VARIABLES = %w[
+        GIT_OBJECT_DIRECTORY
+        GIT_ALTERNATE_OBJECT_DIRECTORIES
+      ].freeze
+
+      def self.set(env)
+        return unless RequestStore.active?
+
+        RequestStore.store[:gitlab_git_env] = whitelist_git_env(env)
+      end
+
+      def self.all
+        return {} unless RequestStore.active?
+
+        RequestStore.fetch(:gitlab_git_env) { {} }
+      end
+
+      def self.[](key)
+        all[key]
+      end
+
+      def self.whitelist_git_env(env)
+        env.select { |key, _| WHITELISTED_GIT_VARIABLES.include?(key.to_s) }.with_indifferent_access
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb
new file mode 100644
index 00000000000..f4e3b5e5129
--- /dev/null
+++ b/lib/gitlab/git/gitmodules_parser.rb
@@ -0,0 +1,77 @@
+module Gitlab
+  module Git
+    class GitmodulesParser
+      def initialize(content)
+        @content = content
+      end
+
+      # Parses the contents of a .gitmodules file and returns a hash of
+      # submodule information, indexed by path.
+      def parse
+        reindex_by_path(get_submodules_by_name)
+      end
+
+      private
+
+      class State
+        def initialize
+          @result = {}
+          @current_submodule = nil
+        end
+
+        def start_section(section)
+          # In some .gitmodules files (e.g. nodegit's), a header
+          # with the same name appears multiple times; we want to
+          # accumulate the configs across these
+          @current_submodule = @result[section] || { 'name' => section }
+          @result[section] = @current_submodule
+        end
+
+        def set_attribute(attr, value)
+          @current_submodule[attr] = value
+        end
+
+        def section_started?
+          !@current_submodule.nil?
+        end
+
+        def submodules_by_name
+          @result
+        end
+      end
+
+      def get_submodules_by_name
+        iterator = State.new
+
+        @content.split("\n").each_with_object(iterator) do |text, iterator|
+          next if text =~ /^\s*#/
+
+          if text =~ /\A\[submodule "(?<name>[^"]+)"\]\z/
+            iterator.start_section($~[:name])
+          else
+            next unless iterator.section_started?
+
+            next unless text =~ /\A\s*(?<key>\w+)\s*=\s*(?<value>.*)\z/
+
+            value = $~[:value].chomp
+            iterator.set_attribute($~[:key], value)
+          end
+        end
+
+        iterator.submodules_by_name
+      end
+
+      def reindex_by_path(submodules_by_name)
+        # Convert from an indexed by name to an array indexed by path
+        # If a submodule doesn't have a path, it is considered bogus
+        # and is ignored
+        submodules_by_name.each_with_object({}) do |(name, data), results|
+          path = data.delete 'path'
+          next unless path
+
+          results[path] = data
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb
new file mode 100644
index 00000000000..1add037fa5f
--- /dev/null
+++ b/lib/gitlab/git/index.rb
@@ -0,0 +1,145 @@
+module Gitlab
+  module Git
+    class Index
+      IndexError = Class.new(StandardError)
+
+      DEFAULT_MODE = 0o100644
+
+      ACTIONS = %w(create create_dir update move delete).freeze
+
+      attr_reader :repository, :raw_index
+
+      def initialize(repository)
+        @repository = repository
+        @raw_index = repository.rugged.index
+      end
+
+      delegate :read_tree, :get, to: :raw_index
+
+      def write_tree
+        raw_index.write_tree(repository.rugged)
+      end
+
+      def dir_exists?(path)
+        raw_index.find { |entry| entry[:path].start_with?("#{path}/") }
+      end
+
+      def create(options)
+        options = normalize_options(options)
+
+        if get(options[:file_path])
+          raise IndexError, "A file with this name already exists"
+        end
+
+        add_blob(options)
+      end
+
+      def create_dir(options)
+        options = normalize_options(options)
+
+        if get(options[:file_path])
+          raise IndexError, "A file with this name already exists"
+        end
+
+        if dir_exists?(options[:file_path])
+          raise IndexError, "A directory with this name already exists"
+        end
+
+        options = options.dup
+        options[:file_path] += '/.gitkeep'
+        options[:content] = ''
+
+        add_blob(options)
+      end
+
+      def update(options)
+        options = normalize_options(options)
+
+        file_entry = get(options[:file_path])
+        unless file_entry
+          raise IndexError, "A file with this name doesn't exist"
+        end
+
+        add_blob(options, mode: file_entry[:mode])
+      end
+
+      def move(options)
+        options = normalize_options(options)
+
+        file_entry = get(options[:previous_path])
+        unless file_entry
+          raise IndexError, "A file with this name doesn't exist"
+        end
+
+        if get(options[:file_path])
+          raise IndexError, "A file with this name already exists"
+        end
+
+        raw_index.remove(options[:previous_path])
+
+        add_blob(options, mode: file_entry[:mode])
+      end
+
+      def delete(options)
+        options = normalize_options(options)
+
+        unless get(options[:file_path])
+          raise IndexError, "A file with this name doesn't exist"
+        end
+
+        raw_index.remove(options[:file_path])
+      end
+
+      private
+
+      def normalize_options(options)
+        options = options.dup
+        options[:file_path] = normalize_path(options[:file_path]) if options[:file_path]
+        options[:previous_path] = normalize_path(options[:previous_path]) if options[:previous_path]
+        options
+      end
+
+      def normalize_path(path)
+        unless path
+          raise IndexError, "You must provide a file path"
+        end
+
+        pathname = Gitlab::Git::PathHelper.normalize_path(path.dup)
+
+        pathname.each_filename do |segment|
+          if segment == '..'
+            raise IndexError, 'Path cannot include directory traversal'
+          end
+
+          unless segment =~ Gitlab::Regex.file_name_regex
+            raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}"
+          end
+        end
+
+        pathname.to_s
+      end
+
+      def add_blob(options, mode: nil)
+        content = options[:content]
+        unless content
+          raise IndexError, "You must provide content"
+        end
+
+        content = Base64.decode64(content) if options[:encoding] == 'base64'
+
+        detect = CharlockHolmes::EncodingDetector.new.detect(content)
+        unless detect && detect[:type] == :binary
+          # When writing to the repo directly as we are doing here,
+          # the `core.autocrlf` config isn't taken into account.
+          content.gsub!("\r\n", "\n") if repository.autocrlf
+        end
+
+        oid = repository.rugged.write(content, :blob)
+
+        raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE)
+      rescue Rugged::IndexError => e
+        raise IndexError, e.message
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb
new file mode 100644
index 00000000000..0148cd8df05
--- /dev/null
+++ b/lib/gitlab/git/path_helper.rb
@@ -0,0 +1,16 @@
+module Gitlab
+  module Git
+    class PathHelper
+      class << self
+        def normalize_path(filename)
+          # Strip all leading slashes so that //foo -> foo
+          filename[/^\/*/] = ''
+
+          # Expand relative paths (e.g. foo/../bar)
+          filename = Pathname.new(filename)
+          filename.relative_path_from(Pathname.new(''))
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb
new file mode 100644
index 00000000000..df9ca3ee5ac
--- /dev/null
+++ b/lib/gitlab/git/popen.rb
@@ -0,0 +1,26 @@
+require 'open3'
+
+module Gitlab
+  module Git
+    module Popen
+      def popen(cmd, path)
+        unless cmd.is_a?(Array)
+          raise "System commands must be given as an array of strings"
+        end
+
+        vars = { "PWD" => path }
+        options = { chdir: path }
+
+        @cmd_output = ""
+        @cmd_status = 0
+        Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
+          @cmd_output << stdout.read
+          @cmd_output << stderr.read
+          @cmd_status = wait_thr.value.exitstatus
+        end
+
+        [@cmd_output, @cmd_status]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb
new file mode 100644
index 00000000000..ebf7393dc61
--- /dev/null
+++ b/lib/gitlab/git/ref.rb
@@ -0,0 +1,49 @@
+module Gitlab
+  module Git
+    class Ref
+      include Gitlab::EncodingHelper
+
+      # Branch or tag name
+      # without "refs/tags|heads" prefix
+      attr_reader :name
+
+      # Target sha.
+      # Usually it is commit sha but in case
+      # when tag reference on other tag it can be tag sha
+      attr_reader :target
+
+      # Dereferenced target
+      # Commit object to which the Ref points to
+      attr_reader :dereferenced_target
+
+      # Extract branch name from full ref path
+      #
+      # Ex.
+      #   Ref.extract_branch_name('refs/heads/master') #=> 'master'
+      def self.extract_branch_name(str)
+        str.gsub(/\Arefs\/heads\//, '')
+      end
+
+      def self.dereference_object(object)
+        object = object.target while object.is_a?(Rugged::Tag::Annotation)
+
+        object
+      end
+
+      def initialize(repository, name, target)
+        encode! name
+        @name = name.gsub(/\Arefs\/(tags|heads)\//, '')
+        @dereferenced_target = Gitlab::Git::Commit.find(repository, target)
+        @target = if target.respond_to?(:oid)
+                    target.oid
+                  elsif target.respond_to?(:name)
+                    target.name
+                  elsif target.is_a? String
+                    target
+                  else
+                    nil
+                  end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
new file mode 100644
index 00000000000..0a0c6f76cd3
--- /dev/null
+++ b/lib/gitlab/git/repository.rb
@@ -0,0 +1,1169 @@
+# Gitlab::Git::Repository is a wrapper around native Rugged::Repository object
+require 'tempfile'
+require 'forwardable'
+require "rubygems/package"
+
+module Gitlab
+  module Git
+    class Repository
+      include Gitlab::Git::Popen
+
+      ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[
+        GIT_OBJECT_DIRECTORY
+        GIT_ALTERNATE_OBJECT_DIRECTORIES
+      ].freeze
+      SEARCH_CONTEXT_LINES = 3
+
+      NoRepository = Class.new(StandardError)
+      InvalidBlobName = Class.new(StandardError)
+      InvalidRef = Class.new(StandardError)
+
+      # Full path to repo
+      attr_reader :path
+
+      # Directory name of repo
+      attr_reader :name
+
+      # Rugged repo object
+      attr_reader :rugged
+
+      attr_reader :storage
+
+      # 'path' must be the path to a _bare_ git repository, e.g.
+      # /path/to/my-repo.git
+      def initialize(storage, relative_path)
+        @storage = storage
+        @relative_path = relative_path
+
+        storage_path = Gitlab.config.repositories.storages[@storage]['path']
+        @path = File.join(storage_path, @relative_path)
+        @name = @relative_path.split("/").last
+        @attributes = Gitlab::Git::Attributes.new(path)
+      end
+
+      delegate  :empty?,
+                :bare?,
+                to: :rugged
+
+      # Default branch in the repository
+      def root_ref
+        @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.default_branch_name
+          else
+            discover_default_branch
+          end
+        end
+      end
+
+      # Alias to old method for compatibility
+      def raw
+        rugged
+      end
+
+      def rugged
+        @rugged ||= Rugged::Repository.new(path, alternates: alternate_object_directories)
+      rescue Rugged::RepositoryError, Rugged::OSError
+        raise NoRepository.new('no repository for such path')
+      end
+
+      # Returns an Array of branch names
+      # sorted by name ASC
+      def branch_names
+        gitaly_migrate(:branch_names) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.branch_names
+          else
+            branches.map(&:name)
+          end
+        end
+      end
+
+      # Returns an Array of Branches
+      def branches(filter: nil, sort_by: nil)
+        branches = rugged.branches.each(filter).map do |rugged_ref|
+          begin
+            Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target)
+          rescue Rugged::ReferenceError
+            # Omit invalid branch
+          end
+        end.compact
+
+        sort_branches(branches, sort_by)
+      end
+
+      def reload_rugged
+        @rugged = nil
+      end
+
+      # Directly find a branch with a simple name (e.g. master)
+      #
+      # force_reload causes a new Rugged repository to be instantiated
+      #
+      # This is to work around a bug in libgit2 that causes in-memory refs to
+      # be stale/invalid when packed-refs is changed.
+      # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333
+      def find_branch(name, force_reload = false)
+        reload_rugged if force_reload
+
+        rugged_ref = rugged.branches[name]
+        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref
+      end
+
+      def local_branches(sort_by: nil)
+        gitaly_migrate(:local_branches) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.local_branches(sort_by: sort_by).map do |gitaly_branch|
+              Gitlab::Git::Branch.new(self, gitaly_branch.name, gitaly_branch)
+            end
+          else
+            branches(filter: :local, sort_by: sort_by)
+          end
+        end
+      end
+
+      # Returns the number of valid branches
+      def branch_count
+        gitaly_migrate(:branch_names) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.count_branch_names
+          else
+            rugged.branches.count do |ref|
+              begin
+                ref.name && ref.target # ensures the branch is valid
+
+                true
+              rescue Rugged::ReferenceError
+                false
+              end
+            end
+          end
+        end
+      end
+
+      # Returns the number of valid tags
+      def tag_count
+        gitaly_migrate(:tag_names) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.count_tag_names
+          else
+            rugged.tags.count
+          end
+        end
+      end
+
+      # Returns an Array of tag names
+      def tag_names
+        gitaly_migrate(:tag_names) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.tag_names
+          else
+            rugged.tags.map { |t| t.name }
+          end
+        end
+      end
+
+      # Returns an Array of Tags
+      def tags
+        rugged.references.each("refs/tags/*").map do |ref|
+          message = nil
+
+          if ref.target.is_a?(Rugged::Tag::Annotation)
+            tag_message = ref.target.message
+
+            if tag_message.respond_to?(:chomp)
+              message = tag_message.chomp
+            end
+          end
+
+          Gitlab::Git::Tag.new(self, ref.name, ref.target, message)
+        end.sort_by(&:name)
+      end
+
+      # Returns true if the given tag exists
+      #
+      # name - The name of the tag as a String.
+      def tag_exists?(name)
+        !!rugged.tags[name]
+      end
+
+      # Returns true if the given branch exists
+      #
+      # name - The name of the branch as a String.
+      def branch_exists?(name)
+        rugged.branches.exists?(name)
+
+      # If the branch name is invalid (e.g. ".foo") Rugged will raise an error.
+      # Whatever code calls this method shouldn't have to deal with that so
+      # instead we just return `false` (which is true since a branch doesn't
+      # exist when it has an invalid name).
+      rescue Rugged::ReferenceError
+        false
+      end
+
+      # Returns an Array of branch and tag names
+      def ref_names
+        branch_names + tag_names
+      end
+
+      # Deprecated. Will be removed in 5.2
+      def heads
+        rugged.references.each("refs/heads/*").map do |head|
+          Gitlab::Git::Ref.new(self, head.name, head.target)
+        end.sort_by(&:name)
+      end
+
+      def has_commits?
+        !empty?
+      end
+
+      def repo_exists?
+        !!rugged
+      end
+
+      # Discovers the default branch based on the repository's available branches
+      #
+      # - If no branches are present, returns nil
+      # - If one branch is present, returns its name
+      # - If two or more branches are present, returns current HEAD or master or first branch
+      def discover_default_branch
+        names = branch_names
+
+        return if names.empty?
+
+        return names[0] if names.length == 1
+
+        if rugged_head
+          extracted_name = Ref.extract_branch_name(rugged_head.name)
+
+          return extracted_name if names.include?(extracted_name)
+        end
+
+        if names.include?('master')
+          'master'
+        else
+          names[0]
+        end
+      end
+
+      def rugged_head
+        rugged.head
+      rescue Rugged::ReferenceError
+        nil
+      end
+
+      def archive_prefix(ref, sha)
+        project_name = self.name.chomp('.git')
+        "#{project_name}-#{ref.tr('/', '-')}-#{sha}"
+      end
+
+      def archive_metadata(ref, storage_path, format = "tar.gz")
+        ref ||= root_ref
+        commit = Gitlab::Git::Commit.find(self, ref)
+        return {} if commit.nil?
+
+        prefix = archive_prefix(ref, commit.id)
+
+        {
+          'RepoPath' => path,
+          'ArchivePrefix' => prefix,
+          'ArchivePath' => archive_file_path(prefix, storage_path, format),
+          'CommitId' => commit.id
+        }
+      end
+
+      def archive_file_path(name, storage_path, format = "tar.gz")
+        # Build file path
+        return nil unless name
+
+        extension =
+          case format
+          when "tar.bz2", "tbz", "tbz2", "tb2", "bz2"
+            "tar.bz2"
+          when "tar"
+            "tar"
+          when "zip"
+            "zip"
+          else
+            # everything else should fall back to tar.gz
+            "tar.gz"
+          end
+
+        file_name = "#{name}.#{extension}"
+        File.join(storage_path, self.name, file_name)
+      end
+
+      # Return repo size in megabytes
+      def size
+        size = popen(%w(du -sk), path).first.strip.to_i
+        (size.to_f / 1024).round(2)
+      end
+
+      # Returns an array of BlobSnippets for files at the specified +ref+ that
+      # contain the +query+ string.
+      def search_files(query, ref = nil)
+        greps = []
+        ref ||= root_ref
+
+        populated_index(ref).each do |entry|
+          # Discard submodules
+          next if submodule?(entry)
+
+          blob = Gitlab::Git::Blob.raw(self, entry[:oid])
+
+          # Skip binary files
+          next if blob.data.encoding == Encoding::ASCII_8BIT
+
+          blob.load_all_data!(self)
+          greps += build_greps(blob.data, query, ref, entry[:path])
+        end
+
+        greps
+      end
+
+      # Use the Rugged Walker API to build an array of commits.
+      #
+      # Usage.
+      #   repo.log(
+      #     ref: 'master',
+      #     path: 'app/models',
+      #     limit: 10,
+      #     offset: 5,
+      #     after: Time.new(2016, 4, 21, 14, 32, 10)
+      #   )
+      #
+      def log(options)
+        default_options = {
+          limit: 10,
+          offset: 0,
+          path: nil,
+          follow: false,
+          skip_merges: false,
+          disable_walk: false,
+          after: nil,
+          before: nil
+        }
+
+        options = default_options.merge(options)
+        options[:limit] ||= 0
+        options[:offset] ||= 0
+        actual_ref = options[:ref] || root_ref
+        begin
+          sha = sha_from_ref(actual_ref)
+        rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
+          # Return an empty array if the ref wasn't found
+          return []
+        end
+
+        if log_using_shell?(options)
+          log_by_shell(sha, options)
+        else
+          log_by_walk(sha, options)
+        end
+      end
+
+      def log_using_shell?(options)
+        options[:path].present? ||
+          options[:disable_walk] ||
+          options[:skip_merges] ||
+          options[:after] ||
+          options[:before]
+      end
+
+      def log_by_walk(sha, options)
+        walk_options = {
+          show: sha,
+          sort: Rugged::SORT_NONE,
+          limit: options[:limit],
+          offset: options[:offset]
+        }
+        Rugged::Walker.walk(rugged, walk_options).to_a
+      end
+
+      def log_by_shell(sha, options)
+        limit = options[:limit].to_i
+        offset = options[:offset].to_i
+        use_follow_flag = options[:follow] && options[:path].present?
+
+        # We will perform the offset in Ruby because --follow doesn't play well with --skip.
+        # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520
+        offset_in_ruby = use_follow_flag && options[:offset].present?
+        limit += offset if offset_in_ruby
+
+        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log]
+        cmd << "--max-count=#{limit}"
+        cmd << '--format=%H'
+        cmd << "--skip=#{offset}" unless offset_in_ruby
+        cmd << '--follow' if use_follow_flag
+        cmd << '--no-merges' if options[:skip_merges]
+        cmd << "--after=#{options[:after].iso8601}" if options[:after]
+        cmd << "--before=#{options[:before].iso8601}" if options[:before]
+        cmd << sha
+
+        # :path can be a string or an array of strings
+        if options[:path].present?
+          cmd << '--'
+          cmd += Array(options[:path])
+        end
+
+        raw_output = IO.popen(cmd) { |io| io.read }
+        lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines
+
+        lines.map! { |c| Rugged::Commit.new(rugged, c.strip) }
+      end
+
+      def count_commits(options)
+        cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list]
+        cmd << "--after=#{options[:after].iso8601}" if options[:after]
+        cmd << "--before=#{options[:before].iso8601}" if options[:before]
+        cmd += %W[--count #{options[:ref]}]
+        cmd += %W[-- #{options[:path]}] if options[:path].present?
+
+        raw_output = IO.popen(cmd) { |io| io.read }
+
+        raw_output.to_i
+      end
+
+      def sha_from_ref(ref)
+        rev_parse_target(ref).oid
+      end
+
+      # Return the object that +revspec+ points to.  If +revspec+ is an
+      # annotated tag, then return the tag's target instead.
+      def rev_parse_target(revspec)
+        obj = rugged.rev_parse(revspec)
+        Ref.dereference_object(obj)
+      end
+
+      # Return a collection of Rugged::Commits between the two revspec arguments.
+      # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for
+      # a detailed list of valid arguments.
+      def commits_between(from, to)
+        walker = Rugged::Walker.new(rugged)
+        walker.sorting(Rugged::SORT_NONE | Rugged::SORT_REVERSE)
+
+        sha_from = sha_from_ref(from)
+        sha_to = sha_from_ref(to)
+
+        walker.push(sha_to)
+        walker.hide(sha_from)
+
+        commits = walker.to_a
+        walker.reset
+
+        commits
+      end
+
+      # Counts the amount of commits between `from` and `to`.
+      def count_commits_between(from, to)
+        commits_between(from, to).size
+      end
+
+      # Returns the SHA of the most recent common ancestor of +from+ and +to+
+      def merge_base_commit(from, to)
+        rugged.merge_base(from, to)
+      end
+
+      # Returns true is +from+ is direct ancestor to +to+, otherwise false
+      def is_ancestor?(from, to)
+        gitaly_commit_client.is_ancestor(from, to)
+      end
+
+      # Return an array of Diff objects that represent the diff
+      # between +from+ and +to+.  See Diff::filter_diff_options for the allowed
+      # diff options.  The +options+ hash can also include :break_rewrites to
+      # split larger rewrites into delete/add pairs.
+      def diff(from, to, options = {}, *paths)
+        Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options)
+      end
+
+      # Returns a RefName for a given SHA
+      def ref_name_for_sha(ref_path, sha)
+        raise ArgumentError, "sha can't be empty" unless sha.present?
+
+        gitaly_migrate(:find_ref_name) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.find_ref_name(sha, ref_path)
+          else
+            args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha})
+
+            # Not found -> ["", 0]
+            # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0]
+            Gitlab::Popen.popen(args, @path).first.split.last
+          end
+        end
+      end
+
+      # Returns branch names collection that contains the special commit(SHA1
+      # or name)
+      #
+      # Ex.
+      #   repo.branch_names_contains('master')
+      #
+      def branch_names_contains(commit)
+        branches_contains(commit).map { |c| c.name }
+      end
+
+      # Returns branch collection that contains the special commit(SHA1 or name)
+      #
+      # Ex.
+      #   repo.branch_names_contains('master')
+      #
+      def branches_contains(commit)
+        commit_obj = rugged.rev_parse(commit)
+        parent = commit_obj.parents.first unless commit_obj.parents.empty?
+
+        walker = Rugged::Walker.new(rugged)
+
+        rugged.branches.select do |branch|
+          walker.push(branch.target_id)
+          walker.hide(parent) if parent
+          result = walker.any? { |c| c.oid == commit_obj.oid }
+          walker.reset
+
+          result
+        end
+      end
+
+      # Get refs hash which key is SHA1
+      # and value is a Rugged::Reference
+      def refs_hash
+        # Initialize only when first call
+        if @refs_hash.nil?
+          @refs_hash = Hash.new { |h, k| h[k] = [] }
+
+          rugged.references.each do |r|
+            # Symbolic/remote references may not have an OID; skip over them
+            target_oid = r.target.try(:oid)
+            if target_oid
+              sha = rev_parse_target(target_oid).oid
+              @refs_hash[sha] << r
+            end
+          end
+        end
+        @refs_hash
+      end
+
+      # Lookup for rugged object by oid or ref name
+      def lookup(oid_or_ref_name)
+        rugged.rev_parse(oid_or_ref_name)
+      end
+
+      # Return hash with submodules info for this repository
+      #
+      # Ex.
+      #   {
+      #     "current_path/rack"  => {
+      #       "name" => "original_path/rack",
+      #       "id" => "c67be4624545b4263184c4a0e8f887efd0a66320",
+      #       "url" => "git://github.com/chneukirchen/rack.git"
+      #     },
+      #     "encoding" => {
+      #       "id" => ....
+      #     }
+      #   }
+      #
+      def submodules(ref)
+        commit = rev_parse_target(ref)
+        return {} unless commit
+
+        begin
+          content = blob_content(commit, ".gitmodules")
+        rescue InvalidBlobName
+          return {}
+        end
+
+        parser = GitmodulesParser.new(content)
+        fill_submodule_ids(commit, parser.parse)
+      end
+
+      # Return total commits count accessible from passed ref
+      def commit_count(ref)
+        walker = Rugged::Walker.new(rugged)
+        walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
+        oid = rugged.rev_parse_oid(ref)
+        walker.push(oid)
+        walker.count
+      end
+
+      # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or
+      # tag name or a commit SHA.  Valid +reset_type+ values are:
+      #
+      #  [:soft]
+      #    the head will be moved to the commit.
+      #  [:mixed]
+      #    will trigger a +:soft+ reset, plus the index will be replaced
+      #    with the content of the commit tree.
+      #  [:hard]
+      #    will trigger a +:mixed+ reset and the working directory will be
+      #    replaced with the content of the index. (Untracked and ignored files
+      #    will be left alone)
+      delegate :reset, to: :rugged
+
+      # Mimic the `git clean` command and recursively delete untracked files.
+      # Valid keys that can be passed in the +options+ hash are:
+      #
+      # :d - Remove untracked directories
+      # :f - Remove untracked directories that are managed by a different
+      #      repository
+      # :x - Remove ignored files
+      #
+      # The value in +options+ must evaluate to true for an option to take
+      # effect.
+      #
+      # Examples:
+      #
+      #   repo.clean(d: true, f: true) # Enable the -d and -f options
+      #
+      #   repo.clean(d: false, x: true) # -x is enabled, -d is not
+      def clean(options = {})
+        strategies = [:remove_untracked]
+        strategies.push(:force) if options[:f]
+        strategies.push(:remove_ignored) if options[:x]
+
+        # TODO: implement this method
+      end
+
+      # Check out the specified ref. Valid options are:
+      #
+      #  :b - Create a new branch at +start_point+ and set HEAD to the new
+      #       branch.
+      #
+      #  * These options are passed to the Rugged::Repository#checkout method:
+      #
+      #  :progress ::
+      #    A callback that will be executed for checkout progress notifications.
+      #    Up to 3 parameters are passed on each execution:
+      #
+      #    - The path to the last updated file (or +nil+ on the very first
+      #      invocation).
+      #    - The number of completed checkout steps.
+      #    - The number of total checkout steps to be performed.
+      #
+      #  :notify ::
+      #    A callback that will be executed for each checkout notification
+      #    types specified with +:notify_flags+. Up to 5 parameters are passed
+      #    on each execution:
+      #
+      #    - An array containing the +:notify_flags+ that caused the callback
+      #      execution.
+      #    - The path of the current file.
+      #    - A hash describing the baseline blob (or +nil+ if it does not
+      #      exist).
+      #    - A hash describing the target blob (or +nil+ if it does not exist).
+      #    - A hash describing the workdir blob (or +nil+ if it does not
+      #      exist).
+      #
+      #  :strategy ::
+      #    A single symbol or an array of symbols representing the strategies
+      #    to use when performing the checkout. Possible values are:
+      #
+      #    :none ::
+      #      Perform a dry run (default).
+      #
+      #    :safe ::
+      #      Allow safe updates that cannot overwrite uncommitted data.
+      #
+      #    :safe_create ::
+      #      Allow safe updates plus creation of missing files.
+      #
+      #    :force ::
+      #      Allow all updates to force working directory to look like index.
+      #
+      #    :allow_conflicts ::
+      #      Allow checkout to make safe updates even if conflicts are found.
+      #
+      #    :remove_untracked ::
+      #      Remove untracked files not in index (that are not ignored).
+      #
+      #    :remove_ignored ::
+      #      Remove ignored files not in index.
+      #
+      #    :update_only ::
+      #      Only update existing files, don't create new ones.
+      #
+      #    :dont_update_index ::
+      #      Normally checkout updates index entries as it goes; this stops
+      #      that.
+      #
+      #    :no_refresh ::
+      #      Don't refresh index/config/etc before doing checkout.
+      #
+      #    :disable_pathspec_match ::
+      #      Treat pathspec as simple list of exact match file paths.
+      #
+      #    :skip_locked_directories ::
+      #      Ignore directories in use, they will be left empty.
+      #
+      #    :skip_unmerged ::
+      #      Allow checkout to skip unmerged files (NOT IMPLEMENTED).
+      #
+      #    :use_ours ::
+      #      For unmerged files, checkout stage 2 from index (NOT IMPLEMENTED).
+      #
+      #    :use_theirs ::
+      #      For unmerged files, checkout stage 3 from index (NOT IMPLEMENTED).
+      #
+      #    :update_submodules ::
+      #      Recursively checkout submodules with same options (NOT
+      #      IMPLEMENTED).
+      #
+      #    :update_submodules_if_changed ::
+      #      Recursively checkout submodules if HEAD moved in super repo (NOT
+      #      IMPLEMENTED).
+      #
+      #  :disable_filters ::
+      #    If +true+, filters like CRLF line conversion will be disabled.
+      #
+      #  :dir_mode ::
+      #    Mode for newly created directories. Default: +0755+.
+      #
+      #  :file_mode ::
+      #    Mode for newly created files. Default: +0755+ or +0644+.
+      #
+      #  :file_open_flags ::
+      #    Mode for opening files. Default:
+      #    <code>IO::CREAT | IO::TRUNC | IO::WRONLY</code>.
+      #
+      #  :notify_flags ::
+      #    A single symbol or an array of symbols representing the cases in
+      #    which the +:notify+ callback should be invoked. Possible values are:
+      #
+      #    :none ::
+      #      Do not invoke the +:notify+ callback (default).
+      #
+      #    :conflict ::
+      #      Invoke the callback for conflicting paths.
+      #
+      #    :dirty ::
+      #      Invoke the callback for "dirty" files, i.e. those that do not need
+      #      an update but no longer match the baseline.
+      #
+      #    :updated ::
+      #      Invoke the callback for any file that was changed.
+      #
+      #    :untracked ::
+      #      Invoke the callback for untracked files.
+      #
+      #    :ignored ::
+      #      Invoke the callback for ignored files.
+      #
+      #    :all ::
+      #      Invoke the callback for all these cases.
+      #
+      #  :paths ::
+      #    A glob string or an array of glob strings specifying which paths
+      #    should be taken into account for the checkout operation. +nil+ will
+      #    match all files.  Default: +nil+.
+      #
+      #  :baseline ::
+      #    A Rugged::Tree that represents the current, expected contents of the
+      #    workdir.  Default: +HEAD+.
+      #
+      #  :target_directory ::
+      #    A path to an alternative workdir directory in which the checkout
+      #    should be performed.
+      def checkout(ref, options = {}, start_point = "HEAD")
+        if options[:b]
+          rugged.branches.create(ref, start_point)
+          options.delete(:b)
+        end
+        default_options = { strategy: [:recreate_missing, :safe] }
+        rugged.checkout(ref, default_options.merge(options))
+      end
+
+      # Delete the specified branch from the repository
+      def delete_branch(branch_name)
+        rugged.branches.delete(branch_name)
+      end
+
+      # Create a new branch named **ref+ based on **stat_point+, HEAD by default
+      #
+      # Examples:
+      #   create_branch("feature")
+      #   create_branch("other-feature", "master")
+      def create_branch(ref, start_point = "HEAD")
+        rugged_ref = rugged.branches.create(ref, start_point)
+        Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target)
+      rescue Rugged::ReferenceError => e
+        raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/
+        raise InvalidRef.new("Invalid reference #{start_point}")
+      end
+
+      # Return an array of this repository's remote names
+      def remote_names
+        rugged.remotes.each_name.to_a
+      end
+
+      # Delete the specified remote from this repository.
+      def remote_delete(remote_name)
+        rugged.remotes.delete(remote_name)
+      end
+
+      # Add a new remote to this repository.  Returns a Rugged::Remote object
+      def remote_add(remote_name, url)
+        rugged.remotes.create(remote_name, url)
+      end
+
+      # Update the specified remote using the values in the +options+ hash
+      #
+      # Example
+      # repo.update_remote("origin", url: "path/to/repo")
+      def remote_update(remote_name, options = {})
+        # TODO: Implement other remote options
+        rugged.remotes.set_url(remote_name, options[:url]) if options[:url]
+      end
+
+      # Fetch the specified remote
+      def fetch(remote_name)
+        rugged.remotes[remote_name].fetch
+      end
+
+      # Push +*refspecs+ to the remote identified by +remote_name+.
+      def push(remote_name, *refspecs)
+        rugged.remotes[remote_name].push(refspecs)
+      end
+
+      AUTOCRLF_VALUES = {
+        "true" => true,
+        "false" => false,
+        "input" => :input
+      }.freeze
+
+      def autocrlf
+        AUTOCRLF_VALUES[rugged.config['core.autocrlf']]
+      end
+
+      def autocrlf=(value)
+        rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value]
+      end
+
+      # Returns result like "git ls-files" , recursive and full file path
+      #
+      # Ex.
+      #   repo.ls_files('master')
+      #
+      def ls_files(ref)
+        actual_ref = ref || root_ref
+
+        begin
+          sha_from_ref(actual_ref)
+        rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
+          # Return an empty array if the ref wasn't found
+          return []
+        end
+
+        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree)
+        cmd += %w(-r)
+        cmd += %w(--full-tree)
+        cmd += %w(--full-name)
+        cmd += %W(-- #{actual_ref})
+
+        raw_output = IO.popen(cmd, &:read).split("\n").map do |f|
+          stuff, path = f.split("\t")
+          _mode, type, _sha = stuff.split(" ")
+          path if type == "blob"
+          # Contain only blob type
+        end
+
+        raw_output.compact
+      end
+
+      def copy_gitattributes(ref)
+        begin
+          commit = lookup(ref)
+        rescue Rugged::ReferenceError
+          raise InvalidRef.new("Ref #{ref} is invalid")
+        end
+
+        # Create the paths
+        info_dir_path = File.join(path, 'info')
+        info_attributes_path = File.join(info_dir_path, 'attributes')
+
+        begin
+          # Retrieve the contents of the blob
+          gitattributes_content = blob_content(commit, '.gitattributes')
+        rescue InvalidBlobName
+          # No .gitattributes found. Should now remove any info/attributes and return
+          File.delete(info_attributes_path) if File.exist?(info_attributes_path)
+          return
+        end
+
+        # Create the info directory if needed
+        Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path)
+
+        # Write the contents of the .gitattributes file to info/attributes
+        # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8
+        File.open(info_attributes_path, "wb") do |file|
+          file.write(gitattributes_content)
+        end
+      end
+
+      # Returns the Git attributes for the given file path.
+      #
+      # See `Gitlab::Git::Attributes` for more information.
+      def attributes(path)
+        @attributes.attributes(path)
+      end
+
+      def gitaly_repository
+        Gitlab::GitalyClient::Util.repository(@storage, @relative_path)
+      end
+
+      private
+
+      def alternate_object_directories
+        Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact
+      end
+
+      # Get the content of a blob for a given commit.  If the blob is a commit
+      # (for submodules) then return the blob's OID.
+      def blob_content(commit, blob_name)
+        blob_entry = tree_entry(commit, blob_name)
+
+        unless blob_entry
+          raise InvalidBlobName.new("Invalid blob name: #{blob_name}")
+        end
+
+        case blob_entry[:type]
+        when :commit
+          blob_entry[:oid]
+        when :tree
+          raise InvalidBlobName.new("#{blob_name} is a tree, not a blob")
+        when :blob
+          rugged.lookup(blob_entry[:oid]).content
+        end
+      end
+
+      # Fill in the 'id' field of a submodule hash from its values
+      # as-of +commit+. Return a Hash consisting only of entries
+      # from the submodule hash for which the 'id' field is filled.
+      def fill_submodule_ids(commit, submodule_data)
+        submodule_data.each do |path, data|
+          id = begin
+            blob_content(commit, path)
+          rescue InvalidBlobName
+            nil
+          end
+          data['id'] = id
+        end
+        submodule_data.select { |path, data| data['id'] }
+      end
+
+      # Returns true if +commit+ introduced changes to +path+, using commit
+      # trees to make that determination.  Uses the history simplification
+      # rules that `git log` uses by default, where a commit is omitted if it
+      # is TREESAME to any parent.
+      #
+      # If the +follow+ option is true and the file specified by +path+ was
+      # renamed, then the path value is set to the old path.
+      def commit_touches_path?(commit, path, follow, walker)
+        entry = tree_entry(commit, path)
+
+        if commit.parents.empty?
+          # This is the root commit, return true if it has +path+ in its tree
+          return !entry.nil?
+        end
+
+        num_treesame = 0
+        commit.parents.each do |parent|
+          parent_entry = tree_entry(parent, path)
+
+          # Only follow the first TREESAME parent for merge commits
+          if num_treesame > 0
+            walker.hide(parent)
+            next
+          end
+
+          if entry.nil? && parent_entry.nil?
+            num_treesame += 1
+          elsif entry && parent_entry && entry[:oid] == parent_entry[:oid]
+            num_treesame += 1
+          end
+        end
+
+        case num_treesame
+        when 0
+          detect_rename(commit, commit.parents.first, path) if follow
+          true
+        else false
+        end
+      end
+
+      # Find the entry for +path+ in the tree for +commit+
+      def tree_entry(commit, path)
+        pathname = Pathname.new(path)
+        first = true
+        tmp_entry = nil
+
+        pathname.each_filename do |dir|
+          if first
+            tmp_entry = commit.tree[dir]
+            first = false
+          elsif tmp_entry.nil?
+            return nil
+          else
+            begin
+              tmp_entry = rugged.lookup(tmp_entry[:oid])
+            rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
+              return nil
+            end
+
+            return nil unless tmp_entry.type == :tree
+            tmp_entry = tmp_entry[dir]
+          end
+        end
+
+        tmp_entry
+      end
+
+      # Compare +commit+ and +parent+ for +path+.  If +path+ is a file and was
+      # renamed in +commit+, then set +path+ to the old filename.
+      def detect_rename(commit, parent, path)
+        diff = parent.diff(commit, paths: [path], disable_pathspec_match: true)
+
+        # If +path+ is a filename, not a directory, then we should only have
+        # one delta.  We don't need to follow renames for directories.
+        return nil if diff.each_delta.count > 1
+
+        delta = diff.each_delta.first
+        if delta.added?
+          full_diff = parent.diff(commit)
+          full_diff.find_similar!
+
+          full_diff.each_delta do |full_delta|
+            if full_delta.renamed? && path == full_delta.new_file[:path]
+              # Look for the old path in ancestors
+              path.replace(full_delta.old_file[:path])
+            end
+          end
+        end
+      end
+
+      # Returns true if the index entry has the special file mode that denotes
+      # a submodule.
+      def submodule?(index_entry)
+        index_entry[:mode] == 57344
+      end
+
+      # Return a Rugged::Index that has read from the tree at +ref_name+
+      def populated_index(ref_name)
+        commit = rev_parse_target(ref_name)
+        index = rugged.index
+        index.read_tree(commit.tree)
+        index
+      end
+
+      # Return an array of BlobSnippets for lines in +file_contents+ that match
+      # +query+
+      def build_greps(file_contents, query, ref, filename)
+        # The file_contents string is potentially huge so we make sure to loop
+        # through it one line at a time. This gives Ruby the chance to GC lines
+        # we are not interested in.
+        #
+        # We need to do a little extra work because we are not looking for just
+        # the lines that matches the query, but also for the context
+        # (surrounding lines). We will use Enumerable#each_cons to efficiently
+        # loop through the lines while keeping surrounding lines on hand.
+        #
+        # First, we turn "foo\nbar\nbaz" into
+        # [
+        #  [nil, -3], [nil, -2], [nil, -1],
+        #  ['foo', 0], ['bar', 1], ['baz', 3],
+        #  [nil, 4], [nil, 5], [nil, 6]
+        # ]
+        lines_with_index = Enumerator.new do |yielder|
+          # Yield fake 'before' lines for the first line of file_contents
+          (-SEARCH_CONTEXT_LINES..-1).each do |i|
+            yielder.yield [nil, i]
+          end
+
+          # Yield the actual file contents
+          count = 0
+          file_contents.each_line do |line|
+            line.chomp!
+            yielder.yield [line, count]
+            count += 1
+          end
+
+          # Yield fake 'after' lines for the last line of file_contents
+          (count + 1..count + SEARCH_CONTEXT_LINES).each do |i|
+            yielder.yield [nil, i]
+          end
+        end
+
+        greps = []
+
+        # Loop through consecutive blocks of lines with indexes
+        lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block|
+          # Get the 'middle' line and index from the block
+          line, _ = line_block[SEARCH_CONTEXT_LINES]
+
+          next unless line && line.match(/#{Regexp.escape(query)}/i)
+
+          # Yay, 'line' contains a match!
+          # Get an array with just the context lines (no indexes)
+          match_with_context = line_block.map(&:first)
+          # Remove 'nil' lines in case we are close to the first or last line
+          match_with_context.compact!
+
+          # Get the line number (1-indexed) of the first context line
+          first_context_line_number = line_block[0][1] + 1
+
+          greps << Gitlab::Git::BlobSnippet.new(
+            ref,
+            match_with_context,
+            first_context_line_number,
+            filename
+          )
+        end
+
+        greps
+      end
+
+      # Return the Rugged patches for the diff between +from+ and +to+.
+      def diff_patches(from, to, options = {}, *paths)
+        options ||= {}
+        break_rewrites = options[:break_rewrites]
+        actual_options = Gitlab::Git::Diff.filter_diff_options(options.merge(paths: paths))
+
+        diff = rugged.diff(from, to, actual_options)
+        diff.find_similar!(break_rewrites: break_rewrites)
+        diff.each_patch
+      end
+
+      def sort_branches(branches, sort_by)
+        case sort_by
+        when 'name'
+          branches.sort_by(&:name)
+        when 'updated_desc'
+          branches.sort do |a, b|
+            b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date
+          end
+        when 'updated_asc'
+          branches.sort do |a, b|
+            a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date
+          end
+        else
+          branches
+        end
+      end
+
+      def gitaly_ref_client
+        @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self)
+      end
+
+      def gitaly_commit_client
+        @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self)
+      end
+
+      def gitaly_migrate(method, &block)
+        Gitlab::GitalyClient.migrate(method, &block)
+      rescue GRPC::NotFound => e
+        raise NoRepository.new(e)
+      rescue GRPC::BadStatus => e
+        raise CommandError.new(e)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
new file mode 100644
index 00000000000..a16b0ed76f4
--- /dev/null
+++ b/lib/gitlab/git/rev_list.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module Git
+    class RevList
+      attr_reader :oldrev, :newrev, :path_to_repo
+
+      def initialize(path_to_repo:, newrev:, oldrev: nil)
+        @oldrev = oldrev
+        @newrev = newrev
+        @path_to_repo = path_to_repo
+      end
+
+      # This method returns an array of new references
+      def new_refs
+        execute([*base_args, newrev, '--not', '--all'])
+      end
+
+      # This methods returns an array of missed references
+      def missed_ref
+        execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"])
+      end
+
+      private
+
+      def execute(args)
+        output, status = Gitlab::Popen.popen(args, nil, Gitlab::Git::Env.all.stringify_keys)
+
+        unless status.zero?
+          raise "Got a non-zero exit code while calling out `#{args.join(' ')}`."
+        end
+
+        output.split("\n")
+      end
+
+      def base_args
+        [
+          Gitlab.config.git.bin_path,
+          "--git-dir=#{path_to_repo}",
+          'rev-list'
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb
new file mode 100644
index 00000000000..b5342c3d310
--- /dev/null
+++ b/lib/gitlab/git/tag.rb
@@ -0,0 +1,17 @@
+module Gitlab
+  module Git
+    class Tag < Ref
+      attr_reader :object_sha
+
+      def initialize(repository, name, target, message = nil)
+        super(repository, name, target)
+
+        @message = message
+      end
+
+      def message
+        encode! @message
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
new file mode 100644
index 00000000000..b9afa05c819
--- /dev/null
+++ b/lib/gitlab/git/tree.rb
@@ -0,0 +1,104 @@
+module Gitlab
+  module Git
+    class Tree
+      include Gitlab::EncodingHelper
+
+      attr_accessor :id, :root_id, :name, :path, :type,
+        :mode, :commit_id, :submodule_url
+
+      class << self
+        # Get list of tree objects
+        # for repository based on commit sha and path
+        # Uses rugged for raw objects
+        def where(repository, sha, path = nil)
+          path = nil if path == '' || path == '/'
+
+          commit = repository.lookup(sha)
+          root_tree = commit.tree
+
+          tree = if path
+                   id = find_id_by_path(repository, root_tree.oid, path)
+                   if id
+                     repository.lookup(id)
+                   else
+                     []
+                   end
+                 else
+                   root_tree
+                 end
+
+          tree.map do |entry|
+            new(
+              id: entry[:oid],
+              root_id: root_tree.oid,
+              name: entry[:name],
+              type: entry[:type],
+              mode: entry[:filemode].to_s(8),
+              path: path ? File.join(path, entry[:name]) : entry[:name],
+              commit_id: sha
+            )
+          end
+        end
+
+        # Recursive search of tree id for path
+        #
+        # Ex.
+        #   blog/            # oid: 1a
+        #     app/           # oid: 2a
+        #       models/      # oid: 3a
+        #       views/       # oid: 4a
+        #
+        #
+        # Tree.find_id_by_path(repo, '1a', 'app/models') # => '3a'
+        #
+        def find_id_by_path(repository, root_id, path)
+          root_tree = repository.lookup(root_id)
+          path_arr = path.split('/')
+
+          entry = root_tree.find do |entry|
+            entry[:name] == path_arr[0] && entry[:type] == :tree
+          end
+
+          return nil unless entry
+
+          if path_arr.size > 1
+            path_arr.shift
+            find_id_by_path(repository, entry[:oid], path_arr.join('/'))
+          else
+            entry[:oid]
+          end
+        end
+      end
+
+      def initialize(options)
+        %w(id root_id name path type mode commit_id).each do |key|
+          self.send("#{key}=", options[key.to_sym])
+        end
+      end
+
+      def name
+        encode! @name
+      end
+
+      def dir?
+        type == :tree
+      end
+
+      def file?
+        type == :blob
+      end
+
+      def submodule?
+        type == :commit
+      end
+
+      def readme?
+        name =~ /^readme/i
+      end
+
+      def contributing?
+        name =~ /^contributing/i
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb
new file mode 100644
index 00000000000..7973da2e8f8
--- /dev/null
+++ b/lib/gitlab/git/util.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module Git
+    module Util
+      LINE_SEP = "\n".freeze
+
+      def self.count_lines(string)
+        case string[-1]
+        when nil
+          0
+        when LINE_SEP
+          string.count(LINE_SEP)
+        else
+          string.count(LINE_SEP) + 1
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 799794c0171..0b62911958d 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -2,132 +2,243 @@
 # class return an instance of `GitlabAccessStatus`
 module Gitlab
   class GitAccess
-    DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }
-    PUSH_COMMANDS = %w{ git-receive-pack }
+    UnauthorizedError = Class.new(StandardError)
+    NotFoundError = Class.new(StandardError)
 
-    attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities
+    ERROR_MESSAGES = {
+      upload: 'You are not allowed to upload code for this project.',
+      download: 'You are not allowed to download code from this project.',
+      deploy_key_upload:
+        'This deploy key does not have write access to this project.',
+      no_repo: 'A repository for this project does not exist yet.',
+      project_not_found: 'The project you were looking for could not be found.',
+      account_blocked: 'Your account has been blocked.',
+      command_not_allowed: "The command you're trying to execute is not allowed.",
+      upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.',
+      receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.'
+    }.freeze
 
-    def initialize(actor, project, protocol, authentication_abilities:)
+    DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze
+    PUSH_COMMANDS = %w{ git-receive-pack }.freeze
+    ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS
+
+    attr_reader :actor, :project, :protocol, :authentication_abilities, :redirected_path
+
+    def initialize(actor, project, protocol, authentication_abilities:, redirected_path: nil)
       @actor    = actor
       @project  = project
       @protocol = protocol
+      @redirected_path = redirected_path
       @authentication_abilities = authentication_abilities
-      @user_access = UserAccess.new(user, project: project)
     end
 
     def check(cmd, changes)
-      return build_status_object(false, "Git access over #{protocol.upcase} is not allowed") unless protocol_allowed?
+      check_protocol!
+      check_active_user!
+      check_project_accessibility!
+      check_project_moved!
+      check_command_disabled!(cmd)
+      check_command_existence!(cmd)
+      check_repository_existence!
+
+      case cmd
+      when *DOWNLOAD_COMMANDS
+        check_download_access!
+      when *PUSH_COMMANDS
+        check_push_access!(changes)
+      end
+
+      true
+    end
+
+    def guest_can_download_code?
+      Guest.can?(:download_code, project)
+    end
+
+    def user_can_download_code?
+      authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code)
+    end
+
+    def build_can_download_code?
+      authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code)
+    end
+
+    def protocol_allowed?
+      Gitlab::ProtocolAccess.allowed?(protocol)
+    end
+
+    private
 
-      unless actor
-        return build_status_object(false, "No user or key was provided.")
+    def check_protocol!
+      unless protocol_allowed?
+        raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed"
       end
+    end
+
+    def check_active_user!
+      return if deploy_key?
 
       if user && !user_access.allowed?
-        return build_status_object(false, "Your account has been blocked.")
+        raise UnauthorizedError, ERROR_MESSAGES[:account_blocked]
       end
+    end
 
-      unless project && (user_access.can_read_project? || deploy_key_can_read_project?)
-        return build_status_object(false, 'The project you were looking for could not be found.')
+    def check_project_accessibility!
+      if project.blank? || !can_read_project?
+        raise NotFoundError, ERROR_MESSAGES[:project_not_found]
       end
+    end
 
-      case cmd
-      when *DOWNLOAD_COMMANDS
-        download_access_check
-      when *PUSH_COMMANDS
-        push_access_check(changes)
-      else
-        build_status_object(false, "The command you're trying to execute is not allowed.")
+    def check_project_moved!
+      if redirected_path
+        url = protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo
+        message = <<-MESSAGE.strip_heredoc
+          Project '#{redirected_path}' was moved to '#{project.full_path}'.
+
+          Please update your Git remote and try again:
+
+            git remote set-url origin #{url}
+        MESSAGE
+
+        raise NotFoundError, message
       end
     end
 
-    def download_access_check
-      if user
-        user_download_access_check
-      elsif deploy_key
-        build_status_object(true)
-      else
-        raise 'Wrong actor'
+    def check_command_disabled!(cmd)
+      if upload_pack?(cmd)
+        check_upload_pack_disabled!
+      elsif receive_pack?(cmd)
+        check_receive_pack_disabled!
       end
     end
 
-    def push_access_check(changes)
-      if user
-        user_push_access_check(changes)
-      elsif deploy_key
-        build_status_object(false, "Deploy keys are not allowed to push code.")
-      else
-        raise 'Wrong actor'
+    def check_upload_pack_disabled!
+      if http? && upload_pack_disabled_over_http?
+        raise UnauthorizedError, ERROR_MESSAGES[:upload_pack_disabled_over_http]
       end
     end
 
-    def user_download_access_check
-      unless user_can_download_code? || build_can_download_code?
-        return build_status_object(false, "You are not allowed to download code from this project.")
+    def check_receive_pack_disabled!
+      if http? && receive_pack_disabled_over_http?
+        raise UnauthorizedError, ERROR_MESSAGES[:receive_pack_disabled_over_http]
       end
+    end
 
-      build_status_object(true)
+    def check_command_existence!(cmd)
+      unless ALL_COMMANDS.include?(cmd)
+        raise UnauthorizedError, ERROR_MESSAGES[:command_not_allowed]
+      end
     end
 
-    def user_can_download_code?
-      authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code)
+    def check_repository_existence!
+      unless project.repository.exists?
+        raise UnauthorizedError, ERROR_MESSAGES[:no_repo]
+      end
     end
 
-    def build_can_download_code?
-      authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code)
+    def check_download_access!
+      return if deploy_key?
+
+      passed = user_can_download_code? ||
+        build_can_download_code? ||
+        guest_can_download_code?
+
+      unless passed
+        raise UnauthorizedError, ERROR_MESSAGES[:download]
+      end
     end
 
-    def user_push_access_check(changes)
-      unless authentication_abilities.include?(:push_code)
-        return build_status_object(false, "You are not allowed to upload code for this project.")
+    def check_push_access!(changes)
+      if deploy_key
+        check_deploy_key_push_access!
+      elsif user
+        check_user_push_access!
+      else
+        raise UnauthorizedError, ERROR_MESSAGES[:upload]
       end
 
-      if changes.blank?
-        return build_status_object(true)
+      return if changes.blank? # Allow access.
+
+      check_change_access!(changes)
+    end
+
+    def check_user_push_access!
+      unless authentication_abilities.include?(:push_code)
+        raise UnauthorizedError, ERROR_MESSAGES[:upload]
       end
+    end
 
-      unless project.repository.exists?
-        return build_status_object(false, "A repository for this project does not exist yet.")
+    def check_deploy_key_push_access!
+      unless deploy_key.can_push_to?(project)
+        raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload]
       end
+    end
 
+    def check_change_access!(changes)
       changes_list = Gitlab::ChangesList.new(changes)
 
       # Iterate over all changes to find if user allowed all of them to be applied
       changes_list.each do |change|
-        status = change_access_check(change)
-        unless status.allowed?
-          # If user does not have access to make at least one change - cancel all push
-          return status
-        end
+        # If user does not have access to make at least one change, cancel all
+        # push by allowing the exception to bubble up
+        check_single_change_access(change)
       end
+    end
+
+    def check_single_change_access(change)
+      Checks::ChangeAccess.new(
+        change,
+        user_access: user_access,
+        project: project,
+        skip_authorization: deploy_key?,
+        protocol: protocol
+      ).exec
+    end
 
-      build_status_object(true)
+    def matching_merge_request?(newrev, branch_name)
+      Checks::MatchingMergeRequest.new(newrev, branch_name, project).match?
     end
 
-    def change_access_check(change)
-      Checks::ChangeAccess.new(change, user_access: user_access, project: project).exec
+    def deploy_key
+      actor if deploy_key?
     end
 
-    def protocol_allowed?
-      Gitlab::ProtocolAccess.allowed?(protocol)
+    def deploy_key?
+      actor.is_a?(DeployKey)
     end
 
-    private
+    def ci?
+      actor == :ci
+    end
 
-    def matching_merge_request?(newrev, branch_name)
-      Checks::MatchingMergeRequest.new(newrev, branch_name, project).match?
+    def can_read_project?
+      if deploy_key?
+        deploy_key.has_access_to?(project)
+      elsif user
+        user.can?(:read_project, project)
+      elsif ci?
+        true # allow CI (build without a user) for backwards compatibility
+      end || Guest.can?(:read_project, project)
     end
 
-    def deploy_key
-      actor if actor.is_a?(DeployKey)
+    def http?
+      protocol == 'http'
     end
 
-    def deploy_key_can_read_project?
-      if deploy_key
-        return true if project.public?
-        deploy_key.projects.include?(project)
-      else
-        false
-      end
+    def upload_pack?(command)
+      command == 'git-upload-pack'
+    end
+
+    def receive_pack?(command)
+      command == 'git-receive-pack'
+    end
+
+    def upload_pack_disabled_over_http?
+      !Gitlab.config.gitlab_shell.upload_pack
+    end
+
+    def receive_pack_disabled_over_http?
+      !Gitlab.config.gitlab_shell.receive_pack
     end
 
     protected
@@ -139,15 +250,19 @@ module Gitlab
         case actor
         when User
           actor
-        when DeployKey
-          nil
         when Key
-          actor.user
+          actor.user unless actor.is_a?(DeployKey)
+        when :ci
+          nil
         end
     end
 
-    def build_status_object(status, message = '')
-      Gitlab::GitAccessStatus.new(status, message)
+    def user_access
+      @user_access ||= if ci?
+                         CiAccess.new
+                       else
+                         UserAccess.new(user, project: project)
+                       end
     end
   end
 end
diff --git a/lib/gitlab/git_access_status.rb b/lib/gitlab/git_access_status.rb
deleted file mode 100644
index 09bb01be694..00000000000
--- a/lib/gitlab/git_access_status.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-module Gitlab
-  class GitAccessStatus
-    attr_accessor :status, :message
-    alias_method :allowed?, :status
-
-    def initialize(status, message = '')
-      @status = status
-      @message = message
-    end
-
-    def to_json(opts = nil)
-      { status: @status, message: @message }.to_json(opts)
-    end
-  end
-end
diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb
index f71d3575909..1fe5155c093 100644
--- a/lib/gitlab/git_access_wiki.rb
+++ b/lib/gitlab/git_access_wiki.rb
@@ -1,11 +1,23 @@
 module Gitlab
   class GitAccessWiki < GitAccess
-    def change_access_check(change)
-      if user_access.can_do_action?(:create_wiki)
-        build_status_object(true)
-      else
-        build_status_object(false, "You are not allowed to write to this project's wiki.")
+    ERROR_MESSAGES = {
+      write_to_wiki: "You are not allowed to write to this project's wiki."
+    }.freeze
+
+    def guest_can_download_code?
+      Guest.can?(:download_wiki_code, project)
+    end
+
+    def user_can_download_code?
+      authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code)
+    end
+
+    def check_single_change_access(change)
+      unless user_access.can_do_action?(:create_wiki)
+        raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki]
       end
+
+      true
     end
   end
 end
diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb
index d32bdd86427..742118b76a8 100644
--- a/lib/gitlab/git_post_receive.rb
+++ b/lib/gitlab/git_post_receive.rb
@@ -1,43 +1,30 @@
 module Gitlab
   class GitPostReceive
     include Gitlab::Identifier
-    attr_reader :repo_path, :identifier, :changes, :project
+    attr_reader :project, :identifier, :changes
 
-    def initialize(repo_path, identifier, changes)
-      repo_path.gsub!(/\.git\z/, '')
-      repo_path.gsub!(/\A\//, '')
-
-      @repo_path = repo_path
+    def initialize(project, identifier, changes)
+      @project = project
       @identifier = identifier
       @changes = deserialize_changes(changes)
-
-      retrieve_project_and_type
-    end
-
-    def wiki?
-      @type == :wiki
-    end
-
-    def regular_project?
-      @type == :project
     end
 
     def identify(revision)
       super(identifier, project, revision)
     end
 
-    private
+    def changes_refs
+      return enum_for(:changes_refs) unless block_given?
 
-    def retrieve_project_and_type
-      @type = :project
-      @project = Project.find_with_namespace(@repo_path)
+      changes.each do |change|
+        oldrev, newrev, ref = change.strip.split(' ')
 
-      if @repo_path.end_with?('.wiki') && !@project
-        @type = :wiki
-        @project = Project.find_with_namespace(@repo_path.gsub(/\.wiki\z/, ''))
+        yield oldrev, newrev, ref
       end
     end
 
+    private
+
     def deserialize_changes(changes)
       changes = utf8_encode_changes(changes)
       changes.lines
diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb
index 4d83d8e72a8..0e87ee30c98 100644
--- a/lib/gitlab/git_ref_validator.rb
+++ b/lib/gitlab/git_ref_validator.rb
@@ -5,6 +5,9 @@ module Gitlab
     #
     # Returns true for a valid reference name, false otherwise
     def validate(ref_name)
+      return false if ref_name.start_with?('refs/heads/')
+      return false if ref_name.start_with?('refs/remotes/')
+
       Gitlab::Utils.system_silent(
         %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name}))
     end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
new file mode 100644
index 00000000000..f605c06dfc3
--- /dev/null
+++ b/lib/gitlab/gitaly_client.rb
@@ -0,0 +1,108 @@
+require 'base64'
+
+require 'gitaly'
+
+module Gitlab
+  module GitalyClient
+    module MigrationStatus
+      DISABLED = 1
+      OPT_IN = 2
+      OPT_OUT = 3
+    end
+
+    SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
+
+    MUTEX = Mutex.new
+    private_constant :MUTEX
+
+    def self.stub(name, storage)
+      MUTEX.synchronize do
+        @stubs ||= {}
+        @stubs[storage] ||= {}
+        @stubs[storage][name] ||= begin
+          klass = Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub)
+          addr = address(storage)
+          addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp'
+          klass.new(addr, :this_channel_is_insecure)
+        end
+      end
+    end
+
+    def self.clear_stubs!
+      MUTEX.synchronize do
+        @stubs = nil
+      end
+    end
+
+    def self.address(storage)
+      params = Gitlab.config.repositories.storages[storage]
+      raise "storage not found: #{storage.inspect}" if params.nil?
+
+      address = params['gitaly_address']
+      unless address.present?
+        raise "storage #{storage.inspect} is missing a gitaly_address"
+      end
+
+      unless URI(address).scheme.in?(%w(tcp unix))
+        raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'"
+      end
+
+      address
+    end
+
+    # All Gitaly RPC call sites should use GitalyClient.call. This method
+    # makes sure that per-request authentication headers are set.
+    def self.call(storage, service, rpc, request)
+      metadata = request_metadata(storage)
+      metadata = yield(metadata) if block_given?
+      stub(service, storage).send(rpc, request, metadata)
+    end
+  
+    def self.request_metadata(storage)
+      encoded_token = Base64.strict_encode64(token(storage).to_s)
+      { metadata: { 'authorization' => "Bearer #{encoded_token}" } }
+    end
+
+    def self.token(storage)
+      params = Gitlab.config.repositories.storages[storage]
+      raise "storage not found: #{storage.inspect}" if params.nil?
+
+      params['gitaly_token'].presence || Gitlab.config.gitaly['token']
+    end
+
+    def self.enabled?
+      Gitlab.config.gitaly.enabled
+    end
+
+    def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN)
+      return false if !enabled? || status == MigrationStatus::DISABLED
+
+      feature = Feature.get("gitaly_#{feature}")
+
+      # If the feature hasn't been set, turn it on if it's opt-out
+      return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature)
+
+      if feature.percentage_of_time_value > 0
+        # Probabilistically enable this feature
+        return Random.rand() * 100 < feature.percentage_of_time_value
+      end
+
+      feature.enabled?
+    end
+
+    def self.migrate(feature)
+      is_enabled  = feature_enabled?(feature)
+      metric_name = feature.to_s
+      metric_name += "_gitaly" if is_enabled
+
+      Gitlab::Metrics.measure(metric_name) do
+        yield is_enabled
+      end
+    end
+
+    def self.expected_server_version
+      path = Rails.root.join(SERVER_VERSION_FILE)
+      path.read.chomp
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb
new file mode 100644
index 00000000000..b8877619797
--- /dev/null
+++ b/lib/gitlab/gitaly_client/commit.rb
@@ -0,0 +1,73 @@
+module Gitlab
+  module GitalyClient
+    class Commit
+      # The ID of empty tree.
+      # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012
+      EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze
+
+      def initialize(repository)
+        @gitaly_repo = repository.gitaly_repository
+        @repository = repository
+      end
+
+      def is_ancestor(ancestor_id, child_id)
+        request = Gitaly::CommitIsAncestorRequest.new(
+          repository: @gitaly_repo,
+          ancestor_id: ancestor_id,
+          child_id: child_id
+        )
+
+        GitalyClient.call(@repository.storage, :commit, :commit_is_ancestor, request).value
+      end
+
+      def diff_from_parent(commit, options = {})
+        request_params = commit_diff_request_params(commit, options)
+        request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false)
+        request = Gitaly::CommitDiffRequest.new(request_params)
+        response = GitalyClient.call(@repository.storage, :diff, :commit_diff, request)
+        Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options)
+      end
+
+      def commit_deltas(commit)
+        request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit))
+        response = GitalyClient.call(@repository.storage, :diff, :commit_delta, request)
+        response.flat_map do |msg|
+          msg.deltas.map { |d| Gitlab::Git::Diff.new(d) }
+        end
+      end
+
+      def tree_entry(ref, path, limit = nil)
+        request = Gitaly::TreeEntryRequest.new(
+          repository: @gitaly_repo,
+          revision: ref,
+          path: path.dup.force_encoding(Encoding::ASCII_8BIT),
+          limit: limit.to_i
+        )
+
+        response = GitalyClient.call(@repository.storage, :commit, :tree_entry, request)
+        entry = response.first
+        return unless entry.oid.present?
+
+        if entry.type == :BLOB
+          rest_of_data = response.reduce("") { |memo, msg| memo << msg.data }
+          entry.data += rest_of_data
+        end
+
+        entry
+      end
+
+      private
+
+      def commit_diff_request_params(commit, options = {})
+        parent_id = commit.parents[0]&.id || EMPTY_TREE_ID
+
+        {
+          repository: @gitaly_repo,
+          left_commit_id: parent_id,
+          right_commit_id: commit.id,
+          paths: options.fetch(:paths, [])
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb
new file mode 100644
index 00000000000..1e117b7e74a
--- /dev/null
+++ b/lib/gitlab/gitaly_client/diff.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module GitalyClient
+    class Diff
+      FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch).freeze
+
+      attr_accessor(*FIELDS)
+
+      def initialize(params)
+        params.each do |key, val|
+          public_send(:"#{key}=", val)
+        end
+      end
+
+      def ==(other)
+        FIELDS.all? do |field|
+          public_send(field) == other.public_send(field)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb
new file mode 100644
index 00000000000..65d81dc5d46
--- /dev/null
+++ b/lib/gitlab/gitaly_client/diff_stitcher.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module GitalyClient
+    class DiffStitcher
+      include Enumerable
+
+      def initialize(rpc_response)
+        @rpc_response = rpc_response
+      end
+
+      def each
+        current_diff = nil
+
+        @rpc_response.each do |diff_msg|
+          if current_diff.nil?
+            diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS)
+            # gRPC uses frozen strings by default, and we need to have an unfrozen string as it
+            # gets processed further down the line. So we unfreeze the first chunk of the patch
+            # in case it's the only chunk we receive for this diff.
+            diff_params[:patch] = diff_msg.raw_patch_data.dup
+
+            current_diff = GitalyClient::Diff.new(diff_params)
+          else
+            current_diff.patch += diff_msg.raw_patch_data
+          end
+
+          if diff_msg.end_of_patch
+            yield current_diff
+            current_diff = nil
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notifications.rb
new file mode 100644
index 00000000000..78ed433e6b8
--- /dev/null
+++ b/lib/gitlab/gitaly_client/notifications.rb
@@ -0,0 +1,20 @@
+module Gitlab
+  module GitalyClient
+    class Notifications
+      # 'repository' is a Gitlab::Git::Repository
+      def initialize(repository)
+        @gitaly_repo = repository.gitaly_repository
+        @storage = repository.storage
+      end
+
+      def post_receive
+        GitalyClient.call(
+          @storage,
+          :notifications,
+          :post_receive,
+          Gitaly::PostReceiveRequest.new(repository: @gitaly_repo)
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb
new file mode 100644
index 00000000000..6d5f54dd959
--- /dev/null
+++ b/lib/gitlab/gitaly_client/ref.rb
@@ -0,0 +1,71 @@
+module Gitlab
+  module GitalyClient
+    class Ref
+      # 'repository' is a Gitlab::Git::Repository
+      def initialize(repository)
+        @gitaly_repo = repository.gitaly_repository
+        @storage = repository.storage
+      end
+
+      def default_branch_name
+        request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo)
+        response = GitalyClient.call(@storage, :ref, :find_default_branch_name, request)
+        Gitlab::Git.branch_name(response.name)
+      end
+
+      def branch_names
+        request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo)
+        response = GitalyClient.call(@storage, :ref, :find_all_branch_names, request)
+        consume_refs_response(response, prefix: 'refs/heads/')
+      end
+
+      def tag_names
+        request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo)
+        response = GitalyClient.call(@storage, :ref, :find_all_tag_names, request)
+        consume_refs_response(response, prefix: 'refs/tags/')
+      end
+
+      def find_ref_name(commit_id, ref_prefix)
+        request = Gitaly::FindRefNameRequest.new(
+          repository: @gitaly_repo,
+          commit_id: commit_id,
+          prefix: ref_prefix
+        )
+        GitalyClient.call(@storage, :ref, :find_ref_name, request).name
+      end
+
+      def count_tag_names
+        tag_names.count
+      end
+
+      def count_branch_names
+        branch_names.count
+      end
+
+      def local_branches(sort_by: nil)
+        request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo)
+        request.sort_by = sort_by_param(sort_by) if sort_by
+        response = GitalyClient.call(@storage, :ref, :find_local_branches, request)
+        consume_branches_response(response)
+      end
+
+      private
+
+      def consume_refs_response(response, prefix:)
+        response.flat_map do |r|
+          r.names.map { |name| name.sub(/\A#{Regexp.escape(prefix)}/, '') }
+        end
+      end
+
+      def sort_by_param(sort_by)
+        enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym)
+        raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value
+        enum_value
+      end
+
+      def consume_branches_response(response)
+        response.flat_map { |r| r.branches }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb
new file mode 100644
index 00000000000..f5a4c5493ef
--- /dev/null
+++ b/lib/gitlab/gitaly_client/util.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module GitalyClient
+    module Util
+      class << self
+        def repository(repository_storage, relative_path)
+          Gitaly::Repository.new(
+            storage_name: repository_storage,
+            relative_path: relative_path
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb
index 8cacf4f4925..8c80791e7c9 100644
--- a/lib/gitlab/github_import/base_formatter.rb
+++ b/lib/gitlab/github_import/base_formatter.rb
@@ -1,29 +1,23 @@
 module Gitlab
   module GithubImport
     class BaseFormatter
-      attr_reader :formatter, :project, :raw_data
+      attr_reader :client, :formatter, :project, :raw_data
 
-      def initialize(project, raw_data)
+      def initialize(project, raw_data, client = nil)
         @project = project
         @raw_data = raw_data
+        @client = client
         @formatter = Gitlab::ImportFormatter.new
       end
 
       def create!
-        self.klass.create!(self.attributes)
+        project.public_send(project_association).find_or_create_by!(find_condition) do |record|
+          record.attributes = attributes
+        end
       end
 
-      private
-
-      def gitlab_user_id(github_id)
-        User.joins(:identities).
-          find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s).
-          try(:id)
-      end
-
-      def gitlab_author_id
-        return @gitlab_author_id if defined?(@gitlab_author_id)
-        @gitlab_author_id = gitlab_user_id(raw_data.user.id)
+      def url
+        raw_data.url || ''
       end
     end
   end
diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb
index 4750675ae9d..8aa885fb811 100644
--- a/lib/gitlab/github_import/branch_formatter.rb
+++ b/lib/gitlab/github_import/branch_formatter.rb
@@ -8,7 +8,15 @@ module Gitlab
       end
 
       def valid?
-        repo.present?
+        sha.present? && ref.present?
+      end
+
+      def user
+        raw_data.user&.login || 'unknown'
+      end
+
+      def short_sha
+        Commit.truncate_sha(sha)
       end
 
       private
@@ -18,7 +26,7 @@ module Gitlab
       end
 
       def commit_exists?
-        project.repository.commit(sha).present?
+        project.repository.branch_names_contains(sha).include?(ref)
       end
 
       def short_id
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index 7f424b74efb..7dbeec5b010 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -4,10 +4,13 @@ module Gitlab
       GITHUB_SAFE_REMAINING_REQUESTS = 100
       GITHUB_SAFE_SLEEP_TIME = 500
 
-      attr_reader :access_token
+      attr_reader :access_token, :host, :api_version
 
-      def initialize(access_token)
+      def initialize(access_token, host: nil, api_version: 'v3')
         @access_token = access_token
+        @host = host.to_s.sub(%r{/+\z}, '')
+        @api_version = api_version
+        @users = {}
 
         if access_token
           ::Octokit.auto_paginate = false
@@ -17,7 +20,7 @@ module Gitlab
       def api
         @api ||= ::Octokit::Client.new(
           access_token: access_token,
-          api_endpoint: github_options[:site],
+          api_endpoint: api_endpoint,
           # If there is no config, we're connecting to github.com and we
           # should verify ssl.
           connection_options: {
@@ -62,8 +65,23 @@ module Gitlab
         api.respond_to?(method) || super
       end
 
+      def user(login)
+        return nil unless login.present?
+        return @users[login] if @users.key?(login)
+
+        @users[login] = api.user(login)
+      end
+
       private
 
+      def api_endpoint
+        if host.present? && api_version.present?
+          "#{host}/api/#{api_version}"
+        else
+          github_options[:site]
+        end
+      end
+
       def config
         Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" }
       end
@@ -105,18 +123,20 @@ module Gitlab
         data = api.send(method, *args)
         return data unless data.is_a?(Array)
 
+        last_response = api.last_response
+
         if block_given?
           yield data
-          each_response_page(&block)
+          # api.last_response could change while we're yielding (e.g. fetching labels for each PR)
+          # so we cache our own last response
+          each_response_page(last_response, &block)
         else
-          each_response_page { |page| data.concat(page) }
+          each_response_page(last_response) { |page| data.concat(page) }
           data
         end
       end
 
-      def each_response_page
-        last_response = api.last_response
-
+      def each_response_page(last_response)
         while last_response.rels[:next]
           sleep rate_limit_sleep_time if rate_limit_exceed?
           last_response = last_response.rels[:next].get
diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb
index 2bddcde2b7c..e21922070c1 100644
--- a/lib/gitlab/github_import/comment_formatter.rb
+++ b/lib/gitlab/github_import/comment_formatter.rb
@@ -1,6 +1,8 @@
 module Gitlab
   module GithubImport
     class CommentFormatter < BaseFormatter
+      attr_writer :author_id
+
       def attributes
         {
           project: project,
@@ -17,11 +19,11 @@ module Gitlab
       private
 
       def author
-        raw_data.user.login
+        @author ||= UserFormatter.new(client, raw_data.user)
       end
 
       def author_id
-        gitlab_author_id || project.creator_id
+        author.gitlab_id || project.creator_id
       end
 
       def body
@@ -52,10 +54,10 @@ module Gitlab
       end
 
       def note
-        if gitlab_author_id
+        if author.gitlab_id
           body
         else
-          formatter.author_line(author) + body
+          formatter.author_line(author.login) + body
         end
       end
 
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 4b70f33a851..a8c0b47e786 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -3,7 +3,7 @@ module Gitlab
     class Importer
       include Gitlab::ShellAdapter
 
-      attr_reader :client, :errors, :project, :repo, :repo_url
+      attr_reader :errors, :project, :repo, :repo_url
 
       def initialize(project)
         @project  = project
@@ -11,22 +11,52 @@ module Gitlab
         @repo_url = project.import_url
         @errors   = []
         @labels   = {}
+      end
+
+      def client
+        return @client if defined?(@client)
+        unless credentials
+          raise Projects::ImportService::Error,
+                "Unable to find project import data credentials for project ID: #{@project.id}"
+        end
 
-        if credentials
-          @client = Client.new(credentials[:user])
-        else
-          raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}"
+        opts = {}
+        # Gitea plan to be GitHub compliant
+        if project.gitea_import?
+          uri = URI.parse(project.import_url)
+          host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '')
+          opts = {
+            host: host,
+            api_version: 'v1'
+          }
         end
+
+        @client = Client.new(credentials[:user], opts)
       end
 
       def execute
+        # The ordering of importing is important here due to the way GitHub structures their data
+        # 1. Labels are required by other items while not having a dependency on anything else
+        # so need to be first
+        # 2. Pull requests must come before issues. Every pull request is also an issue but not
+        # all issues are pull requests. Only the issue entity has labels defined in GitHub. GitLab
+        # doesn't structure data like this so we need to make sure that we've created the MRs
+        # before we attempt to add the labels defined in the GitHub issue for the related, already
+        # imported, pull request
         import_labels
         import_milestones
-        import_issues
         import_pull_requests
-        import_comments
+        import_issues
+        import_comments(:issues)
+        import_comments(:pull_requests)
         import_wiki
-        import_releases
+
+        # Gitea doesn't have a Release API yet
+        # See https://github.com/go-gitea/gitea/issues/330
+        unless project.gitea_import?
+          import_releases
+        end
+
         handle_errors
 
         true
@@ -35,7 +65,9 @@ module Gitlab
       private
 
       def credentials
-        @credentials ||= project.import_data.credentials if project.import_data
+        return @credentials if defined?(@credentials)
+
+        @credentials = project.import_data ? project.import_data.credentials : nil
       end
 
       def handle_errors
@@ -48,63 +80,75 @@ module Gitlab
       end
 
       def import_labels
-        client.labels(repo, per_page: 100) do |labels|
+        fetch_resources(:labels, repo, per_page: 100) do |labels|
           labels.each do |raw|
             begin
-              label = LabelFormatter.new(project, raw).create!
-              @labels[label.title] = label.id
+              gh_label = LabelFormatter.new(project, raw)
+              gh_label.create!
             rescue => e
-              errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+              errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message }
             end
           end
         end
+
+        cache_labels!
       end
 
       def import_milestones
-        client.milestones(repo, state: :all, per_page: 100) do |milestones|
+        fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones|
           milestones.each do |raw|
             begin
-              MilestoneFormatter.new(project, raw).create!
+              gh_milestone = MilestoneFormatter.new(project, raw)
+              gh_milestone.create!
             rescue => e
-              errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+              errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message }
             end
           end
         end
       end
 
       def import_issues
-        client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues|
+        fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues|
           issues.each do |raw|
-            gh_issue = IssueFormatter.new(project, raw)
-
-            if gh_issue.valid?
-              begin
-                issue = gh_issue.create!
-                apply_labels(issue, raw)
-              rescue => e
-                errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
-              end
+            gh_issue = IssueFormatter.new(project, raw, client)
+
+            begin
+              issuable =
+                if gh_issue.pull_request?
+                  MergeRequest.find_by(target_project_id: project.id, iid: gh_issue.number)
+                else
+                  gh_issue.create!
+                end
+
+              apply_labels(issuable, raw)
+            rescue => e
+              errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message }
             end
           end
         end
       end
 
       def import_pull_requests
-        client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests|
+        fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests|
           pull_requests.each do |raw|
-            pull_request = PullRequestFormatter.new(project, raw)
-            next unless pull_request.valid?
+            gh_pull_request = PullRequestFormatter.new(project, raw, client)
+
+            next unless gh_pull_request.valid?
 
             begin
-              restore_source_branch(pull_request) unless pull_request.source_branch_exists?
-              restore_target_branch(pull_request) unless pull_request.target_branch_exists?
+              restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists?
+              restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists?
 
-              merge_request = pull_request.create!
-              apply_labels(merge_request, raw)
+              merge_request = gh_pull_request.create!
+
+              # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage
+              if project.gitea_import?
+                apply_labels(merge_request, raw)
+              end
             rescue => e
-              errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message }
+              errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message }
             ensure
-              clean_up_restored_branches(pull_request)
+              clean_up_restored_branches(gh_pull_request)
             end
           end
         end
@@ -113,7 +157,7 @@ module Gitlab
       end
 
       def restore_source_branch(pull_request)
-        project.repository.fetch_ref(repo_url, "pull/#{pull_request.number}/head", pull_request.source_branch_name)
+        project.repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha)
       end
 
       def restore_target_branch(pull_request)
@@ -127,38 +171,58 @@ module Gitlab
       end
 
       def clean_up_restored_branches(pull_request)
+        return if pull_request.opened?
+
         remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists?
         remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists?
       end
 
-      def apply_labels(issuable, raw_issuable)
-        if raw_issuable.labels.count > 0
-          label_ids = raw_issuable.labels
-            .map { |attrs| @labels[attrs.name] }
-            .compact
+      def apply_labels(issuable, raw)
+        return unless raw.labels.count > 0
 
-          issuable.update_attribute(:label_ids, label_ids)
-        end
+        label_ids = raw.labels
+          .map { |attrs| @labels[attrs.name] }
+          .compact
+
+        issuable.update_attribute(:label_ids, label_ids)
       end
 
-      def import_comments
-        client.issues_comments(repo, per_page: 100) do |comments|
-          create_comments(comments, :issue)
-        end
+      def import_comments(issuable_type)
+        resource_type = "#{issuable_type}_comments".to_sym
+
+        # Two notes here:
+        # 1. We don't have a distinctive attribute for comments (unlike issues iid), so we fetch the last inserted note,
+        # compare it against every comment in the current imported page until we find match, and that's where start importing
+        # 2. GH returns comments for _both_ issues and PRs through issues_comments API, while pull_requests_comments returns
+        # only comments on diffs, so select last note not based on noteable_type but on line_code
+        line_code_is = issuable_type == :pull_requests ? 'NOT NULL' : 'NULL'
+        last_note    = project.notes.where("line_code IS #{line_code_is}").last
+
+        fetch_resources(resource_type, repo, per_page: 100) do |comments|
+          if last_note
+            discard_inserted_comments(comments, last_note)
+            last_note = nil
+          end
 
-        client.pull_requests_comments(repo, per_page: 100) do |comments|
-          create_comments(comments, :pull_request)
+          create_comments(comments)
         end
       end
 
-      def create_comments(comments, issuable_type)
+      def create_comments(comments)
         ActiveRecord::Base.no_touching do
           comments.each do |raw|
             begin
-              comment        = CommentFormatter.new(project, raw)
-              issuable_class = issuable_type == :issue ? Issue : MergeRequest
-              iid            = raw.send("#{issuable_type}_url").split('/').last # GH doesn't return parent ID directly
-              issuable       = issuable_class.find_by_iid(iid)
+              comment = CommentFormatter.new(project, raw, client)
+
+              # GH does not return info about comment's parent, so we guess it by checking its URL!
+              *_, parent, iid = URI(raw.html_url).path.split('/')
+
+              issuable = if parent == 'issues'
+                           Issue.find_by(project_id: project.id, iid: iid)
+                         else
+                           MergeRequest.find_by(target_project_id: project.id, iid: iid)
+                         end
+
               next unless issuable
 
               issuable.notes.create!(comment.attributes)
@@ -169,6 +233,24 @@ module Gitlab
         end
       end
 
+      def discard_inserted_comments(comments, last_note)
+        last_note_attrs = nil
+
+        cut_off_index = comments.find_index do |raw|
+          comment           = CommentFormatter.new(project, raw)
+          comment_attrs     = comment.attributes
+          last_note_attrs ||= last_note.slice(*comment_attrs.keys)
+
+          comment_attrs.with_indifferent_access == last_note_attrs
+        end
+
+        # No matching resource in the collection, which means we got halted right on the end of the last page, so all good
+        return unless cut_off_index
+
+        # Otherwise, remove the resources we've already inserted
+        comments.shift(cut_off_index + 1)
+      end
+
       def import_wiki
         unless project.wiki.repository_exists?
           wiki = WikiFormatter.new(project)
@@ -184,17 +266,64 @@ module Gitlab
       end
 
       def import_releases
-        client.releases(repo, per_page: 100) do |releases|
+        fetch_resources(:releases, repo, per_page: 100) do |releases|
           releases.each do |raw|
             begin
               gh_release = ReleaseFormatter.new(project, raw)
               gh_release.create! if gh_release.valid?
             rescue => e
-              errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message }
+              errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message }
             end
           end
         end
       end
+
+      def cache_labels!
+        project.labels.select(:id, :title).find_each do |label|
+          @labels[label.title] = label.id
+        end
+      end
+
+      def fetch_resources(resource_type, *opts)
+        return if imported?(resource_type)
+
+        opts.last[:page] = current_page(resource_type)
+
+        client.public_send(resource_type, *opts) do |resources|
+          yield resources
+          increment_page(resource_type)
+        end
+
+        imported!(resource_type)
+      end
+
+      def imported?(resource_type)
+        Rails.cache.read("#{cache_key_prefix}:#{resource_type}:imported")
+      end
+
+      def imported!(resource_type)
+        Rails.cache.write("#{cache_key_prefix}:#{resource_type}:imported", true, ex: 1.day)
+      end
+
+      def increment_page(resource_type)
+        key = "#{cache_key_prefix}:#{resource_type}:current-page"
+
+        # Rails.cache.increment calls INCRBY directly on the value stored under the key, which is
+        # a serialized ActiveSupport::Cache::Entry, so it will return an error by Redis, hence this ugly work-around
+        page = Rails.cache.read(key)
+        page += 1
+        Rails.cache.write(key, page)
+
+        page
+      end
+
+      def current_page(resource_type)
+        Rails.cache.fetch("#{cache_key_prefix}:#{resource_type}:current-page", ex: 1.day) { 1 }
+      end
+
+      def cache_key_prefix
+        @cache_key_prefix ||= "github-import:#{project.id}"
+      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/github_import/issuable_formatter.rb
new file mode 100644
index 00000000000..27b171d6ddb
--- /dev/null
+++ b/lib/gitlab/github_import/issuable_formatter.rb
@@ -0,0 +1,66 @@
+module Gitlab
+  module GithubImport
+    class IssuableFormatter < BaseFormatter
+      attr_writer :assignee_id, :author_id
+
+      def project_association
+        raise NotImplementedError
+      end
+
+      delegate :number, to: :raw_data
+
+      def find_condition
+        { iid: number }
+      end
+
+      private
+
+      def state
+        raw_data.state == 'closed' ? 'closed' : 'opened'
+      end
+
+      def assigned?
+        raw_data.assignee.present?
+      end
+
+      def author
+        @author ||= UserFormatter.new(client, raw_data.user)
+      end
+
+      def author_id
+        @author_id ||= author.gitlab_id || project.creator_id
+      end
+
+      def assignee
+        if assigned?
+          @assignee ||= UserFormatter.new(client, raw_data.assignee)
+        end
+      end
+
+      def assignee_id
+        return @assignee_id if defined?(@assignee_id)
+
+        @assignee_id = assignee.try(:gitlab_id)
+      end
+
+      def body
+        raw_data.body || ""
+      end
+
+      def description
+        if author.gitlab_id
+          body
+        else
+          formatter.author_line(author.login) + body
+        end
+      end
+
+      def milestone
+        if raw_data.milestone.present?
+          milestone = MilestoneFormatter.new(project, raw_data.milestone)
+          project.milestones.find_by(milestone.find_condition)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb
index 77621de9f4c..977cd0423ba 100644
--- a/lib/gitlab/github_import/issue_formatter.rb
+++ b/lib/gitlab/github_import/issue_formatter.rb
@@ -1,6 +1,6 @@
 module Gitlab
   module GithubImport
-    class IssueFormatter < BaseFormatter
+    class IssueFormatter < IssuableFormatter
       def attributes
         {
           iid: number,
@@ -10,7 +10,7 @@ module Gitlab
           description: description,
           state: state,
           author_id: author_id,
-          assignee_id: assignee_id,
+          assignee_ids: Array(assignee_id),
           created_at: raw_data.created_at,
           updated_at: raw_data.updated_at
         }
@@ -20,58 +20,12 @@ module Gitlab
         raw_data.comments > 0
       end
 
-      def klass
-        Issue
+      def project_association
+        :issues
       end
 
-      def number
-        raw_data.number
-      end
-
-      def valid?
-        raw_data.pull_request.nil?
-      end
-
-      private
-
-      def assigned?
-        raw_data.assignee.present?
-      end
-
-      def assignee_id
-        if assigned?
-          gitlab_user_id(raw_data.assignee.id)
-        end
-      end
-
-      def author
-        raw_data.user.login
-      end
-
-      def author_id
-        gitlab_author_id || project.creator_id
-      end
-
-      def body
-        raw_data.body || ""
-      end
-
-      def description
-        if gitlab_author_id
-          body
-        else
-          formatter.author_line(author) + body
-        end
-      end
-
-      def milestone
-        if raw_data.milestone.present?
-          project.milestones.find_by(iid: raw_data.milestone.number)
-        end
-      end
-
-      def state
-        raw_data.state == 'closed' ? 'closed' : 'opened'
+      def pull_request?
+        raw_data.pull_request.present?
       end
     end
   end
diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb
index 2cad7fca88e..211ccdc51bb 100644
--- a/lib/gitlab/github_import/label_formatter.rb
+++ b/lib/gitlab/github_import/label_formatter.rb
@@ -9,14 +9,18 @@ module Gitlab
         }
       end
 
-      def klass
-        Label
+      def project_association
+        :labels
       end
 
       def create!
-        project.labels.find_or_create_by!(title: title) do |label|
-          label.color = color
-        end
+        params  = attributes.except(:project)
+        service = ::Labels::FindOrCreateService.new(nil, project, params)
+        label   = service.execute(skip_authorization: true)
+
+        raise ActiveRecord::RecordInvalid.new(label) unless label.persisted?
+
+        label
       end
 
       private
diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb
index b2fa524cf5b..dd782eff059 100644
--- a/lib/gitlab/github_import/milestone_formatter.rb
+++ b/lib/gitlab/github_import/milestone_formatter.rb
@@ -3,7 +3,7 @@ module Gitlab
     class MilestoneFormatter < BaseFormatter
       def attributes
         {
-          iid: raw_data.number,
+          iid: number,
           project: project,
           title: raw_data.title,
           description: raw_data.description,
@@ -14,8 +14,20 @@ module Gitlab
         }
       end
 
-      def klass
-        Milestone
+      def project_association
+        :milestones
+      end
+
+      def find_condition
+        { iid: number }
+      end
+
+      def number
+        if project.gitea_import?
+          raw_data.id
+        else
+          raw_data.number
+        end
       end
 
       private
diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb
index a2410068845..a55adc9b1c8 100644
--- a/lib/gitlab/github_import/project_creator.rb
+++ b/lib/gitlab/github_import/project_creator.rb
@@ -1,14 +1,17 @@
 module Gitlab
   module GithubImport
     class ProjectCreator
-      attr_reader :repo, :name, :namespace, :current_user, :session_data
+      include Gitlab::CurrentSettings
 
-      def initialize(repo, name, namespace, current_user, session_data)
+      attr_reader :repo, :name, :namespace, :current_user, :session_data, :type
+
+      def initialize(repo, name, namespace, current_user, session_data, type: 'github')
         @repo = repo
         @name = name
         @namespace = namespace
         @current_user = current_user
         @session_data = session_data
+        @type = type
       end
 
       def execute
@@ -19,7 +22,7 @@ module Gitlab
           description: repo.description,
           namespace_id: namespace.id,
           visibility_level: visibility_level,
-          import_type: "github",
+          import_type: type,
           import_source: repo.full_name,
           import_url: import_url,
           skip_wiki: skip_wiki
@@ -29,11 +32,11 @@ module Gitlab
       private
 
       def import_url
-        repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@")
+        repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@")
       end
 
       def visibility_level
-        repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility
+        repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
       end
 
       #
diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb
index 1408683100f..150afa31432 100644
--- a/lib/gitlab/github_import/pull_request_formatter.rb
+++ b/lib/gitlab/github_import/pull_request_formatter.rb
@@ -1,8 +1,8 @@
 module Gitlab
   module GithubImport
-    class PullRequestFormatter < BaseFormatter
-      delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true
-      delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true
+    class PullRequestFormatter < IssuableFormatter
+      delegate :user, :project, :ref, :repo, :sha, to: :source_branch, prefix: true
+      delegate :user, :exists?, :project, :ref, :repo, :sha, :short_sha, to: :target_branch, prefix: true
 
       def attributes
         {
@@ -20,16 +20,13 @@ module Gitlab
           author_id: author_id,
           assignee_id: assignee_id,
           created_at: raw_data.created_at,
-          updated_at: raw_data.updated_at
+          updated_at: raw_data.updated_at,
+          imported: true
         }
       end
 
-      def klass
-        MergeRequest
-      end
-
-      def number
-        raw_data.number
+      def project_association
+        :merge_requests
       end
 
       def valid?
@@ -41,71 +38,52 @@ module Gitlab
       end
 
       def source_branch_name
-        @source_branch_name ||= begin
-          source_branch_exists? ? source_branch_ref : "pull/#{number}/#{source_branch_ref}"
-        end
-      end
-
-      def target_branch
-        @target_branch ||= BranchFormatter.new(project, raw_data.base)
+        @source_branch_name ||=
+          if cross_project? || !source_branch_exists?
+            source_branch_name_prefixed
+          else
+            source_branch_ref
+          end
       end
 
-      def target_branch_name
-        @target_branch_name ||= begin
-          target_branch_exists? ? target_branch_ref : "pull/#{number}/#{target_branch_ref}"
-        end
+      def source_branch_name_prefixed
+        "gh-#{target_branch_short_sha}/#{number}/#{source_branch_user}/#{source_branch_ref}"
       end
 
-      def url
-        raw_data.url
+      def source_branch_exists?
+        !cross_project? && source_branch.exists?
       end
 
-      private
-
-      def assigned?
-        raw_data.assignee.present?
+      def target_branch
+        @target_branch ||= BranchFormatter.new(project, raw_data.base)
       end
 
-      def assignee_id
-        if assigned?
-          gitlab_user_id(raw_data.assignee.id)
-        end
+      def target_branch_name
+        @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed
       end
 
-      def author
-        raw_data.user.login
+      def target_branch_name_prefixed
+        "gl-#{target_branch_short_sha}/#{number}/#{target_branch_user}/#{target_branch_ref}"
       end
 
-      def author_id
-        gitlab_author_id || project.creator_id
-      end
+      def cross_project?
+        return true if source_branch_repo.nil?
 
-      def body
-        raw_data.body || ""
+        source_branch_repo.id != target_branch_repo.id
       end
 
-      def description
-        if gitlab_author_id
-          body
-        else
-          formatter.author_line(author) + body
-        end
+      def opened?
+        state == 'opened'
       end
 
-      def milestone
-        if raw_data.milestone.present?
-          project.milestones.find_by(iid: raw_data.milestone.number)
-        end
-      end
+      private
 
       def state
-        @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present?
-                     'merged'
-                   elsif raw_data.state == 'closed'
-                     'closed'
-                   else
-                     'opened'
-                   end
+        if raw_data.state == 'closed' && raw_data.merged_at.present?
+          'merged'
+        else
+          super
+        end
       end
     end
   end
diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/github_import/release_formatter.rb
index 73d643b00ad..1ad702a6058 100644
--- a/lib/gitlab/github_import/release_formatter.rb
+++ b/lib/gitlab/github_import/release_formatter.rb
@@ -11,8 +11,12 @@ module Gitlab
         }
       end
 
-      def klass
-        Release
+      def project_association
+        :releases
+      end
+
+      def find_condition
+        { tag: raw_data.tag_name }
       end
 
       def valid?
diff --git a/lib/gitlab/github_import/user_formatter.rb b/lib/gitlab/github_import/user_formatter.rb
new file mode 100644
index 00000000000..04c2964da20
--- /dev/null
+++ b/lib/gitlab/github_import/user_formatter.rb
@@ -0,0 +1,45 @@
+module Gitlab
+  module GithubImport
+    class UserFormatter
+      attr_reader :client, :raw
+
+      delegate :id, :login, to: :raw, allow_nil: true
+
+      def initialize(client, raw)
+        @client = client
+        @raw = raw
+      end
+
+      def gitlab_id
+        return @gitlab_id if defined?(@gitlab_id)
+
+        @gitlab_id = find_by_external_uid || find_by_email
+      end
+
+      private
+
+      def email
+        @email ||= client.user(raw.login).try(:email)
+      end
+
+      def find_by_email
+        return nil unless email
+
+        User.find_by_any_email(email)
+            .try(:id)
+      end
+
+      def find_by_external_uid
+        return nil unless id
+
+        identities = ::Identity.arel_table
+
+        User.select(:id)
+            .joins(:identities).where(identities[:provider].eq(:github)
+            .and(identities[:extern_uid].eq(id)))
+            .first
+            .try(:id)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb
new file mode 100644
index 00000000000..07c0abcce23
--- /dev/null
+++ b/lib/gitlab/gl_repository.rb
@@ -0,0 +1,20 @@
+module Gitlab
+  module GlRepository
+    def self.gl_repository(project, is_wiki)
+      "#{is_wiki ? 'wiki' : 'project'}-#{project.id}"
+    end
+
+    def self.parse(gl_repository)
+      match_data = /\A(project|wiki)-([1-9][0-9]*)\z/.match(gl_repository)
+      unless match_data
+        raise ArgumentError, "Invalid GL Repository \"#{gl_repository}\""
+      end
+
+      type, id = match_data.captures
+      project = Project.find_by(id: id)
+      wiki = type == 'wiki'
+
+      [project, wiki]
+    end
+  end
+end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 2c21804fe7a..319633656ff 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -1,16 +1,27 @@
+# rubocop:disable Metrics/AbcSize
+
 module Gitlab
   module GonHelper
     def add_gon_variables
-      gon.api_version            = API::API.version
-      gon.default_avatar_url     = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s
+      gon.api_version            = 'v4'
+      gon.default_avatar_url     = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s
       gon.max_file_size          = current_application_settings.max_attachment_size
+      gon.asset_host             = ActionController::Base.asset_host
       gon.relative_url_root      = Gitlab.config.gitlab.relative_url_root
       gon.shortcuts_path         = help_page_path('shortcuts')
       gon.user_color_scheme      = Gitlab::ColorSchemes.for_user(current_user).css_class
-      gon.award_menu_url         = emojis_path
+      gon.katex_css_url          = ActionController::Base.helpers.asset_path('katex.css')
+      gon.katex_js_url           = ActionController::Base.helpers.asset_path('katex.js')
+      gon.sentry_dsn             = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled
+      gon.gitlab_url             = Gitlab.config.gitlab.url
+      gon.revision               = Gitlab::REVISION
+      gon.gitlab_logo            = ActionController::Base.helpers.asset_path('gitlab_logo.png')
 
       if current_user
         gon.current_user_id = current_user.id
+        gon.current_username = current_user.username
+        gon.current_user_fullname = current_user.name
+        gon.current_user_avatar_url = current_user.avatar_url
       end
     end
   end
diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb
index 890bd9a3554..b1dbf554e41 100644
--- a/lib/gitlab/google_code_import/client.rb
+++ b/lib/gitlab/google_code_import/client.rb
@@ -14,7 +14,7 @@ module Gitlab
       end
 
       def valid?
-        raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects")
+        raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.key?("projects")
       end
 
       def repos
diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb
index 62da327931f..ab38c0c3e34 100644
--- a/lib/gitlab/google_code_import/importer.rb
+++ b/lib/gitlab/google_code_import/importer.rb
@@ -1,7 +1,23 @@
 module Gitlab
   module GoogleCodeImport
     class Importer
-      attr_reader :project, :repo
+      attr_reader :project, :repo, :closed_statuses
+
+      NICE_LABEL_COLOR_HASH =
+        {
+          'Status: New'        => '#428bca',
+          'Status: Accepted'   => '#5cb85c',
+          'Status: Started'    => '#8e44ad',
+          'Priority: Critical' => '#ffcfcf',
+          'Priority: High'     => '#deffcf',
+          'Priority: Medium'   => '#fff5cc',
+          'Priority: Low'      => '#cfe9ff',
+          'Type: Defect'       => '#d9534f',
+          'Type: Enhancement'  => '#44ad8e',
+          'Type: Task'         => '#4b6dd0',
+          'Type: Review'       => '#8e44ad',
+          'Type: Other'        => '#7f8c8d'
+        }.freeze
 
       def initialize(project)
         @project = project
@@ -79,7 +95,7 @@ module Gitlab
           labels = import_issue_labels(raw_issue)
 
           assignee_id = nil
-          if raw_issue.has_key?("owner")
+          if raw_issue.key?("owner")
             username = user_map[raw_issue["owner"]["name"]]
 
             if username.start_with?("@")
@@ -92,19 +108,17 @@ module Gitlab
           end
 
           issue = Issue.create!(
+            iid:          raw_issue['id'],
             project_id:   project.id,
-            title:        raw_issue["title"],
+            title:        raw_issue['title'],
             description:  body,
             author_id:    project.creator_id,
-            assignee_id:  assignee_id,
-            state:        raw_issue["state"] == "closed" ? "closed" : "opened"
+            assignee_ids: [assignee_id],
+            state:        raw_issue['state'] == 'closed' ? 'closed' : 'opened'
           )
 
-          issue.add_labels_by_names(labels)
-
-          if issue.iid != raw_issue["id"]
-            issue.update_attribute(:iid, raw_issue["id"])
-          end
+          issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true)
+          issue.update_attribute(:label_ids, issue_labels.pluck(:id))
 
           import_issue_comments(issue, comments)
         end
@@ -130,7 +144,7 @@ module Gitlab
       def import_issue_comments(issue, comments)
         Note.transaction do
           while raw_comment = comments.shift
-            next if raw_comment.has_key?("deletedBy")
+            next if raw_comment.key?("deletedBy")
 
             content     = format_content(raw_comment["content"])
             updates     = format_updates(raw_comment["updates"])
@@ -163,45 +177,19 @@ module Gitlab
       end
 
       def nice_label_color(name)
-        case name
-        when /\AComponent:/
-          "#fff39e"
-        when /\AOpSys:/
-          "#e2e2e2"
-        when /\AMilestone:/
-          "#fee3ff"
-
-        when "Status: New"
-          "#428bca"
-        when "Status: Accepted"
-          "#5cb85c"
-        when "Status: Started"
-          "#8e44ad"
-
-        when "Priority: Critical"
-          "#ffcfcf"
-        when "Priority: High"
-          "#deffcf"
-        when "Priority: Medium"
-          "#fff5cc"
-        when "Priority: Low"
-          "#cfe9ff"
-
-        when "Type: Defect"
-          "#d9534f"
-        when "Type: Enhancement"
-          "#44ad8e"
-        when "Type: Task"
-          "#4b6dd0"
-        when "Type: Review"
-          "#8e44ad"
-        when "Type: Other"
-          "#7f8c8d"
-        when *@closed_statuses.map { |s| nice_status_name(s) }
-          "#cfcfcf"
-        else
-          "#e2e2e2"
-        end
+        NICE_LABEL_COLOR_HASH[name] ||
+          case name
+          when /\AComponent:/
+            '#fff39e'
+          when /\AOpSys:/
+            '#e2e2e2'
+          when /\AMilestone:/
+            '#fee3ff'
+          when *closed_statuses.map { |s| nice_status_name(s) }
+            '#cfcfcf'
+          else
+            '#e2e2e2'
+          end
       end
 
       def nice_label_name(name)
@@ -236,8 +224,8 @@ module Gitlab
       end
 
       def create_label(name)
-        color = nice_label_color(name)
-        Label.create!(project_id: project.id, name: name, color: color)
+        params = { name: name, color: nice_label_color(name) }
+        ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true)
       end
 
       def format_content(raw_content)
@@ -247,15 +235,15 @@ module Gitlab
       def format_updates(raw_updates)
         updates = []
 
-        if raw_updates.has_key?("status")
+        if raw_updates.key?("status")
           updates << "*Status: #{raw_updates["status"]}*"
         end
 
-        if raw_updates.has_key?("owner")
+        if raw_updates.key?("owner")
           updates << "*Owner: #{user_map[raw_updates["owner"]]}*"
         end
 
-        if raw_updates.has_key?("cc")
+        if raw_updates.key?("cc")
           cc = raw_updates["cc"].map do |l|
             deleted = l.start_with?("-")
             l = l[1..-1] if deleted
@@ -267,7 +255,7 @@ module Gitlab
           updates << "*Cc: #{cc.join(", ")}*"
         end
 
-        if raw_updates.has_key?("labels")
+        if raw_updates.key?("labels")
           labels = raw_updates["labels"].map do |l|
             deleted = l.start_with?("-")
             l = l[1..-1] if deleted
@@ -279,11 +267,11 @@ module Gitlab
           updates << "*Labels: #{labels.join(", ")}*"
         end
 
-        if raw_updates.has_key?("mergedInto")
+        if raw_updates.key?("mergedInto")
           updates << "*Merged into: ##{raw_updates["mergedInto"]}*"
         end
 
-        if raw_updates.has_key?("blockedOn")
+        if raw_updates.key?("blockedOn")
           blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on|
             format_blocking_updates(raw_blocked_on)
           end
@@ -291,7 +279,7 @@ module Gitlab
           updates << "*Blocked on: #{blocked_ons.join(", ")}*"
         end
 
-        if raw_updates.has_key?("blocking")
+        if raw_updates.key?("blocking")
           blockings = raw_updates["blocking"].map do |raw_blocked_on|
             format_blocking_updates(raw_blocked_on)
           end
@@ -312,7 +300,7 @@ module Gitlab
           if name == project.import_source
             "##{id}"
           else
-            "#{project.namespace.path}/#{name}##{id}"
+            "#{project.namespace.full_path}/#{name}##{id}"
           end
         text = "~~#{text}~~" if deleted
         text
diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb
new file mode 100644
index 00000000000..5a31e56cb30
--- /dev/null
+++ b/lib/gitlab/group_hierarchy.rb
@@ -0,0 +1,111 @@
+module Gitlab
+  # Retrieving of parent or child groups based on a base ActiveRecord relation.
+  #
+  # This class uses recursive CTEs and as a result will only work on PostgreSQL.
+  class GroupHierarchy
+    attr_reader :ancestors_base, :descendants_base, :model
+
+    # ancestors_base - An instance of ActiveRecord::Relation for which to
+    #                  get parent groups.
+    # descendants_base - An instance of ActiveRecord::Relation for which to
+    #                    get child groups. If omitted, ancestors_base is used.
+    def initialize(ancestors_base, descendants_base = ancestors_base)
+      raise ArgumentError.new("Model of ancestors_base does not match model of descendants_base") if ancestors_base.model != descendants_base.model
+
+      @ancestors_base = ancestors_base
+      @descendants_base = descendants_base
+      @model = ancestors_base.model
+    end
+
+    # Returns a relation that includes the ancestors_base set of groups
+    # and all their ancestors (recursively).
+    def base_and_ancestors
+      return ancestors_base unless Group.supports_nested_groups?
+
+      base_and_ancestors_cte.apply_to(model.all)
+    end
+
+    # Returns a relation that includes the descendants_base set of groups
+    # and all their descendants (recursively).
+    def base_and_descendants
+      return descendants_base unless Group.supports_nested_groups?
+
+      base_and_descendants_cte.apply_to(model.all)
+    end
+
+    # Returns a relation that includes the base groups, their ancestors,
+    # and the descendants of the base groups.
+    #
+    # The resulting query will roughly look like the following:
+    #
+    #     WITH RECURSIVE ancestors AS ( ... ),
+    #       descendants AS ( ... )
+    #     SELECT *
+    #     FROM (
+    #       SELECT *
+    #       FROM ancestors namespaces
+    #
+    #       UNION
+    #
+    #       SELECT *
+    #       FROM descendants namespaces
+    #     ) groups;
+    #
+    # Using this approach allows us to further add criteria to the relation with
+    # Rails thinking it's selecting data the usual way.
+    #
+    # If nested groups are not supported, ancestors_base is returned.
+    def all_groups
+      return ancestors_base unless Group.supports_nested_groups?
+
+      ancestors = base_and_ancestors_cte
+      descendants = base_and_descendants_cte
+
+      ancestors_table = ancestors.alias_to(groups_table)
+      descendants_table = descendants.alias_to(groups_table)
+
+      union = SQL::Union.new([model.unscoped.from(ancestors_table),
+                              model.unscoped.from(descendants_table)])
+
+      model
+        .unscoped
+        .with
+        .recursive(ancestors.to_arel, descendants.to_arel)
+        .from("(#{union.to_sql}) #{model.table_name}")
+    end
+
+    private
+
+    def base_and_ancestors_cte
+      cte = SQL::RecursiveCTE.new(:base_and_ancestors)
+
+      cte << ancestors_base.except(:order)
+
+      # Recursively get all the ancestors of the base set.
+      cte << model
+        .from([groups_table, cte.table])
+        .where(groups_table[:id].eq(cte.table[:parent_id]))
+        .except(:order)
+
+      cte
+    end
+
+    def base_and_descendants_cte
+      cte = SQL::RecursiveCTE.new(:base_and_descendants)
+
+      cte << descendants_base.except(:order)
+
+      # Recursively get all the descendants of the base set.
+      cte << model
+        .from([groups_table, cte.table])
+        .where(groups_table[:parent_id].eq(cte.table[:id]))
+        .except(:order)
+
+      cte
+    end
+
+    def groups_table
+      model.arel_table
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb
new file mode 100644
index 00000000000..7de6d4d9367
--- /dev/null
+++ b/lib/gitlab/health_checks/base_abstract_check.rb
@@ -0,0 +1,45 @@
+module Gitlab
+  module HealthChecks
+    module BaseAbstractCheck
+      def name
+        super.demodulize.underscore
+      end
+
+      def human_name
+        name.sub(/_check$/, '').capitalize
+      end
+
+      def readiness
+        raise NotImplementedError
+      end
+
+      def liveness
+        HealthChecks::Result.new(true)
+      end
+
+      def metrics
+        []
+      end
+
+      protected
+
+      def metric(name, value, **labels)
+        Metric.new(name, value, labels)
+      end
+
+      def with_timing(proc)
+        start = Time.now
+        result = proc.call
+        yield result, Time.now.to_f - start.to_f
+      end
+
+      def catch_timeout(seconds, &block)
+        begin
+          Timeout.timeout(seconds.to_i, &block)
+        rescue Timeout::Error => ex
+          ex
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb
new file mode 100644
index 00000000000..fd94984f8a2
--- /dev/null
+++ b/lib/gitlab/health_checks/db_check.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module HealthChecks
+    class DbCheck
+      extend SimpleAbstractCheck
+
+      class << self
+        private
+
+        def metric_prefix
+          'db_ping'
+        end
+
+        def is_successful?(result)
+          result == '1'
+        end
+
+        def check
+          catch_timeout 10.seconds do
+            if Gitlab::Database.postgresql?
+              ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')
+            else
+              ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
new file mode 100644
index 00000000000..e78b7f22e03
--- /dev/null
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -0,0 +1,118 @@
+module Gitlab
+  module HealthChecks
+    class FsShardsCheck
+      extend BaseAbstractCheck
+      RANDOM_STRING = SecureRandom.hex(1000).freeze
+      COMMAND_TIMEOUT = '1'.freeze
+      TIMEOUT_EXECUTABLE = 'timeout'.freeze
+
+      class << self
+        def readiness
+          repository_storages.map do |storage_name|
+            begin
+              tmp_file_path = tmp_file_path(storage_name)
+
+              if !storage_stat_test(storage_name)
+                HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name)
+              elsif !storage_write_test(tmp_file_path)
+                HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name)
+              elsif !storage_read_test(tmp_file_path)
+                HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name)
+              else
+                HealthChecks::Result.new(true, nil, shard: storage_name)
+              end
+            rescue RuntimeError => ex
+              message = "unexpected error #{ex} when checking storage #{storage_name}"
+              Rails.logger.error(message)
+              HealthChecks::Result.new(false, message, shard: storage_name)
+            ensure
+              delete_test_file(tmp_file_path)
+            end
+          end
+        end
+
+        def metrics
+          repository_storages.flat_map do |storage_name|
+            tmp_file_path = tmp_file_path(storage_name)
+            [
+              operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name),
+              operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name),
+              operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name)
+            ].flatten
+          end
+        end
+
+        private
+
+        def operation_metrics(ok_metric, latency_metric, operation, **labels)
+          with_timing operation do |result, elapsed|
+            [
+              metric(latency_metric, elapsed, **labels),
+              metric(ok_metric, result ? 1 : 0, **labels)
+            ]
+          end
+        rescue RuntimeError => ex
+          Rails.logger("unexpected error #{ex} when checking #{ok_metric}")
+          [metric(ok_metric, 0, **labels)]
+        end
+
+        def repository_storages
+          @repository_storage ||= Gitlab::CurrentSettings.current_application_settings.repository_storages
+        end
+
+        def storages_paths
+          @storage_paths ||= Gitlab.config.repositories.storages
+        end
+
+        def exec_with_timeout(cmd_args, *args, &block)
+          Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block)
+        end
+
+        def tmp_file_path(storage_name)
+          Dir::Tmpname.create(%w(fs_shards_check +deleted), path(storage_name)) { |path| path }
+        end
+
+        def path(storage_name)
+          storages_paths&.dig(storage_name, 'path')
+        end
+
+        def storage_stat_test(storage_name)
+          stat_path = File.join(path(storage_name), '.')
+          begin
+            _, status = exec_with_timeout(%W{ stat #{stat_path} })
+            status == 0
+          rescue Errno::ENOENT
+            File.exist?(stat_path) && File::Stat.new(stat_path).readable?
+          end
+        end
+
+        def storage_write_test(tmp_path)
+          _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin|
+            stdin.write(RANDOM_STRING)
+          end
+          status == 0
+        rescue Errno::ENOENT
+          written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT
+          written_bytes == RANDOM_STRING.length
+        end
+
+        def storage_read_test(tmp_path)
+          _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin|
+            stdin.write(RANDOM_STRING)
+          end
+          status == 0
+        rescue Errno::ENOENT
+          file_contents = File.read(tmp_path) rescue Errno::ENOENT
+          file_contents == RANDOM_STRING
+        end
+
+        def delete_test_file(tmp_path)
+          _, status = exec_with_timeout(%W{ rm -f #{tmp_path} })
+          status == 0
+        rescue Errno::ENOENT
+          File.delete(tmp_path) rescue Errno::ENOENT
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb
new file mode 100644
index 00000000000..1a2eab0b005
--- /dev/null
+++ b/lib/gitlab/health_checks/metric.rb
@@ -0,0 +1,3 @@
+module Gitlab::HealthChecks
+  Metric = Struct.new(:name, :value, :labels)
+end
diff --git a/lib/gitlab/health_checks/prometheus_text_format.rb b/lib/gitlab/health_checks/prometheus_text_format.rb
new file mode 100644
index 00000000000..b3c759b4730
--- /dev/null
+++ b/lib/gitlab/health_checks/prometheus_text_format.rb
@@ -0,0 +1,40 @@
+module Gitlab
+  module HealthChecks
+    class PrometheusTextFormat
+      def marshal(metrics)
+        "#{metrics_with_type_declarations(metrics).join("\n")}\n"
+      end
+
+      private
+
+      def metrics_with_type_declarations(metrics)
+        type_declaration_added = {}
+
+        metrics.flat_map do |metric|
+          metric_lines = []
+
+          unless type_declaration_added.key?(metric.name)
+            type_declaration_added[metric.name] = true
+            metric_lines << metric_type_declaration(metric)
+          end
+
+          metric_lines << metric_text(metric)
+        end
+      end
+
+      def metric_type_declaration(metric)
+        "# TYPE #{metric.name} gauge"
+      end
+
+      def metric_text(metric)
+        labels = metric.labels&.map { |key, value| "#{key}=\"#{value}\"" }&.join(',') || ''
+
+        if labels.empty?
+          "#{metric.name} #{metric.value}"
+        else
+          "#{metric.name}{#{labels}} #{metric.value}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb
new file mode 100644
index 00000000000..57bbe5b3ad0
--- /dev/null
+++ b/lib/gitlab/health_checks/redis_check.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module HealthChecks
+    class RedisCheck
+      extend SimpleAbstractCheck
+
+      class << self
+        private
+
+        def metric_prefix
+          'redis_ping'
+        end
+
+        def is_successful?(result)
+          result == 'PONG'
+        end
+
+        def check
+          catch_timeout 10.seconds do
+            Gitlab::Redis.with(&:ping)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb
new file mode 100644
index 00000000000..8086760023e
--- /dev/null
+++ b/lib/gitlab/health_checks/result.rb
@@ -0,0 +1,3 @@
+module Gitlab::HealthChecks
+  Result = Struct.new(:success, :message, :labels)
+end
diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb
new file mode 100644
index 00000000000..fbe1645c1b1
--- /dev/null
+++ b/lib/gitlab/health_checks/simple_abstract_check.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module HealthChecks
+    module SimpleAbstractCheck
+      include BaseAbstractCheck
+
+      def readiness
+        check_result = check
+        if is_successful?(check_result)
+          HealthChecks::Result.new(true)
+        elsif check_result.is_a?(Timeout::Error)
+          HealthChecks::Result.new(false, "#{human_name} check timed out")
+        else
+          HealthChecks::Result.new(false, "unexpected #{human_name} check result: #{check_result}")
+        end
+      end
+
+      def metrics
+        with_timing method(:check) do |result, elapsed|
+          Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result)
+          [
+            metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0),
+            metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0),
+            metric("#{metric_prefix}_latency", elapsed)
+          ]
+        end
+      end
+
+      private
+
+      def metric_prefix
+        raise NotImplementedError
+      end
+
+      def is_successful?(result)
+        raise NotImplementedError
+      end
+
+      def check
+        raise NotImplementedError
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index 9360afedfcb..5408a1a6838 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -1,36 +1,23 @@
 module Gitlab
   class Highlight
     def self.highlight(blob_name, blob_content, repository: nil, plain: false)
-      new(blob_name, blob_content, repository: repository).
-        highlight(blob_content, continue: false, plain: plain)
+      new(blob_name, blob_content, repository: repository)
+        .highlight(blob_content, continue: false, plain: plain)
     end
 
-    def self.highlight_lines(repository, ref, file_name)
-      blob = repository.blob_at(ref, file_name)
-      return [] unless blob
-
-      blob.load_all_data!(repository)
-      highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe)
-    end
+    attr_reader :blob_name
 
     def initialize(blob_name, blob_content, repository: nil)
-      @formatter = Rouge::Formatters::HTMLGitlab.new
+      @formatter = Rouge::Formatters::HTMLGitlab
       @repository = repository
       @blob_name = blob_name
       @blob_content = blob_content
     end
 
     def highlight(text, continue: true, plain: false)
-      if plain
-        hl_lexer = Rouge::Lexers::PlainText
-        continue = false
-      else
-        hl_lexer = self.lexer
-      end
-
-      @formatter.format(hl_lexer.lex(text, continue: continue)).html_safe
-    rescue
-      @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe
+      highlighted_text = highlight_text(text, continue: continue, plain: plain)
+      highlighted_text = link_dependencies(text, highlighted_text) if blob_name
+      highlighted_text
     end
 
     def lexer
@@ -50,5 +37,27 @@ module Gitlab
 
       Rouge::Lexer.find_fancy(language_name)
     end
+
+    def highlight_text(text, continue: true, plain: false)
+      if plain
+        highlight_plain(text)
+      else
+        highlight_rich(text, continue: continue)
+      end
+    end
+
+    def highlight_plain(text)
+      @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe
+    end
+
+    def highlight_rich(text, continue: true)
+      @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe
+    rescue
+      highlight_plain(text)
+    end
+
+    def link_dependencies(text, highlighted_text)
+      Gitlab::DependencyLinker.link(blob_name, text, highlighted_text)
+    end
   end
 end
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
new file mode 100644
index 00000000000..db7cdf4b5c7
--- /dev/null
+++ b/lib/gitlab/i18n.rb
@@ -0,0 +1,54 @@
+module Gitlab
+  module I18n
+    extend self
+
+    AVAILABLE_LANGUAGES = {
+      'en' => 'English',
+      'es' => 'Español',
+      'de' => 'Deutsch',
+      'fr' => 'Français',
+      'pt_BR' => 'Português(Brasil)',
+      'zh_CN' => '简体中文',
+      'zh_HK' => '繁體中文(香港)',
+      'zh_TW' => '繁體中文(臺灣)',
+      'bg' => 'български',
+      'eo' => 'Esperanto'
+    }.freeze
+
+    def available_locales
+      AVAILABLE_LANGUAGES.keys
+    end
+
+    def locale
+      FastGettext.locale
+    end
+
+    def locale=(locale_string)
+      requested_locale = locale_string || ::I18n.default_locale
+      new_locale = FastGettext.set_locale(requested_locale)
+      ::I18n.locale = new_locale
+    end
+
+    def use_default_locale
+      FastGettext.set_locale(::I18n.default_locale)
+      ::I18n.locale = ::I18n.default_locale
+    end
+
+    def with_locale(locale_string)
+      original_locale = locale
+
+      self.locale = locale_string
+      yield
+    ensure
+      self.locale = original_locale
+    end
+
+    def with_user_locale(user, &block)
+      with_locale(user&.preferred_language, &block)
+    end
+
+    def with_default_locale(&block)
+      with_locale(::I18n.default_locale, &block)
+    end
+  end
+end
diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb
index f8809db21aa..94678b6ec40 100644
--- a/lib/gitlab/identifier.rb
+++ b/lib/gitlab/identifier.rb
@@ -21,10 +21,8 @@ module Gitlab
 
       return if !commit || !commit.author_email
 
-      email = commit.author_email
-
-      identify_with_cache(:email, email) do
-        User.find_by(email: email)
+      identify_with_cache(:email, commit.author_email) do
+        commit.author
       end
     end
 
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index 181e288a014..3470a09eaf0 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -3,7 +3,7 @@ module Gitlab
     extend self
 
     # For every version update, the version history in import_export.md has to be kept up to date.
-    VERSION = '0.1.4'
+    VERSION = '0.1.8'.freeze
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)
@@ -35,7 +35,7 @@ module Gitlab
     end
 
     def export_filename(project:)
-      basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.namespace.path}_#{project.path}"
+      basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.full_path.tr('/', '_')}"
 
       "#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
     end
diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb
index b9e4042220a..34169319b26 100644
--- a/lib/gitlab/import_export/attribute_cleaner.rb
+++ b/lib/gitlab/import_export/attribute_cleaner.rb
@@ -1,12 +1,27 @@
 module Gitlab
   module ImportExport
     class AttributeCleaner
-      ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES
+      ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id']
 
-      def self.clean!(relation_hash:)
-        relation_hash.reject! do |key, _value|
-          key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key)
-        end
+      def self.clean(*args)
+        new(*args).clean
+      end
+
+      def initialize(relation_hash:, relation_class:)
+        @relation_hash = relation_hash
+        @relation_class = relation_class
+      end
+
+      def clean
+        @relation_hash.reject do |key, _value|
+          prohibited_key?(key) || !@relation_class.attribute_method?(key)
+        end.except('id')
+      end
+
+      private
+
+      def prohibited_key?(key)
+        key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key)
       end
     end
   end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index f00c7460e82..90942774a2e 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -15,14 +15,6 @@ module Gitlab
         execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all))
       end
 
-      def git_unbundle(repo_path:, bundle_path:)
-        execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path}))
-      end
-
-      def git_restore_hooks
-        execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args)
-      end
-
       def mkdir_p(path)
         FileUtils.mkdir_p(path, mode: DEFAULT_MODE)
         FileUtils.chmod(DEFAULT_MODE, path)
@@ -56,10 +48,6 @@ module Gitlab
         FileUtils.copy_entry(source, destination)
         true
       end
-
-      def repository_storage_paths_args
-        Gitlab.config.repositories.storages.values
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb
index e341c4d9cf8..788eedf2686 100644
--- a/lib/gitlab/import_export/error.rb
+++ b/lib/gitlab/import_export/error.rb
@@ -1,5 +1,5 @@
 module Gitlab
   module ImportExport
-    class Error < StandardError; end
+    Error = Class.new(StandardError)
   end
 end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 113895ba22c..ffd17118c91 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -43,6 +43,14 @@ module Gitlab
 
         raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result
 
+        remove_symlinks!
+      end
+
+      def remove_symlinks!
+        Dir["#{@shared.export_path}/**/*"].each do |path|
+          FileUtils.rm(path) if File.lstat(path).symlink?
+        end
+
         true
       end
     end
diff --git a/lib/gitlab/import_export/hash_util.rb b/lib/gitlab/import_export/hash_util.rb
new file mode 100644
index 00000000000..d4adeeb3797
--- /dev/null
+++ b/lib/gitlab/import_export/hash_util.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module ImportExport
+    class HashUtil
+      def self.deep_symbolize_array!(array)
+        return if array.blank?
+
+        array.map! do |hash|
+          hash.deep_symbolize_keys!
+
+          yield(hash) if block_given?
+
+          hash
+        end
+      end
+
+      def self.deep_symbolize_array_with_date!(array)
+        self.deep_symbolize_array!(array) do |hash|
+          hash.select { |k, _v| k.to_s.end_with?('_date') }.each do |key, value|
+            hash[key] = Time.zone.parse(value)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index bb9d1080330..1860352c96d 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -1,15 +1,18 @@
 # Model relationships to be included in the project import/export
 project_tree:
-  - :labels
+  - labels:
+      :priorities
   - milestones:
     - :events
   - issues:
     - :events
+    - :timelogs
     - notes:
       - :author
       - :events
     - label_links:
-      - :label
+      - label:
+          :priorities
     - milestone:
       - :events
   - snippets:
@@ -23,33 +26,34 @@ project_tree:
     - notes:
       - :author
       - :events
-    - :merge_request_diff
+    - merge_request_diff:
+      - :merge_request_diff_files
     - :events
+    - :timelogs
     - label_links:
-      - :label
+      - label:
+          :priorities
     - milestone:
       - :events
   - pipelines:
     - notes:
       - :author
       - :events
+    - :stages
     - :statuses
-  - :variables
   - :triggers
-  - :deploy_keys
+  - :pipeline_schedules
   - :services
   - :hooks
   - protected_branches:
     - :merge_access_levels
     - :push_access_levels
+  - protected_tags:
+    - :create_access_levels
   - :project_feature
 
 # Only include the following attributes for the models specified.
 included_attributes:
-  project:
-    - :description
-    - :visibility_level
-    - :archived
   user:
     - :id
     - :email
@@ -59,21 +63,63 @@ included_attributes:
 
 # Do not include the following attributes for the models specified.
 excluded_attributes:
+  project:
+    - :name
+    - :path
+    - :namespace_id
+    - :creator_id
+    - :import_url
+    - :import_status
+    - :avatar
+    - :import_type
+    - :import_source
+    - :import_error
+    - :mirror
+    - :runners_token
+    - :repository_storage
+    - :repository_read_only
+    - :lfs_enabled
+    - :import_jid
+    - :created_at
+    - :updated_at
+    - :import_jid
+    - :import_jid
+    - :id
+    - :star_count
+    - :last_activity_at
+    - :last_repository_updated_at
+    - :last_repository_check_at
   snippets:
     - :expired_at
   merge_request_diff:
     - :st_diffs
+  merge_request_diff_files:
+    - :diff
   issues:
     - :milestone_id
   merge_requests:
     - :milestone_id
+    - :ref_fetched
   award_emoji:
     - :awardable_id
+  statuses:
+    - :trace
+    - :token
 
 methods:
+  labels:
+    - :type
+  label:
+    - :type
   statuses:
     - :type
   services:
     - :type
   merge_request_diff:
     - :utf8_st_diffs
+  merge_request_diff_files:
+    - :utf8_diff
+  merge_requests:
+    - :diff_head_sha
+  project:
+    - :description_html
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index e9ee47fc090..fbdd74788bc 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -9,7 +9,7 @@ module Gitlab
       end
 
       def execute
-        if import_file && check_version! && [project_tree, avatar_restorer, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
+        if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore)
           project_tree.restored_project
         else
           raise Projects::ImportService::Error.new(@shared.errors.join(', '))
@@ -56,7 +56,7 @@ module Gitlab
       end
 
       def path_with_namespace
-        File.join(@project.namespace.path, @project.path)
+        File.join(@project.namespace.full_path, @project.path)
       end
 
       def repo_path
diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb
index 0cc10f40087..b48f63bcd7e 100644
--- a/lib/gitlab/import_export/json_hash_builder.rb
+++ b/lib/gitlab/import_export/json_hash_builder.rb
@@ -65,11 +65,17 @@ module Gitlab
       # +value+ existing model to be included in the hash
       # +parsed_hash+ the original hash
       def parse_hash(value)
+        return nil if already_contains_methods?(value)
+
         @attributes_finder.parse(value) do |hash|
           { include: hash_or_merge(value, hash) }
         end
       end
 
+      def already_contains_methods?(value)
+        value.is_a?(Hash) && value.values.detect { |val| val[:methods]}
+      end
+
       # Adds new model configuration to an existing hash with key +current_key+
       # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
       #
@@ -77,7 +83,9 @@ module Gitlab
       # +value+ existing model to be included in the hash
       # +json_config_hash+ the original hash containing the root model
       def add_model_value(current_key, value, json_config_hash)
-        @attributes_finder.parse(value) { |hash| value = { value => hash } }
+        @attributes_finder.parse(value) do |hash|
+          value = { value => hash } unless value.is_a?(Hash)
+        end
 
         add_to_array(current_key, json_config_hash, value)
       end
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
index 36c4cf6efa0..8b8e48aac76 100644
--- a/lib/gitlab/import_export/members_mapper.rb
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -1,13 +1,10 @@
 module Gitlab
   module ImportExport
     class MembersMapper
-      attr_reader :missing_author_ids
-
       def initialize(exported_members:, user:, project:)
-        @exported_members = exported_members
+        @exported_members = user.admin? ? exported_members : []
         @user = user
         @project = project
-        @missing_author_ids = []
 
         # This needs to run first, as second call would be from #map
         # which means project members already exist.
@@ -35,16 +32,21 @@ module Gitlab
         @user.id
       end
 
+      def include?(old_author_id)
+        map.keys.include?(old_author_id) && map[old_author_id] != default_user_id
+      end
+
       private
 
       def missing_keys_tracking_hash
         Hash.new do |_, key|
-          @missing_author_ids << key
           default_user_id
         end
       end
 
       def ensure_default_member!
+        @project.project_members.destroy_all
+
         ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true)
       end
 
@@ -55,11 +57,16 @@ module Gitlab
       end
 
       def member_hash(member)
-        member.except('id').merge(source_id: @project.id, importing: true)
+        parsed_hash(member).merge('source_id' => @project.id, 'importing' => true)
+      end
+
+      def parsed_hash(member)
+        Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys,
+                                                     relation_class: ProjectMember)
       end
 
       def find_project_user_query(member)
-        user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email']))
+        user_arel[:email].eq(member['user']['email']).or(user_arel[:username].eq(member['user']['username']))
       end
 
       def user_arel
diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb
new file mode 100644
index 00000000000..c20adc20bfd
--- /dev/null
+++ b/lib/gitlab/import_export/merge_request_parser.rb
@@ -0,0 +1,41 @@
+module Gitlab
+  module ImportExport
+    class MergeRequestParser
+      FORKED_PROJECT_ID = -1
+
+      def initialize(project, diff_head_sha, merge_request, relation_hash)
+        @project = project
+        @diff_head_sha = diff_head_sha
+        @merge_request = merge_request
+        @relation_hash = relation_hash
+      end
+
+      def parse!
+        if fork_merge_request? && @diff_head_sha
+          @merge_request.source_project_id = @relation_hash['project_id']
+
+          fetch_ref unless branch_exists?(@merge_request.source_branch)
+          create_target_branch unless branch_exists?(@merge_request.target_branch)
+        end
+
+        @merge_request
+      end
+
+      def create_target_branch
+        @project.repository.create_branch(@merge_request.target_branch, @merge_request.target_branch_sha)
+      end
+
+      def fetch_ref
+        @project.repository.fetch_ref(@project.repository.path, @diff_head_sha, @merge_request.source_branch)
+      end
+
+      def branch_exists?(branch_name)
+        @project.repository.branch_exists?(branch_name)
+      end
+
+      def fork_merge_request?
+        @relation_hash['source_project_id'] == FORKED_PROJECT_ID
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 5a109f24f9f..84ab1977dfa 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -9,8 +9,14 @@ module Gitlab
       end
 
       def restore
-        json = IO.read(@path)
-        @tree_hash = ActiveSupport::JSON.decode(json)
+        begin
+          json = IO.read(@path)
+          @tree_hash = ActiveSupport::JSON.decode(json)
+        rescue => e
+          Rails.logger.error("Import/Export error: #{e.message}")
+          raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
+        end
+
         @project_members = @tree_hash.delete('project_members')
 
         ActiveRecord::Base.no_touching do
@@ -46,7 +52,11 @@ module Gitlab
           create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash)
 
           relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
-          relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s])
+          relation_hash_list = @tree_hash[relation_key.to_s]
+
+          next unless relation_hash_list
+
+          relation_hash = create_relation(relation_key, relation_hash_list)
           saved << restored_project.append_or_update_attribute(relation_key, relation_hash)
         end
         saved.all?
@@ -61,14 +71,14 @@ module Gitlab
       def restore_project
         return @project unless @tree_hash
 
-        @project.update(project_params)
+        @project.update_columns(project_params)
         @project
       end
 
       def project_params
         @tree_hash.reject do |key, value|
           # return params that are not 1 to many or 1 to 1 relations
-          value.is_a?(Array) || key == key.singularize
+          value.respond_to?(:each) && !Project.column_names.include?(key)
         end
       end
 
@@ -110,14 +120,18 @@ module Gitlab
       def create_relation(relation, relation_hash_list)
         relation_array = [relation_hash_list].flatten.map do |relation_hash|
           Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
-                                                       relation_hash: relation_hash,
+                                                       relation_hash: parsed_relation_hash(relation_hash),
                                                        members_mapper: members_mapper,
                                                        user: @user,
-                                                       project_id: restored_project.id)
-        end
+                                                       project: restored_project)
+        end.compact
 
         relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
       end
+
+      def parsed_relation_hash(relation_hash)
+        relation_hash.merge!('group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id)
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
index 2fbf437ec26..3473b466936 100644
--- a/lib/gitlab/import_export/project_tree_saver.rb
+++ b/lib/gitlab/import_export/project_tree_saver.rb
@@ -5,8 +5,9 @@ module Gitlab
 
       attr_reader :full_path
 
-      def initialize(project:, shared:)
+      def initialize(project:, current_user:, shared:)
         @project = project
+        @current_user = current_user
         @shared = shared
         @full_path = File.join(@shared.export_path, ImportExport.project_filename)
       end
@@ -24,7 +25,35 @@ module Gitlab
       private
 
       def project_json_tree
-        @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree)
+        project_json['project_members'] += group_members_json
+
+        project_json.to_json
+      end
+
+      def project_json
+        @project_json ||= @project.as_json(reader.project_tree)
+      end
+
+      def reader
+        @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
+      end
+
+      def group_members_json
+        group_members.as_json(reader.group_members_tree).each do |group_member|
+          group_member['source_type'] = 'Project' # Make group members project members of the future import
+        end
+      end
+
+      def group_members
+        return [] unless @current_user.can?(:admin_group, @project.group)
+
+        # We need `.where.not(user_id: nil)` here otherwise when a group has an
+        # invitee, it would make the following query return 0 rows since a NULL
+        # user_id would be present in the subquery
+        # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
+        non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
+
+        GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
       end
     end
   end
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
index 5021a1a14ce..eb7f5120592 100644
--- a/lib/gitlab/import_export/reader.rb
+++ b/lib/gitlab/import_export/reader.rb
@@ -15,12 +15,19 @@ module Gitlab
       # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
       # for outputting a project in JSON format, including its relations and sub relations.
       def project_tree
-        @attributes_finder.find_included(:project).merge(include: build_hash(@tree))
+        attributes = @attributes_finder.find(:project)
+        project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {}
+
+        project_attributes.merge(include: build_hash(@tree))
       rescue => e
         @shared.error(e)
         false
       end
 
+      def group_members_tree
+        @attributes_finder.find_included(:project_members).merge(include: @attributes_finder.find(:user))
+      end
+
       private
 
       # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 9300f789e1b..20580459046 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -3,35 +3,41 @@ module Gitlab
     class RelationFactory
       OVERRIDES = { snippets: :project_snippets,
                     pipelines: 'Ci::Pipeline',
+                    stages: 'Ci::Stage',
                     statuses: 'commit_status',
-                    variables: 'Ci::Variable',
                     triggers: 'Ci::Trigger',
+                    pipeline_schedules: 'Ci::PipelineSchedule',
                     builds: 'Ci::Build',
                     hooks: 'ProjectHook',
                     merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
-                    push_access_levels: 'ProtectedBranch::PushAccessLevel' }.freeze
+                    push_access_levels: 'ProtectedBranch::PushAccessLevel',
+                    create_access_levels: 'ProtectedTag::CreateAccessLevel',
+                    labels: :project_labels,
+                    priorities: :label_priorities,
+                    label: :project_label }.freeze
 
-      USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze
+      USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze
 
-      PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze
+      PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
 
       BUILD_MODELS = %w[Ci::Build commit_status].freeze
 
       IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
 
-      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze
+      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze
 
-      FINDER_ATTRIBUTES = %w[title project_id].freeze
+      TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze
 
       def self.create(*args)
         new(*args).create
       end
 
-      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:)
+      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:)
         @relation_name = OVERRIDES[relation_sym] || relation_sym
-        @relation_hash = relation_hash.except('id', 'noteable_id').merge('project_id' => project_id)
+        @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project.id)
         @members_mapper = members_mapper
         @user = user
+        @project = project
         @imported_object_retries = 0
       end
 
@@ -39,6 +45,8 @@ module Gitlab
       # the relation_hash, updating references with new object IDs, mapping users using
       # the "members_mapper" object, also updating notes if required.
       def create
+        return nil if unknown_service?
+
         setup_models
 
         generate_imported_object
@@ -56,9 +64,14 @@ module Gitlab
 
         update_user_references
         update_project_references
-        reset_ci_tokens if @relation_name == 'Ci::Trigger'
+
+        handle_group_label if group_label?
+        reset_tokens!
+        remove_encrypted_attributes!
+
         @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data']
-        set_st_diffs if @relation_name == :merge_request_diff
+        set_st_diff_commits if @relation_name == :merge_request_diff
+        set_diff if @relation_name == :merge_request_diff_files
       end
 
       def update_user_references
@@ -75,17 +88,13 @@ module Gitlab
       # is left.
       def set_note_author
         old_author_id = @relation_hash['author_id']
-
-        # Users with admin access can map users
-        @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id
-
         author = @relation_hash.delete('author')
 
-        update_note_for_missing_author(author['name']) if missing_author?(old_author_id)
+        update_note_for_missing_author(author['name']) unless has_author?(old_author_id)
       end
 
-      def missing_author?(old_author_id)
-        !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id)
+      def has_author?(old_author_id)
+        admin_user? && @members_mapper.include?(old_author_id)
       end
 
       def missing_author_note(updated_at, author_name)
@@ -94,12 +103,15 @@ module Gitlab
       end
 
       def generate_imported_object
-        if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes
-          trace = @relation_hash.delete('trace')
+        if BUILD_MODELS.include?(@relation_name)
+          @relation_hash.delete('trace') # old export files have trace
+          @relation_hash.delete('token')
+
           imported_object do |object|
-            object.trace = trace
             object.commit_id = nil
           end
+        elsif @relation_name == :merge_requests
+          MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse!
         else
           imported_object
         end
@@ -110,12 +122,11 @@ module Gitlab
 
         # If source and target are the same, populate them with the new project ID.
         if @relation_hash['source_project_id']
-          @relation_hash['source_project_id'] = same_source_and_target? ? project_id : -1
+          @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID
         end
 
         # project_id may not be part of the export, but we always need to populate it if required.
         @relation_hash['project_id'] = project_id
-        @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id']
         @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id']
       end
 
@@ -123,11 +134,36 @@ module Gitlab
         @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
       end
 
-      def reset_ci_tokens
-        return unless Gitlab::ImportExport.reset_tokens?
+      def group_label?
+        @relation_hash['type'] == 'GroupLabel'
+      end
+
+      def handle_group_label
+        # If there's no group, move the label to a project label
+        if @relation_hash['group_id']
+          @relation_hash['project_id'] = nil
+          @relation_name = :group_label
+        else
+          @relation_hash['type'] = 'ProjectLabel'
+        end
+      end
+
+      def reset_tokens!
+        return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s)
 
         # If we import/export a project to the same instance, tokens will have to be reset.
-        @relation_hash['token'] = nil
+        # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across.
+        relation_class.attribute_names.select { |name| name.include?('token') }.each do |token|
+          @relation_hash[token] = nil
+        end
+      end
+
+      def remove_encrypted_attributes!
+        return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any?
+
+        relation_class.encrypted_attributes.each_key do |key|
+          @relation_hash[key.to_s] = nil
+        end
       end
 
       def relation_class
@@ -137,6 +173,7 @@ module Gitlab
       def imported_object
         yield(existing_or_new_object) if block_given?
         existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing)
+
         existing_or_new_object
       rescue ActiveRecord::RecordNotUnique
         # as the operation is not atomic, retry in the unlikely scenario an INSERT is
@@ -151,19 +188,23 @@ module Gitlab
       end
 
       def admin_user?
-        @user.is_admin?
+        @user.admin?
       end
 
       def parsed_relation_hash
-        @parsed_relation_hash ||= begin
-          Gitlab::ImportExport::AttributeCleaner.clean!(relation_hash: @relation_hash)
-
-          @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) }
-        end
+        @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash,
+                                                                               relation_class: relation_class)
       end
 
-      def set_st_diffs
+      def set_st_diff_commits
         @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs')
+
+        HashUtil.deep_symbolize_array!(@relation_hash['st_diffs'])
+        HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits'])
+      end
+
+      def set_diff
+        @relation_hash['diff'] = @relation_hash.delete('utf8_diff')
       end
 
       def existing_or_new_object
@@ -171,11 +212,9 @@ module Gitlab
         # Otherwise always create the record, skipping the extra SELECT clause.
         @existing_or_new_object ||= begin
           if EXISTING_OBJECT_CHECK.include?(@relation_name)
-            events = parsed_relation_hash.delete('events')
+            attribute_hash = attribute_hash_for(['events'])
 
-            unless events.blank?
-              existing_object.assign_attributes(events: events)
-            end
+            existing_object.assign_attributes(attribute_hash) if attribute_hash.any?
 
             existing_object
           else
@@ -184,17 +223,48 @@ module Gitlab
         end
       end
 
+      def attribute_hash_for(attributes)
+        attributes.inject({}) do |hash, value|
+          hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value]
+          hash
+        end
+      end
+
       def existing_object
         @existing_object ||=
           begin
-            finder_hash = parsed_relation_hash.slice(*FINDER_ATTRIBUTES)
-            existing_object = relation_class.find_or_create_by(finder_hash)
+            existing_object = find_or_create_object!
+
             # Done in two steps, as MySQL behaves differently than PostgreSQL using
             # the +find_or_create_by+ method and does not return the ID the second time.
-            existing_object.update(parsed_relation_hash)
+            existing_object.update!(parsed_relation_hash)
             existing_object
           end
       end
+
+      def unknown_service?
+        @relation_name == :services && parsed_relation_hash['type'] &&
+          !Object.const_defined?(parsed_relation_hash['type'])
+      end
+
+      def find_or_create_object!
+        finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id]
+        finder_hash = parsed_relation_hash.slice(*finder_attributes)
+
+        if label?
+          label = relation_class.find_or_initialize_by(finder_hash)
+          parsed_relation_hash.delete('priorities') if label.persisted?
+
+          label.save!
+          label
+        else
+          relation_class.find_or_create_by(finder_hash)
+        end
+      end
+
+      def label?
+        @relation_name.to_s.include?('label')
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index 48a9a6fa5e2..c824d3ea9fc 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -2,6 +2,7 @@ module Gitlab
   module ImportExport
     class RepoRestorer
       include Gitlab::ImportExport::CommandLineUtil
+      include Gitlab::ShellAdapter
 
       def initialize(project:, shared:, path_to_bundle:)
         @project = project
@@ -12,29 +13,11 @@ module Gitlab
       def restore
         return true unless File.exist?(@path_to_bundle)
 
-        mkdir_p(path_to_repo)
-
-        git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks
+        gitlab_shell.import_repository(@project.repository_storage_path, @project.path_with_namespace, @path_to_bundle)
       rescue => e
         @shared.error(e)
         false
       end
-
-      private
-
-      def path_to_repo
-        @project.repository.path_to_repo
-      end
-
-      def repo_restore_hooks
-        return true if wiki?
-
-        git_restore_hooks
-      end
-
-      def wiki?
-        @project.class.name == 'ProjectWiki'
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb
index fc08082fc86..bd3c3ee3b2f 100644
--- a/lib/gitlab/import_export/version_checker.rb
+++ b/lib/gitlab/import_export/version_checker.rb
@@ -24,12 +24,19 @@ module Gitlab
       end
 
       def verify_version!(version)
-        if Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version)
+        if different_version?(version)
           raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}")
         else
           true
         end
       end
+
+      def different_version?(version)
+        Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version)
+      rescue => e
+        Rails.logger.error("Import/Export error: #{e.message}")
+        raise Gitlab::ImportExport::Error.new('Incorrect VERSION format')
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index 94261b7eeed..52276cbcd9a 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -5,23 +5,38 @@
 #
 module Gitlab
   module ImportSources
-    extend CurrentSettings
+    ImportSource = Struct.new(:name, :title, :importer)
+
+    ImportTable = [
+      ImportSource.new('github',         'GitHub',        Gitlab::GithubImport::Importer),
+      ImportSource.new('bitbucket',      'Bitbucket',     Gitlab::BitbucketImport::Importer),
+      ImportSource.new('gitlab',         'GitLab.com',    Gitlab::GitlabImport::Importer),
+      ImportSource.new('google_code',    'Google Code',   Gitlab::GoogleCodeImport::Importer),
+      ImportSource.new('fogbugz',        'FogBugz',       Gitlab::FogbugzImport::Importer),
+      ImportSource.new('git',            'Repo by URL',   nil),
+      ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer),
+      ImportSource.new('gitea',          'Gitea',         Gitlab::GithubImport::Importer)
+    ].freeze
 
     class << self
+      def options
+        @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }]
+      end
+
       def values
-        options.values
+        @values ||= ImportTable.map(&:name)
       end
 
-      def options
-        {
-          'GitHub'        => 'github',
-          'Bitbucket'     => 'bitbucket',
-          'GitLab.com'    => 'gitlab',
-          'Google Code'   => 'google_code',
-          'FogBugz'       => 'fogbugz',
-          'Repo by URL'   => 'git',
-          'GitLab export' => 'gitlab_project'
-        }
+      def importer_names
+        @importer_names ||= ImportTable.select(&:importer).map(&:name)
+      end
+
+      def importer(name)
+        ImportTable.find { |import_source| import_source.name == name }.importer
+      end
+
+      def title(name)
+        options.key(name)
       end
     end
   end
diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb
index d7be50bd437..c9122a23568 100644
--- a/lib/gitlab/incoming_email.rb
+++ b/lib/gitlab/incoming_email.rb
@@ -1,14 +1,27 @@
 module Gitlab
   module IncomingEmail
-    class << self
-      FALLBACK_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze
+    UNSUBSCRIBE_SUFFIX = '+unsubscribe'.freeze
+    WILDCARD_PLACEHOLDER = '%{key}'.freeze
 
+    class << self
       def enabled?
         config.enabled && config.address
       end
 
+      def supports_wildcard?
+        config.address && config.address.include?(WILDCARD_PLACEHOLDER)
+      end
+
+      def supports_issue_creation?
+        enabled? && supports_wildcard?
+      end
+
       def reply_address(key)
-        config.address.gsub('%{key}', key)
+        config.address.sub(WILDCARD_PLACEHOLDER, key)
+      end
+
+      def unsubscribe_address(key)
+        config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}")
       end
 
       def key_from_address(address)
@@ -22,10 +35,14 @@ module Gitlab
       end
 
       def key_from_fallback_message_id(mail_id)
-        match = mail_id.match(FALLBACK_MESSAGE_ID_REGEX)
-        return unless match
+        message_id_regexp = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\z/
 
-        match[1]
+        mail_id[message_id_regexp, 1]
+      end
+
+      def scan_fallback_references(references)
+        # It's looking for each <...>
+        references.scan(/(?!<)[^<>]+(?=>)/)
       end
 
       def config
@@ -39,7 +56,7 @@ module Gitlab
         return nil unless wildcard_address
 
         regex = Regexp.escape(wildcard_address)
-        regex = regex.gsub(Regexp.escape('%{key}'), "(.+)")
+        regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)')
         Regexp.new(regex).freeze
       end
     end
diff --git a/lib/gitlab/issuable_sorter.rb b/lib/gitlab/issuable_sorter.rb
new file mode 100644
index 00000000000..d392214867a
--- /dev/null
+++ b/lib/gitlab/issuable_sorter.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module IssuableSorter
+    class << self
+      def sort(project, issuables, &sort_key)
+        grouped_items = issuables.group_by do |issuable|
+          if issuable.project.id == project.id
+            :project_ref
+          elsif issuable.project.namespace.id == project.namespace.id
+            :namespace_ref
+          else
+            :full_ref
+          end
+        end
+
+        natural_sort_issuables(grouped_items[:project_ref], project) +
+          natural_sort_issuables(grouped_items[:namespace_ref], project) +
+          natural_sort_issuables(grouped_items[:full_ref], project)
+      end
+
+      private
+
+      def natural_sort_issuables(issuables, project)
+        VersionSorter.sort(issuables || []) do |issuable|
+          issuable.to_reference(project)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/issues_labels.rb b/lib/gitlab/issues_labels.rb
index 1bec6088292..b8ca7f2f55f 100644
--- a/lib/gitlab/issues_labels.rb
+++ b/lib/gitlab/issues_labels.rb
@@ -18,8 +18,8 @@ module Gitlab
           { title: "enhancement", color: green }
         ]
 
-        labels.each do |label|
-          project.labels.create(label)
+        labels.each do |params|
+          ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true)
         end
       end
     end
diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb
new file mode 100644
index 00000000000..208f0e1bbea
--- /dev/null
+++ b/lib/gitlab/job_waiter.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  # JobWaiter can be used to wait for a number of Sidekiq jobs to complete.
+  class JobWaiter
+    # The sleep interval between checking keys, in seconds.
+    INTERVAL = 0.1
+
+    # jobs - The job IDs to wait for.
+    def initialize(jobs)
+      @jobs = jobs
+    end
+
+    # Waits for all the jobs to be completed.
+    #
+    # timeout - The maximum amount of seconds to block the caller for. This
+    #           ensures we don't indefinitely block a caller in case a job takes
+    #           long to process, or is never processed.
+    def wait(timeout = 10)
+      start = Time.current
+
+      while (Time.current - start) <= timeout
+        break if SidekiqStatus.all_completed?(@jobs)
+
+        sleep(INTERVAL) # to not overload Redis too much.
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb
new file mode 100644
index 00000000000..c56c1a4322f
--- /dev/null
+++ b/lib/gitlab/kubernetes.rb
@@ -0,0 +1,80 @@
+module Gitlab
+  # Helper methods to do with Kubernetes network services & resources
+  module Kubernetes
+    # This is the comand that is run to start a terminal session. Kubernetes
+    # expects `command=foo&command=bar, not `command[]=foo&command[]=bar`
+    EXEC_COMMAND = URI.encode_www_form(
+      ['sh', '-c', 'bash || sh'].map { |value| ['command', value] }
+    )
+
+    # Filters an array of pods (as returned by the kubernetes API) by their labels
+    def filter_by_label(items, labels = {})
+      items.select do |item|
+        metadata = item.fetch("metadata", {})
+        item_labels = metadata.fetch("labels", nil)
+        next unless item_labels
+
+        labels.all? { |k, v| item_labels[k.to_s] == v }
+      end
+    end
+
+    # Converts a pod (as returned by the kubernetes API) into a terminal
+    def terminals_for_pod(api_url, namespace, pod)
+      metadata = pod.fetch("metadata", {})
+      status   = pod.fetch("status", {})
+      spec     = pod.fetch("spec", {})
+
+      containers = spec["containers"]
+      pod_name   = metadata["name"]
+      phase      = status["phase"]
+
+      return unless containers.present? && pod_name.present? && phase == "Running"
+
+      created_at = DateTime.parse(metadata["creationTimestamp"]) rescue nil
+
+      containers.map do |container|
+        {
+          selectors:    { pod: pod_name, container: container["name"] },
+          url:          container_exec_url(api_url, namespace, pod_name, container["name"]),
+          subprotocols: ['channel.k8s.io'],
+          headers:      Hash.new { |h, k| h[k] = [] },
+          created_at:   created_at
+        }
+      end
+    end
+
+    def add_terminal_auth(terminal, token:, max_session_time:, ca_pem: nil)
+      terminal[:headers]['Authorization'] << "Bearer #{token}"
+      terminal[:max_session_time] = max_session_time
+      terminal[:ca_pem] = ca_pem if ca_pem.present?
+    end
+
+    def container_exec_url(api_url, namespace, pod_name, container_name)
+      url = URI.parse(api_url)
+      url.path = [
+        url.path.sub(%r{/+\z}, ''),
+        'api', 'v1',
+        'namespaces', ERB::Util.url_encode(namespace),
+        'pods', ERB::Util.url_encode(pod_name),
+        'exec'
+      ].join('/')
+
+      url.query = {
+        container: container_name,
+        tty: true,
+        stdin: true,
+        stdout: true,
+        stderr: true
+      }.to_query + '&' + EXEC_COMMAND
+
+      case url.scheme
+      when 'http'
+        url.scheme = 'ws'
+      when 'https'
+        url.scheme = 'wss'
+      end
+
+      url.to_s
+    end
+  end
+end
diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb
index 7e06bd2b0fb..8779577258b 100644
--- a/lib/gitlab/ldap/access.rb
+++ b/lib/gitlab/ldap/access.rb
@@ -16,8 +16,8 @@ module Gitlab
       def self.allowed?(user)
         self.open(user) do |access|
           if access.allowed?
-            user.last_credential_check_at = Time.now
-            user.save
+            Users::UpdateService.new(user, last_credential_check_a: Time.now).execute
+
             true
           else
             false
@@ -34,21 +34,21 @@ module Gitlab
       def allowed?
         if ldap_user
           unless ldap_config.active_directory
-            user.activate if user.ldap_blocked?
+            unblock_user(user, 'is available again') if user.ldap_blocked?
             return true
           end
 
           # Block user in GitLab if he/she was blocked in AD
           if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter)
-            user.ldap_block
+            block_user(user, 'is disabled in Active Directory')
             false
           else
-            user.activate if user.ldap_blocked?
+            unblock_user(user, 'is not disabled anymore') if user.ldap_blocked?
             true
           end
         else
           # Block the user if they no longer exist in LDAP/AD
-          user.ldap_block
+          block_user(user, 'does not exist anymore')
           false
         end
       end
@@ -64,6 +64,24 @@ module Gitlab
       def ldap_user
         @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter)
       end
+
+      def block_user(user, reason)
+        user.ldap_block
+
+        Gitlab::AppLogger.info(
+          "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \
+          "blocking Gitlab user \"#{user.name}\" (#{user.email})"
+        )
+      end
+
+      def unblock_user(user, reason)
+        user.activate
+
+        Gitlab::AppLogger.info(
+          "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \
+          "unblocking Gitlab user \"#{user.name}\" (#{user.email})"
+        )
+      end
     end
   end
 end
diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb
index 8b38cfaefb6..7b05290e5cc 100644
--- a/lib/gitlab/ldap/adapter.rb
+++ b/lib/gitlab/ldap/adapter.rb
@@ -89,9 +89,7 @@ module Gitlab
       end
 
       def user_filter(filter = nil)
-        if config.user_filter.present?
-          user_filter = Net::LDAP::Filter.construct(config.user_filter)
-        end
+        user_filter = config.constructed_user_filter if config.user_filter.present?
 
         if user_filter && filter
           Net::LDAP::Filter.join(filter, user_filter)
diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb
index bf4dd9542d5..95378e5a769 100644
--- a/lib/gitlab/ldap/auth_hash.rb
+++ b/lib/gitlab/ldap/auth_hash.rb
@@ -25,7 +25,7 @@ module Gitlab
       end
 
       def get_raw(key)
-        auth_hash.extra[:raw_info][key]
+        auth_hash.extra[:raw_info][key] if auth_hash.extra
       end
 
       def ldap_config
diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb
index bad683c6511..4745311402c 100644
--- a/lib/gitlab/ldap/authentication.rb
+++ b/lib/gitlab/ldap/authentication.rb
@@ -54,11 +54,9 @@ module Gitlab
 
         # Apply LDAP user filter if present
         if config.user_filter.present?
-          filter = Net::LDAP::Filter.join(
-            filter,
-            Net::LDAP::Filter.construct(config.user_filter)
-          )
+          filter = Net::LDAP::Filter.join(filter, config.constructed_user_filter)
         end
+
         filter
       end
 
diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb
index f9bb5775323..6fdf68641e2 100644
--- a/lib/gitlab/ldap/config.rb
+++ b/lib/gitlab/ldap/config.rb
@@ -13,7 +13,7 @@ module Gitlab
       end
 
       def self.providers
-        servers.map {|server| server['provider_name'] }
+        servers.map { |server| server['provider_name'] }
       end
 
       def self.valid_provider?(provider)
@@ -38,13 +38,31 @@ module Gitlab
       end
 
       def adapter_options
-        {
-          host: options['host'],
-          port: options['port'],
+        opts = base_options.merge(
           encryption: encryption
-        }.tap do |options|
-          options.merge!(auth_options) if has_auth?
+        )
+
+        opts.merge!(auth_options) if has_auth?
+
+        opts
+      end
+
+      def omniauth_options
+        opts = base_options.merge(
+          base: base,
+          method: options['method'],
+          filter: omniauth_user_filter,
+          name_proc: name_proc
+        )
+
+        if has_auth?
+          opts.merge!(
+            bind_dn: options['bind_dn'],
+            password: options['password']
+          )
         end
+
+        opts
       end
 
       def base
@@ -68,6 +86,10 @@ module Gitlab
         options['user_filter']
       end
 
+      def constructed_user_filter
+        @constructed_user_filter ||= Net::LDAP::Filter.construct(user_filter)
+      end
+
       def group_base
         options['group_base']
       end
@@ -85,15 +107,48 @@ module Gitlab
       end
 
       def attributes
-        options['attributes']
+        default_attributes.merge(options['attributes'])
       end
 
       def timeout
         options['timeout'].to_i
       end
 
+      def has_auth?
+        options['password'] || options['bind_dn']
+      end
+
+      def allow_username_or_email_login
+        options['allow_username_or_email_login']
+      end
+
+      def name_proc
+        if allow_username_or_email_login
+          proc { |name| name.gsub(/@.*\z/, '') }
+        else
+          proc { |name| name }
+        end
+      end
+
+      def default_attributes
+        {
+          'username'    => %w(uid userid sAMAccountName),
+          'email'       => %w(mail email userPrincipalName),
+          'name'        => 'cn',
+          'first_name'  => 'givenName',
+          'last_name'   => 'sn'
+        }
+      end
+
       protected
 
+      def base_options
+        {
+          host: options['host'],
+          port: options['port']
+        }
+      end
+
       def base_config
         Gitlab.config.ldap
       end
@@ -123,8 +178,14 @@ module Gitlab
         }
       end
 
-      def has_auth?
-        options['password'] || options['bind_dn']
+      def omniauth_user_filter
+        uid_filter = Net::LDAP::Filter.eq(uid, '%{username}')
+
+        if user_filter.present?
+          Net::LDAP::Filter.join(uid_filter, constructed_user_filter).to_s
+        else
+          uid_filter.to_s
+        end
       end
     end
   end
diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb
index b81f3e8e8f5..43eb73250b7 100644
--- a/lib/gitlab/ldap/person.rb
+++ b/lib/gitlab/ldap/person.rb
@@ -28,7 +28,7 @@ module Gitlab
       end
 
       def name
-        entry.cn.first
+        attribute_value(:name).first
       end
 
       def uid
@@ -40,12 +40,10 @@ module Gitlab
       end
 
       def email
-        entry.try(:mail)
+        attribute_value(:email)
       end
 
-      def dn
-        entry.dn
-      end
+      delegate :dn, to: :entry
 
       private
 
@@ -56,6 +54,19 @@ module Gitlab
       def config
         @config ||= Gitlab::LDAP::Config.new(provider)
       end
+
+      # Using the LDAP attributes configuration, find and return the first
+      # attribute with a value. For example, by default, when given 'email',
+      # this method looks for 'mail', 'email' and 'userPrincipalName' and
+      # returns the first with a value.
+      def attribute_value(attribute)
+        attributes = Array(config.attributes[attribute.to_s])
+        selected_attr = attributes.find { |attr| entry.respond_to?(attr) }
+
+        return nil unless selected_attr
+
+        entry.public_send(selected_attr)
+      end
     end
   end
 end
diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb
index b84c81f1a6c..39180dc17d9 100644
--- a/lib/gitlab/ldap/user.rb
+++ b/lib/gitlab/ldap/user.rb
@@ -1,5 +1,3 @@
-require 'gitlab/o_auth/user'
-
 # LDAP extension for User model
 #
 # * Find or create user from omniauth.auth data
@@ -12,9 +10,9 @@ module Gitlab
       class << self
         def find_by_uid_and_provider(uid, provider)
           # LDAP distinguished name is case-insensitive
-          identity = ::Identity.
-            where(provider: provider).
-            iwhere(extern_uid: uid).last
+          identity = ::Identity
+            .where(provider: provider)
+            .iwhere(extern_uid: uid).last
           identity && identity.user
         end
       end
@@ -43,11 +41,6 @@ module Gitlab
 
       def update_user_attributes
         if persisted?
-          if auth_hash.has_email?
-            gl_user.skip_reconfirmation!
-            gl_user.email = auth_hash.email
-          end
-
           # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved.
           identity = gl_user.identities.find { |identity|  identity.provider == auth_hash.provider }
           identity ||= gl_user.identities.build(provider: auth_hash.provider)
@@ -57,10 +50,6 @@ module Gitlab
           # For an existing identity with no change in DN, this line changes nothing.
           identity.extern_uid = auth_hash.uid
         end
-
-        gl_user.ldap_email = auth_hash.has_email?
-
-        gl_user
       end
 
       def changed?
@@ -71,6 +60,10 @@ module Gitlab
         ldap_config.block_auto_created_users
       end
 
+      def sync_email_from_provider?
+        true
+      end
+
       def allowed?
         Gitlab::LDAP::Access.allowed?(gl_user)
       end
diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb
index 12999a90a29..3503fac40e8 100644
--- a/lib/gitlab/mail_room.rb
+++ b/lib/gitlab/mail_room.rb
@@ -31,9 +31,15 @@ module Gitlab
         config[:ssl] = false if config[:ssl].nil?
         config[:start_tls] = false if config[:start_tls].nil?
         config[:mailbox] = 'inbox' if config[:mailbox].nil?
+        config[:idle_timeout] = 60 if config[:idle_timeout].nil?
 
         if config[:enabled] && config[:address]
-          config[:redis_url] = Gitlab::Redis.new(rails_env).url
+          gitlab_redis = Gitlab::Redis.new(rails_env)
+          config[:redis_url] = gitlab_redis.url
+
+          if gitlab_redis.sentinels?
+            config[:sentinels] = gitlab_redis.sentinels
+          end
         end
 
         config
diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb
index dda371e6554..49285e35251 100644
--- a/lib/gitlab/markup_helper.rb
+++ b/lib/gitlab/markup_helper.rb
@@ -1,6 +1,11 @@
 module Gitlab
   module MarkupHelper
-    module_function
+    extend self
+
+    MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze
+    ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze
+    OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze
+    EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS
 
     # Public: Determines if a given filename is compatible with GitHub::Markup.
     #
@@ -8,10 +13,7 @@ module Gitlab
     #
     # Returns boolean
     def markup?(filename)
-      gitlab_markdown?(filename) ||
-        asciidoc?(filename) ||
-        filename.downcase.end_with?(*%w(.textile .rdoc .org .creole .wiki
-                                        .mediawiki .rst))
+      EXTENSIONS.include?(extension(filename))
     end
 
     # Public: Determines if a given filename is compatible with
@@ -21,7 +23,7 @@ module Gitlab
     #
     # Returns boolean
     def gitlab_markdown?(filename)
-      filename.downcase.end_with?(*%w(.mdown .mkd .mkdn .md .markdown))
+      MARKDOWN_EXTENSIONS.include?(extension(filename))
     end
 
     # Public: Determines if the given filename has AsciiDoc extension.
@@ -30,7 +32,7 @@ module Gitlab
     #
     # Returns boolean
     def asciidoc?(filename)
-      filename.downcase.end_with?(*%w(.adoc .ad .asciidoc))
+      ASCIIDOC_EXTENSIONS.include?(extension(filename))
     end
 
     # Public: Determines if the given filename is plain text.
@@ -39,12 +41,17 @@ module Gitlab
     #
     # Returns boolean
     def plain?(filename)
-      filename.downcase.end_with?('.txt') ||
-        filename.casecmp('readme').zero?
+      extension(filename) == 'txt' || filename.casecmp('readme').zero?
     end
 
     def previewable?(filename)
       markup?(filename)
     end
+
+    private
+
+    def extension(filename)
+      File.extname(filename).downcase.delete('.')
+    end
   end
 end
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 3d1ba33ec68..4779755bb22 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -1,158 +1,10 @@
 module Gitlab
   module Metrics
-    extend Gitlab::CurrentSettings
-
-    RAILS_ROOT   = Rails.root.to_s
-    METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
-    PATH_REGEX   = /^#{RAILS_ROOT}\/?/
-
-    def self.settings
-      @settings ||= {
-        enabled:               current_application_settings[:metrics_enabled],
-        pool_size:             current_application_settings[:metrics_pool_size],
-        timeout:               current_application_settings[:metrics_timeout],
-        method_call_threshold: current_application_settings[:metrics_method_call_threshold],
-        host:                  current_application_settings[:metrics_host],
-        port:                  current_application_settings[:metrics_port],
-        sample_interval:       current_application_settings[:metrics_sample_interval] || 15,
-        packet_size:           current_application_settings[:metrics_packet_size] || 1
-      }
-    end
+    extend Gitlab::Metrics::InfluxDb
+    extend Gitlab::Metrics::Prometheus
 
     def self.enabled?
-      settings[:enabled] || false
-    end
-
-    def self.mri?
-      RUBY_ENGINE == 'ruby'
-    end
-
-    def self.method_call_threshold
-      # This is memoized since this method is called for every instrumented
-      # method. Loading data from an external cache on every method call slows
-      # things down too much.
-      @method_call_threshold ||= settings[:method_call_threshold]
-    end
-
-    def self.pool
-      @pool
-    end
-
-    def self.submit_metrics(metrics)
-      prepared = prepare_metrics(metrics)
-
-      pool.with do |connection|
-        prepared.each_slice(settings[:packet_size]) do |slice|
-          begin
-            connection.write_points(slice)
-          rescue StandardError
-          end
-        end
-      end
-    end
-
-    def self.prepare_metrics(metrics)
-      metrics.map do |hash|
-        new_hash = hash.symbolize_keys
-
-        new_hash[:tags].each do |key, value|
-          if value.blank?
-            new_hash[:tags].delete(key)
-          else
-            new_hash[:tags][key] = escape_value(value)
-          end
-        end
-
-        new_hash
-      end
-    end
-
-    def self.escape_value(value)
-      value.to_s.gsub('=', '\\=')
-    end
-
-    # Measures the execution time of a block.
-    #
-    # Example:
-    #
-    #     Gitlab::Metrics.measure(:find_by_username_duration) do
-    #       User.find_by_username(some_username)
-    #     end
-    #
-    # name - The name of the field to store the execution time in.
-    #
-    # Returns the value yielded by the supplied block.
-    def self.measure(name)
-      trans = current_transaction
-
-      return yield unless trans
-
-      real_start = Time.now.to_f
-      cpu_start = System.cpu_time
-
-      retval = yield
-
-      cpu_stop = System.cpu_time
-      real_stop = Time.now.to_f
-
-      real_time = (real_stop - real_start) * 1000.0
-      cpu_time = cpu_stop - cpu_start
-
-      trans.increment("#{name}_real_time", real_time)
-      trans.increment("#{name}_cpu_time", cpu_time)
-      trans.increment("#{name}_call_count", 1)
-
-      retval
-    end
-
-    # Adds a tag to the current transaction (if any)
-    #
-    # name - The name of the tag to add.
-    # value - The value of the tag.
-    def self.tag_transaction(name, value)
-      trans = current_transaction
-
-      trans.add_tag(name, value) if trans
-    end
-
-    # Sets the action of the current transaction (if any)
-    #
-    # action - The name of the action.
-    def self.action=(action)
-      trans = current_transaction
-
-      trans.action = action if trans
-    end
-
-    # Tracks an event.
-    #
-    # See `Gitlab::Metrics::Transaction#add_event` for more details.
-    def self.add_event(*args)
-      trans = current_transaction
-
-      trans.add_event(*args) if trans
-    end
-
-    # Returns the prefix to use for the name of a series.
-    def self.series_prefix
-      @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
-    end
-
-    # When enabled this should be set before being used as the usual pattern
-    # "@foo ||= bar" is _not_ thread-safe.
-    if enabled?
-      @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
-        host = settings[:host]
-        port = settings[:port]
-
-        InfluxDB::Client.
-          new(udp: { host: host, port: port })
-      end
-    end
-
-    # Allow access from other metrics related middlewares
-    def self.current_transaction
-      Transaction.current
+      influx_metrics_enabled? || prometheus_metrics_enabled?
     end
   end
 end
diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb
new file mode 100644
index 00000000000..d7c56463aac
--- /dev/null
+++ b/lib/gitlab/metrics/influx_db.rb
@@ -0,0 +1,170 @@
+module Gitlab
+  module Metrics
+    module InfluxDb
+      extend Gitlab::CurrentSettings
+      extend self
+
+      MUTEX = Mutex.new
+      private_constant :MUTEX
+
+      def influx_metrics_enabled?
+        settings[:enabled] || false
+      end
+
+      RAILS_ROOT = Rails.root.to_s
+      METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
+      PATH_REGEX = /^#{RAILS_ROOT}\/?/
+
+      def settings
+        @settings ||= {
+          enabled: current_application_settings[:metrics_enabled],
+          pool_size: current_application_settings[:metrics_pool_size],
+          timeout: current_application_settings[:metrics_timeout],
+          method_call_threshold: current_application_settings[:metrics_method_call_threshold],
+          host: current_application_settings[:metrics_host],
+          port: current_application_settings[:metrics_port],
+          sample_interval: current_application_settings[:metrics_sample_interval] || 15,
+          packet_size: current_application_settings[:metrics_packet_size] || 1
+        }
+      end
+
+      def mri?
+        RUBY_ENGINE == 'ruby'
+      end
+
+      def method_call_threshold
+        # This is memoized since this method is called for every instrumented
+        # method. Loading data from an external cache on every method call slows
+        # things down too much.
+        @method_call_threshold ||= settings[:method_call_threshold]
+      end
+
+      def submit_metrics(metrics)
+        prepared = prepare_metrics(metrics)
+
+        pool&.with do |connection|
+          prepared.each_slice(settings[:packet_size]) do |slice|
+            begin
+              connection.write_points(slice)
+            rescue StandardError
+            end
+          end
+        end
+      rescue Errno::EADDRNOTAVAIL, SocketError => ex
+        Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.')
+        Gitlab::EnvironmentLogger.error(ex)
+      end
+
+      def prepare_metrics(metrics)
+        metrics.map do |hash|
+          new_hash = hash.symbolize_keys
+
+          new_hash[:tags].each do |key, value|
+            if value.blank?
+              new_hash[:tags].delete(key)
+            else
+              new_hash[:tags][key] = escape_value(value)
+            end
+          end
+
+          new_hash
+        end
+      end
+
+      def escape_value(value)
+        value.to_s.gsub('=', '\\=')
+      end
+
+      # Measures the execution time of a block.
+      #
+      # Example:
+      #
+      #     Gitlab::Metrics.measure(:find_by_username_duration) do
+      #       User.find_by_username(some_username)
+      #     end
+      #
+      # name - The name of the field to store the execution time in.
+      #
+      # Returns the value yielded by the supplied block.
+      def measure(name)
+        trans = current_transaction
+
+        return yield unless trans
+
+        real_start = Time.now.to_f
+        cpu_start = System.cpu_time
+
+        retval = yield
+
+        cpu_stop = System.cpu_time
+        real_stop = Time.now.to_f
+
+        real_time = (real_stop - real_start) * 1000.0
+        cpu_time = cpu_stop - cpu_start
+
+        trans.increment("#{name}_real_time", real_time)
+        trans.increment("#{name}_cpu_time", cpu_time)
+        trans.increment("#{name}_call_count", 1)
+
+        retval
+      end
+
+      # Adds a tag to the current transaction (if any)
+      #
+      # name - The name of the tag to add.
+      # value - The value of the tag.
+      def tag_transaction(name, value)
+        trans = current_transaction
+
+        trans&.add_tag(name, value)
+      end
+
+      # Sets the action of the current transaction (if any)
+      #
+      # action - The name of the action.
+      def action=(action)
+        trans = current_transaction
+
+        trans&.action = action
+      end
+
+      # Tracks an event.
+      #
+      # See `Gitlab::Metrics::Transaction#add_event` for more details.
+      def add_event(*args)
+        trans = current_transaction
+
+        trans&.add_event(*args)
+      end
+
+      # Returns the prefix to use for the name of a series.
+      def series_prefix
+        @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
+      end
+
+      # Allow access from other metrics related middlewares
+      def current_transaction
+        Transaction.current
+      end
+
+      # When enabled this should be set before being used as the usual pattern
+      # "@foo ||= bar" is _not_ thread-safe.
+      def pool
+        if influx_metrics_enabled?
+          if @pool.nil?
+            MUTEX.synchronize do
+              @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
+                host = settings[:host]
+                port = settings[:port]
+
+                InfluxDB::Client
+                  .new(udp: { host: host, port: port })
+              end
+            end
+          end
+          @pool
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 4b7a791e497..6aa38542cb4 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -143,11 +143,12 @@ module Gitlab
         # signature this would break things. As a result we'll make sure the
         # generated method _only_ accepts regular arguments if the underlying
         # method also accepts them.
-        if method.arity == 0
-          args_signature = ''
-        else
-          args_signature = '*args'
-        end
+        args_signature =
+          if method.arity == 0
+            ''
+          else
+            '*args'
+          end
 
         proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1
           def #{name}(#{args_signature})
diff --git a/lib/gitlab/metrics/null_metric.rb b/lib/gitlab/metrics/null_metric.rb
new file mode 100644
index 00000000000..3b5a2907195
--- /dev/null
+++ b/lib/gitlab/metrics/null_metric.rb
@@ -0,0 +1,10 @@
+module Gitlab
+  module Metrics
+    # Mocks ::Prometheus::Client::Metric and all derived metrics
+    class NullMetric
+      def method_missing(name, *args, &block)
+        nil
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb
new file mode 100644
index 00000000000..9d314a56e58
--- /dev/null
+++ b/lib/gitlab/metrics/prometheus.rb
@@ -0,0 +1,55 @@
+require 'prometheus/client'
+
+module Gitlab
+  module Metrics
+    module Prometheus
+      include Gitlab::CurrentSettings
+
+      def metrics_folder_present?
+        ENV.has_key?('prometheus_multiproc_dir') &&
+          ::Dir.exist?(ENV['prometheus_multiproc_dir']) &&
+          ::File.writable?(ENV['prometheus_multiproc_dir'])
+      end
+
+      def prometheus_metrics_enabled?
+        return @prometheus_metrics_enabled if defined?(@prometheus_metrics_enabled)
+
+        @prometheus_metrics_enabled = prometheus_metrics_enabled_unmemoized
+      end
+
+      def registry
+        @registry ||= ::Prometheus::Client.registry
+      end
+
+      def counter(name, docstring, base_labels = {})
+        provide_metric(name) || registry.counter(name, docstring, base_labels)
+      end
+
+      def summary(name, docstring, base_labels = {})
+        provide_metric(name) || registry.summary(name, docstring, base_labels)
+      end
+
+      def gauge(name, docstring, base_labels = {})
+        provide_metric(name) || registry.gauge(name, docstring, base_labels)
+      end
+
+      def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS)
+        provide_metric(name) || registry.histogram(name, docstring, base_labels, buckets)
+      end
+
+      def provide_metric(name)
+        if prometheus_metrics_enabled?
+          registry.get(name)
+        else
+          NullMetric.new
+        end
+      end
+
+      private
+
+      def prometheus_metrics_enabled_unmemoized
+        metrics_folder_present? && current_application_settings[:prometheus_metrics_enabled] || false
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index 01c96a6fe96..adc0db1a874 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -2,8 +2,8 @@ module Gitlab
   module Metrics
     # Rack middleware for tracking Rails and Grape requests.
     class RackMiddleware
-      CONTROLLER_KEY = 'action_controller.instance'
-      ENDPOINT_KEY   = 'api.endpoint'
+      CONTROLLER_KEY = 'action_controller.instance'.freeze
+      ENDPOINT_KEY   = 'api.endpoint'.freeze
       CONTENT_TYPES = {
         'text/html' => :html,
         'text/plain' => :txt,
@@ -14,7 +14,7 @@ module Gitlab
         'image/jpeg' => :jpeg,
         'image/gif' => :gif,
         'image/svg+xml' => :svg
-      }
+      }.freeze
 
       def initialize(app)
         @app = app
@@ -70,8 +70,19 @@ module Gitlab
 
       def tag_endpoint(trans, env)
         endpoint = env[ENDPOINT_KEY]
-        path = endpoint_paths_cache[endpoint.route.route_method][endpoint.route.route_path]
-        trans.action = "Grape##{endpoint.route.route_method} #{path}"
+
+        begin
+          route = endpoint.route
+        rescue
+          # endpoint.route is calling env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info]
+          # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response
+          # so we're rescuing exceptions and bailing out
+        end
+
+        if route
+          path = endpoint_paths_cache[route.request_method][route.path]
+          trans.action = "Grape##{route.request_method} #{path}"
+        end
       end
 
       private
diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
index 2e9dd4645e3..d435a33e9c7 100644
--- a/lib/gitlab/metrics/subscribers/action_view.rb
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -5,7 +5,7 @@ module Gitlab
       class ActionView < ActiveSupport::Subscriber
         attach_to :action_view
 
-        SERIES = 'views'
+        SERIES = 'views'.freeze
 
         def render_template(event)
           track(event) if current_transaction
diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb
index 287b7a83547..aba3e0df382 100644
--- a/lib/gitlab/metrics/system.rb
+++ b/lib/gitlab/metrics/system.rb
@@ -11,7 +11,7 @@ module Gitlab
           mem   = 0
           match = File.read('/proc/self/status').match(/VmRSS:\s+(\d+)/)
 
-          if match and match[1]
+          if match && match[1]
             mem = match[1].to_f * 1024
           end
 
@@ -34,13 +34,13 @@ module Gitlab
       # THREAD_CPUTIME is not supported on OS X
       if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID)
         def self.cpu_time
-          Process.
-            clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond)
+          Process
+            .clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond)
         end
       else
         def self.cpu_time
-          Process.
-            clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond)
+          Process
+            .clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond)
         end
       end
 
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 7bc16181be6..4f9fb1c7853 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -5,7 +5,7 @@ module Gitlab
       THREAD_KEY = :_gitlab_metrics_transaction
 
       # The series to store events (e.g. Git pushes) in.
-      EVENT_SERIES = 'events'
+      EVENT_SERIES = 'events'.freeze
 
       attr_reader :tags, :values, :method, :metrics
 
diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index 5764ab15652..6023fa1820f 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -30,21 +30,69 @@ module Gitlab
       end
 
       def go_body(request)
-        base_url = Gitlab.config.gitlab.url
-        # Go subpackages may be in the form of namespace/project/path1/path2/../pathN
-        # We can just ignore the paths and leave the namespace/project
-        path_info = request.env["PATH_INFO"]
-        path_info.sub!(/^\//, '')
-        project_path = path_info.split('/').first(2).join('/')
-        request_url = URI.join(base_url, project_path)
-        domain_path = strip_url(request_url.to_s)
+        project_url = URI.join(Gitlab.config.gitlab.url, project_path(request))
+        import_prefix = strip_url(project_url.to_s)
 
-        "<!DOCTYPE html><html><head><meta content='#{domain_path} git #{request_url}.git' name='go-import'></head></html>\n"
+        "<!DOCTYPE html><html><head><meta content='#{import_prefix} git #{project_url}.git' name='go-import'></head></html>\n"
       end
 
       def strip_url(url)
         url.gsub(/\Ahttps?:\/\//, '')
       end
+
+      def project_path(request)
+        path_info = request.env["PATH_INFO"]
+        path_info.sub!(/^\//, '')
+
+        # Go subpackages may be in the form of `namespace/project/path1/path2/../pathN`.
+        # In a traditional project with a single namespace, this would denote repo
+        # `namespace/project` with subpath `path1/path2/../pathN`, but with nested
+        # groups, this could also be `namespace/project/path1` with subpath
+        # `path2/../pathN`, for example.
+
+        # We find all potential project paths out of the path segments
+        path_segments = path_info.split('/')
+        simple_project_path = path_segments.first(2).join('/')
+
+        # If the path is at most 2 segments long, it is a simple `namespace/project` path and we're done
+        return simple_project_path if path_segments.length <= 2
+
+        project_paths = []
+        begin
+          project_paths << path_segments.join('/')
+          path_segments.pop
+        end while path_segments.length >= 2
+
+        # We see if a project exists with any of these potential paths
+        project = project_for_paths(project_paths, request)
+
+        if project
+          # If a project is found and the user has access, we return the full project path
+          project.full_path
+        else
+          # If not, we return the first two components as if it were a simple `namespace/project` path,
+          # so that we don't reveal the existence of a nested project the user doesn't have access to.
+          # This means that for an unauthenticated request to `group/subgroup/project/subpackage`
+          # for a private `group/subgroup/project` with subpackage path `subpackage`, GitLab will respond
+          # as if the user is looking for project `group/subgroup`, with subpackage path `project/subpackage`.
+          # Since `go get` doesn't authenticate by default, this means that
+          # `go get gitlab.com/group/subgroup/project/subpackage` will not work for private projects.
+          # `go get gitlab.com/group/subgroup/project.git/subpackage` will work, since Go is smart enough
+          # to figure that out. `import 'gitlab.com/...'` behaves the same as `go get`.
+          simple_project_path
+        end
+      end
+
+      def project_for_paths(paths, request)
+        project = Project.where_full_path_in(paths).first
+        return unless Ability.allowed?(current_user(request), :read_project, project)
+
+        project
+      end
+
+      def current_user(request)
+        request.env['warden']&.authenticate
+      end
     end
   end
 end
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
new file mode 100644
index 00000000000..fee741b47be
--- /dev/null
+++ b/lib/gitlab/middleware/multipart.rb
@@ -0,0 +1,103 @@
+# Gitlab::Middleware::Multipart - a Rack::Multipart replacement
+#
+# Rack::Multipart leaves behind tempfiles in /tmp and uses valuable Ruby
+# process time to copy files around. This alternative solution uses
+# gitlab-workhorse to clean up the tempfiles and puts the tempfiles in a
+# location where copying should not be needed.
+#
+# When gitlab-workhorse finds files in a multipart MIME body it sends
+# a signed message via a request header. This message lists the names of
+# the multipart entries that gitlab-workhorse filtered out of the
+# multipart structure and saved to tempfiles. Workhorse adds new entries
+# in the multipart structure with paths to the tempfiles.
+#
+# The job of this Rack middleware is to detect and decode the message
+# from workhorse. If present, it walks the Rack 'params' hash for the
+# current request, opens the respective tempfiles, and inserts the open
+# Ruby File objects in the params hash where Rack::Multipart would have
+# put them. The goal is that application code deeper down can keep
+# working the way it did with Rack::Multipart without changes.
+#
+# CAVEAT: the code that modifies the params hash is a bit complex. It is
+# conceivable that certain Rack params structures will not be modified
+# correctly. We are not aware of such bugs at this time though.
+#
+
+module Gitlab
+  module Middleware
+    class Multipart
+      RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS'.freeze
+
+      class Handler
+        def initialize(env, message)
+          @request = Rack::Request.new(env)
+          @rewritten_fields = message['rewritten_fields']
+          @open_files = []
+        end
+
+        def with_open_files
+          @rewritten_fields.each do |field, tmp_path|
+            parsed_field = Rack::Utils.parse_nested_query(field)
+            raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1
+
+            key, value = parsed_field.first
+            if value.nil?
+              value = open_file(tmp_path)
+              @open_files << value
+            else
+              value = decorate_params_value(value, @request.params[key], tmp_path)
+            end
+            @request.update_param(key, value)
+          end
+
+          yield
+        ensure
+          @open_files.each(&:close)
+        end
+
+        # This function calls itself recursively
+        def decorate_params_value(path_hash, value_hash, tmp_path)
+          unless path_hash.is_a?(Hash) && path_hash.count == 1
+            raise "invalid path: #{path_hash.inspect}"
+          end
+          path_key, path_value = path_hash.first
+
+          unless value_hash.is_a?(Hash) && value_hash[path_key]
+            raise "invalid value hash: #{value_hash.inspect}"
+          end
+
+          case path_value
+          when nil
+            value_hash[path_key] = open_file(tmp_path)
+            @open_files << value_hash[path_key]
+            value_hash
+          when Hash
+            decorate_params_value(path_value, value_hash[path_key], tmp_path)
+            value_hash
+          else
+            raise "unexpected path value: #{path_value.inspect}"
+          end
+        end
+
+        def open_file(path)
+          ::UploadedFile.new(path, File.basename(path), 'application/octet-stream')
+        end
+      end
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        encoded_message = env.delete(RACK_ENV_KEY)
+        return @app.call(env) if encoded_message.blank?
+
+        message = Gitlab::Workhorse.decode_jwt(encoded_message)[0]
+
+        Handler.new(env, message).with_open_files do
+          @app.call(env)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/middleware/webpack_proxy.rb b/lib/gitlab/middleware/webpack_proxy.rb
new file mode 100644
index 00000000000..6105d165810
--- /dev/null
+++ b/lib/gitlab/middleware/webpack_proxy.rb
@@ -0,0 +1,24 @@
+# This Rack middleware is intended to proxy the webpack assets directory to the
+# webpack-dev-server.  It is only intended for use in development.
+
+module Gitlab
+  module Middleware
+    class WebpackProxy < Rack::Proxy
+      def initialize(app = nil, opts = {})
+        @proxy_host = opts.fetch(:proxy_host, 'localhost')
+        @proxy_port = opts.fetch(:proxy_port, 3808)
+        @proxy_path = opts[:proxy_path] if opts[:proxy_path]
+
+        super(app, backend: "http://#{@proxy_host}:#{@proxy_port}", **opts)
+      end
+
+      def perform_request(env)
+        if @proxy_path && env['PATH_INFO'].start_with?("/#{@proxy_path}")
+          super(env)
+        else
+          @app.call(env)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb
index 9ad7a38d505..ac9d66c836d 100644
--- a/lib/gitlab/o_auth/provider.rb
+++ b/lib/gitlab/o_auth/provider.rb
@@ -22,7 +22,11 @@ module Gitlab
       def self.config_for(name)
         name = name.to_s
         if ldap_provider?(name)
-          Gitlab::LDAP::Config.new(name).options
+          if Gitlab::LDAP::Config.valid_provider?(name)
+            Gitlab::LDAP::Config.new(name).options
+          else
+            nil
+          end
         else
           Gitlab.config.omniauth.providers.find { |provider| provider.name == name }
         end
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index 0a91d3918d5..b3f453e506d 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -5,13 +5,14 @@
 #
 module Gitlab
   module OAuth
-    class SignupDisabledError < StandardError; end
+    SignupDisabledError = Class.new(StandardError)
 
     class User
       attr_accessor :auth_hash, :gl_user
 
       def initialize(auth_hash)
         self.auth_hash = auth_hash
+        update_email
       end
 
       def persisted?
@@ -29,17 +30,16 @@ module Gitlab
       def save(provider = 'OAuth')
         unauthorized_to_create unless gl_user
 
-        if needs_blocking?
-          gl_user.save!
-          gl_user.block
-        else
-          gl_user.save!
-        end
+        block_after_save = needs_blocking?
+
+        Users::UpdateService.new(gl_user).execute!
+
+        gl_user.block if block_after_save
 
         log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}"
         gl_user
       rescue ActiveRecord::RecordInvalid => e
-        log.info "(#{provider}) Error saving user: #{gl_user.errors.full_messages}"
+        log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}"
         return self, e.record.errors
       end
 
@@ -102,6 +102,8 @@ module Gitlab
         Gitlab::LDAP::Config.providers.each do |provider|
           adapter = Gitlab::LDAP::Adapter.new(provider)
           @ldap_person = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter)
+          # The `uid` might actually be a DN. Try it next.
+          @ldap_person ||= Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter)
           break if @ldap_person
         end
         @ldap_person
@@ -146,10 +148,8 @@ module Gitlab
       end
 
       def build_new_user
-        user = ::User.new(user_attributes)
-        user.skip_confirmation!
-        user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider)
-        user
+        user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true)
+        Users::BuildService.new(nil, user_params).execute(skip_authorization: true)
       end
 
       def user_attributes
@@ -175,6 +175,22 @@ module Gitlab
         }
       end
 
+      def sync_email_from_provider?
+        auth_hash.provider.to_s == Gitlab.config.omniauth.sync_email_from_provider.to_s
+      end
+
+      def update_email
+        if auth_hash.has_email? && sync_email_from_provider?
+          if persisted?
+            gl_user.skip_reconfirmation!
+            gl_user.email = auth_hash.email
+          end
+
+          gl_user.external_email = true
+          gl_user.email_provider = auth_hash.provider
+        end
+      end
+
       def log
         Gitlab::AppLogger
       end
diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb
new file mode 100644
index 00000000000..962ff4d3985
--- /dev/null
+++ b/lib/gitlab/optimistic_locking.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module OptimisticLocking
+    module_function
+
+    def retry_lock(subject, retries = 100, &block)
+      loop do
+        begin
+          ActiveRecord::Base.transaction do
+            return yield(subject)
+          end
+        rescue ActiveRecord::StaleObjectError
+          retries -= 1
+          raise unless retries >= 0
+          subject.reload
+        end
+      end
+    end
+
+    alias_method :retry_optimistic_lock, :retry_lock
+  end
+end
diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb
index 4e2f8ed5587..fc3f21233dd 100644
--- a/lib/gitlab/other_markup.rb
+++ b/lib/gitlab/other_markup.rb
@@ -4,18 +4,13 @@ module Gitlab
     # Public: Converts the provided markup into HTML.
     #
     # input         - the source text in a markup format
-    # context       - a Hash with the template context:
-    #                 :commit
-    #                 :project
-    #                 :project_wiki
-    #                 :requested_path
-    #                 :ref
     #
     def self.render(file_name, input, context)
-      html = GitHub::Markup.render(file_name, input).
-        force_encoding(input.encoding)
+      html = GitHub::Markup.render(file_name, input)
+        .force_encoding(input.encoding)
+      context[:pipeline] = :markup
 
-      html = Banzai.post_process(html, context)
+      html = Banzai.render(html, context)
 
       html.html_safe
     end
diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb
new file mode 100644
index 00000000000..0d541935bc6
--- /dev/null
+++ b/lib/gitlab/otp_key_rotator.rb
@@ -0,0 +1,87 @@
+module Gitlab
+  # The +otp_key_base+ param is used to encrypt the User#otp_secret attribute.
+  #
+  # When +otp_key_base+ is changed, it invalidates the current encrypted values
+  # of User#otp_secret. This class can be used to decrypt all the values with
+  # the old key, encrypt them with the new key, and and update the database
+  # with the new values.
+  #
+  # For persistence between runs, a CSV file is used with the following columns:
+  #
+  #   user_id, old_value, new_value
+  #
+  # Only the encrypted values are stored in this file.
+  #
+  # As users may have their 2FA settings changed at any time, this is only
+  # guaranteed to be safe if run offline.
+  class OtpKeyRotator
+    HEADERS = %w[user_id old_value new_value].freeze
+
+    attr_reader :filename
+
+    # Create a new rotator. +filename+ is used to store values by +calculate!+,
+    # and to update the database with new and old values in +apply!+ and
+    # +rollback!+, respectively.
+    def initialize(filename)
+      @filename = filename
+    end
+
+    def rotate!(old_key:, new_key:)
+      old_key ||= Gitlab::Application.secrets.otp_key_base
+
+      raise ArgumentError.new("Old key is the same as the new key") if old_key == new_key
+      raise ArgumentError.new("New key is too short! Must be 256 bits") if new_key.size < 64
+
+      write_csv do |csv|
+        ActiveRecord::Base.transaction do
+          User.with_two_factor.in_batches do |relation|
+            rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt)
+            rows.each do |row|
+              user = %i[id ciphertext iv salt].zip(row).to_h
+              new_value = reencrypt(user, old_key, new_key)
+
+              User.where(id: user[:id]).update_all(encrypted_otp_secret: new_value)
+              csv << [user[:id], user[:ciphertext], new_value]
+            end
+          end
+        end
+      end
+    end
+
+    def rollback!
+      ActiveRecord::Base.transaction do
+        CSV.foreach(filename, headers: HEADERS, return_headers: false) do |row|
+          User.where(id: row['user_id']).update_all(encrypted_otp_secret: row['old_value'])
+        end
+      end
+    end
+
+    private
+
+    attr_reader :old_key, :new_key
+
+    def otp_secret_settings
+      @otp_secret_settings ||= User.encrypted_attributes[:otp_secret]
+    end
+
+    def reencrypt(user, old_key, new_key)
+      original = user[:ciphertext].unpack("m").join
+      opts = {
+        iv: user[:iv].unpack("m").join,
+        salt: user[:salt].unpack("m").join,
+        algorithm: otp_secret_settings[:algorithm],
+        insecure_mode: otp_secret_settings[:insecure_mode]
+      }
+
+      decrypted = Encryptor.decrypt(original, opts.merge(key: old_key))
+      encrypted = Encryptor.encrypt(decrypted, opts.merge(key: new_key))
+      [encrypted].pack("m")
+    end
+
+    def write_csv(&blk)
+      File.open(filename, "w") do |file|
+        yield CSV.new(file, headers: HEADERS, write_headers: false)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb
new file mode 100644
index 00000000000..fb215f27cbd
--- /dev/null
+++ b/lib/gitlab/pages_transfer.rb
@@ -0,0 +1,7 @@
+module Gitlab
+  class PagesTransfer < ProjectTransfer
+    def root_dir
+      Gitlab.config.pages.path
+    end
+  end
+end
diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
new file mode 100644
index 00000000000..10eb99fb461
--- /dev/null
+++ b/lib/gitlab/path_regex.rb
@@ -0,0 +1,266 @@
+module Gitlab
+  module PathRegex
+    extend self
+
+    # All routes that appear on the top level must be listed here.
+    # This will make sure that groups cannot be created with these names
+    # as these routes would be masked by the paths already in place.
+    #
+    # Example:
+    #   /api/api-project
+    #
+    #  the path `api` shouldn't be allowed because it would be masked by `api/*`
+    #
+    TOP_LEVEL_ROUTES = %w[
+      -
+      .well-known
+      abuse_reports
+      admin
+      all
+      api
+      assets
+      autocomplete
+      ci
+      dashboard
+      explore
+      files
+      groups
+      health_check
+      help
+      hooks
+      import
+      invites
+      issues
+      jwt
+      koding
+      member
+      merge_requests
+      new
+      notes
+      notification_settings
+      oauth
+      profile
+      projects
+      public
+      repository
+      robots.txt
+      s
+      search
+      sent_notifications
+      services
+      snippets
+      system
+      teams
+      u
+      unicorn_test
+      unsubscribes
+      uploads
+      users
+    ].freeze
+
+    # This list should contain all words following `/*namespace_id/:project_id` in
+    # routes that contain a second wildcard.
+    #
+    # Example:
+    #   /*namespace_id/:project_id/badges/*ref/build
+    #
+    # If `badges` was allowed as a project/group name, we would not be able to access the
+    # `badges` route for those projects:
+    #
+    # Consider a namespace with path `foo/bar` and a project called `badges`.
+    # The route to the build badge would then be `/foo/bar/badges/badges/master/build.svg`
+    #
+    # When accessing this path the route would be matched to the `badges` path
+    # with the following params:
+    #   - namespace_id: `foo`
+    #   - project_id: `bar`
+    #   - ref: `badges/master`
+    #
+    # Failing to find the project, this would result in a 404.
+    #
+    # By rejecting `badges` the router can _count_ on the fact that `badges` will
+    # be preceded by the `namespace/project`.
+    PROJECT_WILDCARD_ROUTES = %w[
+      -
+      badges
+      blame
+      blob
+      builds
+      commits
+      create
+      create_dir
+      edit
+      environments/folders
+      files
+      find_file
+      gitlab-lfs/objects
+      info/lfs/objects
+      new
+      preview
+      raw
+      refs
+      tree
+      update
+      wikis
+    ].freeze
+
+    # These are all the paths that follow `/groups/*id/ or `/groups/*group_id`
+    # We need to reject these because we have a `/groups/*id` page that is the same
+    # as the `/*id`.
+    #
+    # If we would allow a subgroup to be created with the name `activity` then
+    # this group would not be accessible through `/groups/parent/activity` since
+    # this would map to the activity-page of its parent.
+    GROUP_ROUTES = %w[
+      activity
+      analytics
+      audit_events
+      avatar
+      edit
+      group_members
+      hooks
+      issues
+      labels
+      ldap
+      ldap_group_links
+      merge_requests
+      milestones
+      notification_setting
+      pipeline_quota
+      projects
+      subgroups
+    ].freeze
+
+    ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES
+    ILLEGAL_GROUP_PATH_WORDS = (PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze
+
+    # The namespace regex is used in JavaScript to validate usernames in the "Register" form. However, Javascript
+    # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`.
+    # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to
+    # allow non-regex validations, etc), `NAMESPACE_FORMAT_REGEX_JS` serves as a Javascript-compatible version of
+    # `NAMESPACE_FORMAT_REGEX`, with the negative lookbehind assertion removed. This means that the client-side validation
+    # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation.
+    PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze
+    NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze
+
+    NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze
+    NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/.freeze
+    PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/.freeze
+    FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/)*#{NAMESPACE_FORMAT_REGEX}}.freeze
+
+    def root_namespace_route_regex
+      @root_namespace_route_regex ||= begin
+        illegal_words = Regexp.new(Regexp.union(TOP_LEVEL_ROUTES).source, Regexp::IGNORECASE)
+
+        single_line_regexp %r{
+          (?!(#{illegal_words})/)
+          #{NAMESPACE_FORMAT_REGEX}
+        }x
+      end
+    end
+
+    def full_namespace_route_regex
+      @full_namespace_route_regex ||= begin
+        illegal_words = Regexp.new(Regexp.union(ILLEGAL_GROUP_PATH_WORDS).source, Regexp::IGNORECASE)
+
+        single_line_regexp %r{
+          #{root_namespace_route_regex}
+          (?:
+            /
+            (?!#{illegal_words}/)
+            #{NAMESPACE_FORMAT_REGEX}
+          )*
+        }x
+      end
+    end
+
+    def project_route_regex
+      @project_route_regex ||= begin
+        illegal_words = Regexp.new(Regexp.union(ILLEGAL_PROJECT_PATH_WORDS).source, Regexp::IGNORECASE)
+
+        single_line_regexp %r{
+          (?!(#{illegal_words})/)
+          #{PROJECT_PATH_FORMAT_REGEX}
+        }x
+      end
+    end
+
+    def project_git_route_regex
+      @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze
+    end
+
+    def root_namespace_path_regex
+      @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z}
+    end
+
+    def full_namespace_path_regex
+      @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z}
+    end
+
+    def project_path_regex
+      @project_path_regex ||= %r{\A#{project_route_regex}/\z}
+    end
+
+    def full_project_path_regex
+      @full_project_path_regex ||= %r{\A#{full_namespace_route_regex}/#{project_route_regex}/\z}
+    end
+
+    def full_namespace_format_regex
+      @namespace_format_regex ||= /A#{FULL_NAMESPACE_FORMAT_REGEX}\z/.freeze
+    end
+
+    def namespace_format_regex
+      @namespace_format_regex ||= /\A#{NAMESPACE_FORMAT_REGEX}\z/.freeze
+    end
+
+    def namespace_format_message
+      "can contain only letters, digits, '_', '-' and '.'. " \
+      "Cannot start with '-' or end in '.', '.git' or '.atom'." \
+    end
+
+    def project_path_format_regex
+      @project_path_format_regex ||= /\A#{PROJECT_PATH_FORMAT_REGEX}\z/.freeze
+    end
+
+    def project_path_format_message
+      "can contain only letters, digits, '_', '-' and '.'. " \
+      "Cannot start with '-', end in '.git' or end in '.atom'" \
+    end
+
+    def archive_formats_regex
+      #                           |zip|tar|    tar.gz    |         tar.bz2         |
+      @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze
+    end
+
+    def git_reference_regex
+      # Valid git ref regex, see:
+      # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html
+
+      @git_reference_regex ||= single_line_regexp %r{
+        (?!
+           (?# doesn't begins with)
+           \/|                    (?# rule #6)
+           (?# doesn't contain)
+           .*(?:
+              [\/.]\.|            (?# rule #1,3)
+              \/\/|               (?# rule #6)
+              @\{|                (?# rule #8)
+              \\                  (?# rule #9)
+           )
+        )
+        [^\000-\040\177~^:?*\[]+  (?# rule #4-5)
+        (?# doesn't end with)
+        (?<!\.lock)               (?# rule #1)
+        (?<![\/.])                (?# rule #6-7)
+      }x
+    end
+
+    private
+
+    def single_line_regexp(regex)
+      # Turns a multiline extended regexp into a single line one,
+      # beacuse `rake routes` breaks on multiline regexes.
+      Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze
+    end
+  end
+end
diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb
new file mode 100644
index 00000000000..163a40ad306
--- /dev/null
+++ b/lib/gitlab/performance_bar.rb
@@ -0,0 +1,7 @@
+module Gitlab
+  module PerformanceBar
+    def self.enabled?
+      Feature.enabled?('gitlab_performance_bar')
+    end
+  end
+end
diff --git a/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb b/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb
new file mode 100644
index 00000000000..d939a6ea18d
--- /dev/null
+++ b/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb
@@ -0,0 +1,22 @@
+# This solves a bug with a X-Senfile header that wouldn't be set properly, see
+# https://github.com/peek/peek-performance_bar/pull/27
+module Gitlab
+  module PerformanceBar
+    module PeekPerformanceBarWithRackBody
+      def call(env)
+        @env = env
+        reset_stats
+
+        @total_requests += 1
+        first_request if @total_requests == 1
+
+        env['process.request_start'] = @start.to_f
+        env['process.total_requests'] = total_requests
+
+        status, headers, body = @app.call(env)
+        body = Rack::BodyProxy.new(body) { record_request }
+        [status, headers, body]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb
new file mode 100644
index 00000000000..574ae8731a5
--- /dev/null
+++ b/lib/gitlab/performance_bar/peek_query_tracker.rb
@@ -0,0 +1,39 @@
+# Inspired by https://github.com/peek/peek-pg/blob/master/lib/peek/views/pg.rb
+module Gitlab
+  module PerformanceBar
+    module PeekQueryTracker
+      def sorted_queries
+        PEEK_DB_CLIENT.query_details
+          .sort { |a, b| b[:duration] <=> a[:duration] }
+      end
+
+      def results
+        super.merge(queries: sorted_queries)
+      end
+
+      private
+
+      def setup_subscribers
+        super
+
+        # Reset each counter when a new request starts
+        before_request do
+          PEEK_DB_CLIENT.query_details = []
+        end
+
+        subscribe('sql.active_record') do |_, start, finish, _, data|
+          if RequestStore.active? && RequestStore.store[:peek_enabled]
+            track_query(data[:sql].strip, data[:binds], start, finish)
+          end
+        end
+      end
+
+      def track_query(raw_query, bindings, start, finish)
+        query = Gitlab::Sherlock::Query.new(raw_query, start, finish)
+        query_info = { duration: '%.3f' % query.duration, sql: query.formatted_query }
+
+        PEEK_DB_CLIENT.query_details << query_info
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb
new file mode 100644
index 00000000000..f0c50584f07
--- /dev/null
+++ b/lib/gitlab/polling_interval.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  class PollingInterval
+    include Gitlab::CurrentSettings
+
+    HEADER_NAME = 'Poll-Interval'.freeze
+
+    def self.set_header(response, interval:)
+      if polling_enabled?
+        multiplier = current_application_settings.polling_interval_multiplier
+        value = (interval * multiplier).to_i
+      else
+        value = -1
+      end
+
+      response.headers[HEADER_NAME] = value.to_s
+    end
+
+    def self.polling_enabled?
+      !current_application_settings.polling_interval_multiplier.zero?
+    end
+  end
+end
diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb
index cc74bb29087..4bc5cda8cb5 100644
--- a/lib/gitlab/popen.rb
+++ b/lib/gitlab/popen.rb
@@ -5,13 +5,13 @@ module Gitlab
   module Popen
     extend self
 
-    def popen(cmd, path = nil)
+    def popen(cmd, path = nil, vars = {})
       unless cmd.is_a?(Array)
         raise "System commands must be given as an array of strings"
       end
 
       path ||= Dir.pwd
-      vars = { "PWD" => path }
+      vars['PWD'] = path
       options = { chdir: path }
 
       unless File.directory?(path)
diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb
new file mode 100644
index 00000000000..15b8beacf60
--- /dev/null
+++ b/lib/gitlab/project_authorizations/with_nested_groups.rb
@@ -0,0 +1,125 @@
+module Gitlab
+  module ProjectAuthorizations
+    # Calculating new project authorizations when supporting nested groups.
+    #
+    # This class relies on Common Table Expressions to efficiently get all data,
+    # including data for nested groups. As a result this class can only be used
+    # on PostgreSQL.
+    class WithNestedGroups
+      attr_reader :user
+
+      # user - The User object for which to calculate the authorizations.
+      def initialize(user)
+        @user = user
+      end
+
+      def calculate
+        cte = recursive_cte
+        cte_alias = cte.table.alias(Group.table_name)
+        projects = Project.arel_table
+        links = ProjectGroupLink.arel_table
+
+        relations = [
+          # The project a user has direct access to.
+          user.projects.select_for_project_authorization,
+
+          # The personal projects of the user.
+          user.personal_projects.select_as_master_for_project_authorization,
+
+          # Projects that belong directly to any of the groups the user has
+          # access to.
+          Namespace
+            .unscoped
+            .select([alias_as_column(projects[:id], 'project_id'),
+                     cte_alias[:access_level]])
+            .from(cte_alias)
+            .joins(:projects),
+
+          # Projects shared with any of the namespaces the user has access to.
+          Namespace
+            .unscoped
+            .select([links[:project_id],
+                     least(cte_alias[:access_level],
+                           links[:group_access],
+                           'access_level')])
+            .from(cte_alias)
+            .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id')
+            .joins('INNER JOIN projects ON projects.id = project_group_links.project_id')
+            .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id')
+            .where('p_ns.share_with_group_lock IS FALSE')
+        ]
+
+        union = Gitlab::SQL::Union.new(relations)
+
+        ProjectAuthorization
+          .unscoped
+          .with
+          .recursive(cte.to_arel)
+          .select_from_union(union)
+      end
+
+      private
+
+      # Builds a recursive CTE that gets all the groups the current user has
+      # access to, including any nested groups.
+      def recursive_cte
+        cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte)
+        members = Member.arel_table
+        namespaces = Namespace.arel_table
+
+        # Namespaces the user is a member of.
+        cte << user.groups
+          .select([namespaces[:id], members[:access_level]])
+          .except(:order)
+
+        # Sub groups of any groups the user is a member of.
+        cte << Group.select([namespaces[:id],
+                             greatest(members[:access_level],
+                                      cte.table[:access_level], 'access_level')])
+          .joins(join_cte(cte))
+          .joins(join_members)
+          .except(:order)
+
+        cte
+      end
+
+      # Builds a LEFT JOIN to join optional memberships onto the CTE.
+      def join_members
+        members = Member.arel_table
+        namespaces = Namespace.arel_table
+
+        cond = members[:source_id]
+          .eq(namespaces[:id])
+          .and(members[:source_type].eq('Namespace'))
+          .and(members[:requested_at].eq(nil))
+          .and(members[:user_id].eq(user.id))
+
+        Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond))
+      end
+
+      # Builds an INNER JOIN to join namespaces onto the CTE.
+      def join_cte(cte)
+        namespaces = Namespace.arel_table
+        cond = cte.table[:id].eq(namespaces[:parent_id])
+
+        Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond))
+      end
+
+      def greatest(left, right, column_alias)
+        sql_function('GREATEST', [left, right], column_alias)
+      end
+
+      def least(left, right, column_alias)
+        sql_function('LEAST', [left, right], column_alias)
+      end
+
+      def sql_function(name, args, column_alias)
+        alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias)
+      end
+
+      def alias_as_column(value, alias_to)
+        Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb
new file mode 100644
index 00000000000..ad87540e6c2
--- /dev/null
+++ b/lib/gitlab/project_authorizations/without_nested_groups.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module ProjectAuthorizations
+    # Calculating new project authorizations when not supporting nested groups.
+    class WithoutNestedGroups
+      attr_reader :user
+
+      # user - The User object for which to calculate the authorizations.
+      def initialize(user)
+        @user = user
+      end
+
+      def calculate
+        relations = [
+          # Projects the user is a direct member of
+          user.projects.select_for_project_authorization,
+
+          # Personal projects
+          user.personal_projects.select_as_master_for_project_authorization,
+
+          # Projects of groups the user is a member of
+          user.groups_projects.select_for_project_authorization,
+
+          # Projects shared with groups the user is a member of
+          user.groups.joins(:shared_projects).select_for_project_authorization
+        ]
+
+        union = Gitlab::SQL::Union.new(relations)
+
+        ProjectAuthorization
+          .unscoped
+          .select_from_union(union)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 5b9cfaeb2f8..561aa9e162c 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -5,11 +5,7 @@ module Gitlab
     def initialize(current_user, project, query, repository_ref = nil)
       @current_user = current_user
       @project = project
-      @repository_ref = if repository_ref.present?
-                          repository_ref
-                        else
-                          nil
-                        end
+      @repository_ref = repository_ref.presence || project.default_branch
       @query = query
     end
 
@@ -44,40 +40,95 @@ module Gitlab
       @commits_count ||= commits.count
     end
 
+    def self.parse_search_result(result)
+      ref = nil
+      filename = nil
+      basename = nil
+      startline = 0
+
+      result.each_line.each_with_index do |line, index|
+        if line =~ /^.*:.*:\d+:/
+          ref, filename, startline = line.split(':')
+          startline = startline.to_i - index
+          extname = Regexp.escape(File.extname(filename))
+          basename = filename.sub(/#{extname}$/, '')
+          break
+        end
+      end
+
+      data = ""
+
+      result.each_line do |line|
+        data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '')
+      end
+
+      FoundBlob.new(
+        filename: filename,
+        basename: basename,
+        ref: ref,
+        startline: startline,
+        data: data
+      )
+    end
+
+    def single_commit_result?
+      commits_count == 1 && total_result_count == 1
+    end
+
+    def total_result_count
+      issues_count + merge_requests_count + milestones_count + notes_count + blobs_count + wiki_blobs_count + commits_count
+    end
+
     private
 
     def blobs
-      if project.empty_repo? || query.blank?
-        []
-      else
-        project.repository.search_files(query, repository_ref)
-      end
+      return [] unless Ability.allowed?(@current_user, :download_code, @project)
+
+      @blobs ||= Gitlab::FileFinder.new(project, repository_ref).find(query)
     end
 
     def wiki_blobs
-      if project.wiki_enabled? && query.present?
-        project_wiki = ProjectWiki.new(project)
+      return [] unless Ability.allowed?(@current_user, :read_wiki, @project)
 
-        unless project_wiki.empty?
-          project_wiki.search_files(query)
+      @wiki_blobs ||= begin
+        if project.wiki_enabled? && query.present?
+          project_wiki = ProjectWiki.new(project)
+
+          unless project_wiki.empty?
+            project_wiki.search_files(query)
+          else
+            []
+          end
         else
           []
         end
-      else
-        []
       end
     end
 
     def notes
-      project.notes.user.search(query, as_user: @current_user).order('updated_at DESC')
+      @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC')
     end
 
     def commits
-      if project.empty_repo? || query.blank?
-        []
-      else
-        project.repository.find_commits_by_message(query).compact
-      end
+      @commits ||= find_commits(query)
+    end
+
+    def find_commits(query)
+      return [] unless Ability.allowed?(@current_user, :download_code, @project)
+
+      commits = find_commits_by_message(query)
+      commit_by_sha = find_commit_by_sha(query)
+      commits |= [commit_by_sha] if commit_by_sha
+      commits
+    end
+
+    def find_commits_by_message(query)
+      project.repository.find_commits_by_message(query)
+    end
+
+    def find_commit_by_sha(query)
+      key = query.strip
+      project.repository.commit(key) if Commit.valid_hash?(key)
     end
 
     def project_ids_relation
diff --git a/lib/gitlab/project_transfer.rb b/lib/gitlab/project_transfer.rb
new file mode 100644
index 00000000000..1bba0b78e2f
--- /dev/null
+++ b/lib/gitlab/project_transfer.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  class ProjectTransfer
+    def move_project(project_path, namespace_path_was, namespace_path)
+      new_namespace_folder = File.join(root_dir, namespace_path)
+      FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder)
+      from = File.join(root_dir, namespace_path_was, project_path)
+      to = File.join(root_dir, namespace_path, project_path)
+      move(from, to, "")
+    end
+
+    def rename_project(path_was, path, namespace_path)
+      base_dir = File.join(root_dir, namespace_path)
+      move(path_was, path, base_dir)
+    end
+
+    def rename_namespace(path_was, path)
+      move(path_was, path)
+    end
+
+    def root_dir
+      raise NotImplementedError
+    end
+
+    private
+
+    def move(path_was, path, base_dir = nil)
+      base_dir = root_dir unless base_dir
+      from = File.join(base_dir, path_was)
+      to = File.join(base_dir, path)
+      FileUtils.mv(from, to)
+    rescue Errno::ENOENT
+      false
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb
new file mode 100644
index 00000000000..cb95daf2260
--- /dev/null
+++ b/lib/gitlab/prometheus/additional_metrics_parser.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Prometheus
+    module AdditionalMetricsParser
+      extend self
+
+      def load_groups_from_yaml
+        additional_metrics_raw.map(&method(:group_from_entry))
+      end
+
+      private
+
+      def validate!(obj)
+        raise ParsingError.new(obj.errors.full_messages.join('\n')) unless obj.valid?
+      end
+
+      def group_from_entry(entry)
+        entry[:name] = entry.delete(:group)
+        entry[:metrics]&.map! do |entry|
+          Metric.new(entry).tap(&method(:validate!))
+        end
+
+        MetricGroup.new(entry).tap(&method(:validate!))
+      end
+
+      def additional_metrics_raw
+        load_yaml_file&.map(&:deep_symbolize_keys).freeze
+      end
+
+      def load_yaml_file
+        @loaded_yaml_file ||= YAML.load_file(Rails.root.join('config/prometheus/additional_metrics.yml'))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb
new file mode 100644
index 00000000000..f54b2c6aaff
--- /dev/null
+++ b/lib/gitlab/prometheus/metric.rb
@@ -0,0 +1,16 @@
+module Gitlab
+  module Prometheus
+    class Metric
+      include ActiveModel::Model
+
+      attr_accessor :title, :required_metrics, :weight, :y_label, :queries
+
+      validates :title, :required_metrics, :weight, :y_label, :queries, presence: true
+
+      def initialize(params = {})
+        super(params)
+        @y_label ||= 'Values'
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb
new file mode 100644
index 00000000000..729fef34b35
--- /dev/null
+++ b/lib/gitlab/prometheus/metric_group.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module Prometheus
+    class MetricGroup
+      include ActiveModel::Model
+
+      attr_accessor :name, :priority, :metrics
+      validates :name, :priority, :metrics, presence: true
+
+      def self.all
+        AdditionalMetricsParser.load_groups_from_yaml
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb
new file mode 100644
index 00000000000..49cc0e16080
--- /dev/null
+++ b/lib/gitlab/prometheus/parsing_error.rb
@@ -0,0 +1,5 @@
+module Gitlab
+  module Prometheus
+    ParsingError = Class.new(StandardError)
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
new file mode 100644
index 00000000000..67c69d9ccf3
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Prometheus
+    module Queries
+      class AdditionalMetricsDeploymentQuery < BaseQuery
+        include QueryAdditionalMetrics
+
+        def query(deployment_id)
+          Deployment.find_by(id: deployment_id).try do |deployment|
+            query_context = {
+              environment_slug: deployment.environment.slug,
+              environment_filter: %{container_name!="POD",environment="#{deployment.environment.slug}"},
+              timeframe_start: (deployment.created_at - 30.minutes).to_f,
+              timeframe_end: (deployment.created_at + 30.minutes).to_f
+            }
+
+            query_metrics(query_context)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
new file mode 100644
index 00000000000..b5a679ddd79
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Prometheus
+    module Queries
+      class AdditionalMetricsEnvironmentQuery < BaseQuery
+        include QueryAdditionalMetrics
+
+        def query(environment_id)
+          Environment.find_by(id: environment_id).try do |environment|
+            query_context = {
+              environment_slug: environment.slug,
+              environment_filter: %{container_name!="POD",environment="#{environment.slug}"},
+              timeframe_start: 8.hours.ago.to_f,
+              timeframe_end: Time.now.to_f
+            }
+
+            query_metrics(query_context)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb
new file mode 100644
index 00000000000..c60828165bd
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/base_query.rb
@@ -0,0 +1,26 @@
+module Gitlab
+  module Prometheus
+    module Queries
+      class BaseQuery
+        attr_accessor :client
+        delegate :query_range, :query, :label_values, :series, to: :client, prefix: true
+
+        def raw_memory_usage_query(environment_slug)
+          %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20}
+        end
+
+        def raw_cpu_usage_query(environment_slug)
+          %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[2m])) * 100}
+        end
+
+        def initialize(client)
+          @client = client
+        end
+
+        def query(*args)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb
new file mode 100644
index 00000000000..170f483540e
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/deployment_query.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module Prometheus
+    module Queries
+      class DeploymentQuery < BaseQuery
+        def query(deployment_id)
+          Deployment.find_by(id: deployment_id).try do |deployment|
+            environment_slug = deployment.environment.slug
+
+            memory_query = raw_memory_usage_query(environment_slug)
+            memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))}
+            cpu_query = raw_cpu_usage_query(environment_slug)
+            cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100}
+
+            timeframe_start = (deployment.created_at - 30.minutes).to_f
+            timeframe_end = (deployment.created_at + 30.minutes).to_f
+
+            {
+              memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end),
+              memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f),
+              memory_after: client_query(memory_avg_query, time: timeframe_end),
+
+              cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end),
+              cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f),
+              cpu_after: client_query(cpu_avg_query, time: timeframe_end)
+            }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb
new file mode 100644
index 00000000000..66f29d95177
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/environment_query.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module Prometheus
+    module Queries
+      class EnvironmentQuery < BaseQuery
+        def query(environment_id)
+          Environment.find_by(id: environment_id).try do |environment|
+            environment_slug = environment.slug
+            timeframe_start = 8.hours.ago.to_f
+            timeframe_end = Time.now.to_f
+
+            memory_query = raw_memory_usage_query(environment_slug)
+            cpu_query = raw_cpu_usage_query(environment_slug)
+
+            {
+              memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end),
+              memory_current: client_query(memory_query, time: timeframe_end),
+              cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end),
+              cpu_current: client_query(cpu_query, time: timeframe_end)
+            }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb
new file mode 100644
index 00000000000..d4894c87f8d
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb
@@ -0,0 +1,80 @@
+module Gitlab
+  module Prometheus
+    module Queries
+      class MatchedMetricsQuery < BaseQuery
+        MAX_QUERY_ITEMS = 40.freeze
+
+        def query
+          groups_data.map do |group, data|
+            {
+              group: group.name,
+              priority: group.priority,
+              active_metrics: data[:active_metrics],
+              metrics_missing_requirements: data[:metrics_missing_requirements]
+            }
+          end
+        end
+
+        private
+
+        def groups_data
+          metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all)
+          lookup = active_series_lookup(metrics_groups)
+
+          groups = {}
+
+          metrics_groups.each do |group|
+            groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 }
+            active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) }
+
+            groups[group][:active_metrics] += active_metrics
+            groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics
+          end
+
+          groups
+        end
+
+        def active_series_lookup(metric_groups)
+          timeframe_start = 8.hours.ago
+          timeframe_end = Time.now
+
+          series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq
+
+          lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series|
+            client_series(*batched_series, start: timeframe_start, stop: timeframe_end)
+              .select(&method(:has_matching_label))
+              .map { |series_info| [series_info['__name__'], true] }
+          end
+          lookup.to_h
+        end
+
+        def has_matching_label(series_info)
+          series_info.key?('environment')
+        end
+
+        def available_metrics
+          @available_metrics ||= client_label_values || []
+        end
+
+        def filter_active_metrics(metric_group)
+          metric_group.metrics.select! do |metric|
+            metric.required_metrics.all?(&available_metrics.method(:include?))
+          end
+          metric_group
+        end
+
+        def groups_with_active_metrics(metric_groups)
+          metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? }
+        end
+
+        def metrics_with_required_series(metric_groups)
+          metric_groups.flat_map do |group|
+            group.metrics.select do |metric|
+              metric.required_metrics.all?(&available_metrics.method(:include?))
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
new file mode 100644
index 00000000000..e44be770544
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
@@ -0,0 +1,73 @@
+module Gitlab
+  module Prometheus
+    module Queries
+      module QueryAdditionalMetrics
+        def query_metrics(query_context)
+          query_processor = method(:process_query).curry[query_context]
+
+          groups = matched_metrics.map do |group|
+            metrics = group.metrics.map do |metric|
+              {
+                title: metric.title,
+                weight: metric.weight,
+                y_label: metric.y_label,
+                queries: metric.queries.map(&query_processor).select(&method(:query_with_result))
+              }
+            end
+
+            {
+              group: group.name,
+              priority: group.priority,
+              metrics: metrics.select(&method(:metric_with_any_queries))
+            }
+          end
+
+          groups.select(&method(:group_with_any_metrics))
+        end
+
+        private
+
+        def metric_with_any_queries(metric)
+          metric[:queries]&.count&.> 0
+        end
+
+        def group_with_any_metrics(group)
+          group[:metrics]&.count&.> 0
+        end
+
+        def query_with_result(query)
+          query[:result]&.any? do |item|
+            item&.[](:values)&.any? || item&.[](:value)&.any?
+          end
+        end
+
+        def process_query(context, query)
+          query_with_result = query.dup
+          result =
+            if query.key?(:query_range)
+              client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end])
+            else
+              client_query(query[:query] % context, time: context[:timeframe_end])
+            end
+          query_with_result[:result] = result&.map(&:deep_symbolize_keys)
+          query_with_result
+        end
+
+        def available_metrics
+          @available_metrics ||= client_label_values || []
+        end
+
+        def matched_metrics
+          result = Gitlab::Prometheus::MetricGroup.all.map do |group|
+            group.metrics.select! do |metric|
+              metric.required_metrics.all?(&available_metrics.method(:include?))
+            end
+            group
+          end
+
+          result.select { |group| group.metrics.any? }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
new file mode 100644
index 00000000000..aa94614bf18
--- /dev/null
+++ b/lib/gitlab/prometheus_client.rb
@@ -0,0 +1,84 @@
+module Gitlab
+  PrometheusError = Class.new(StandardError)
+
+  # Helper methods to interact with Prometheus network services & resources
+  class PrometheusClient
+    attr_reader :api_url
+
+    def initialize(api_url:)
+      @api_url = api_url
+    end
+
+    def ping
+      json_api_get('query', query: '1')
+    end
+
+    def query(query, time: Time.now)
+      get_result('vector') do
+        json_api_get('query', query: query, time: time.to_f)
+      end
+    end
+
+    def query_range(query, start: 8.hours.ago, stop: Time.now)
+      get_result('matrix') do
+        json_api_get('query_range',
+          query: query,
+          start: start.to_f,
+          end: stop.to_f,
+          step: 1.minute.to_i)
+      end
+    end
+
+    def label_values(name = '__name__')
+      json_api_get("label/#{name}/values")
+    end
+
+    def series(*matches, start: 8.hours.ago, stop: Time.now)
+      json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f)
+    end
+
+    private
+
+    def json_api_get(type, args = {})
+      get(join_api_url(type, args))
+    rescue Errno::ECONNREFUSED
+      raise PrometheusError, 'Connection refused'
+    end
+
+    def join_api_url(type, args = {})
+      url = URI.parse(api_url)
+    rescue URI::Error
+      raise PrometheusError, "Invalid API URL: #{api_url}"
+    else
+      url.path = [url.path.sub(%r{/+\z}, ''), 'api', 'v1', type].join('/')
+      url.query = args.to_query
+
+      url.to_s
+    end
+
+    def get(url)
+      handle_response(HTTParty.get(url))
+    rescue SocketError
+      raise PrometheusError, "Can't connect to #{url}"
+    rescue OpenSSL::SSL::SSLError
+      raise PrometheusError, "#{url} contains invalid SSL data"
+    rescue HTTParty::Error
+      raise PrometheusError, "Network connection error"
+    end
+
+    def handle_response(response)
+      if response.code == 200 && response['status'] == 'success'
+        response['data'] || {}
+      elsif response.code == 400
+        raise PrometheusError, response['error'] || 'Bad data received'
+      else
+        raise PrometheusError, "#{response.code} - #{response.body}"
+      end
+    end
+
+    def get_result(expected_type)
+      data = yield
+      data['result'] if data['resultType'] == expected_type
+    end
+  end
+end
diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb
new file mode 100644
index 00000000000..3937d9c153a
--- /dev/null
+++ b/lib/gitlab/quick_actions/command_definition.rb
@@ -0,0 +1,89 @@
+module Gitlab
+  module QuickActions
+    class CommandDefinition
+      attr_accessor :name, :aliases, :description, :explanation, :params,
+        :condition_block, :parse_params_block, :action_block
+
+      def initialize(name, attributes = {})
+        @name = name
+
+        @aliases = attributes[:aliases] || []
+        @description = attributes[:description] || ''
+        @explanation = attributes[:explanation] || ''
+        @params = attributes[:params] || []
+        @condition_block = attributes[:condition_block]
+        @parse_params_block = attributes[:parse_params_block]
+        @action_block = attributes[:action_block]
+      end
+
+      def all_names
+        [name, *aliases]
+      end
+
+      def noop?
+        action_block.nil?
+      end
+
+      def available?(opts)
+        return true unless condition_block
+
+        context = OpenStruct.new(opts)
+        context.instance_exec(&condition_block)
+      end
+
+      def explain(context, opts, arg)
+        return unless available?(opts)
+
+        if explanation.respond_to?(:call)
+          execute_block(explanation, context, arg)
+        else
+          explanation
+        end
+      end
+
+      def execute(context, opts, arg)
+        return if noop? || !available?(opts)
+
+        execute_block(action_block, context, arg)
+      end
+
+      def to_h(opts)
+        context = OpenStruct.new(opts)
+
+        desc = description
+        if desc.respond_to?(:call)
+          desc = context.instance_exec(&desc) rescue ''
+        end
+
+        prms = params
+        if prms.respond_to?(:call)
+          prms = Array(context.instance_exec(&prms)) rescue params
+        end
+
+        {
+          name: name,
+          aliases: aliases,
+          description: desc,
+          params: prms
+        }
+      end
+
+      private
+
+      def execute_block(block, context, arg)
+        if arg.present?
+          parsed = parse_params(arg, context)
+          context.instance_exec(parsed, &block)
+        elsif block.arity == 0
+          context.instance_exec(&block)
+        end
+      end
+
+      def parse_params(arg, context)
+        return arg unless parse_params_block
+
+        context.instance_exec(arg, &parse_params_block)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/quick_actions/dsl.rb
index 50b0937d267..a4a97236ffc 100644
--- a/lib/gitlab/slash_commands/dsl.rb
+++ b/lib/gitlab/quick_actions/dsl.rb
@@ -1,5 +1,5 @@
 module Gitlab
-  module SlashCommands
+  module QuickActions
     module Dsl
       extend ActiveSupport::Concern
 
@@ -14,7 +14,7 @@ module Gitlab
       end
 
       class_methods do
-        # Allows to give a description to the next slash command.
+        # Allows to give a description to the next quick action.
         # This description is shown in the autocomplete menu.
         # It accepts a block that will be evaluated with the context given to
         # `CommandDefintion#to_h`.
@@ -31,7 +31,7 @@ module Gitlab
           @description = block_given? ? block : text
         end
 
-        # Allows to define params for the next slash command.
+        # Allows to define params for the next quick action.
         # These params are shown in the autocomplete menu.
         #
         # Example:
@@ -40,8 +40,24 @@ module Gitlab
         #   command :command_key do |arguments|
         #     # Awesome code block
         #   end
-        def params(*params)
-          @params = params
+        def params(*params, &block)
+          @params = block_given? ? block : params
+        end
+
+        # Allows to give an explanation of what the command will do when
+        # executed. This explanation is shown when rendering the Markdown
+        # preview.
+        #
+        # Example:
+        #
+        #   explanation do |arguments|
+        #     "Adds label(s) #{arguments.join(' ')}"
+        #   end
+        #   command :command_key do |arguments|
+        #     # Awesome code block
+        #   end
+        def explanation(text = '', &block)
+          @explanation = block_given? ? block : text
         end
 
         # Allows to define conditions that must be met in order for the command
@@ -61,6 +77,24 @@ module Gitlab
           @condition_block = block
         end
 
+        # Allows to perform initial parsing of parameters. The result is passed
+        # both to `command` and `explanation` blocks, instead of the raw
+        # parameters.
+        # It accepts a block that will be evaluated with the context given to
+        # `CommandDefintion#to_h`.
+        #
+        # Example:
+        #
+        #   parse_params do |raw|
+        #     raw.strip
+        #   end
+        #   command :command_key do |parsed|
+        #     # Awesome code block
+        #   end
+        def parse_params(&block)
+          @parse_params_block = block
+        end
+
         # Registers a new command which is recognizeable from body of email or
         # comment.
         # It accepts aliases and takes a block.
@@ -75,11 +109,13 @@ module Gitlab
 
           definition = CommandDefinition.new(
             name,
-            aliases:          aliases,
-            description:      @description,
-            params:           @params,
-            condition_block:  @condition_block,
-            action_block:     block
+            aliases: aliases,
+            description: @description,
+            explanation: @explanation,
+            params: @params,
+            condition_block: @condition_block,
+            parse_params_block: @parse_params_block,
+            action_block: block
           )
 
           self.command_definitions << definition
@@ -89,8 +125,14 @@ module Gitlab
           end
 
           @description = nil
+          @explanation = nil
           @params = nil
           @condition_block = nil
+          @parse_params_block = nil
+        end
+
+        def definition_by_name(name)
+          command_definitions_by_name[name.to_sym]
         end
       end
     end
diff --git a/lib/gitlab/slash_commands/extractor.rb b/lib/gitlab/quick_actions/extractor.rb
index a672e5e4855..09576be7156 100644
--- a/lib/gitlab/slash_commands/extractor.rb
+++ b/lib/gitlab/quick_actions/extractor.rb
@@ -1,10 +1,10 @@
 module Gitlab
-  module SlashCommands
+  module QuickActions
     # This class takes an array of commands that should be extracted from a
     # given text.
     #
     # ```
-    # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels])
+    # extractor = Gitlab::QuickActions::Extractor.new([:open, :assign, :labels])
     # ```
     class Extractor
       attr_reader :command_definitions
@@ -24,7 +24,7 @@ module Gitlab
       #
       # Usage:
       # ```
-      # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels])
+      # extractor = Gitlab::QuickActions::Extractor.new([:open, :assign, :labels])
       # msg = %(hello\n/labels ~foo ~"bar baz"\nworld)
       # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']]
       # msg #=> "hello\nworld"
@@ -103,7 +103,7 @@ module Gitlab
               (?<cmd>#{Regexp.union(names)})
               (?:
                 [ ]
-                (?<arg>[^\/\n]*)
+                (?<arg>[^\n]*)
               )?
               (?:\n|$)
             )
diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb
index 70e7f25d518..4bc76ea033f 100644
--- a/lib/gitlab/recaptcha.rb
+++ b/lib/gitlab/recaptcha.rb
@@ -10,5 +10,9 @@ module Gitlab
         true
       end
     end
+
+    def self.enabled?
+      current_application_settings.recaptcha_enabled
+    end
   end
 end
diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb
index c649da8c426..bc5370de32a 100644
--- a/lib/gitlab/redis.rb
+++ b/lib/gitlab/redis.rb
@@ -1,27 +1,18 @@
 # This file should not have any direct dependency on Rails environment
 # please require all dependencies below:
 require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/module/delegation'
 
 module Gitlab
   class Redis
-    CACHE_NAMESPACE = 'cache:gitlab'
-    SESSION_NAMESPACE = 'session:gitlab'
-    SIDEKIQ_NAMESPACE = 'resque:gitlab'
-    MAILROOM_NAMESPACE = 'mail_room:gitlab'
-    DEFAULT_REDIS_URL = 'redis://localhost:6379'
-    CONFIG_FILE = File.expand_path('../../config/resque.yml', __dir__)
+    CACHE_NAMESPACE = 'cache:gitlab'.freeze
+    SESSION_NAMESPACE = 'session:gitlab'.freeze
+    SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze
+    MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze
+    DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze
 
     class << self
-      # Do NOT cache in an instance variable. Result may be mutated by caller.
-      def params
-        new.params
-      end
-
-      # Do NOT cache in an instance variable. Result may be mutated by caller.
-      # @deprecated Use .params instead to get sentinel support
-      def url
-        new.url
-      end
+      delegate :params, :url, to: :new
 
       def with
         @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) }
@@ -42,13 +33,17 @@ module Gitlab
         return @_raw_config if defined?(@_raw_config)
 
         begin
-          @_raw_config = File.read(CONFIG_FILE).freeze
+          @_raw_config = ERB.new(File.read(config_file)).result.freeze
         rescue Errno::ENOENT
           @_raw_config = false
         end
 
         @_raw_config
       end
+
+      def config_file
+        ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__)
+      end
     end
 
     def initialize(rails_env = nil)
@@ -63,6 +58,14 @@ module Gitlab
       raw_config_hash[:url]
     end
 
+    def sentinels
+      raw_config_hash[:sentinels]
+    end
+
+    def sentinels?
+      sentinels && !sentinels.empty?
+    end
+
     private
 
     def redis_store_options
diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb
index 11c0b01f0dc..7668ecacc4b 100644
--- a/lib/gitlab/reference_extractor.rb
+++ b/lib/gitlab/reference_extractor.rb
@@ -1,13 +1,12 @@
 module Gitlab
   # Extract possible GFM references from an arbitrary String for further processing.
   class ReferenceExtractor < Banzai::ReferenceExtractor
-    REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range)
+    REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user).freeze
     attr_accessor :project, :current_user, :author
 
     def initialize(project, current_user = nil)
       @project = project
       @current_user = current_user
-
       @references = {}
 
       super()
@@ -21,6 +20,11 @@ module Gitlab
       super(type, project, current_user)
     end
 
+    def reset_memoized_values
+      @references = {}
+      super()
+    end
+
     REFERABLES.each do |type|
       define_method("#{type}s") do
         @references[type] ||= references(type)
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 776bbcbb5d0..b706434217d 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -2,17 +2,6 @@ module Gitlab
   module Regex
     extend self
 
-    NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])'.freeze
-
-    def namespace_regex
-      @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze
-    end
-
-    def namespace_regex_message
-      "can contain only letters, digits, '_', '-' and '.'. " \
-      "Cannot start with '-' or end in '.'." \
-    end
-
     def namespace_name_regex
       @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze
     end
@@ -22,85 +11,56 @@ module Gitlab
     end
 
     def project_name_regex
-      @project_name_regex ||= /\A[\p{Alnum}_][\p{Alnum}\p{Pd}_\. ]*\z/.freeze
+      @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9c0}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9c0}_\. ]*\z/.freeze
     end
 
     def project_name_regex_message
-      "can contain only letters, digits, '_', '.', dash and space. " \
-      "It must start with letter, digit or '_'."
-    end
-
-    def project_path_regex
-      @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git|\.atom)\z/.freeze
-    end
-
-    def project_path_regex_message
-      "can contain only letters, digits, '_', '-' and '.'. " \
-      "Cannot start with '-', end in '.git' or end in '.atom'" \
+      "can contain only letters, digits, emojis, '_', '.', dash, space. " \
+      "It must start with letter, digit, emoji or '_'."
     end
 
     def file_name_regex
-      @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze
+      @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze
     end
 
     def file_name_regex_message
-      "can contain only letters, digits, '_', '-', '@' and '.'."
-    end
-
-    def file_path_regex
-      @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/\@]*\z/.freeze
+      "can contain only letters, digits, '_', '-', '@', '+' and '.'."
     end
 
-    def file_path_regex_message
-      "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'."
+    def container_registry_reference_regex
+      Gitlab::PathRegex.git_reference_regex
     end
 
-    def directory_traversal_regex
-      @directory_traversal_regex ||= /\.{2}/.freeze
+    ##
+    # Docker Distribution Registry 2.4.1 repository name rules
+    #
+    def container_repository_name_regex
+      @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z}
     end
 
-    def directory_traversal_regex_message
-      "cannot include directory traversal."
+    def environment_name_regex
+      @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze
     end
 
-    def archive_formats_regex
-      #                           |zip|tar|    tar.gz    |         tar.bz2         |
-      @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze
+    def environment_name_regex_message
+      "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces"
     end
 
-    def git_reference_regex
-      # Valid git ref regex, see:
-      # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html
-
-      @git_reference_regex ||= %r{
-        (?!
-           (?# doesn't begins with)
-           \/|                    (?# rule #6)
-           (?# doesn't contain)
-           .*(?:
-              [\/.]\.|            (?# rule #1,3)
-              \/\/|               (?# rule #6)
-              @\{|                (?# rule #8)
-              \\                  (?# rule #9)
-           )
-        )
-        [^\000-\040\177~^:?*\[]+  (?# rule #4-5)
-        (?# doesn't end with)
-        (?<!\.lock)               (?# rule #1)
-        (?<![\/.])                (?# rule #6-7)
-      }x.freeze
+    def kubernetes_namespace_regex
+      /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/
     end
 
-    def container_registry_reference_regex
-      git_reference_regex
+    def kubernetes_namespace_regex_message
+      "can contain only letters, digits or '-', and cannot start or end with '-'"
     end
 
-    def environment_name_regex
-      @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze
+    def environment_slug_regex
+      @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze
     end
 
-    def environment_name_regex_message
-      "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces"
+    def environment_slug_regex_message
+      "can contain only lowercase letters, digits, and '-'. " \
+      "Must start with a letter, and cannot end with '-'"
     end
   end
 end
diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb
new file mode 100644
index 00000000000..3591fa9145e
--- /dev/null
+++ b/lib/gitlab/repo_path.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module RepoPath
+    NotFoundError = Class.new(StandardError)
+
+    def self.parse(repo_path)
+      wiki = false
+      project_path = strip_storage_path(repo_path.sub(/\.git\z/, ''), fail_on_not_found: false)
+      project, was_redirected = find_project(project_path)
+
+      if project_path.end_with?('.wiki') && project.nil?
+        project, was_redirected = find_project(project_path.chomp('.wiki'))
+        wiki = true
+      end
+
+      redirected_path = project_path if was_redirected
+
+      [project, wiki, redirected_path]
+    end
+
+    def self.strip_storage_path(repo_path, fail_on_not_found: true)
+      result = repo_path
+
+      storage = Gitlab.config.repositories.storages.values.find do |params|
+        repo_path.start_with?(params['path'])
+      end
+
+      if storage
+        result = result.sub(storage['path'], '')
+      elsif fail_on_not_found
+        raise NotFoundError.new("No known storage path matches #{repo_path.inspect}")
+      end
+
+      result.sub(/\A\/*/, '')
+    end
+
+    def self.find_project(project_path)
+      project = Project.find_by_full_path(project_path, follow_redirects: true)
+      was_redirected = project && project.full_path.casecmp(project_path) != 0
+
+      [project, was_redirected]
+    end
+  end
+end
diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb
new file mode 100644
index 00000000000..fef536ecb0b
--- /dev/null
+++ b/lib/gitlab/request_context.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  class RequestContext
+    class << self
+      def client_ip
+        RequestStore[:client_ip]
+      end
+    end
+
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      req = Rack::Request.new(env)
+
+      RequestStore[:client_ip] = req.ip
+
+      @app.call(env)
+    end
+  end
+end
diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb
index 8130e55351e..0c9ab759e81 100644
--- a/lib/gitlab/request_profiler.rb
+++ b/lib/gitlab/request_profiler.rb
@@ -2,7 +2,7 @@ require 'fileutils'
 
 module Gitlab
   module RequestProfiler
-    PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles"
+    PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles".freeze
 
     def profile_token
       Rails.cache.fetch('profile-token') do
diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb
index 786e1d49f5e..ef42b0557e0 100644
--- a/lib/gitlab/request_profiler/middleware.rb
+++ b/lib/gitlab/request_profiler/middleware.rb
@@ -1,5 +1,4 @@
 require 'ruby-prof'
-require_dependency 'gitlab/request_profiler'
 
 module Gitlab
   module RequestProfiler
@@ -20,7 +19,7 @@ module Gitlab
         header_token = env['HTTP_X_PROFILE_TOKEN']
         return unless header_token.present?
 
-        profile_token = RequestProfiler.profile_token
+        profile_token = Gitlab::RequestProfiler.profile_token
         return unless profile_token.present?
 
         header_token == profile_token
diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb
new file mode 100644
index 00000000000..877aa6e6a28
--- /dev/null
+++ b/lib/gitlab/route_map.rb
@@ -0,0 +1,50 @@
+module Gitlab
+  class RouteMap
+    FormatError = Class.new(StandardError)
+
+    def initialize(data)
+      begin
+        entries = YAML.safe_load(data)
+      rescue
+        raise FormatError, 'Route map is not valid YAML'
+      end
+
+      raise FormatError, 'Route map is not an array' unless entries.is_a?(Array)
+
+      @map = entries.map { |entry| parse_entry(entry) }
+    end
+
+    def public_path_for_source_path(path)
+      mapping = @map.find { |mapping| mapping[:source] === path }
+      return unless mapping
+
+      path.sub(mapping[:source], mapping[:public])
+    end
+
+    private
+
+    def parse_entry(entry)
+      raise FormatError, 'Route map entry is not a hash' unless entry.is_a?(Hash)
+      raise FormatError, 'Route map entry does not have a source key' unless entry.key?('source')
+      raise FormatError, 'Route map entry does not have a public key' unless entry.key?('public')
+
+      source_pattern = entry['source']
+      public_path = entry['public']
+
+      if source_pattern.start_with?('/') && source_pattern.end_with?('/')
+        source_pattern = source_pattern[1...-1].gsub('\/', '/')
+
+        begin
+          source_pattern = /\A#{source_pattern}\z/
+        rescue RegexpError => e
+          raise FormatError, "Route map entry source is not a valid regular expression: #{e}"
+        end
+      end
+
+      {
+        source: source_pattern,
+        public: public_path
+      }
+    end
+  end
+end
diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb
new file mode 100644
index 00000000000..36d1a8a6f64
--- /dev/null
+++ b/lib/gitlab/routes/legacy_builds.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Routes
+    class LegacyBuilds
+      def initialize(map)
+        @map = map
+      end
+
+      def draw
+        @map.instance_eval do
+          resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do
+            collection do
+              resources :artifacts, only: [], controller: 'build_artifacts' do
+                collection do
+                  get :latest_succeeded,
+                    path: '*ref_name_and_path',
+                    format: false
+                end
+              end
+            end
+
+            member do
+              get :raw
+            end
+
+            resource :artifacts, only: [], controller: 'build_artifacts' do
+              get :download
+              get :browse, path: 'browse(/*path)', format: false
+              get :file, path: 'file/*path', format: false
+              get :raw, path: 'raw/*path', format: false
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb
index 5132177de51..632e2d87500 100644
--- a/lib/gitlab/routing.rb
+++ b/lib/gitlab/routing.rb
@@ -1,5 +1,11 @@
 module Gitlab
   module Routing
+    extend ActiveSupport::Concern
+
+    included do
+      include Gitlab::Routing.url_helpers
+    end
+
     # Returns the URL helpers Module.
     #
     # This method caches the output as Rails' "url_helpers" method creates an
diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb
index f253dc7477e..8a7cc690046 100644
--- a/lib/gitlab/saml/user.rb
+++ b/lib/gitlab/saml/user.rb
@@ -28,11 +28,12 @@ module Gitlab
         if external_users_enabled? && @user
           # Check if there is overlap between the user's groups and the external groups
           # setting then set user as external or internal.
-          if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty?
-            @user.external = false
-          else
-            @user.external = true
-          end
+          @user.external =
+            if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty?
+              false
+            else
+              true
+            end
         end
 
         @user
diff --git a/lib/gitlab/sanitizers/svg/whitelist.rb b/lib/gitlab/sanitizers/svg/whitelist.rb
index 7b6b70d8dbc..d50f826f924 100644
--- a/lib/gitlab/sanitizers/svg/whitelist.rb
+++ b/lib/gitlab/sanitizers/svg/whitelist.rb
@@ -6,18 +6,19 @@ module Gitlab
     module SVG
       class Whitelist
         ALLOWED_ELEMENTS = %w[
-        a altGlyph altGlyphDef altGlyphItem animate
-        animateColor animateMotion animateTransform circle clipPath color-profile
-        cursor defs desc ellipse feBlend feColorMatrix feComponentTransfer
-        feComposite feConvolveMatrix feDiffuseLighting feDisplacementMap
-        feDistantLight feFlood feFuncA feFuncB feFuncG feFuncR feGaussianBlur
-        feImage feMerge feMergeNode feMorphology feOffset fePointLight
-        feSpecularLighting feSpotLight feTile feTurbulence filter font font-face
-        font-face-format font-face-name font-face-src font-face-uri foreignObject
-        g glyph glyphRef hkern image line linearGradient marker mask metadata
-        missing-glyph mpath path pattern polygon polyline radialGradient rect
-        script set stop style svg switch symbol text textPath title tref tspan use
-        view vkern].freeze
+          a altGlyph altGlyphDef altGlyphItem animate
+          animateColor animateMotion animateTransform circle clipPath color-profile
+          cursor defs desc ellipse feBlend feColorMatrix feComponentTransfer
+          feComposite feConvolveMatrix feDiffuseLighting feDisplacementMap
+          feDistantLight feFlood feFuncA feFuncB feFuncG feFuncR feGaussianBlur
+          feImage feMerge feMergeNode feMorphology feOffset fePointLight
+          feSpecularLighting feSpotLight feTile feTurbulence filter font font-face
+          font-face-format font-face-name font-face-src font-face-uri foreignObject
+          g glyph glyphRef hkern image line linearGradient marker mask metadata
+          missing-glyph mpath path pattern polygon polyline radialGradient rect
+          script set stop style svg switch symbol text textPath title tref tspan use
+          view vkern
+        ].freeze
 
         ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS = %w[svg].freeze
 
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 2690938fe82..efe8095beea 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -1,5 +1,26 @@
 module Gitlab
   class SearchResults
+    class FoundBlob
+      attr_reader :id, :filename, :basename, :ref, :startline, :data
+
+      def initialize(opts = {})
+        @id = opts.fetch(:id, nil)
+        @filename = opts.fetch(:filename, nil)
+        @basename = opts.fetch(:basename, nil)
+        @ref = opts.fetch(:ref, nil)
+        @startline = opts.fetch(:startline, nil)
+        @data = opts.fetch(:data, nil)
+      end
+
+      def path
+        filename
+      end
+
+      def no_highlighting?
+        false
+      end
+    end
+
     attr_reader :current_user, :query
 
     # Limit search results by passed projects
@@ -43,6 +64,10 @@ module Gitlab
       @milestones_count ||= milestones.count
     end
 
+    def single_commit_result?
+      false
+    end
+
     private
 
     def projects
@@ -50,13 +75,14 @@ module Gitlab
     end
 
     def issues
-      issues = Issue.visible_to_user(current_user).where(project_id: project_ids_relation)
+      issues = IssuesFinder.new(current_user).execute.where(project_id: project_ids_relation)
 
-      if query =~ /#(\d+)\z/
-        issues = issues.where(iid: $1)
-      else
-        issues = issues.full_search(query)
-      end
+      issues =
+        if query =~ /#(\d+)\z/
+          issues.where(iid: $1)
+        else
+          issues.full_search(query)
+        end
 
       issues.order('updated_at DESC')
     end
@@ -68,12 +94,13 @@ module Gitlab
     end
 
     def merge_requests
-      merge_requests = MergeRequest.in_projects(project_ids_relation)
-      if query =~ /[#!](\d+)\z/
-        merge_requests = merge_requests.where(iid: $1)
-      else
-        merge_requests = merge_requests.full_search(query)
-      end
+      merge_requests = MergeRequestsFinder.new(current_user).execute.in_projects(project_ids_relation)
+      merge_requests =
+        if query =~ /[#!](\d+)\z/
+          merge_requests.where(iid: $1)
+        else
+          merge_requests.full_search(query)
+        end
       merge_requests.order('updated_at DESC')
     end
 
diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb
index 7cf506ebe64..823f697f51c 100644
--- a/lib/gitlab/seeder.rb
+++ b/lib/gitlab/seeder.rb
@@ -1,24 +1,23 @@
+module DeliverNever
+  def deliver_later
+    self
+  end
+end
+
 module Gitlab
   class Seeder
     def self.quiet
       mute_mailer
       SeedFu.quiet = true
+
       yield
+
       SeedFu.quiet = false
       puts "\nOK".color(:green)
     end
 
-    def self.by_user(user)
-      yield
-    end
-
     def self.mute_mailer
-      code = <<-eos
-def Notify.deliver_later
-  self
-end
-      eos
-      eval(code)
+      ActionMailer::MessageDelivery.prepend(DeliverNever)
     end
   end
 end
diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb
index 117fc508135..2442c2ded3b 100644
--- a/lib/gitlab/sentry.rb
+++ b/lib/gitlab/sentry.rb
@@ -11,7 +11,7 @@ module Gitlab
         Raven.user_context(
           id: current_user.id,
           email: current_user.email,
-          username: current_user.username,
+          username: current_user.username
         )
       end
     end
diff --git a/lib/gitlab/serializer/ci/variables.rb b/lib/gitlab/serializer/ci/variables.rb
new file mode 100644
index 00000000000..c059c454eac
--- /dev/null
+++ b/lib/gitlab/serializer/ci/variables.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Serializer
+    module Ci
+      # This serializer could make sure our YAML variables' keys and values
+      # are always strings. This is more for legacy build data because
+      # from now on we convert them into strings before saving to database.
+      module Variables
+        extend self
+
+        def load(string)
+          return unless string
+
+          object = YAML.safe_load(string, [Symbol])
+
+          object.map do |variable|
+            variable[:key] = variable[:key].to_s
+            variable
+          end
+        end
+
+        def dump(object)
+          YAML.dump(object)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/serializer/pagination.rb b/lib/gitlab/serializer/pagination.rb
new file mode 100644
index 00000000000..9c92b83dddc
--- /dev/null
+++ b/lib/gitlab/serializer/pagination.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Serializer
+    class Pagination
+      InvalidResourceError = Class.new(StandardError)
+      include ::API::Helpers::Pagination
+
+      def initialize(request, response)
+        @request = request
+        @response = response
+      end
+
+      def paginate(resource)
+        if resource.respond_to?(:page)
+          super(resource)
+        else
+          raise InvalidResourceError
+        end
+      end
+
+      private
+
+      # Methods needed by `API::Helpers::Pagination`
+      #
+
+      attr_reader :request
+
+      def params
+        @request.query_parameters
+      end
+
+      def header(header, value)
+        @response.headers[header] = value
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/shell.rb
index d0060fbaca1..22554236c38 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -2,7 +2,7 @@ require 'securerandom'
 
 module Gitlab
   class Shell
-    class Error < StandardError; end
+    Error = Class.new(StandardError)
 
     KeyAdder = Struct.new(:io) do
       def add_key(id, key)
@@ -30,12 +30,12 @@ module Gitlab
       end
 
       def version_required
-        @version_required ||= File.read(Rails.root.
-                                        join('GITLAB_SHELL_VERSION')).strip
+        @version_required ||= File.read(Rails.root
+                                        .join('GITLAB_SHELL_VERSION')).strip
       end
 
       def strip_key(key)
-        key.split(/ /)[0, 2].join(' ')
+        key.split(/[ ]+/)[0, 2].join(' ')
       end
 
       private
@@ -47,8 +47,8 @@ module Gitlab
 
         unless File.size?(secret_file)
           # Generate a new token of 16 random hexadecimal characters and store it in secret_file.
-          token = SecureRandom.hex(16)
-          File.write(secret_file, token)
+          @secret_token = SecureRandom.hex(16)
+          File.write(secret_file, @secret_token)
         end
 
         link_path = File.join(shell_path, '.gitlab_shell_secret')
@@ -80,8 +80,30 @@ module Gitlab
     #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
     #
     def import_repository(storage, name, url)
-      output, status = Popen::popen([gitlab_shell_projects_path, 'import-project',
-                                     storage, "#{name}.git", url, '900'])
+      # Timeout should be less than 900 ideally, to prevent the memory killer
+      # to silently kill the process without knowing we are timing out here.
+      output, status = Popen.popen([gitlab_shell_projects_path, 'import-project',
+                                    storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"])
+      raise Error, output unless status.zero?
+      true
+    end
+
+    # Fetch remote for repository
+    #
+    # name - project path with namespace
+    # remote - remote name
+    # forced - should we use --force flag?
+    # no_tags - should we use --no-tags flag?
+    #
+    # Ex.
+    #   fetch_remote("gitlab/gitlab-ci", "upstream")
+    #
+    def fetch_remote(storage, name, remote, forced: false, no_tags: false)
+      args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
+      args << '--force' if forced
+      args << '--no-tags' if no_tags
+
+      output, status = Popen.popen(args)
       raise Error, output unless status.zero?
       true
     end
@@ -127,19 +149,6 @@ module Gitlab
                                    'rm-project', storage, "#{name}.git"])
     end
 
-    # Gc repository
-    #
-    # storage - project storage path
-    # path - project path with namespace
-    #
-    # Ex.
-    #   gc("/path/to/storage", "gitlab/gitlab-ci")
-    #
-    def gc(storage, path)
-      Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc',
-                                   storage, "#{path}.git"])
-    end
-
     # Add new key to gitlab-shell
     #
     # Ex.
@@ -156,7 +165,7 @@ module Gitlab
     #   batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") }
     def batch_add_keys(&block)
       IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io|
-        block.call(KeyAdder.new(io))
+        yield(KeyAdder.new(io))
       end
     end
 
@@ -185,7 +194,10 @@ module Gitlab
     #   add_namespace("/path/to/storage", "gitlab")
     #
     def add_namespace(storage, name)
-      FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name)
+      path = full_path(storage, name)
+      FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
+    rescue Errno::EEXIST => e
+      Rails.logger.warn("Directory exists as a file: #{e} at: #{path}")
     end
 
     # Remove directory from repositories storage
diff --git a/lib/gitlab/backend/shell_adapter.rb b/lib/gitlab/shell_adapter.rb
index fbe2a7a0d72..fbe2a7a0d72 100644
--- a/lib/gitlab/backend/shell_adapter.rb
+++ b/lib/gitlab/shell_adapter.rb
diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb
index aa1468bff6b..b5f9d040047 100644
--- a/lib/gitlab/sherlock/line_profiler.rb
+++ b/lib/gitlab/sherlock/line_profiler.rb
@@ -77,8 +77,8 @@ module Gitlab
             line_samples << LineSample.new(duration, events)
           end
 
-          samples << FileSample.
-            new(file, line_samples, total_duration, total_events)
+          samples << FileSample
+            .new(file, line_samples, total_duration, total_events)
         end
 
         samples
diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb
index 4917c4ae2ac..948bf5e6528 100644
--- a/lib/gitlab/sherlock/query.rb
+++ b/lib/gitlab/sherlock/query.rb
@@ -94,20 +94,21 @@ module Gitlab
       private
 
       def raw_explain(query)
-        if Gitlab::Database.postgresql?
-          explain = "EXPLAIN ANALYZE #{query};"
-        else
-          explain = "EXPLAIN #{query};"
-        end
+        explain =
+          if Gitlab::Database.postgresql?
+            "EXPLAIN ANALYZE #{query};"
+          else
+            "EXPLAIN #{query};"
+          end
 
         ActiveRecord::Base.connection.execute(explain)
       end
 
       def format_sql(query)
-        query.each_line.
-          map { |line| line.strip }.
-          join("\n").
-          gsub(PREFIX_NEWLINE) { "\n#{$1} " }
+        query.each_line
+          .map { |line| line.strip }
+          .join("\n")
+          .gsub(PREFIX_NEWLINE) { "\n#{$1} " }
       end
     end
   end
diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb
new file mode 100644
index 00000000000..ca8d3271541
--- /dev/null
+++ b/lib/gitlab/sidekiq_status.rb
@@ -0,0 +1,102 @@
+module Gitlab
+  # The SidekiqStatus module and its child classes can be used for checking if a
+  # Sidekiq job has been processed or not.
+  #
+  # To check if a job has been completed, simply pass the job ID to the
+  # `completed?` method:
+  #
+  #     job_id = SomeWorker.perform_async(...)
+  #
+  #     if Gitlab::SidekiqStatus.completed?(job_id)
+  #       ...
+  #     end
+  #
+  # For each job ID registered a separate key is stored in Redis, making lookups
+  # much faster than using Sidekiq's built-in job finding/status API. These keys
+  # expire after a certain period of time to prevent storing too many keys in
+  # Redis.
+  module SidekiqStatus
+    STATUS_KEY = 'gitlab-sidekiq-status:%s'.freeze
+
+    # The default time (in seconds) after which a status key is expired
+    # automatically. The default of 30 minutes should be more than sufficient
+    # for most jobs.
+    DEFAULT_EXPIRATION = 30.minutes.to_i
+
+    # Starts tracking of the given job.
+    #
+    # jid - The Sidekiq job ID
+    # expire - The expiration time of the Redis key.
+    def self.set(jid, expire = DEFAULT_EXPIRATION)
+      Sidekiq.redis do |redis|
+        redis.set(key_for(jid), 1, ex: expire)
+      end
+    end
+
+    # Stops the tracking of the given job.
+    #
+    # jid - The Sidekiq job ID to remove.
+    def self.unset(jid)
+      Sidekiq.redis do |redis|
+        redis.del(key_for(jid))
+      end
+    end
+
+    # Returns true if all the given job have been completed.
+    #
+    # job_ids - The Sidekiq job IDs to check.
+    #
+    # Returns true or false.
+    def self.all_completed?(job_ids)
+      self.num_running(job_ids).zero?
+    end
+
+    # Returns the number of jobs that are running.
+    #
+    # job_ids - The Sidekiq job IDs to check.
+    def self.num_running(job_ids)
+      responses = self.job_status(job_ids)
+
+      responses.select(&:present?).count
+    end
+
+    # Returns the number of jobs that have completed.
+    #
+    # job_ids - The Sidekiq job IDs to check.
+    def self.num_completed(job_ids)
+      job_ids.size - self.num_running(job_ids)
+    end
+
+    # Returns the job status for each of the given job IDs.
+    #
+    # job_ids - The Sidekiq job IDs to check.
+    #
+    # Returns an array of true or false indicating job completion.
+    # true = job is still running
+    # false = job completed
+    def self.job_status(job_ids)
+      keys = job_ids.map { |jid| key_for(jid) }
+
+      Sidekiq.redis do |redis|
+        redis.pipelined do
+          keys.each { |key| redis.exists(key) }
+        end
+      end
+    end
+
+    # Returns the JIDs that are completed
+    #
+    # job_ids - The Sidekiq job IDs to check.
+    #
+    # Returns an array of completed JIDs
+    def self.completed_jids(job_ids)
+      Sidekiq.redis do |redis|
+        job_ids.reject { |jid| redis.exists(key_for(jid)) }
+      end
+    end
+
+    def self.key_for(jid)
+      STATUS_KEY % jid
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb
new file mode 100644
index 00000000000..00983b3284a
--- /dev/null
+++ b/lib/gitlab/sidekiq_status/client_middleware.rb
@@ -0,0 +1,12 @@
+module Gitlab
+  module SidekiqStatus
+    class ClientMiddleware
+      def call(_, job, _, _)
+        status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION
+
+        Gitlab::SidekiqStatus.set(job['jid'], status_expiration)
+        yield
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_status/server_middleware.rb b/lib/gitlab/sidekiq_status/server_middleware.rb
new file mode 100644
index 00000000000..ceab10b8301
--- /dev/null
+++ b/lib/gitlab/sidekiq_status/server_middleware.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module SidekiqStatus
+    class ServerMiddleware
+      def call(worker, job, queue)
+        ret = yield
+
+        Gitlab::SidekiqStatus.unset(job['jid'])
+
+        ret
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb
new file mode 100644
index 00000000000..d4d39a888e7
--- /dev/null
+++ b/lib/gitlab/sidekiq_throttler.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  class SidekiqThrottler
+    class << self
+      def execute!
+        if Gitlab::CurrentSettings.sidekiq_throttling_enabled?
+          Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue|
+            Sidekiq::Queue[queue].limit = queue_limit
+          end
+        end
+      end
+
+      private
+
+      def queue_limit
+        @queue_limit ||=
+          begin
+            factor = Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_factor
+            (factor * Sidekiq.options[:concurrency]).ceil
+          end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb
new file mode 100644
index 00000000000..cc3c9a50555
--- /dev/null
+++ b/lib/gitlab/slash_commands/base_command.rb
@@ -0,0 +1,47 @@
+module Gitlab
+  module SlashCommands
+    class BaseCommand
+      QUERY_LIMIT = 5
+
+      def self.match(_text)
+        raise NotImplementedError
+      end
+
+      def self.help_message
+        raise NotImplementedError
+      end
+
+      def self.available?(_project)
+        raise NotImplementedError
+      end
+
+      def self.allowed?(_user, _ability)
+        true
+      end
+
+      def self.can?(object, action, subject)
+        Ability.allowed?(object, action, subject)
+      end
+
+      def execute(_)
+        raise NotImplementedError
+      end
+
+      def collection
+        raise NotImplementedError
+      end
+
+      attr_accessor :project, :current_user, :params
+
+      def initialize(project, user, params = {})
+        @project, @current_user, @params = project, user, params.dup
+      end
+
+      private
+
+      def find_by_iid(iid)
+        collection.find_by(iid: iid)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb
new file mode 100644
index 00000000000..a78408b0519
--- /dev/null
+++ b/lib/gitlab/slash_commands/command.rb
@@ -0,0 +1,44 @@
+module Gitlab
+  module SlashCommands
+    class Command < BaseCommand
+      COMMANDS = [
+        Gitlab::SlashCommands::IssueShow,
+        Gitlab::SlashCommands::IssueNew,
+        Gitlab::SlashCommands::IssueSearch,
+        Gitlab::SlashCommands::Deploy
+      ].freeze
+
+      def execute
+        command, match = match_command
+
+        if command
+          if command.allowed?(project, current_user)
+            command.new(project, current_user, params).execute(match)
+          else
+            Gitlab::SlashCommands::Presenters::Access.new.access_denied
+          end
+        else
+          Gitlab::SlashCommands::Help.new(project, current_user, params).execute(available_commands, params[:text])
+        end
+      end
+
+      def match_command
+        match = nil
+        service =
+          available_commands.find do |klass|
+            match = klass.match(params[:text])
+          end
+
+        [service, match]
+      end
+
+      private
+
+      def available_commands
+        COMMANDS.select do |klass|
+          klass.available?(project)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/slash_commands/command_definition.rb
deleted file mode 100644
index 60d35be2599..00000000000
--- a/lib/gitlab/slash_commands/command_definition.rb
+++ /dev/null
@@ -1,57 +0,0 @@
-module Gitlab
-  module SlashCommands
-    class CommandDefinition
-      attr_accessor :name, :aliases, :description, :params, :condition_block, :action_block
-
-      def initialize(name, attributes = {})
-        @name = name
-
-        @aliases         = attributes[:aliases] || []
-        @description     = attributes[:description] || ''
-        @params          = attributes[:params] || []
-        @condition_block = attributes[:condition_block]
-        @action_block    = attributes[:action_block]
-      end
-
-      def all_names
-        [name, *aliases]
-      end
-
-      def noop?
-        action_block.nil?
-      end
-
-      def available?(opts)
-        return true unless condition_block
-
-        context = OpenStruct.new(opts)
-        context.instance_exec(&condition_block)
-      end
-
-      def execute(context, opts, arg)
-        return if noop? || !available?(opts)
-
-        if arg.present?
-          context.instance_exec(arg, &action_block)
-        elsif action_block.arity == 0
-          context.instance_exec(&action_block)
-        end
-      end
-
-      def to_h(opts)
-        desc = description
-        if desc.respond_to?(:call)
-          context = OpenStruct.new(opts)
-          desc = context.instance_exec(&desc) rescue ''
-        end
-
-        {
-          name: name,
-          aliases: aliases,
-          description: desc,
-          params: params
-        }
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/slash_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb
new file mode 100644
index 00000000000..e71eb15d604
--- /dev/null
+++ b/lib/gitlab/slash_commands/deploy.rb
@@ -0,0 +1,50 @@
+module Gitlab
+  module SlashCommands
+    class Deploy < BaseCommand
+      def self.match(text)
+        /\Adeploy\s+(?<from>\S+.*)\s+to+\s+(?<to>\S+.*)\z/.match(text)
+      end
+
+      def self.help_message
+        'deploy <environment> to <target-environment>'
+      end
+
+      def self.available?(project)
+        project.builds_enabled?
+      end
+
+      def self.allowed?(project, user)
+        can?(user, :create_deployment, project)
+      end
+
+      def execute(match)
+        from = match[:from]
+        to = match[:to]
+
+        actions = find_actions(from, to)
+
+        if actions.none?
+          Gitlab::SlashCommands::Presenters::Deploy.new(nil).no_actions
+        elsif actions.one?
+          action = play!(from, to, actions.first)
+          Gitlab::SlashCommands::Presenters::Deploy.new(action).present(from, to)
+        else
+          Gitlab::SlashCommands::Presenters::Deploy.new(actions).too_many_actions
+        end
+      end
+
+      private
+
+      def play!(from, to, action)
+        action.play(current_user)
+      end
+
+      def find_actions(from, to)
+        environment = project.environments.find_by(name: from)
+        return [] unless environment
+
+        environment.actions_for(to).select(&:starts_environment?)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/help.rb b/lib/gitlab/slash_commands/help.rb
new file mode 100644
index 00000000000..81f3707e03e
--- /dev/null
+++ b/lib/gitlab/slash_commands/help.rb
@@ -0,0 +1,28 @@
+module Gitlab
+  module SlashCommands
+    class Help < BaseCommand
+      # This class has to be used last, as it always matches. It has to match
+      # because other commands were not triggered and we want to show the help
+      # command
+      def self.match(_text)
+        true
+      end
+
+      def self.help_message
+        'help'
+      end
+
+      def self.allowed?(_project, _user)
+        true
+      end
+
+      def execute(commands, text)
+        Gitlab::SlashCommands::Presenters::Help.new(commands).present(trigger, text)
+      end
+
+      def trigger
+        params[:command]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/issue_command.rb b/lib/gitlab/slash_commands/issue_command.rb
new file mode 100644
index 00000000000..87ea19b8806
--- /dev/null
+++ b/lib/gitlab/slash_commands/issue_command.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  module SlashCommands
+    class IssueCommand < BaseCommand
+      def self.available?(project)
+        project.issues_enabled? && project.default_issues_tracker?
+      end
+
+      def collection
+        IssuesFinder.new(current_user, project_id: project.id).execute
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb
new file mode 100644
index 00000000000..25f965e843d
--- /dev/null
+++ b/lib/gitlab/slash_commands/issue_new.rb
@@ -0,0 +1,42 @@
+module Gitlab
+  module SlashCommands
+    class IssueNew < IssueCommand
+      def self.match(text)
+        # we can not match \n with the dot by passing the m modifier as than
+        # the title and description are not seperated
+        /\Aissue\s+(new|create)\s+(?<title>[^\n]*)\n*(?<description>(.|\n)*)/.match(text)
+      end
+
+      def self.help_message
+        'issue new <title> *`⇧ Shift`*+*`↵ Enter`* <description>'
+      end
+
+      def self.allowed?(project, user)
+        can?(user, :create_issue, project)
+      end
+
+      def execute(match)
+        title = match[:title]
+        description = match[:description].to_s.rstrip
+
+        issue = create_issue(title: title, description: description)
+
+        if issue.persisted?
+          presenter(issue).present
+        else
+          presenter(issue).display_errors
+        end
+      end
+
+      private
+
+      def create_issue(title:, description:)
+        Issues::CreateService.new(project, current_user, title: title, description: description).execute
+      end
+
+      def presenter(issue)
+        Gitlab::SlashCommands::Presenters::IssueNew.new(issue)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/issue_search.rb b/lib/gitlab/slash_commands/issue_search.rb
new file mode 100644
index 00000000000..acba84b54b4
--- /dev/null
+++ b/lib/gitlab/slash_commands/issue_search.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module SlashCommands
+    class IssueSearch < IssueCommand
+      def self.match(text)
+        /\Aissue\s+search\s+(?<query>.*)/.match(text)
+      end
+
+      def self.help_message
+        "issue search <your query>"
+      end
+
+      def execute(match)
+        issues = collection.search(match[:query]).limit(QUERY_LIMIT)
+
+        if issues.present?
+          Presenters::IssueSearch.new(issues).present
+        else
+          Presenters::Access.new(issues).not_found
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/issue_show.rb b/lib/gitlab/slash_commands/issue_show.rb
new file mode 100644
index 00000000000..ffa5184e5cb
--- /dev/null
+++ b/lib/gitlab/slash_commands/issue_show.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module SlashCommands
+    class IssueShow < IssueCommand
+      def self.match(text)
+        /\Aissue\s+show\s+#{Issue.reference_prefix}?(?<iid>\d+)/.match(text)
+      end
+
+      def self.help_message
+        "issue show <id>"
+      end
+
+      def execute(match)
+        issue = find_by_iid(match[:iid])
+
+        if issue
+          Gitlab::SlashCommands::Presenters::IssueShow.new(issue).present
+        else
+          Gitlab::SlashCommands::Presenters::Access.new.not_found
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb
new file mode 100644
index 00000000000..1a817eb735b
--- /dev/null
+++ b/lib/gitlab/slash_commands/presenters/access.rb
@@ -0,0 +1,40 @@
+module Gitlab
+  module SlashCommands
+    module Presenters
+      class Access < Presenters::Base
+        def access_denied
+          ephemeral_response(text: "Whoops! This action is not allowed. This incident will be [reported](https://xkcd.com/838/).")
+        end
+
+        def not_found
+          ephemeral_response(text: "404 not found! GitLab couldn't find what you were looking for! :boom:")
+        end
+
+        def authorize
+          message =
+            if @resource
+              ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{@resource})."
+            else
+              ":sweat_smile: Couldn't identify you, nor can I autorize you!"
+            end
+
+          ephemeral_response(text: message)
+        end
+
+        def unknown_command(commands)
+          ephemeral_response(text: help_message(trigger))
+        end
+
+        private
+
+        def help_message(trigger)
+          header_with_list("Command not found, these are the commands you can use", full_commands(trigger))
+        end
+
+        def full_commands(trigger)
+          @resource.map { |command| "#{trigger} #{command.help_message}" }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb
new file mode 100644
index 00000000000..27696436574
--- /dev/null
+++ b/lib/gitlab/slash_commands/presenters/base.rb
@@ -0,0 +1,77 @@
+module Gitlab
+  module SlashCommands
+    module Presenters
+      class Base
+        include Gitlab::Routing.url_helpers
+
+        def initialize(resource = nil)
+          @resource = resource
+        end
+
+        def display_errors
+          message = header_with_list("The action was not successful, because:", @resource.errors.full_messages)
+
+          ephemeral_response(text: message)
+        end
+
+        private
+
+        def header_with_list(header, items)
+          message = [header]
+
+          items.each do |item|
+            message << "- #{item}"
+          end
+
+          message.join("\n")
+        end
+
+        def ephemeral_response(message)
+          response = {
+            response_type: :ephemeral,
+            status: 200
+          }.merge(message)
+
+          format_response(response)
+        end
+
+        def in_channel_response(message)
+          response = {
+            response_type: :in_channel,
+            status: 200
+          }.merge(message)
+
+          format_response(response)
+        end
+
+        def format_response(response)
+          response[:text] = format(response[:text]) if response.key?(:text)
+
+          if response.key?(:attachments)
+            response[:attachments].each do |attachment|
+              attachment[:pretext] = format(attachment[:pretext]) if attachment[:pretext]
+              attachment[:text] = format(attachment[:text]) if attachment[:text]
+            end
+          end
+
+          response
+        end
+
+        # Convert Markdown to slacks format
+        def format(string)
+          Slack::Notifier::LinkFormatter.format(string)
+        end
+
+        def resource_url
+          url_for(
+            [
+              @resource.project.namespace.becomes(Namespace),
+              @resource.project,
+              @resource
+            ]
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/presenters/deploy.rb b/lib/gitlab/slash_commands/presenters/deploy.rb
new file mode 100644
index 00000000000..b8dc77bd37b
--- /dev/null
+++ b/lib/gitlab/slash_commands/presenters/deploy.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module SlashCommands
+    module Presenters
+      class Deploy < Presenters::Base
+        def present(from, to)
+          message = "Deployment started from #{from} to #{to}. [Follow its progress](#{resource_url})."
+
+          in_channel_response(text: message)
+        end
+
+        def no_actions
+          ephemeral_response(text: "No action found to be executed")
+        end
+
+        def too_many_actions
+          ephemeral_response(text: "Too many actions defined")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb
new file mode 100644
index 00000000000..ea611a4d629
--- /dev/null
+++ b/lib/gitlab/slash_commands/presenters/help.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module SlashCommands
+    module Presenters
+      class Help < Presenters::Base
+        def present(trigger, text)
+          ephemeral_response(text: help_message(trigger, text))
+        end
+
+        private
+
+        def help_message(trigger, text)
+          return "No commands available :thinking_face:" unless @resource.present?
+
+          if text.start_with?('help')
+            header_with_list("Available commands", full_commands(trigger))
+          else
+            header_with_list("Unknown command, these commands are available", full_commands(trigger)) 
+          end
+        end
+
+        def full_commands(trigger)
+          @resource.map { |command| "#{trigger} #{command.help_message}" }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/presenters/issue_base.rb b/lib/gitlab/slash_commands/presenters/issue_base.rb
new file mode 100644
index 00000000000..341f2aabdd0
--- /dev/null
+++ b/lib/gitlab/slash_commands/presenters/issue_base.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module SlashCommands
+    module Presenters
+      module IssueBase
+        def color(issuable)
+          issuable.open? ? '#38ae67' : '#d22852'
+        end
+
+        def status_text(issuable)
+          issuable.open? ? 'Open' : 'Closed'
+        end
+
+        def project
+          @resource.project
+        end
+
+        def author
+          @resource.author
+        end
+
+        def fields
+          [
+            {
+              title: "Assignee",
+              value: @resource.assignees.any? ? @resource.assignees.first.name : "_None_",
+              short: true
+            },
+            {
+              title: "Milestone",
+              value: @resource.milestone ? @resource.milestone.title : "_None_",
+              short: true
+            },
+            {
+              title: "Labels",
+              value: @resource.labels.any? ? @resource.label_names.join(', ') : "_None_",
+              short: true
+            }
+          ]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb
new file mode 100644
index 00000000000..86490a39cc1
--- /dev/null
+++ b/lib/gitlab/slash_commands/presenters/issue_new.rb
@@ -0,0 +1,50 @@
+module Gitlab
+  module SlashCommands
+    module Presenters
+      class IssueNew < Presenters::Base
+        include Presenters::IssueBase
+
+        def present
+          in_channel_response(new_issue)
+        end
+
+        private
+
+        def new_issue
+          {
+            attachments: [
+              {
+                title:        "#{@resource.title} · #{@resource.to_reference}",
+                title_link:   resource_url,
+                author_name:  author.name,
+                author_icon:  author.avatar_url,
+                fallback:     "New issue #{@resource.to_reference}: #{@resource.title}",
+                pretext:      pretext,
+                color:        color(@resource),
+                fields:       fields,
+                mrkdwn_in: [
+                  :title,
+                  :pretext,
+                  :text,
+                  :fields
+                ]
+              }
+            ]
+          }
+        end
+
+        def pretext
+          "I created an issue on #{author_profile_link}'s behalf: **#{@resource.to_reference}** in #{project_link}"
+        end
+
+        def project_link
+          "[#{project.name_with_namespace}](#{project.web_url})"
+        end
+
+        def author_profile_link
+          "[#{author.to_reference}](#{url_for(author)})"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/presenters/issue_search.rb b/lib/gitlab/slash_commands/presenters/issue_search.rb
new file mode 100644
index 00000000000..4e27d668685
--- /dev/null
+++ b/lib/gitlab/slash_commands/presenters/issue_search.rb
@@ -0,0 +1,47 @@
+module Gitlab
+  module SlashCommands
+    module Presenters
+      class IssueSearch < Presenters::Base
+        include Presenters::IssueBase
+
+        def present
+          text = if @resource.count >= 5
+                   "Here are the first 5 issues I found:"
+                 elsif @resource.one?
+                   "Here is the only issue I found:"
+                 else
+                   "Here are the #{@resource.count} issues I found:"
+                 end
+
+          ephemeral_response(text: text, attachments: attachments)
+        end
+
+        private
+
+        def attachments
+          @resource.map do |issue|
+            url = "[#{issue.to_reference}](#{url_for([namespace, project, issue])})"
+
+            {
+              color: color(issue),
+              fallback: "#{issue.to_reference} #{issue.title}",
+              text: "#{url} · #{issue.title} (#{status_text(issue)})",
+
+              mrkdwn_in: [
+                :text
+              ]
+            }
+          end
+        end
+
+        def project
+          @project ||= @resource.first.project
+        end
+
+        def namespace
+          @namespace ||= project.namespace.becomes(Namespace)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/presenters/issue_show.rb b/lib/gitlab/slash_commands/presenters/issue_show.rb
new file mode 100644
index 00000000000..c99316df667
--- /dev/null
+++ b/lib/gitlab/slash_commands/presenters/issue_show.rb
@@ -0,0 +1,61 @@
+module Gitlab
+  module SlashCommands
+    module Presenters
+      class IssueShow < Presenters::Base
+        include Presenters::IssueBase
+
+        def present
+          if @resource.confidential?
+            ephemeral_response(show_issue)
+          else
+            in_channel_response(show_issue)
+          end
+        end
+
+        private
+
+        def show_issue
+          {
+            attachments: [
+              {
+                title:        "#{@resource.title} · #{@resource.to_reference}",
+                title_link:   resource_url,
+                author_name:  author.name,
+                author_icon:  author.avatar_url,
+                fallback:     "Issue #{@resource.to_reference}: #{@resource.title}",
+                pretext:      pretext,
+                text:         text,
+                color:        color(@resource),
+                fields:       fields,
+                mrkdwn_in: [
+                  :pretext,
+                  :text,
+                  :fields
+                ]
+              }
+            ]
+          }
+        end
+
+        def text
+          message = "**#{status_text(@resource)}**"
+
+          if @resource.upvotes.zero? && @resource.downvotes.zero? && @resource.user_notes_count.zero?
+            return message
+          end
+
+          message << " · "
+          message << ":+1: #{@resource.upvotes} " unless @resource.upvotes.zero?
+          message << ":-1: #{@resource.downvotes} " unless @resource.downvotes.zero?
+          message << ":speech_balloon: #{@resource.user_notes_count}" unless @resource.user_notes_count.zero?
+
+          message
+        end
+
+        def pretext
+          "Issue *#{@resource.to_reference}* from #{project.name_with_namespace}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/result.rb b/lib/gitlab/slash_commands/result.rb
new file mode 100644
index 00000000000..7021b4b01b2
--- /dev/null
+++ b/lib/gitlab/slash_commands/result.rb
@@ -0,0 +1,5 @@
+module Gitlab
+  module SlashCommands
+    Result = Struct.new(:type, :message)
+  end
+end
diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb
index 9e01f02029c..b85f70e450e 100644
--- a/lib/gitlab/snippet_search_results.rb
+++ b/lib/gitlab/snippet_search_results.rb
@@ -31,11 +31,11 @@ module Gitlab
     private
 
     def snippet_titles
-      limit_snippets.search(query).order('updated_at DESC')
+      limit_snippets.search(query).order('updated_at DESC').includes(:author)
     end
 
     def snippet_blobs
-      limit_snippets.search_code(query).order('updated_at DESC')
+      limit_snippets.search_code(query).order('updated_at DESC').includes(:author)
     end
 
     def default_scope
diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb
new file mode 100644
index 00000000000..16ec002f139
--- /dev/null
+++ b/lib/gitlab/sql/recursive_cte.rb
@@ -0,0 +1,62 @@
+module Gitlab
+  module SQL
+    # Class for easily building recursive CTE statements.
+    #
+    # Example:
+    #
+    #     cte = RecursiveCTE.new(:my_cte_name)
+    #     ns = Arel::Table.new(:namespaces)
+    #
+    #     cte << Namespace.
+    #       where(ns[:parent_id].eq(some_namespace_id))
+    #
+    #     cte << Namespace.
+    #       from([ns, cte.table]).
+    #       where(ns[:parent_id].eq(cte.table[:id]))
+    #
+    #     Namespace.with.
+    #       recursive(cte.to_arel).
+    #       from(cte.alias_to(ns))
+    class RecursiveCTE
+      attr_reader :table
+
+      # name - The name of the CTE as a String or Symbol.
+      def initialize(name)
+        @table = Arel::Table.new(name)
+        @queries = []
+      end
+
+      # Adds a query to the body of the CTE.
+      #
+      # relation - The relation object to add to the body of the CTE.
+      def <<(relation)
+        @queries << relation
+      end
+
+      # Returns the Arel relation for this CTE.
+      def to_arel
+        sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries).to_sql)
+
+        Arel::Nodes::As.new(table, Arel::Nodes::Grouping.new(sql))
+      end
+
+      # Returns an "AS" statement that aliases the CTE name as the given table
+      # name. This allows one to trick ActiveRecord into thinking it's selecting
+      # from an actual table, when in reality it's selecting from a CTE.
+      #
+      # alias_table - The Arel table to use as the alias.
+      def alias_to(alias_table)
+        Arel::Nodes::As.new(table, alias_table)
+      end
+
+      # Applies the CTE to the given relation, returning a new one that will
+      # query from it.
+      def apply_to(relation)
+        relation.except(:where)
+          .with
+          .recursive(to_arel)
+          .from(alias_to(relation.model.arel_table))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb
index 1cd89b3a9c4..222021e8802 100644
--- a/lib/gitlab/sql/union.rb
+++ b/lib/gitlab/sql/union.rb
@@ -22,9 +22,7 @@ module Gitlab
         # By using "unprepared_statements" we remove the usage of placeholders
         # (thus fixing this problem), at a slight performance cost.
         fragments = ActiveRecord::Base.connection.unprepared_statement do
-          @relations.map do |rel|
-            rel.reorder(nil).to_sql
-          end
+          @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?)
         end
 
         fragments.join("\nUNION\n")
diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb
new file mode 100644
index 00000000000..94fba0a221a
--- /dev/null
+++ b/lib/gitlab/string_range_marker.rb
@@ -0,0 +1,102 @@
+module Gitlab
+  class StringRangeMarker
+    attr_accessor :raw_line, :rich_line
+
+    def initialize(raw_line, rich_line = raw_line)
+      @raw_line = raw_line
+      @rich_line = ERB::Util.html_escape(rich_line)
+    end
+
+    def mark(marker_ranges)
+      return rich_line unless marker_ranges
+
+      rich_marker_ranges = []
+      marker_ranges.each do |range|
+        # Map the inline-diff range based on the raw line to character positions in the rich line
+        rich_positions = position_mapping[range].flatten
+        # Turn the array of character positions into ranges
+        rich_marker_ranges.concat(collapse_ranges(rich_positions))
+      end
+
+      offset = 0
+      # Mark each range
+      rich_marker_ranges.each_with_index do |range, i|
+        offset_range = (range.begin + offset)..(range.end + offset)
+        original_text = rich_line[offset_range]
+
+        text = yield(original_text, left: i == 0, right: i == rich_marker_ranges.length - 1)
+
+        rich_line[offset_range] = text
+
+        offset += text.length - original_text.length
+      end
+
+      rich_line.html_safe
+    end
+
+    private
+
+    # Mapping of character positions in the raw line, to the rich (highlighted) line
+    def position_mapping
+      @position_mapping ||= begin
+        mapping = []
+        rich_pos = 0
+        (0..raw_line.length).each do |raw_pos|
+          rich_char = rich_line[rich_pos]
+
+          # The raw and rich lines are the same except for HTML tags,
+          # so skip over any `<...>` segment
+          while rich_char == '<'
+            until rich_char == '>'
+              rich_pos += 1
+              rich_char = rich_line[rich_pos]
+            end
+
+            rich_pos += 1
+            rich_char = rich_line[rich_pos]
+          end
+
+          # multi-char HTML entities in the rich line correspond to a single character in the raw line
+          if rich_char == '&'
+            multichar_mapping = [rich_pos]
+            until rich_char == ';'
+              rich_pos += 1
+              multichar_mapping << rich_pos
+              rich_char = rich_line[rich_pos]
+            end
+
+            mapping[raw_pos] = multichar_mapping
+          else
+            mapping[raw_pos] = rich_pos
+          end
+
+          rich_pos += 1
+        end
+
+        mapping
+      end
+    end
+
+    # Takes an array of integers, and returns an array of ranges covering the same integers
+    def collapse_ranges(positions)
+      return [] if positions.empty?
+      ranges = []
+
+      start = prev = positions[0]
+      range = start..prev
+      positions[1..-1].each do |pos|
+        if pos == prev + 1
+          range = start..pos
+          prev = pos
+        else
+          ranges << range
+          start = prev = pos
+          range = start..prev
+        end
+      end
+      ranges << range
+
+      ranges
+    end
+  end
+end
diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb
new file mode 100644
index 00000000000..7ebf1c0428c
--- /dev/null
+++ b/lib/gitlab/string_regex_marker.rb
@@ -0,0 +1,13 @@
+module Gitlab
+  class StringRegexMarker < StringRangeMarker
+    def mark(regex, group: 0, &block)
+      regex_match = raw_line.match(regex)
+      return rich_line unless regex_match
+
+      begin_index, end_index = regex_match.offset(group)
+      name_range = begin_index..(end_index - 1)
+
+      super([name_range], &block)
+    end
+  end
+end
diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb
new file mode 100644
index 00000000000..20b054b0bd8
--- /dev/null
+++ b/lib/gitlab/template/dockerfile_template.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module Template
+    class DockerfileTemplate < BaseTemplate
+      def content
+        explanation = "# This file is a template, and might need editing before it works on your project."
+        [explanation, super].join("\n")
+      end
+
+      class << self
+        def extension
+          '.Dockerfile'
+        end
+
+        def categories
+          {
+            "General" => ''
+          }
+        end
+
+        def base_dir
+          Rails.root.join('vendor/Dockerfile')
+        end
+
+        def finder(project = nil)
+          Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb
index 22c39436cb2..cb7957e2af9 100644
--- a/lib/gitlab/template/finders/repo_template_finder.rb
+++ b/lib/gitlab/template/finders/repo_template_finder.rb
@@ -4,7 +4,7 @@ module Gitlab
     module Finders
       class RepoTemplateFinder < BaseTemplateFinder
         # Raised when file is not found
-        class FileNotFoundError < StandardError; end
+        FileNotFoundError = Class.new(StandardError)
 
         def initialize(project, base_dir, extension, categories = {})
           @categories     = categories
diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb
index 8d1a1ed54c9..fd040148a1e 100644
--- a/lib/gitlab/template/gitlab_ci_yml_template.rb
+++ b/lib/gitlab/template/gitlab_ci_yml_template.rb
@@ -13,8 +13,9 @@ module Gitlab
 
         def categories
           {
-            "General" => '',
-            "Pages" => 'Pages'
+            'General' => '',
+            'Pages' => 'Pages',
+            'Auto deploy' => 'autodeploy'
           }
         end
 
@@ -25,6 +26,11 @@ module Gitlab
         def finder(project = nil)
           Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories)
         end
+
+        def dropdown_names(context)
+          categories = context == 'autodeploy' ? ['Auto deploy'] : %w(General Pages)
+          super().slice(*categories)
+        end
       end
     end
   end
diff --git a/lib/gitlab/testing/request_blocker_middleware.rb b/lib/gitlab/testing/request_blocker_middleware.rb
new file mode 100644
index 00000000000..aa67fa08577
--- /dev/null
+++ b/lib/gitlab/testing/request_blocker_middleware.rb
@@ -0,0 +1,61 @@
+# rubocop:disable Style/ClassVars
+
+# This is inspired by http://www.salsify.com/blog/engineering/tearing-capybara-ajax-tests
+# Rack middleware that keeps track of the number of active requests and can block new requests.
+module Gitlab
+  module Testing
+    class RequestBlockerMiddleware
+      @@num_active_requests = Concurrent::AtomicFixnum.new(0)
+      @@block_requests = Concurrent::AtomicBoolean.new(false)
+
+      # Returns the number of requests the server is currently processing.
+      def self.num_active_requests
+        @@num_active_requests.value
+      end
+
+      # Prevents the server from accepting new requests. Any new requests will return an HTTP
+      # 503 status.
+      def self.block_requests!
+        @@block_requests.value = true
+      end
+
+      # Allows the server to accept requests again.
+      def self.allow_requests!
+        @@block_requests.value = false
+      end
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        increment_active_requests
+        if block_requests?
+          block_request(env)
+        else
+          @app.call(env)
+        end
+      ensure
+        decrement_active_requests
+      end
+
+      private
+
+      def block_requests?
+        @@block_requests.true?
+      end
+
+      def block_request(env)
+        [503, {}, []]
+      end
+
+      def increment_active_requests
+        @@num_active_requests.increment
+      end
+
+      def decrement_active_requests
+        @@num_active_requests.decrement
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
deleted file mode 100644
index d4020af76f9..00000000000
--- a/lib/gitlab/themes.rb
+++ /dev/null
@@ -1,87 +0,0 @@
-module Gitlab
-  # Module containing GitLab's application theme definitions and helper methods
-  # for accessing them.
-  module Themes
-    extend self
-
-    # Theme ID used when no `default_theme` configuration setting is provided.
-    APPLICATION_DEFAULT = 2
-
-    # Struct class representing a single Theme
-    Theme = Struct.new(:id, :name, :css_class)
-
-    # All available Themes
-    THEMES = [
-      Theme.new(1, 'Graphite', 'ui_graphite'),
-      Theme.new(2, 'Charcoal', 'ui_charcoal'),
-      Theme.new(3, 'Green',    'ui_green'),
-      Theme.new(4, 'Gray',     'ui_gray'),
-      Theme.new(5, 'Violet',   'ui_violet'),
-      Theme.new(6, 'Blue',     'ui_blue')
-    ].freeze
-
-    # Convenience method to get a space-separated String of all the theme
-    # classes that might be applied to the `body` element
-    #
-    # Returns a String
-    def body_classes
-      THEMES.collect(&:css_class).uniq.join(' ')
-    end
-
-    # Get a Theme by its ID
-    #
-    # If the ID is invalid, returns the default Theme.
-    #
-    # id - Integer ID
-    #
-    # Returns a Theme
-    def by_id(id)
-      THEMES.detect { |t| t.id == id } || default
-    end
-
-    # Returns the number of defined Themes
-    def count
-      THEMES.size
-    end
-
-    # Get the default Theme
-    #
-    # Returns a Theme
-    def default
-      by_id(default_id)
-    end
-
-    # Iterate through each Theme
-    #
-    # Yields the Theme object
-    def each(&block)
-      THEMES.each(&block)
-    end
-
-    # Get the Theme for the specified user, or the default
-    #
-    # user - User record
-    #
-    # Returns a Theme
-    def for_user(user)
-      if user
-        by_id(user.theme_id)
-      else
-        default
-      end
-    end
-
-    private
-
-    def default_id
-      id = Gitlab.config.gitlab.default_theme.to_i
-
-      # Prevent an invalid configuration setting from causing an infinite loop
-      if id < THEMES.first.id || id > THEMES.last.id
-        APPLICATION_DEFAULT
-      else
-        id
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb
new file mode 100644
index 00000000000..d615c24149a
--- /dev/null
+++ b/lib/gitlab/time_tracking_formatter.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module TimeTrackingFormatter
+    extend self
+
+    def parse(string)
+      with_custom_config do
+        string.sub!(/\A-/, '')
+
+        seconds = ChronicDuration.parse(string, default_unit: 'hours') rescue nil
+        seconds *= -1 if seconds && Regexp.last_match
+        seconds
+      end
+    end
+
+    def output(seconds)
+      with_custom_config do
+        ChronicDuration.output(seconds, format: :short, limit_to_hours: false, weeks: true) rescue nil
+      end
+    end
+
+    def with_custom_config
+      # We may want to configure it through project settings in a future version.
+      ChronicDuration.hours_per_day = 8
+      ChronicDuration.days_per_week = 5
+
+      result = yield
+
+      ChronicDuration.hours_per_day = 24
+      ChronicDuration.days_per_week = 7
+
+      result
+    end
+  end
+end
diff --git a/lib/gitlab/update_path_error.rb b/lib/gitlab/update_path_error.rb
new file mode 100644
index 00000000000..8947ecfb92e
--- /dev/null
+++ b/lib/gitlab/update_path_error.rb
@@ -0,0 +1,3 @@
+module Gitlab
+  UpdatePathError = Class.new(StandardError)
+end
diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb
index f3567f3ef85..961df0468a4 100644
--- a/lib/gitlab/upgrader.rb
+++ b/lib/gitlab/upgrader.rb
@@ -46,7 +46,7 @@ module Gitlab
       git_tags = fetch_git_tags
       git_tags = git_tags.select { |version| version =~ /v\d+\.\d+\.\d+\Z/ }
       git_versions = git_tags.map { |tag| Gitlab::VersionInfo.parse(tag.match(/v\d+\.\d+\.\d+/).to_s) }
-      "v#{git_versions.sort.last.to_s}"
+      "v#{git_versions.sort.last}"
     end
 
     def fetch_git_tags
@@ -59,15 +59,18 @@ module Gitlab
         "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash),
         "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch),
         "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}),
-        "Install gems" => %W(bundle),
-        "Migrate DB" => %W(bundle exec rake db:migrate),
-        "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile),
-        "Clear cache" => %W(bundle exec rake cache:clear)
+        "Install gems" => %w(bundle),
+        "Migrate DB" => %w(bundle exec rake db:migrate),
+        "Recompile assets" => %w(bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile),
+        "Clear cache" => %w(bundle exec rake cache:clear)
       }
     end
 
     def env
-      { 'RAILS_ENV' => 'production' }
+      {
+        'RAILS_ENV' => 'production',
+        'NODE_ENV' => 'production'
+      }
     end
 
     def upgrade
diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb
index be8fcc7b2d2..b5f41240529 100644
--- a/lib/gitlab/uploads_transfer.rb
+++ b/lib/gitlab/uploads_transfer.rb
@@ -1,35 +1,7 @@
 module Gitlab
-  class UploadsTransfer
-    def move_project(project_path, namespace_path_was, namespace_path)
-      new_namespace_folder = File.join(root_dir, namespace_path)
-      FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder)
-      from = File.join(root_dir, namespace_path_was, project_path)
-      to = File.join(root_dir, namespace_path, project_path)
-      move(from, to, "")
-    end
-
-    def rename_project(path_was, path, namespace_path)
-      base_dir = File.join(root_dir, namespace_path)
-      move(path_was, path, base_dir)
-    end
-
-    def rename_namespace(path_was, path)
-      move(path_was, path)
-    end
-
-    private
-
-    def move(path_was, path, base_dir = nil)
-      base_dir = root_dir unless base_dir
-      from = File.join(base_dir, path_was)
-      to = File.join(base_dir, path)
-      FileUtils.mv(from, to)
-    rescue Errno::ENOENT
-      false
-    end
-
+  class UploadsTransfer < ProjectTransfer
     def root_dir
-      File.join(Rails.root, "public", "uploads")
+      File.join(CarrierWave.root, FileUploader.base_dir)
     end
   end
 end
diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb
new file mode 100644
index 00000000000..7e14a566696
--- /dev/null
+++ b/lib/gitlab/url_blocker.rb
@@ -0,0 +1,59 @@
+require 'resolv'
+
+module Gitlab
+  class UrlBlocker
+    class << self
+      # Used to specify what hosts and port numbers should be prohibited for project
+      # imports.
+      VALID_PORTS = [22, 80, 443].freeze
+
+      def blocked_url?(url)
+        return false if url.nil?
+
+        blocked_ips = ["127.0.0.1", "::1", "0.0.0.0"]
+        blocked_ips.concat(Socket.ip_address_list.map(&:ip_address))
+
+        begin
+          uri = Addressable::URI.parse(url)
+          # Allow imports from the GitLab instance itself but only from the configured ports
+          return false if internal?(uri)
+
+          return true if blocked_port?(uri.port)
+
+          server_ips = Resolv.getaddresses(uri.hostname)
+          return true if (blocked_ips & server_ips).any?
+        rescue Addressable::URI::InvalidURIError
+          return true
+        end
+
+        false
+      end
+
+      private
+
+      def blocked_port?(port)
+        return false if port.blank?
+
+        port < 1024 && !VALID_PORTS.include?(port)
+      end
+
+      def internal?(uri)
+        internal_web?(uri) || internal_shell?(uri)
+      end
+
+      def internal_web?(uri)
+        uri.hostname == config.gitlab.host &&
+          (uri.port.blank? || uri.port == config.gitlab.port)
+      end
+
+      def internal_shell?(uri)
+        uri.hostname == config.gitlab_shell.ssh_host &&
+          (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port)
+      end
+
+      def config
+        Gitlab.config
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index 99d0c28e749..23af9318d1a 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -24,6 +24,8 @@ module Gitlab
         wiki_page_url
       when ProjectSnippet
         project_snippet_url(object)
+      when Snippet
+        personal_snippet_url(object)
       else
         raise NotImplementedError.new("No URL builder defined for #{object.class}")
       end
@@ -59,7 +61,12 @@ module Gitlab
 
       elsif object.for_snippet?
         snippet = Snippet.find(object.noteable_id)
-        project_snippet_url(snippet, anchor: dom_id(object))
+
+        if snippet.is_a?(PersonalSnippet)
+          snippet_url(snippet, anchor: dom_id(object))
+        else
+          project_snippet_url(snippet, anchor: dom_id(object))
+        end
       end
     end
 
diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb
index 19dad699edf..c81dc7e30d0 100644
--- a/lib/gitlab/url_sanitizer.rb
+++ b/lib/gitlab/url_sanitizer.rb
@@ -1,7 +1,7 @@
 module Gitlab
   class UrlSanitizer
     def self.sanitize(content)
-      regexp = URI::Parser.new.make_regexp(['http', 'https', 'ssh', 'git'])
+      regexp = URI::Parser.new.make_regexp(%w(http https ssh git))
 
       content.gsub(regexp) { |url| new(url).masked_url }
     rescue Addressable::URI::InvalidURIError
@@ -9,6 +9,8 @@ module Gitlab
     end
 
     def self.valid?(url)
+      return false unless url
+
       Addressable::URI.parse(url.strip)
 
       true
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
new file mode 100644
index 00000000000..38dc82493cf
--- /dev/null
+++ b/lib/gitlab/usage_data.rb
@@ -0,0 +1,67 @@
+module Gitlab
+  class UsageData
+    include Gitlab::CurrentSettings
+
+    class << self
+      def data(force_refresh: false)
+        Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data }
+      end
+
+      def uncached_data
+        license_usage_data.merge(system_usage_data)
+      end
+
+      def to_json(force_refresh: false)
+        data(force_refresh: force_refresh).to_json
+      end
+
+      def system_usage_data
+        {
+          counts: {
+            boards: Board.count,
+            ci_builds: ::Ci::Build.count,
+            ci_pipelines: ::Ci::Pipeline.count,
+            ci_runners: ::Ci::Runner.count,
+            ci_triggers: ::Ci::Trigger.count,
+            ci_pipeline_schedules: ::Ci::PipelineSchedule.count,
+            deploy_keys: DeployKey.count,
+            deployments: Deployment.count,
+            environments: Environment.count,
+            in_review_folder: Environment.in_review_folder.count,
+            groups: Group.count,
+            issues: Issue.count,
+            keys: Key.count,
+            labels: Label.count,
+            lfs_objects: LfsObject.count,
+            merge_requests: MergeRequest.count,
+            milestones: Milestone.count,
+            notes: Note.count,
+            pages_domains: PagesDomain.count,
+            projects: Project.count,
+            projects_prometheus_active: PrometheusService.active.count,
+            protected_branches: ProtectedBranch.count,
+            releases: Release.count,
+            snippets: Snippet.count,
+            todos: Todo.count,
+            uploads: Upload.count,
+            web_hooks: WebHook.count
+          }
+        }
+      end
+
+      def license_usage_data
+        usage_data = {
+          uuid: current_application_settings.uuid,
+          hostname: Gitlab.config.gitlab.host,
+          version: Gitlab::VERSION,
+          active_user_count: User.active.count,
+          recorded_at: Time.now,
+          mattermost_enabled: Gitlab.config.mattermost.enabled,
+          edition: 'CE'
+        }
+
+        usage_data
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb
index 9858d2e7d83..3b922da7ced 100644
--- a/lib/gitlab/user_access.rb
+++ b/lib/gitlab/user_access.rb
@@ -8,6 +8,8 @@ module Gitlab
     end
 
     def can_do_action?(action)
+      return false unless can_access_git?
+
       @permission_cache ||= {}
       @permission_cache[action] ||= user.can?(action, project)
     end
@@ -17,7 +19,7 @@ module Gitlab
     end
 
     def allowed?
-      return false if user.blank? || user.blocked?
+      return false unless can_access_git?
 
       if user.requires_ldap_check? && user.try_obtain_ldap_lease
         return false unless Gitlab::LDAP::Access.allowed?(user)
@@ -26,34 +28,58 @@ module Gitlab
       true
     end
 
+    def can_create_tag?(ref)
+      return false unless can_access_git?
+
+      if ProtectedTag.protected?(project, ref)
+        project.protected_tags.protected_ref_accessible_to?(ref, user, action: :create)
+      else
+        user.can?(:push_code, project)
+      end
+    end
+
+    def can_delete_branch?(ref)
+      return false unless can_access_git?
+
+      if ProtectedBranch.protected?(project, ref)
+        user.can?(:delete_protected_branch, project)
+      else
+        user.can?(:push_code, project)
+      end
+    end
+
     def can_push_to_branch?(ref)
-      return false unless user
+      return false unless can_access_git?
 
-      if project.protected_branch?(ref)
+      if ProtectedBranch.protected?(project, ref)
         return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user)
 
-        access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten
-        access_levels.any? { |access_level| access_level.check_access(user) }
+        project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push)
       else
         user.can?(:push_code, project)
       end
     end
 
     def can_merge_to_branch?(ref)
-      return false unless user
+      return false unless can_access_git?
 
-      if project.protected_branch?(ref)
-        access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten
-        access_levels.any? { |access_level| access_level.check_access(user) }
+      if ProtectedBranch.protected?(project, ref)
+        project.protected_branches.protected_ref_accessible_to?(ref, user, action: :merge)
       else
         user.can?(:push_code, project)
       end
     end
 
     def can_read_project?
-      return false unless user
+      return false unless can_access_git?
 
       user.can?(:read_project, project)
     end
+
+    private
+
+    def can_access_git?
+      user && user.can?(:access_git)
+    end
   end
 end
diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb
new file mode 100644
index 00000000000..eb36ab9fded
--- /dev/null
+++ b/lib/gitlab/user_activities.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  class UserActivities
+    include Enumerable
+
+    KEY = 'users:activities'.freeze
+    BATCH_SIZE = 500
+
+    def self.record(key, time = Time.now)
+      Gitlab::Redis.with do |redis|
+        redis.hset(KEY, key, time.to_i)
+      end
+    end
+
+    def delete(*keys)
+      Gitlab::Redis.with do |redis|
+        redis.hdel(KEY, keys)
+      end
+    end
+
+    def each
+      cursor = 0
+      loop do
+        cursor, pairs =
+          Gitlab::Redis.with do |redis|
+            redis.hscan(KEY, cursor, count: BATCH_SIZE)
+          end
+
+        Hash[pairs].each { |pair| yield pair }
+
+        break if cursor == '0'
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index e59ead5d76c..fa182c4deda 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -13,5 +13,21 @@ module Gitlab
     def force_utf8(str)
       str.force_encoding(Encoding::UTF_8)
     end
+
+    def to_boolean(value)
+      return value if [true, false].include?(value)
+      return true if value =~ /^(true|t|yes|y|1|on)$/i
+      return false if value =~ /^(false|f|no|n|0|off)$/i
+
+      nil
+    end
+
+    def boolean_to_yes_no(bool)
+      if bool
+        'Yes'
+      else
+        'No'
+      end
+    end
   end
 end
diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb
new file mode 100644
index 00000000000..dbfe0941e4d
--- /dev/null
+++ b/lib/gitlab/view/presenter/base.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module View
+    module Presenter
+      CannotOverrideMethodError = Class.new(StandardError)
+
+      module Base
+        extend ActiveSupport::Concern
+
+        include Gitlab::Routing
+        include Gitlab::Allowable
+
+        attr_reader :subject
+
+        def can?(user, action, overriden_subject = nil)
+          super(user, action, overriden_subject || subject)
+        end
+
+        class_methods do
+          def presenter?
+            true
+          end
+
+          def presents(name)
+            define_method(name) { subject }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb
new file mode 100644
index 00000000000..387ff0f5d43
--- /dev/null
+++ b/lib/gitlab/view/presenter/delegated.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module View
+    module Presenter
+      class Delegated < SimpleDelegator
+        include Gitlab::View::Presenter::Base
+
+        def initialize(subject, **attributes)
+          @subject = subject
+
+          attributes.each do |key, value|
+            if subject.respond_to?(key)
+              raise CannotOverrideMethodError.new("#{subject} already respond to #{key}!")
+            end
+
+            define_singleton_method(key) { value }
+          end
+
+          super(subject)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/view/presenter/factory.rb b/lib/gitlab/view/presenter/factory.rb
new file mode 100644
index 00000000000..d172d61e2c9
--- /dev/null
+++ b/lib/gitlab/view/presenter/factory.rb
@@ -0,0 +1,24 @@
+module Gitlab
+  module View
+    module Presenter
+      class Factory
+        def initialize(subject, **attributes)
+          @subject = subject
+          @attributes = attributes
+        end
+
+        def fabricate!
+          presenter_class.new(subject, attributes)
+        end
+
+        private
+
+        attr_reader :subject, :attributes
+
+        def presenter_class
+          "#{subject.class.name}Presenter".constantize
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/view/presenter/simple.rb b/lib/gitlab/view/presenter/simple.rb
new file mode 100644
index 00000000000..b7653a0f3cc
--- /dev/null
+++ b/lib/gitlab/view/presenter/simple.rb
@@ -0,0 +1,17 @@
+module Gitlab
+  module View
+    module Presenter
+      class Simple
+        include Gitlab::View::Presenter::Base
+
+        def initialize(subject, **attributes)
+          @subject = subject
+
+          attributes.each do |key, value|
+            define_singleton_method(key) { value }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb
index 9462f3368e6..48f3d950779 100644
--- a/lib/gitlab/visibility_level.rb
+++ b/lib/gitlab/visibility_level.rb
@@ -11,8 +11,11 @@ module Gitlab
     included do
       scope :public_only,               -> { where(visibility_level: PUBLIC) }
       scope :public_and_internal_only,  -> { where(visibility_level: [PUBLIC, INTERNAL] ) }
+      scope :non_public_only,           -> { where.not(visibility_level: PUBLIC) }
 
-      scope :public_to_user, -> (user) { user && !user.external ? public_and_internal_only : public_only }
+      scope :public_to_user, -> (user = nil) do
+        where(visibility_level: VisibilityLevel.levels_for_user(user))
+      end
     end
 
     PRIVATE  = 0 unless const_defined?(:PRIVATE)
@@ -20,15 +23,37 @@ module Gitlab
     PUBLIC   = 20 unless const_defined?(:PUBLIC)
 
     class << self
-      def values
-        options.values
+      delegate :values, to: :options
+
+      def levels_for_user(user = nil)
+        return [PUBLIC] unless user
+
+        if user.full_private_access?
+          [PRIVATE, INTERNAL, PUBLIC]
+        elsif user.external?
+          [PUBLIC]
+        else
+          [INTERNAL, PUBLIC]
+        end
+      end
+
+      def string_values
+        string_options.keys
       end
 
       def options
         {
-          'Private'  => PRIVATE,
-          'Internal' => INTERNAL,
-          'Public'   => PUBLIC
+          N_('VisibilityLevel|Private')  => PRIVATE,
+          N_('VisibilityLevel|Internal') => INTERNAL,
+          N_('VisibilityLevel|Public')   => PUBLIC
+        }
+      end
+
+      def string_options
+        {
+          'private'  => PRIVATE,
+          'internal' => INTERNAL,
+          'public'   => PUBLIC
         }
       end
 
@@ -40,7 +65,7 @@ module Gitlab
       end
 
       def allowed_for?(user, level)
-        user.is_admin? || allowed_level?(level.to_i)
+        user.admin? || allowed_level?(level.to_i)
       end
 
       # Return true if the specified level is allowed for the current user.
@@ -60,7 +85,7 @@ module Gitlab
       end
 
       def valid_level?(level)
-        options.has_value?(level)
+        options.value?(level)
       end
 
       def level_name(level)
@@ -71,18 +96,39 @@ module Gitlab
 
         level_name
       end
+
+      def level_value(level)
+        return level.to_i if level.to_i.to_s == level.to_s && string_options.key(level.to_i)
+        string_options[level] || PRIVATE
+      end
+
+      def string_level(level)
+        string_options.key(level)
+      end
     end
 
     def private?
-      visibility_level_field == PRIVATE
+      visibility_level_value == PRIVATE
     end
 
     def internal?
-      visibility_level_field == INTERNAL
+      visibility_level_value == INTERNAL
     end
 
     def public?
-      visibility_level_field == PUBLIC
+      visibility_level_value == PUBLIC
+    end
+
+    def visibility_level_value
+      self[visibility_level_field]
+    end
+
+    def visibility
+      Gitlab::VisibilityLevel.string_level(visibility_level_value)
+    end
+
+    def visibility=(level)
+      self[visibility_level_field] = Gitlab::VisibilityLevel.level_value(level)
     end
   end
 end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 5d33f98e89e..f96ee69096d 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -1,31 +1,61 @@
 require 'base64'
 require 'json'
 require 'securerandom'
+require 'uri'
 
 module Gitlab
   class Workhorse
-    SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'
-    VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'
-    INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'
-    INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'
+    SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'.freeze
+    VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'.freeze
+    INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'.freeze
+    INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'.freeze
+    NOTIFICATION_CHANNEL = 'workhorse:notifications'.freeze
 
     # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32
     # bytes https://tools.ietf.org/html/rfc4868#section-2.6
     SECRET_LENGTH = 32
 
     class << self
-      def git_http_ok(repository, user)
-        {
+      def git_http_ok(repository, is_wiki, user, action)
+        project = repository.project
+        repo_path = repository.path_to_repo
+        params = {
           GL_ID: Gitlab::GlId.gl_id(user),
-          RepoPath: repository.path_to_repo,
+          GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki),
+          RepoPath: repo_path
         }
+
+        if Gitlab.config.gitaly.enabled
+          server = {
+            address: Gitlab::GitalyClient.address(project.repository_storage),
+            token: Gitlab::GitalyClient.token(project.repository_storage)
+          }
+          params[:Repository] = repository.gitaly_repository.to_h
+
+          feature_enabled = case action.to_s
+                            when 'git_receive_pack'
+                              Gitlab::GitalyClient.feature_enabled?(:post_receive_pack)
+                            when 'git_upload_pack'
+                              Gitlab::GitalyClient.feature_enabled?(:post_upload_pack)
+                            when 'info_refs'
+                              true
+                            else
+                              raise "Unsupported action: #{action}"
+                            end
+          if feature_enabled
+            params[:GitalyAddress] = server[:address] # This field will be deprecated
+            params[:GitalyServer] = server
+          end
+        end
+
+        params
       end
 
       def lfs_upload_ok(oid, size)
         {
           StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload",
           LfsOid: oid,
-          LfsSize: size,
+          LfsSize: size
         }
       end
 
@@ -36,7 +66,7 @@ module Gitlab
       def send_git_blob(repository, blob)
         params = {
           'RepoPath' => repository.path_to_repo,
-          'BlobId' => blob.id,
+          'BlobId' => blob.id
         }
 
         [
@@ -95,6 +125,20 @@ module Gitlab
         ]
       end
 
+      def terminal_websocket(terminal)
+        details = {
+          'Terminal' => {
+            'Subprotocols' => terminal[:subprotocols],
+            'Url' => terminal[:url],
+            'Header' => terminal[:headers],
+            'MaxSessionTime' => terminal[:max_session_time]
+          }
+        }
+        details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.key?(:ca_pem)
+
+        details
+      end
+
       def version
         path = Rails.root.join(VERSION_FILE)
         path.readable? ? path.read.chomp : 'unknown'
@@ -111,22 +155,38 @@ module Gitlab
       def write_secret
         bytes = SecureRandom.random_bytes(SECRET_LENGTH)
         File.open(secret_path, 'w:BINARY', 0600) do |f|
-          f.chmod(0600)
+          f.chmod(0600) # If the file already existed, the '0600' passed to 'open' above was a no-op.
           f.write(Base64.strict_encode64(bytes))
         end
       end
 
       def verify_api_request!(request_headers)
+        decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER])
+      end
+
+      def decode_jwt(encoded_message)
         JWT.decode(
-          request_headers[INTERNAL_API_REQUEST_HEADER],
+          encoded_message,
           secret,
           true,
-          { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' },
+          { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }
         )
       end
 
       def secret_path
-        Rails.root.join('.gitlab_workhorse_secret')
+        Gitlab.config.workhorse.secret_file
+      end
+
+      def set_key_and_notify(key, value, expire: nil, overwrite: true)
+        Gitlab::Redis.with do |redis|
+          result = redis.set(key, value, ex: expire, nx: !overwrite)
+          if result
+            redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}")
+            value
+          else
+            redis.get(key)
+          end
+        end
       end
 
       protected
diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb
new file mode 100644
index 00000000000..3d60618006c
--- /dev/null
+++ b/lib/mattermost/client.rb
@@ -0,0 +1,51 @@
+module Mattermost
+  ClientError = Class.new(Mattermost::Error)
+
+  class Client
+    attr_reader :user
+
+    def initialize(user)
+      @user = user
+    end
+
+    def with_session(&blk)
+      Mattermost::Session.new(user).with_session(&blk)
+    end
+
+    private
+
+    # Should be used in a session manually
+    def get(session, path, options = {})
+      json_response session.get(path, options)
+    end
+
+    # Should be used in a session manually
+    def post(session, path, options = {})
+      json_response session.post(path, options)
+    end
+
+    def session_get(path, options = {})
+      with_session do |session|
+        get(session, path, options)
+      end
+    end
+
+    def session_post(path, options = {})
+      with_session do |session|
+        post(session, path, options)
+      end
+    end
+
+    def json_response(response)
+      json_response = JSON.parse(response.body)
+
+      unless response.success?
+        raise Mattermost::ClientError.new(json_response['message'] || 'Undefined error')
+      end
+
+      json_response
+    rescue JSON::JSONError
+      raise Mattermost::ClientError.new('Cannot parse response')
+    end
+  end
+end
diff --git a/lib/mattermost/command.rb b/lib/mattermost/command.rb
new file mode 100644
index 00000000000..33e450d7f0a
--- /dev/null
+++ b/lib/mattermost/command.rb
@@ -0,0 +1,10 @@
+module Mattermost
+  class Command < Client
+    def create(params)
+      response = session_post("/api/v3/teams/#{params[:team_id]}/commands/create",
+        body: params.to_json)
+
+      response['token']
+    end
+  end
+end
diff --git a/lib/mattermost/error.rb b/lib/mattermost/error.rb
new file mode 100644
index 00000000000..dee6deb7974
--- /dev/null
+++ b/lib/mattermost/error.rb
@@ -0,0 +1,3 @@
+module Mattermost
+  Error = Class.new(StandardError)
+end
diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb
new file mode 100644
index 00000000000..688a79c0441
--- /dev/null
+++ b/lib/mattermost/session.rb
@@ -0,0 +1,160 @@
+module Mattermost
+  class NoSessionError < Mattermost::Error
+    def message
+      'No session could be set up, is Mattermost configured with Single Sign On?'
+    end
+  end
+
+  ConnectionError = Class.new(Mattermost::Error)
+
+  # This class' prime objective is to obtain a session token on a Mattermost
+  # instance with SSO configured where this GitLab instance is the provider.
+  #
+  # The process depends on OAuth, but skips a step in the authentication cycle.
+  # For example, usually a user would click the 'login in GitLab' button on
+  # Mattermost, which would yield a 302 status code and redirects you to GitLab
+  # to approve the use of your account on Mattermost. Which would trigger a
+  # callback so Mattermost knows this request is approved and gets the required
+  # data to create the user account etc.
+  #
+  # This class however skips the button click, and also the approval phase to
+  # speed up the process and keep it without manual action and get a session
+  # going.
+  class Session
+    include Doorkeeper::Helpers::Controller
+    include HTTParty
+
+    LEASE_TIMEOUT = 60
+
+    base_uri Settings.mattermost.host
+
+    attr_accessor :current_resource_owner, :token
+
+    def initialize(current_user)
+      @current_resource_owner = current_user
+    end
+
+    def with_session
+      with_lease do
+        raise Mattermost::NoSessionError unless create
+
+        begin
+          yield self
+        rescue Errno::ECONNREFUSED
+          raise Mattermost::NoSessionError
+        ensure
+          destroy
+        end
+      end
+    end
+
+    # Next methods are needed for Doorkeeper
+    def pre_auth
+      @pre_auth ||= Doorkeeper::OAuth::PreAuthorization.new(
+        Doorkeeper.configuration, server.client_via_uid, params)
+    end
+
+    def authorization
+      @authorization ||= strategy.request
+    end
+
+    def strategy
+      @strategy ||= server.authorization_request(pre_auth.response_type)
+    end
+
+    def request
+      @request ||= OpenStruct.new(parameters: params)
+    end
+
+    def params
+      Rack::Utils.parse_query(oauth_uri.query).symbolize_keys
+    end
+
+    def get(path, options = {})
+      handle_exceptions do
+        self.class.get(path, options.merge(headers: @headers))
+      end
+    end
+
+    def post(path, options = {})
+      handle_exceptions do
+        self.class.post(path, options.merge(headers: @headers))
+      end
+    end
+
+    private
+
+    def create
+      return unless oauth_uri
+      return unless token_uri
+
+      @token = request_token
+      @headers = {
+        Authorization: "Bearer #{@token}"
+      }
+
+      @token
+    end
+
+    def destroy
+      post('/api/v3/users/logout')
+    end
+
+    def oauth_uri
+      return @oauth_uri if defined?(@oauth_uri)
+
+      @oauth_uri = nil
+
+      response = get("/api/v3/oauth/gitlab/login", follow_redirects: false)
+      return unless 300 <= response.code && response.code < 400
+
+      redirect_uri = response.headers['location']
+      return unless redirect_uri
+
+      @oauth_uri = URI.parse(redirect_uri)
+    end
+
+    def token_uri
+      @token_uri ||=
+        if oauth_uri
+          authorization.authorize.redirect_uri if pre_auth.authorizable?
+        end
+    end
+
+    def request_token
+      response = get(token_uri, follow_redirects: false)
+
+      if 200 <= response.code && response.code < 400
+        response.headers['token']
+      end
+    end
+
+    def with_lease
+      lease_uuid = lease_try_obtain
+      raise NoSessionError unless lease_uuid
+
+      begin
+        yield
+      ensure
+        Gitlab::ExclusiveLease.cancel(lease_key, lease_uuid)
+      end
+    end
+
+    def lease_key
+      "mattermost:session"
+    end
+
+    def lease_try_obtain
+      lease = ::Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT)
+      lease.try_obtain
+    end
+
+    def handle_exceptions
+      yield
+    rescue HTTParty::Error => e
+      raise Mattermost::ConnectionError.new(e.message)
+    rescue Errno::ECONNREFUSED => e
+      raise Mattermost::ConnectionError.new(e.message)
+    end
+  end
+end
diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb
new file mode 100644
index 00000000000..2cdbbdece16
--- /dev/null
+++ b/lib/mattermost/team.rb
@@ -0,0 +1,18 @@
+module Mattermost
+  class Team < Client
+    # Returns **all** teams for an admin
+    def all
+      session_get('/api/v3/teams/all').values
+    end
+
+    # Creates a team on the linked Mattermost instance, the team admin will be the
+    # `current_user` passed to the Mattermost::Client instance
+    def create(name:, display_name:, type:)
+      session_post('/api/v3/teams/create', body: {
+        name: name,
+        display_name: display_name,
+        type: type
+      }.to_json)
+    end
+  end
+end
diff --git a/lib/microsoft_teams/activity.rb b/lib/microsoft_teams/activity.rb
new file mode 100644
index 00000000000..d2c420efdaf
--- /dev/null
+++ b/lib/microsoft_teams/activity.rb
@@ -0,0 +1,19 @@
+module MicrosoftTeams
+  class Activity
+    def initialize(title:, subtitle:, text:, image:)
+      @title = title
+      @subtitle = subtitle
+      @text = text
+      @image = image
+    end
+
+    def prepare
+      {
+        'activityTitle' => @title,
+        'activitySubtitle' => @subtitle,
+        'activityText' => @text,
+        'activityImage' => @image
+      }
+    end
+  end
+end
diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb
new file mode 100644
index 00000000000..3bef68a1bcb
--- /dev/null
+++ b/lib/microsoft_teams/notifier.rb
@@ -0,0 +1,46 @@
+module MicrosoftTeams
+  class Notifier
+    def initialize(webhook)
+      @webhook = webhook
+      @header = { 'Content-type' => 'application/json' }
+    end
+
+    def ping(options = {})
+      result = false
+
+      begin
+        response = HTTParty.post(
+          @webhook.to_str,
+          headers: @header,
+          body: body(options)
+        )
+
+        result = true if response
+      rescue HTTParty::Error, StandardError => error
+        Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}")
+      end
+
+      result
+    end
+
+    private
+
+    def body(options = {})
+      result = { 'sections' => [] }
+
+      result['title'] = options[:title]
+      result['summary'] = options[:pretext]
+      result['sections'] << MicrosoftTeams::Activity.new(options[:activity]).prepare
+
+      attachments = options[:attachments]
+      unless attachments.blank?
+        result['sections'] << {
+          'title' => 'Details',
+          'facts' => [{ 'name' => 'Attachments', 'value' => attachments }]
+        }
+      end
+
+      result.to_json
+    end
+  end
+end
diff --git a/lib/omni_auth/strategies/bitbucket.rb b/lib/omni_auth/strategies/bitbucket.rb
new file mode 100644
index 00000000000..5a7d67c2390
--- /dev/null
+++ b/lib/omni_auth/strategies/bitbucket.rb
@@ -0,0 +1,41 @@
+require 'omniauth-oauth2'
+
+module OmniAuth
+  module Strategies
+    class Bitbucket < OmniAuth::Strategies::OAuth2
+      option :name, 'bitbucket'
+
+      option :client_options, {
+        site: 'https://bitbucket.org',
+        authorize_url: 'https://bitbucket.org/site/oauth2/authorize',
+        token_url: 'https://bitbucket.org/site/oauth2/access_token'
+      }
+
+      uid do
+        raw_info['username']
+      end
+
+      info do
+        {
+          name: raw_info['display_name'],
+          avatar: raw_info['links']['avatar']['href'],
+          email: primary_email
+        }
+      end
+
+      def raw_info
+        @raw_info ||= access_token.get('api/2.0/user').parsed
+      end
+
+      def primary_email
+        primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] }
+        primary && primary['email'] || nil
+      end
+
+      def emails
+        email_response = access_token.get('api/2.0/user/emails').parsed
+        @emails ||= email_response && email_response['values'] || nil
+      end
+    end
+  end
+end
diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb
new file mode 100644
index 00000000000..99f9c2c9b04
--- /dev/null
+++ b/lib/peek/rblineprof/custom_controller_helpers.rb
@@ -0,0 +1,96 @@
+module Peek
+  module Rblineprof
+    module CustomControllerHelpers
+      extend ActiveSupport::Concern
+
+      # This will become useless once https://github.com/peek/peek-rblineprof/pull/5
+      # is merged
+      def pygmentize(file_name, code, lexer = nil)
+        if lexer.present?
+          Gitlab::Highlight.highlight(file_name, code)
+        else
+          "<pre>#{Rack::Utils.escape_html(code)}</pre>"
+        end
+      end
+
+      # rubocop:disable all
+      def inject_rblineprof
+        ret = nil
+        profile = lineprof(rblineprof_profiler_regex) do
+          ret = yield
+        end
+
+        if response.content_type =~ %r|text/html|
+          sort = params[:lineprofiler_sort]
+          mode = params[:lineprofiler_mode] || 'cpu'
+          min  = (params[:lineprofiler_min] || 5).to_i * 1000
+          summary = params[:lineprofiler_summary]
+
+          # Sort each file by the longest calculated time
+          per_file = profile.map do |file, lines|
+            total, child, excl, total_cpu, child_cpu, excl_cpu = lines[0]
+
+            wall = summary == 'exclusive' ? excl : total
+            cpu  = summary == 'exclusive' ? excl_cpu : total_cpu
+            idle = summary == 'exclusive' ? (excl - excl_cpu) : (total - total_cpu)
+
+            [
+              file, lines,
+              wall, cpu, idle,
+              sort == 'idle' ? idle : sort == 'cpu' ? cpu : wall
+            ]
+          end.sort_by{ |a,b,c,d,e,f| -f }
+
+          output = ''
+          per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort|
+
+            output << "<div class='peek-rblineprof-file'><div class='heading'>"
+
+            show_src = file_sort > min
+            tmpl = show_src ? "<a href='#' class='js-lineprof-file'>%s</a>" : "%s"
+
+            if mode == 'cpu'
+              output << sprintf("<span class='duration'>% 8.1fms + % 8.1fms</span> #{tmpl}", file_cpu / 1000.0, file_idle / 1000.0, file_name.sub(Rails.root.to_s + '/', ''))
+            else
+              output << sprintf("<span class='duration'>% 8.1fms</span> #{tmpl}", file_wall/1000.0, file_name.sub(Rails.root.to_s + '/', ''))
+            end
+
+            output << "</div>" # .heading
+
+            next unless show_src
+
+            output << "<div class='data'>"
+            code = []
+            times = []
+            File.readlines(file_name).each_with_index do |line, i|
+              code << line
+              wall, cpu, calls = lines[i + 1]
+
+              if calls && calls > 0
+                if mode == 'cpu'
+                  idle = wall - cpu
+                  times << sprintf("% 8.1fms + % 8.1fms (% 5d)", cpu / 1000.0, idle / 1000.0, calls)
+                else
+                  times << sprintf("% 8.1fms (% 5d)", wall / 1000.0, calls)
+                end
+              else
+                times << ' '
+              end
+            end
+            output << "<pre class='duration'>#{times.join("\n")}</pre>"
+            # The following line was changed from
+            # https://github.com/peek/peek-rblineprof/blob/8d3b7a283a27de2f40abda45974516693d882258/lib/peek/rblineprof/controller_helpers.rb#L125
+            # This will become useless once https://github.com/peek/peek-rblineprof/pull/16
+            # is merged and is implemented.
+            output << "<pre class='code highlight white'>#{pygmentize(file_name, code.join, 'ruby')}</pre>"
+            output << "</div></div>" # .data then .peek-rblineprof-file
+          end
+
+          response.body += "<div class='peek-rblineprof-modal' id='line-profile'>#{output}</div>".html_safe
+        end
+
+        ret
+      end
+    end
+  end
+end
diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb
index 4edfd015074..be0d97370d0 100644
--- a/lib/rouge/formatters/html_gitlab.rb
+++ b/lib/rouge/formatters/html_gitlab.rb
@@ -5,10 +5,10 @@ module Rouge
 
       # Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance.
       #
-      # [+linenostart+]     The line number for the first line (default: 1).
-      def initialize(linenostart: 1)
-        @linenostart = linenostart
-        @line_number = linenostart
+      # [+tag+]     The tag (language) of the lexer used to generate the formatted tokens
+      def initialize(tag: nil)
+        @line_number = 1
+        @tag = tag
       end
 
       def stream(tokens, &b)
@@ -17,7 +17,7 @@ module Rouge
           yield "\n" unless is_first
           is_first = false
 
-          yield %(<span id="LC#{@line_number}" class="line">)
+          yield %(<span id="LC#{@line_number}" class="line" lang="#{@tag}">)
           line.each { |token, value| yield span(token, value.chomp) }
           yield %(</span>)
 
diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb
new file mode 100644
index 00000000000..939b23a3421
--- /dev/null
+++ b/lib/rouge/lexers/math.rb
@@ -0,0 +1,9 @@
+module Rouge
+  module Lexers
+    class Math < PlainText
+      title "A passthrough lexer used for LaTeX input"
+      desc "PLEASE REFACTOR - this should be handled by SyntaxHighlightFilter"
+      tag 'math'
+    end
+  end
+end
diff --git a/lib/rouge/lexers/plantuml.rb b/lib/rouge/lexers/plantuml.rb
new file mode 100644
index 00000000000..63c461764fc
--- /dev/null
+++ b/lib/rouge/lexers/plantuml.rb
@@ -0,0 +1,9 @@
+module Rouge
+  module Lexers
+    class Plantuml < PlainText
+      title "A passthrough lexer used for PlantUML input"
+      desc "PLEASE REFACTOR - this should be handled by SyntaxHighlightFilter"
+      tag 'plantuml'
+    end
+  end
+end
diff --git a/lib/support/deploy/deploy.sh b/lib/support/deploy/deploy.sh
index adea4c7a747..ab46c47d8f5 100755
--- a/lib/support/deploy/deploy.sh
+++ b/lib/support/deploy/deploy.sh
@@ -31,8 +31,8 @@ echo 'Deploy: Bundle and migrate'
 sudo -u git -H bundle --without aws development test mysql --deployment
 
 sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production
-sudo -u git -H bundle exec rake assets:clean RAILS_ENV=production
-sudo -u git -H bundle exec rake assets:precompile RAILS_ENV=production
+sudo -u git -H bundle exec rake gitlab:assets:clean RAILS_ENV=production
+sudo -u git -H bundle exec rake gitlab:assets:compile RAILS_ENV=production
 sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production
 
 # return stashed changes (if necessary)
diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab
index 31b00ff128a..c5f93336346 100755
--- a/lib/support/init.d/gitlab
+++ b/lib/support/init.d/gitlab
@@ -42,7 +42,16 @@ gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse 2> /dev/null && pwd)
 gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid"
 gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public"
 gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log"
+gitlab_pages_enabled=false
+gitlab_pages_dir=$(cd $app_root/../gitlab-pages 2> /dev/null && pwd)
+gitlab_pages_pid_path="$pid_path/gitlab-pages.pid"
+gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090"
+gitlab_pages_log="$app_root/log/gitlab-pages.log"
 shell_path="/bin/bash"
+gitaly_enabled=true
+gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd)
+gitaly_pid_path="$pid_path/gitaly.pid"
+gitaly_log="$app_root/log/gitaly.log"
 
 # Read configuration variable file if it is present
 test -f /etc/default/gitlab && . /etc/default/gitlab
@@ -89,13 +98,27 @@ check_pids(){
       mpid=0
     fi
   fi
+  if [ "$gitlab_pages_enabled" = true ]; then
+    if [ -f "$gitlab_pages_pid_path" ]; then
+      gppid=$(cat "$gitlab_pages_pid_path")
+    else
+      gppid=0
+    fi
+  fi
+  if [ "$gitaly_enabled" = true ]; then
+    if [ -f "$gitaly_pid_path" ]; then
+      gapid=$(cat "$gitaly_pid_path")
+    else
+      gapid=0
+    fi
+  fi
 }
 
 ## Called when we have started the two processes and are waiting for their pid files.
 wait_for_pids(){
   # We are sleeping a bit here mostly because sidekiq is slow at writing its pid
   i=0;
-  while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do
+  while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; } || { [ "$gitlab_pages_enabled" = true ] && [ ! -f $gitlab_pages_pid_path ]; } || { [ "$gitaly_enabled" = true ] && [ ! -f $gitaly_pid_path ]; }; do
     sleep 0.1;
     i=$((i+1))
     if [ $((i%10)) = 0 ]; then
@@ -144,7 +167,23 @@ check_status(){
       mail_room_status="-1"
     fi
   fi
-  if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then
+  if [ "$gitlab_pages_enabled" = true ]; then
+    if [ $gppid -ne 0 ]; then
+      kill -0 "$gppid" 2>/dev/null
+      gitlab_pages_status="$?"
+    else
+      gitlab_pages_status="-1"
+    fi
+  fi
+  if [ "$gitaly_enabled" = true ]; then
+    if [ $gapid -ne 0 ]; then
+      kill -0 "$gapid" 2>/dev/null
+      gitaly_status="$?"
+    else
+      gitaly_status="-1"
+    fi
+  fi
+  if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; } && { [ "$gitlab_pages_enabled" != true ] || [ $gitlab_pages_status = 0 ]; } && { [ "$gitaly_enabled" != true ] || [ $gitaly_status = 0 ]; }; then
     gitlab_status=0
   else
     # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html
@@ -186,12 +225,26 @@ check_stale_pids(){
       exit 1
     fi
   fi
+  if [ "$gitlab_pages_enabled" = true ] && [ "$gppid" != "0" ] && [ "$gitlab_pages_status" != "0" ]; then
+    echo "Removing stale GitLab Pages job dispatcher pid. This is most likely caused by GitLab Pages crashing the last time it ran."
+    if ! rm "$gitlab_pages_pid_path"; then
+      echo "Unable to remove stale pid, exiting"
+      exit 1
+    fi
+  fi
+  if [ "$gitaly_enabled" = true ] && [ "$gapid" != "0" ] && [ "$gitaly_status" != "0" ]; then
+    echo "Removing stale Gitaly pid. This is most likely caused by Gitaly crashing the last time it ran."
+    if ! rm "$gitaly_pid_path"; then
+      echo "Unable to remove stale pid, exiting"
+      exit 1
+    fi
+  fi
 }
 
 ## If no parts of the service is running, bail out.
 exit_if_not_running(){
   check_stale_pids
-  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then
+  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then
     echo "GitLab is not running."
     exit
   fi
@@ -213,6 +266,12 @@ start_gitlab() {
   if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then
     echo "Starting GitLab MailRoom"
   fi
+  if [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" != "0" ]; then
+    echo "Starting GitLab Pages"
+  fi
+  if [ "$gitaly_enabled" = true ] && [ "$gitaly_status" != "0" ]; then
+    echo "Starting Gitaly"
+  fi
 
   # Then check if the service is running. If it is: don't start again.
   if [ "$web_status" = "0" ]; then
@@ -252,6 +311,25 @@ start_gitlab() {
     fi
   fi
 
+  if [ "$gitlab_pages_enabled" = true ]; then
+    if [ "$gitlab_pages_status" = "0" ]; then
+      echo "The GitLab Pages is already running with pid $spid, not restarting"
+    else
+      $app_root/bin/daemon_with_pidfile $gitlab_pages_pid_path  \
+          $gitlab_pages_dir/gitlab-pages $gitlab_pages_options \
+        >> $gitlab_pages_log 2>&1 &
+    fi
+  fi
+
+  if [ "$gitaly_enabled" = true ]; then
+    if [ "$gitaly_status" = "0" ]; then
+      echo "Gitaly is already running with pid $gapid, not restarting"
+    else
+      $app_root/bin/daemon_with_pidfile $gitaly_pid_path \
+          $gitaly_dir/gitaly $gitaly_dir/config.toml >> $gitaly_log 2>&1 &
+    fi
+  fi
+
   # Wait for the pids to be planted
   wait_for_pids
   # Finally check the status to tell wether or not GitLab is running
@@ -278,13 +356,21 @@ stop_gitlab() {
     echo "Shutting down GitLab MailRoom"
     RAILS_ENV=$RAILS_ENV bin/mail_room stop
   fi
+  if [ "$gitlab_pages_status" = "0" ]; then
+    echo "Shutting down gitlab-pages"
+    kill -- $(cat $gitlab_pages_pid_path)
+  fi
+  if [ "$gitaly_status" = "0" ]; then
+    echo "Shutting down Gitaly"
+    kill -- $(cat $gitaly_pid_path)
+  fi
 
   # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script.
-  while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do
+  while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; do
     sleep 1
     check_status
     printf "."
-    if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then
+    if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then
       printf "\n"
       break
     fi
@@ -298,6 +384,8 @@ stop_gitlab() {
   if [ "$mail_room_enabled" = true ]; then
     rm "$mail_room_pid_path" 2>/dev/null
   fi
+  rm -f "$gitlab_pages_pid_path"
+  rm -f "$gitaly_pid_path"
 
   print_status
 }
@@ -305,7 +393,7 @@ stop_gitlab() {
 ## Prints the status of GitLab and its components.
 print_status() {
   check_status
-  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then
+  if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then
     echo "GitLab is not running."
     return
   fi
@@ -331,7 +419,21 @@ print_status() {
         printf "The GitLab MailRoom email processor is \033[31mnot running\033[0m.\n"
     fi
   fi
-  if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; }; then
+  if [ "$gitlab_pages_enabled" = true ]; then
+    if [ "$gitlab_pages_status" = "0" ]; then
+        echo "The GitLab Pages with pid $mpid is running."
+    else
+        printf "The GitLab Pages is \033[31mnot running\033[0m.\n"
+    fi
+  fi
+  if [ "$gitaly_enabled" = true ]; then
+    if [ "$gitaly_status" = "0" ]; then
+        echo "Gitaly with pid $gapid is running."
+    else
+        printf "Gitaly is \033[31mnot running\033[0m.\n"
+    fi
+  fi
+  if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" = "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" = "0" ]; }; then
     printf "GitLab and all its components are \033[32mup and running\033[0m.\n"
   fi
 }
@@ -362,7 +464,7 @@ reload_gitlab(){
 ## Restarts Sidekiq and Unicorn.
 restart_gitlab(){
   check_status
-  if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then
+  if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; then
     stop_gitlab
   fi
   start_gitlab
diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example
index cc8617b72ca..295c79fccfc 100755..100644
--- a/lib/support/init.d/gitlab.default.example
+++ b/lib/support/init.d/gitlab.default.example
@@ -47,6 +47,30 @@ gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid"
 gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $socket_path/gitlab.socket -documentRoot $app_root/public"
 gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log"
 
+# The GitLab Pages Daemon needs either a separate IP address on which it will
+# listen or use different ports than 80 or 443 that will be forwarded to GitLab
+# Pages Daemon.
+#
+# To enable HTTP support for custom domains add the `-listen-http` directive
+# in `gitlab_pages_options` below.
+# The value of -listen-http must be set to `gitlab.yml > pages > external_http`
+# as well. For example:
+#
+#   -listen-http 1.1.1.1:80 -listen-http [2001::1]:80
+#
+# To enable HTTPS support for custom domains add the `-listen-https`,
+# `-root-cert` and `-root-key` directives in `gitlab_pages_options` below.
+# The value of -listen-https must be set to `gitlab.yml > pages > external_https`
+# as well. For example:
+#
+#   -listen-https 1.1.1.1:443 -listen-http [2001::1]:443 -root-cert /path/to/example.com.crt -root-key /path/to/example.com.key
+#
+# The -pages-domain must be specified the same as in `gitlab.yml > pages > host`.
+# Set `gitlab_pages_enabled=true` if you want to enable the Pages feature.
+gitlab_pages_enabled=false
+gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090"
+gitlab_pages_log="$app_root/log/gitlab-pages.log"
+
 # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled.
 # This is required for the Reply by email feature.
 # The default is "false"
@@ -60,3 +84,9 @@ mail_room_pid_path="$pid_path/mail_room.pid"
 # shell other than "bash"
 # The default is "/bin/bash"
 shell_path="/bin/bash"
+
+# This variable controls whether the init script starts/stops Gitaly
+gitaly_enabled=true
+gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd)
+gitaly_pid_path="$pid_path/gitaly.pid"
+gitaly_log="$app_root/log/gitaly.log"
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index d521de28e8a..f25e66d54c8 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -20,6 +20,11 @@ upstream gitlab-workhorse {
   server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
 }
 
+map $http_upgrade $connection_upgrade_gitlab {
+    default upgrade;
+    ''      close;
+}
+
 ## Normal HTTP host
 server {
   ## Either remove "default_server" from the listen line below,
@@ -33,6 +38,13 @@ server {
 
   ## See app/controllers/application_controller.rb for headers set
 
+  ## Real IP Module Config
+  ## http://nginx.org/en/docs/http/ngx_http_realip_module.html
+  real_ip_header X-Real-IP; ## X-Real-IP or X-Forwarded-For or proxy_protocol
+  real_ip_recursive off;    ## If you enable 'on'
+  ## If you have a trusted IP address, uncomment it and set it
+  # set_real_ip_from YOUR_TRUSTED_ADDRESS; ## Replace this with something like 192.168.1.0/24
+
   ## Individual nginx logs for this GitLab vhost
   access_log  /var/log/nginx/gitlab_access.log;
   error_log   /var/log/nginx/gitlab_error.log;
@@ -53,6 +65,8 @@ server {
     proxy_set_header    X-Real-IP           $remote_addr;
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
     proxy_set_header    X-Forwarded-Proto   $scheme;
+    proxy_set_header    Upgrade             $http_upgrade;
+    proxy_set_header    Connection          $connection_upgrade_gitlab;
 
     proxy_pass http://gitlab-workhorse;
   }
diff --git a/lib/support/nginx/gitlab-pages b/lib/support/nginx/gitlab-pages
new file mode 100644
index 00000000000..d9746c5c1aa
--- /dev/null
+++ b/lib/support/nginx/gitlab-pages
@@ -0,0 +1,28 @@
+## GitLab
+##
+
+## Pages serving host
+server {
+  listen 0.0.0.0:80;
+  listen [::]:80 ipv6only=on;
+
+  ## Replace this with something like pages.gitlab.com
+  server_name ~^.*\.YOUR_GITLAB_PAGES\.DOMAIN$;
+
+  ## Individual nginx logs for GitLab pages
+  access_log  /var/log/nginx/gitlab_pages_access.log;
+  error_log   /var/log/nginx/gitlab_pages_error.log;
+
+  location / {
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   $scheme;
+    # The same address as passed to GitLab Pages: `-listen-proxy`
+    proxy_pass          http://localhost:8090/;
+  }
+
+  # Define custom error pages
+  error_page 403 /403.html;
+  error_page 404 /404.html;
+}
diff --git a/lib/support/nginx/gitlab-pages-ssl b/lib/support/nginx/gitlab-pages-ssl
new file mode 100644
index 00000000000..a1ccf266835
--- /dev/null
+++ b/lib/support/nginx/gitlab-pages-ssl
@@ -0,0 +1,77 @@
+## GitLab
+##
+
+## Redirects all HTTP traffic to the HTTPS host
+server {
+  ## Either remove "default_server" from the listen line below,
+  ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab
+  ## to be served if you visit any address that your server responds to, eg.
+  ## the ip address of the server (http://x.x.x.x/)
+  listen 0.0.0.0:80;
+  listen [::]:80 ipv6only=on;
+
+  ## Replace this with something like pages.gitlab.com
+  server_name ~^.*\.YOUR_GITLAB_PAGES\.DOMAIN$;
+  server_tokens off; ## Don't show the nginx version number, a security best practice
+
+  return 301  https://$http_host$request_uri;
+
+  access_log  /var/log/nginx/gitlab_pages_access.log;
+  error_log   /var/log/nginx/gitlab_pages_access.log;
+}
+
+## Pages serving host
+server {
+  listen 0.0.0.0:443 ssl;
+  listen [::]:443 ipv6only=on ssl http2;
+
+  ## Replace this with something like pages.gitlab.com
+  server_name ~^.*\.YOUR_GITLAB_PAGES\.DOMAIN$;
+  server_tokens off; ## Don't show the nginx version number, a security best practice
+
+  ## Strong SSL Security
+  ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
+  ssl on;
+  ssl_certificate /etc/nginx/ssl/gitlab-pages.crt;
+  ssl_certificate_key /etc/nginx/ssl/gitlab-pages.key;
+
+  # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs
+  ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
+  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+  ssl_prefer_server_ciphers on;
+  ssl_session_cache shared:SSL:10m;
+  ssl_session_timeout 5m;
+
+  ## See app/controllers/application_controller.rb for headers set
+
+  ## [Optional] If your certficate has OCSP, enable OCSP stapling to reduce the overhead and latency of running SSL.
+  ## Replace with your ssl_trusted_certificate. For more info see:
+  ## - https://medium.com/devops-programming/4445f4862461
+  ## - https://www.ruby-forum.com/topic/4419319
+  ## - https://www.digitalocean.com/community/tutorials/how-to-configure-ocsp-stapling-on-apache-and-nginx
+  # ssl_stapling on;
+  # ssl_stapling_verify on;
+  # ssl_trusted_certificate /etc/nginx/ssl/stapling.trusted.crt;
+
+  ## [Optional] Generate a stronger DHE parameter:
+  ##   sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096
+  ##
+  # ssl_dhparam /etc/ssl/certs/dhparam.pem;
+
+  ## Individual nginx logs for GitLab pages
+  access_log  /var/log/nginx/gitlab_pages_access.log;
+  error_log   /var/log/nginx/gitlab_pages_error.log;
+
+  location / {
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   $scheme;
+    # The same address as passed to GitLab Pages: `-listen-proxy`
+    proxy_pass          http://localhost:8090/;
+  }
+
+  # Define custom error pages
+  error_page 403 /403.html;
+  error_page 404 /404.html;
+}
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index bf014b56cf6..2b40da18bab 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -24,6 +24,11 @@ upstream gitlab-workhorse {
   server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
 }
 
+map $http_upgrade $connection_upgrade_gitlab_ssl {
+    default upgrade;
+    ''      close;
+}
+
 ## Redirects all HTTP traffic to the HTTPS host
 server {
   ## Either remove "default_server" from the listen line below,
@@ -77,6 +82,16 @@ server {
   ##
   # ssl_dhparam /etc/ssl/certs/dhparam.pem;
 
+  ## [Optional] Enable HTTP Strict Transport Security
+  # add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+
+  ## Real IP Module Config
+  ## http://nginx.org/en/docs/http/ngx_http_realip_module.html
+  real_ip_header X-Real-IP; ## X-Real-IP or X-Forwarded-For or proxy_protocol
+  real_ip_recursive off;    ## If you enable 'on'
+  ## If you have a trusted IP address, uncomment it and set it
+  # set_real_ip_from YOUR_TRUSTED_ADDRESS; ## Replace this with something like 192.168.1.0/24
+
   ## Individual nginx logs for this GitLab vhost
   access_log  /var/log/nginx/gitlab_access.log;
   error_log   /var/log/nginx/gitlab_error.log;
@@ -98,6 +113,9 @@ server {
     proxy_set_header    X-Forwarded-Ssl     on;
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
     proxy_set_header    X-Forwarded-Proto   $scheme;
+    proxy_set_header    Upgrade             $http_upgrade;
+    proxy_set_header    Connection          $connection_upgrade_gitlab_ssl;
+
     proxy_pass http://gitlab-workhorse;
   }
 
diff --git a/lib/system_check.rb b/lib/system_check.rb
new file mode 100644
index 00000000000..466c39904fa
--- /dev/null
+++ b/lib/system_check.rb
@@ -0,0 +1,21 @@
+# Library to perform System Checks
+#
+# Every Check is implemented as its own class inherited from SystemCheck::BaseCheck
+# Execution coordination and boilerplate output is done by the SystemCheck::SimpleExecutor
+#
+# This structure decouples checks from Rake tasks and facilitates unit-testing
+module SystemCheck
+  # Executes a bunch of checks for specified component
+  #
+  # @param [String] component name of the component relative to the checks being executed
+  # @param [Array<BaseCheck>] checks classes of corresponding checks to be executed in the same order
+  def self.run(component, checks = [])
+    executor = SimpleExecutor.new(component)
+
+    checks.each do |check|
+      executor << check
+    end
+
+    executor.execute
+  end
+end
diff --git a/lib/system_check/app/active_users_check.rb b/lib/system_check/app/active_users_check.rb
new file mode 100644
index 00000000000..1d72c8d6903
--- /dev/null
+++ b/lib/system_check/app/active_users_check.rb
@@ -0,0 +1,17 @@
+module SystemCheck
+  module App
+    class ActiveUsersCheck < SystemCheck::BaseCheck
+      set_name 'Active users:'
+
+      def multi_check
+        active_users = User.active.count
+
+        if active_users > 0
+          $stdout.puts active_users.to_s.color(:green)
+        else
+          $stdout.puts active_users.to_s.color(:red)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/database_config_exists_check.rb b/lib/system_check/app/database_config_exists_check.rb
new file mode 100644
index 00000000000..d1fae192350
--- /dev/null
+++ b/lib/system_check/app/database_config_exists_check.rb
@@ -0,0 +1,25 @@
+module SystemCheck
+  module App
+    class DatabaseConfigExistsCheck < SystemCheck::BaseCheck
+      set_name 'Database config exists?'
+
+      def check?
+        database_config_file = Rails.root.join('config', 'database.yml')
+
+        File.exist?(database_config_file)
+      end
+
+      def show_error
+        try_fixing_it(
+          'Copy config/database.yml.<your db> to config/database.yml',
+          'Check that the information in config/database.yml is correct'
+        )
+        for_more_information(
+          'doc/install/databases.md',
+          'http://guides.rubyonrails.org/getting_started.html#configuring-a-database'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb
new file mode 100644
index 00000000000..198867f7ac6
--- /dev/null
+++ b/lib/system_check/app/git_config_check.rb
@@ -0,0 +1,42 @@
+module SystemCheck
+  module App
+    class GitConfigCheck < SystemCheck::BaseCheck
+      OPTIONS = {
+        'core.autocrlf' => 'input'
+      }.freeze
+
+      set_name 'Git configured correctly?'
+
+      def check?
+        correct_options = OPTIONS.map do |name, value|
+          run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value
+        end
+
+        correct_options.all?
+      end
+
+      # Tries to configure git itself
+      #
+      # Returns true if all subcommands were successful (according to their exit code)
+      # Returns false if any or all subcommands failed.
+      def repair!
+        return false unless is_gitlab_user?
+
+        command_success = OPTIONS.map do |name, value|
+          system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value}))
+        end
+
+        command_success.all?
+      end
+
+      def show_error
+        try_fixing_it(
+          sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{OPTIONS['core.autocrlf']}\"")
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb
new file mode 100644
index 00000000000..c388682dfb4
--- /dev/null
+++ b/lib/system_check/app/git_version_check.rb
@@ -0,0 +1,29 @@
+module SystemCheck
+  module App
+    class GitVersionCheck < SystemCheck::BaseCheck
+      set_name -> { "Git version >= #{self.required_version} ?" }
+      set_check_pass -> { "yes (#{self.current_version})" }
+
+      def self.required_version
+        @required_version ||= Gitlab::VersionInfo.new(2, 7, 3)
+      end
+
+      def self.current_version
+        @current_version ||= Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version)))
+      end
+
+      def check?
+        self.class.current_version.valid? && self.class.required_version <= self.class.current_version
+      end
+
+      def show_error
+        $stdout.puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\""
+
+        try_fixing_it(
+          "Update your git to a version >= #{self.class.required_version} from #{self.class.current_version}"
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/gitlab_config_exists_check.rb b/lib/system_check/app/gitlab_config_exists_check.rb
new file mode 100644
index 00000000000..247aa0994e4
--- /dev/null
+++ b/lib/system_check/app/gitlab_config_exists_check.rb
@@ -0,0 +1,24 @@
+module SystemCheck
+  module App
+    class GitlabConfigExistsCheck < SystemCheck::BaseCheck
+      set_name 'GitLab config exists?'
+
+      def check?
+        gitlab_config_file = Rails.root.join('config', 'gitlab.yml')
+
+        File.exist?(gitlab_config_file)
+      end
+
+      def show_error
+        try_fixing_it(
+          'Copy config/gitlab.yml.example to config/gitlab.yml',
+          'Update config/gitlab.yml to match your setup'
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/gitlab_config_up_to_date_check.rb b/lib/system_check/app/gitlab_config_up_to_date_check.rb
new file mode 100644
index 00000000000..c609e48e133
--- /dev/null
+++ b/lib/system_check/app/gitlab_config_up_to_date_check.rb
@@ -0,0 +1,30 @@
+module SystemCheck
+  module App
+    class GitlabConfigUpToDateCheck < SystemCheck::BaseCheck
+      set_name 'GitLab config up to date?'
+      set_skip_reason "can't check because of previous errors"
+
+      def skip?
+        gitlab_config_file = Rails.root.join('config', 'gitlab.yml')
+        !File.exist?(gitlab_config_file)
+      end
+
+      def check?
+        # omniauth or ldap could have been deleted from the file
+        !Gitlab.config['git_host']
+      end
+
+      def show_error
+        try_fixing_it(
+          'Back-up your config/gitlab.yml',
+          'Copy config/gitlab.yml.example to config/gitlab.yml',
+          'Update config/gitlab.yml to match your setup'
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/init_script_exists_check.rb b/lib/system_check/app/init_script_exists_check.rb
new file mode 100644
index 00000000000..d246e058e86
--- /dev/null
+++ b/lib/system_check/app/init_script_exists_check.rb
@@ -0,0 +1,27 @@
+module SystemCheck
+  module App
+    class InitScriptExistsCheck < SystemCheck::BaseCheck
+      set_name 'Init script exists?'
+      set_skip_reason 'skipped (omnibus-gitlab has no init script)'
+
+      def skip?
+        omnibus_gitlab?
+      end
+
+      def check?
+        script_path = '/etc/init.d/gitlab'
+        File.exist?(script_path)
+      end
+
+      def show_error
+        try_fixing_it(
+          'Install the init script'
+        )
+        for_more_information(
+          see_installation_guide_section 'Install Init Script'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb
new file mode 100644
index 00000000000..015c7ed1731
--- /dev/null
+++ b/lib/system_check/app/init_script_up_to_date_check.rb
@@ -0,0 +1,43 @@
+module SystemCheck
+  module App
+    class InitScriptUpToDateCheck < SystemCheck::BaseCheck
+      SCRIPT_PATH = '/etc/init.d/gitlab'.freeze
+
+      set_name 'Init script up-to-date?'
+      set_skip_reason 'skipped (omnibus-gitlab has no init script)'
+
+      def skip?
+        omnibus_gitlab?
+      end
+
+      def multi_check
+        recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab')
+
+        unless File.exist?(SCRIPT_PATH)
+          $stdout.puts "can't check because of previous errors".color(:magenta)
+          return
+        end
+
+        recipe_content = File.read(recipe_path)
+        script_content = File.read(SCRIPT_PATH)
+
+        if recipe_content == script_content
+          $stdout.puts 'yes'.color(:green)
+        else
+          $stdout.puts 'no'.color(:red)
+          show_error
+        end
+      end
+
+      def show_error
+        try_fixing_it(
+          'Re-download the init script'
+        )
+        for_more_information(
+          see_installation_guide_section 'Install Init Script'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/log_writable_check.rb b/lib/system_check/app/log_writable_check.rb
new file mode 100644
index 00000000000..3e0c436d6ee
--- /dev/null
+++ b/lib/system_check/app/log_writable_check.rb
@@ -0,0 +1,28 @@
+module SystemCheck
+  module App
+    class LogWritableCheck < SystemCheck::BaseCheck
+      set_name 'Log directory writable?'
+
+      def check?
+        File.writable?(log_path)
+      end
+
+      def show_error
+        try_fixing_it(
+          "sudo chown -R gitlab #{log_path}",
+          "sudo chmod -R u+rwX #{log_path}"
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+
+      private
+
+      def log_path
+        Rails.root.join('log')
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/migrations_are_up_check.rb b/lib/system_check/app/migrations_are_up_check.rb
new file mode 100644
index 00000000000..5eedbacce77
--- /dev/null
+++ b/lib/system_check/app/migrations_are_up_check.rb
@@ -0,0 +1,20 @@
+module SystemCheck
+  module App
+    class MigrationsAreUpCheck < SystemCheck::BaseCheck
+      set_name 'All migrations up?'
+
+      def check?
+        migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status))
+
+        migration_status !~ /down\s+\d{14}/
+      end
+
+      def show_error
+        try_fixing_it(
+          sudo_gitlab('bundle exec rake db:migrate RAILS_ENV=production')
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/orphaned_group_members_check.rb b/lib/system_check/app/orphaned_group_members_check.rb
new file mode 100644
index 00000000000..2b46d36fe51
--- /dev/null
+++ b/lib/system_check/app/orphaned_group_members_check.rb
@@ -0,0 +1,20 @@
+module SystemCheck
+  module App
+    class OrphanedGroupMembersCheck < SystemCheck::BaseCheck
+      set_name 'Database contains orphaned GroupMembers?'
+      set_check_pass 'no'
+      set_check_fail 'yes'
+
+      def check?
+        !GroupMember.where('user_id not in (select id from users)').exists?
+      end
+
+      def show_error
+        try_fixing_it(
+          'You can delete the orphaned records using something along the lines of:',
+          sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'")
+        )
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/projects_have_namespace_check.rb b/lib/system_check/app/projects_have_namespace_check.rb
new file mode 100644
index 00000000000..a6ec9f7665c
--- /dev/null
+++ b/lib/system_check/app/projects_have_namespace_check.rb
@@ -0,0 +1,37 @@
+module SystemCheck
+  module App
+    class ProjectsHaveNamespaceCheck < SystemCheck::BaseCheck
+      set_name 'Projects have namespace:'
+      set_skip_reason "can't check, you have no projects"
+
+      def skip?
+        !Project.exists?
+      end
+
+      def multi_check
+        $stdout.puts ''
+
+        Project.find_each(batch_size: 100) do |project|
+          $stdout.print sanitized_message(project)
+
+          if project.namespace
+            $stdout.puts 'yes'.color(:green)
+          else
+            $stdout.puts 'no'.color(:red)
+            show_error
+          end
+        end
+      end
+
+      def show_error
+        try_fixing_it(
+          "Migrate global projects"
+        )
+        for_more_information(
+          "doc/update/5.4-to-6.0.md in section \"#global-projects\""
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb
new file mode 100644
index 00000000000..a0610e73576
--- /dev/null
+++ b/lib/system_check/app/redis_version_check.rb
@@ -0,0 +1,25 @@
+module SystemCheck
+  module App
+    class RedisVersionCheck < SystemCheck::BaseCheck
+      MIN_REDIS_VERSION = '2.8.0'.freeze
+      set_name "Redis version >= #{MIN_REDIS_VERSION}?"
+
+      def check?
+        redis_version = run_command(%w(redis-cli --version))
+        redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/)
+
+        redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(MIN_REDIS_VERSION))
+      end
+
+      def show_error
+        try_fixing_it(
+          "Update your redis server to a version >= #{MIN_REDIS_VERSION}"
+        )
+        for_more_information(
+          'gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb
new file mode 100644
index 00000000000..fd82f5f8a4a
--- /dev/null
+++ b/lib/system_check/app/ruby_version_check.rb
@@ -0,0 +1,27 @@
+module SystemCheck
+  module App
+    class RubyVersionCheck < SystemCheck::BaseCheck
+      set_name -> { "Ruby version >= #{self.required_version} ?" }
+      set_check_pass -> { "yes (#{self.current_version})" }
+
+      def self.required_version
+        @required_version ||= Gitlab::VersionInfo.new(2, 3, 3)
+      end
+
+      def self.current_version
+        @current_version ||= Gitlab::VersionInfo.parse(run_command(%w(ruby --version)))
+      end
+
+      def check?
+        self.class.current_version.valid? && self.class.required_version <= self.class.current_version
+      end
+
+      def show_error
+        try_fixing_it(
+          "Update your ruby to a version >= #{self.class.required_version} from #{self.class.current_version}"
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/tmp_writable_check.rb b/lib/system_check/app/tmp_writable_check.rb
new file mode 100644
index 00000000000..99a75e57abf
--- /dev/null
+++ b/lib/system_check/app/tmp_writable_check.rb
@@ -0,0 +1,28 @@
+module SystemCheck
+  module App
+    class TmpWritableCheck < SystemCheck::BaseCheck
+      set_name 'Tmp directory writable?'
+
+      def check?
+        File.writable?(tmp_path)
+      end
+
+      def show_error
+        try_fixing_it(
+          "sudo chown -R gitlab #{tmp_path}",
+          "sudo chmod -R u+rwX #{tmp_path}"
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+
+      private
+
+      def tmp_path
+        Rails.root.join('tmp')
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/uploads_directory_exists_check.rb b/lib/system_check/app/uploads_directory_exists_check.rb
new file mode 100644
index 00000000000..7026d0ba075
--- /dev/null
+++ b/lib/system_check/app/uploads_directory_exists_check.rb
@@ -0,0 +1,21 @@
+module SystemCheck
+  module App
+    class UploadsDirectoryExistsCheck < SystemCheck::BaseCheck
+      set_name 'Uploads directory exists?'
+
+      def check?
+        File.directory?(Rails.root.join('public/uploads'))
+      end
+
+      def show_error
+        try_fixing_it(
+          "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads"
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/uploads_path_permission_check.rb b/lib/system_check/app/uploads_path_permission_check.rb
new file mode 100644
index 00000000000..7df6c060254
--- /dev/null
+++ b/lib/system_check/app/uploads_path_permission_check.rb
@@ -0,0 +1,36 @@
+module SystemCheck
+  module App
+    class UploadsPathPermissionCheck < SystemCheck::BaseCheck
+      set_name 'Uploads directory has correct permissions?'
+      set_skip_reason 'skipped (no uploads folder found)'
+
+      def skip?
+        !File.directory?(rails_uploads_path)
+      end
+
+      def check?
+        File.stat(uploads_fullpath).mode == 040700
+      end
+
+      def show_error
+        try_fixing_it(
+          "sudo chmod 700 #{uploads_fullpath}"
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+
+      private
+
+      def rails_uploads_path
+        Rails.root.join('public/uploads')
+      end
+
+      def uploads_fullpath
+        File.realpath(rails_uploads_path)
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/uploads_path_tmp_permission_check.rb b/lib/system_check/app/uploads_path_tmp_permission_check.rb
new file mode 100644
index 00000000000..b276a81eac1
--- /dev/null
+++ b/lib/system_check/app/uploads_path_tmp_permission_check.rb
@@ -0,0 +1,40 @@
+module SystemCheck
+  module App
+    class UploadsPathTmpPermissionCheck < SystemCheck::BaseCheck
+      set_name 'Uploads directory tmp has correct permissions?'
+      set_skip_reason 'skipped (no tmp uploads folder yet)'
+
+      def skip?
+        !File.directory?(uploads_fullpath) || !Dir.exist?(upload_path_tmp)
+      end
+
+      def check?
+        # If tmp upload dir has incorrect permissions, assume others do as well
+        # Verify drwx------ permissions
+        File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp)
+      end
+
+      def show_error
+        try_fixing_it(
+          "sudo chown -R #{gitlab_user} #{uploads_fullpath}",
+          "sudo find #{uploads_fullpath} -type f -exec chmod 0644 {} \\;",
+          "sudo find #{uploads_fullpath} -type d -not -path #{uploads_fullpath} -exec chmod 0700 {} \\;"
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+
+      private
+
+      def upload_path_tmp
+        File.join(uploads_fullpath, 'tmp')
+      end
+
+      def uploads_fullpath
+        File.realpath(Rails.root.join('public/uploads'))
+      end
+    end
+  end
+end
diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb
new file mode 100644
index 00000000000..5dcb3f0886b
--- /dev/null
+++ b/lib/system_check/base_check.rb
@@ -0,0 +1,129 @@
+module SystemCheck
+  # Base class for Checks. You must inherit from here
+  # and implement the methods below when necessary
+  class BaseCheck
+    include ::SystemCheck::Helpers
+
+    # Define a custom term for when check passed
+    #
+    # @param [String] term used when check passed (default: 'yes')
+    def self.set_check_pass(term)
+      @check_pass = term
+    end
+
+    # Define a custom term for when check failed
+    #
+    # @param [String] term used when check failed (default: 'no')
+    def self.set_check_fail(term)
+      @check_fail = term
+    end
+
+    # Define the name of the SystemCheck that will be displayed during execution
+    #
+    # @param [String] name of the check
+    def self.set_name(name)
+      @name = name
+    end
+
+    # Define the reason why we skipped the SystemCheck
+    #
+    # This is only used if subclass implements `#skip?`
+    #
+    # @param [String] reason to be displayed
+    def self.set_skip_reason(reason)
+      @skip_reason = reason
+    end
+
+    # Term to be displayed when check passed
+    #
+    # @return [String] term when check passed ('yes' if not re-defined in a subclass)
+    def self.check_pass
+      call_or_return(@check_pass) || 'yes'
+    end
+
+    ## Term to be displayed when check failed
+    #
+    # @return [String] term when check failed ('no' if not re-defined in a subclass)
+    def self.check_fail
+      call_or_return(@check_fail) || 'no'
+    end
+
+    # Name of the SystemCheck defined by the subclass
+    #
+    # @return [String] the name
+    def self.display_name
+      call_or_return(@name) || self.name
+    end
+
+    # Skip reason defined by the subclass
+    #
+    # @return [String] the reason
+    def self.skip_reason
+      call_or_return(@skip_reason) || 'skipped'
+    end
+
+    # Does the check support automatically repair routine?
+    #
+    # @return [Boolean] whether check implemented `#repair!` method or not
+    def can_repair?
+      self.class.instance_methods(false).include?(:repair!)
+    end
+
+    def can_skip?
+      self.class.instance_methods(false).include?(:skip?)
+    end
+
+    def is_multi_check?
+      self.class.instance_methods(false).include?(:multi_check)
+    end
+
+    # Execute the check routine
+    #
+    # This is where you should implement the main logic that will return
+    # a boolean at the end
+    #
+    # You should not print any output to STDOUT here, use the specific methods instead
+    #
+    # @return [Boolean] whether check passed or failed
+    def check?
+      raise NotImplementedError
+    end
+
+    # Execute a custom check that cover multiple unities
+    #
+    # When using multi_check you have to provide the output yourself
+    def multi_check
+      raise NotImplementedError
+    end
+
+    # Prints troubleshooting instructions
+    #
+    # This is where you should print detailed information for any error found during #check?
+    #
+    # You may use helper methods to help format the output:
+    #
+    # @see #try_fixing_it
+    # @see #fix_and_rerun
+    # @see #for_more_infromation
+    def show_error
+      raise NotImplementedError
+    end
+
+    # When implemented by a subclass, will attempt to fix the issue automatically
+    def repair!
+      raise NotImplementedError
+    end
+
+    # When implemented by a subclass, will evaluate whether check should be skipped or not
+    #
+    # @return [Boolean] whether or not this check should be skipped
+    def skip?
+      raise NotImplementedError
+    end
+
+    def self.call_or_return(input)
+      input.respond_to?(:call) ? input.call : input
+    end
+    private_class_method :call_or_return
+  end
+end
diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb
new file mode 100644
index 00000000000..c42ae4fe4c4
--- /dev/null
+++ b/lib/system_check/helpers.rb
@@ -0,0 +1,75 @@
+require 'tasks/gitlab/task_helpers'
+
+module SystemCheck
+  module Helpers
+    include ::Gitlab::TaskHelpers
+
+    # Display a message telling to fix and rerun the checks
+    def fix_and_rerun
+      $stdout.puts '  Please fix the error above and rerun the checks.'.color(:red)
+    end
+
+    # Display a formatted list of references (documentation or links) where to find more information
+    #
+    # @param [Array<String>] sources one or more references (documentation or links)
+    def for_more_information(*sources)
+      $stdout.puts '  For more information see:'.color(:blue)
+      sources.each do |source|
+        $stdout.puts "  #{source}"
+      end
+    end
+
+    def see_installation_guide_section(section)
+      "doc/install/installation.md in section \"#{section}\""
+    end
+
+    # @deprecated This will no longer be used when all checks were executed using SystemCheck
+    def finished_checking(component)
+      $stdout.puts ''
+      $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}"
+      $stdout.puts ''
+    end
+
+    # @deprecated This will no longer be used when all checks were executed using SystemCheck
+    def start_checking(component)
+      $stdout.puts "Checking #{component.color(:yellow)} ..."
+      $stdout.puts ''
+    end
+
+    # Display a formatted list of instructions on how to fix the issue identified by the #check?
+    #
+    # @param [Array<String>] steps one or short sentences with help how to fix the issue
+    def try_fixing_it(*steps)
+      steps = steps.shift if steps.first.is_a?(Array)
+
+      $stdout.puts '  Try fixing it:'.color(:blue)
+      steps.each do |step|
+        $stdout.puts "  #{step}"
+      end
+    end
+
+    def sanitized_message(project)
+      if should_sanitize?
+        "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... "
+      else
+        "#{project.name_with_namespace.color(:yellow)} ... "
+      end
+    end
+
+    def should_sanitize?
+      if ENV['SANITIZE'] == 'true'
+        true
+      else
+        false
+      end
+    end
+
+    def omnibus_gitlab?
+      Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails'
+    end
+
+    def sudo_gitlab(command)
+      "sudo -u #{gitlab_user} -H #{command}"
+    end
+  end
+end
diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb
new file mode 100644
index 00000000000..e5986612908
--- /dev/null
+++ b/lib/system_check/simple_executor.rb
@@ -0,0 +1,101 @@
+module SystemCheck
+  # Simple Executor is current default executor for GitLab
+  # It is a simple port from display logic in the old check.rake
+  #
+  # There is no concurrency level and the output is progressively
+  # printed into the STDOUT
+  #
+  # @attr_reader [Array<BaseCheck>] checks classes of corresponding checks to be executed in the same order
+  # @attr_reader [String] component name of the component relative to the checks being executed
+  class SimpleExecutor
+    attr_reader :checks
+    attr_reader :component
+
+    # @param [String] component name of the component relative to the checks being executed
+    def initialize(component)
+      raise ArgumentError unless component.is_a? String
+
+      @component = component
+      @checks = Set.new
+    end
+
+    # Add a check to be executed
+    #
+    # @param [BaseCheck] check class
+    def <<(check)
+      raise ArgumentError unless check < BaseCheck
+      @checks << check
+    end
+
+    # Executes defined checks in the specified order and outputs confirmation or error information
+    def execute
+      start_checking(component)
+
+      @checks.each do |check|
+        run_check(check)
+      end
+
+      finished_checking(component)
+    end
+
+    # Executes a single check
+    #
+    # @param [SystemCheck::BaseCheck] check_klass
+    def run_check(check_klass)
+      $stdout.print "#{check_klass.display_name} ... "
+
+      check = check_klass.new
+
+      # When implements skip method, we run it first, and if true, skip the check
+      if check.can_skip? && check.skip?
+        $stdout.puts check_klass.skip_reason.color(:magenta)
+        return
+      end
+
+      # When implements a multi check, we don't control the output
+      if check.is_multi_check?
+        check.multi_check
+        return
+      end
+
+      if check.check?
+        $stdout.puts check_klass.check_pass.color(:green)
+      else
+        $stdout.puts check_klass.check_fail.color(:red)
+
+        if check.can_repair?
+          $stdout.print 'Trying to fix error automatically. ...'
+          if check.repair!
+            $stdout.puts 'Success'.color(:green)
+            return
+          else
+            $stdout.puts 'Failed'.color(:red)
+          end
+        end
+
+        check.show_error
+      end
+    rescue StandardError => e
+      $stdout.puts "Exception: #{e.message}".color(:red)
+    end
+
+    private
+
+    # Prints header content for the series of checks to be executed for this component
+    #
+    # @param [String] component name of the component relative to the checks being executed
+    def start_checking(component)
+      $stdout.puts "Checking #{component.color(:yellow)} ..."
+      $stdout.puts ''
+    end
+
+    # Prints footer content for the series of checks executed for this component
+    #
+    # @param [String] component name of the component relative to the checks being executed
+    def finished_checking(component)
+      $stdout.puts ''
+      $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}"
+      $stdout.puts ''
+    end
+  end
+end
diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake
index d5a402907d8..99b3168d9eb 100644
--- a/lib/tasks/brakeman.rake
+++ b/lib/tasks/brakeman.rake
@@ -2,7 +2,7 @@ desc 'Security check via brakeman'
 task :brakeman do
   # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge
   # requests are welcome!
-  if system(*%W(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z))
+  if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z))
     puts 'Security check succeed'
   else
     puts 'Security check failed'
diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index a95a3455a4a..125a3d560d6 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -1,7 +1,7 @@
 namespace :cache do
   namespace :clear do
     REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000
-    REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan
+    REDIS_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan
 
     desc "GitLab | Clear redis cache"
     task redis: :environment do
@@ -21,13 +21,8 @@ namespace :cache do
       end
     end
 
-    desc "GitLab | Clear database cache (in the background)"
-    task db: :environment do
-      ClearDatabaseCacheWorker.perform_async
-    end
-
-    task all: [:db, :redis]
+    task all: [:redis]
   end
 
-  task clear: 'cache:clear:all'
+  task clear: 'cache:clear:redis'
 end
diff --git a/lib/tasks/ci/.gitkeep b/lib/tasks/ci/.gitkeep
deleted file mode 100644
index e69de29bb2d..00000000000
--- a/lib/tasks/ci/.gitkeep
+++ /dev/null
diff --git a/lib/tasks/config_lint.rake b/lib/tasks/config_lint.rake
new file mode 100644
index 00000000000..ddbcf1e1eb8
--- /dev/null
+++ b/lib/tasks/config_lint.rake
@@ -0,0 +1,25 @@
+module ConfigLint
+  def self.run(files)
+    failures = files.reject do |file|
+      yield(file)
+    end
+
+    if failures.present?
+      puts failures
+      exit failures.count
+    end
+  end
+end
+
+desc "Checks syntax for shell scripts and nginx config files in 'lib/support/'"
+task :config_lint do
+  shell_scripts = [
+    'lib/support/init.d/gitlab',
+    'lib/support/init.d/gitlab.default.example',
+    'lib/support/deploy/deploy.sh'
+  ]
+
+  ConfigLint.run(shell_scripts) do |file|
+    Kernel.system('bash', '-n', file)
+  end
+end
diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake
index 6f27972c4e4..e65609d7001 100644
--- a/lib/tasks/dev.rake
+++ b/lib/tasks/dev.rake
@@ -2,14 +2,9 @@ task dev: ["dev:setup"]
 
 namespace :dev do
   desc "GitLab | Setup developer environment (db, fixtures)"
-  task :setup => :environment do
+  task setup: :environment do
     ENV['force'] = 'yes'
     Rake::Task["gitlab:setup"].invoke
     Rake::Task["gitlab:shell:setup"].invoke
   end
-
-  desc 'GitLab | Start/restart foreman and watch for changes'
-  task :foreman => :environment do
-    sh 'rerun --dir app,config,lib -- foreman start'
-  end
 end
diff --git a/lib/tasks/downtime_check.rake b/lib/tasks/downtime_check.rake
index afe5d42910c..557f4fef10b 100644
--- a/lib/tasks/downtime_check.rake
+++ b/lib/tasks/downtime_check.rake
@@ -1,10 +1,10 @@
 desc 'Checks if migrations in a branch require downtime'
 task downtime_check: :environment do
-  if defined?(Gitlab::License)
-    repo = 'gitlab-ee'
-  else
-    repo = 'gitlab-ce'
-  end
+  repo = if defined?(Gitlab::License)
+           'gitlab-ee'
+         else
+           'gitlab-ce'
+         end
 
   `git fetch https://gitlab.com/gitlab-org/#{repo}.git --depth 1`
 
diff --git a/lib/tasks/ee_compat_check.rake b/lib/tasks/ee_compat_check.rake
new file mode 100644
index 00000000000..f494fa5c5c2
--- /dev/null
+++ b/lib/tasks/ee_compat_check.rake
@@ -0,0 +1,4 @@
+desc 'Checks if the branch would apply cleanly to EE'
+task ee_compat_check: :environment do
+  Rake::Task['gitlab:dev:ee_compat_check'].invoke
+end
diff --git a/lib/tasks/eslint.rake b/lib/tasks/eslint.rake
new file mode 100644
index 00000000000..51f5d768102
--- /dev/null
+++ b/lib/tasks/eslint.rake
@@ -0,0 +1,8 @@
+unless Rails.env.production?
+  desc "GitLab | Run ESLint"
+  task eslint: ['yarn:check'] do
+    unless system('yarn run eslint')
+      abort('rake eslint failed')
+    end
+  end
+end
diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake
index e9587595fef..7ad2b2e4d39 100644
--- a/lib/tasks/flay.rake
+++ b/lib/tasks/flay.rake
@@ -1,6 +1,6 @@
 desc 'Code duplication analyze via flay'
 task :flay do
-  output = %x(bundle exec flay --mass 35 app/ lib/gitlab/)
+  output = `bundle exec flay --mass 35 app/ lib/gitlab/`
 
   if output.include? "Similar code found"
     puts output
diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake
index 993112aee3b..87ca39b079b 100644
--- a/lib/tasks/gemojione.rake
+++ b/lib/tasks/gemojione.rake
@@ -1,33 +1,37 @@
 namespace :gemojione do
   desc 'Generates Emoji SHA256 digests'
-  task digests: :environment do
+  task digests: ['yarn:check', 'environment'] do
     require 'digest/sha2'
     require 'json'
 
-    dir = Gemojione.images_path
-    digests = []
-    aliases = Hash.new { |hash, key| hash[key] = [] }
-    aliases_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json')
-
-    JSON.parse(File.read(aliases_path)).each do |alias_name, real_name|
-      aliases[real_name] << alias_name
-    end
-
-    Gitlab::AwardEmoji.emojis.map do |name, emoji_hash|
-      fpath = File.join(dir, "#{emoji_hash['unicode']}.png")
-      digest = Digest::SHA256.file(fpath).hexdigest
+    # We don't have `node_modules` available in built versions of GitLab
+    FileUtils.cp_r(Rails.root.join('node_modules', 'emoji-unicode-version', 'emoji-unicode-version-map.json'), File.join(Rails.root, 'fixtures', 'emojis'))
 
-      digests << { name: name, unicode: emoji_hash['unicode'], digest: digest }
+    dir = Gemojione.images_path
+    resultant_emoji_map = {}
+
+    Gitlab::Emoji.emojis.each do |name, emoji_hash|
+      # Ignore aliases
+      unless Gitlab::Emoji.emojis_aliases.key?(name)
+        fpath = File.join(dir, "#{emoji_hash['unicode']}.png")
+        hash_digest = Digest::SHA256.file(fpath).hexdigest
+
+        entry = {
+          category: emoji_hash['category'],
+          moji: emoji_hash['moji'],
+          description: emoji_hash['description'],
+          unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name),
+          digest: hash_digest
+        }
 
-      aliases[name].each do |alias_name|
-        digests << { name: alias_name, unicode: emoji_hash['unicode'], digest: digest }
+        resultant_emoji_map[name] = entry
       end
     end
 
     out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json')
 
     File.open(out, 'w') do |handle|
-      handle.write(JSON.pretty_generate(digests))
+      handle.write(JSON.pretty_generate(resultant_emoji_map))
     end
   end
 
@@ -55,21 +59,40 @@ namespace :gemojione do
     SPRITESHEET_WIDTH = 860
     SPRITESHEET_HEIGHT = 840
 
+    # Setup a map to rename image files
+    emoji_unicode_string_to_name_map = {}
+    Gitlab::Emoji.emojis.each do |name, emoji_hash|
+      # Ignore aliases
+      unless Gitlab::Emoji.emojis_aliases.key?(name)
+        emoji_unicode_string_to_name_map[emoji_hash['unicode']] = name
+      end
+    end
+
+    # Copy the Gemojione assets to the temporary folder for renaming
+    emoji_dir = "app/assets/images/emoji"
+    FileUtils.rm_rf(emoji_dir)
+    FileUtils.mkdir_p(emoji_dir, mode: 0700)
+    FileUtils.cp_r(File.join(Gemojione.images_path, '.'), emoji_dir)
+    Dir[File.join(emoji_dir, "**/*.png")].each do |png|
+      image_path = png
+      rename_to_named_emoji_image!(emoji_unicode_string_to_name_map, image_path)
+    end
+
     Dir.mktmpdir do |tmpdir|
-      # Copy the Gemojione assets to the temporary folder for resizing
-      FileUtils.cp_r(Gemojione.images_path, tmpdir)
+      FileUtils.cp_r(File.join(emoji_dir, '.'), tmpdir)
 
       Dir.chdir(tmpdir) do
         Dir["**/*.png"].each do |png|
-          resize!(File.join(tmpdir, png), SIZE)
+          tmp_image_path = File.join(tmpdir, png)
+          resize!(tmp_image_path, SIZE)
         end
       end
 
-      style_path = Rails.root.join(*%w(app assets stylesheets pages emojis.scss))
+      style_path = Rails.root.join(*%w(app assets stylesheets framework emoji-sprites.scss))
 
       # Combine the resized assets into a packed sprite and re-generate the SCSS
       SpriteFactory.cssurl = "image-url('$IMAGE')"
-      SpriteFactory.run!(File.join(tmpdir, 'png'), {
+      SpriteFactory.run!(tmpdir, {
         output_style: style_path,
         output_image: "app/assets/images/emoji.png",
         selector:     '.emoji-',
@@ -83,7 +106,7 @@ namespace :gemojione do
       # let's simplify it
       system(%Q(sed -i '' "s/width: #{SIZE}px; height: #{SIZE}px; background: image-url('emoji.png')/background-position:/" #{style_path}))
       system(%Q(sed -i '' "s/ no-repeat//" #{style_path}))
-      system(%Q(sed -i '' "s/ 0px/ 0/" #{style_path}))
+      system(%Q(sed -i '' "s/ 0px/ 0/g" #{style_path}))
 
       # Append a generic rule that applies to all Emojis
       File.open(style_path, 'a') do |f|
@@ -92,6 +115,8 @@ namespace :gemojione do
         .emoji-icon {
           background-image: image-url('emoji.png');
           background-repeat: no-repeat;
+          color: transparent;
+          text-indent: -99em;
           height: #{SIZE}px;
           width: #{SIZE}px;
 
@@ -112,16 +137,17 @@ namespace :gemojione do
     # Now do it again but for Retina
     Dir.mktmpdir do |tmpdir|
       # Copy the Gemojione assets to the temporary folder for resizing
-      FileUtils.cp_r(Gemojione.images_path, tmpdir)
+      FileUtils.cp_r(File.join(emoji_dir, '.'), tmpdir)
 
       Dir.chdir(tmpdir) do
         Dir["**/*.png"].each do |png|
-          resize!(File.join(tmpdir, png), RETINA)
+          tmp_image_path = File.join(tmpdir, png)
+          resize!(tmp_image_path, RETINA)
         end
       end
 
       # Combine the resized assets into a packed sprite and re-generate the SCSS
-      SpriteFactory.run!(File.join(tmpdir), {
+      SpriteFactory.run!(tmpdir, {
         output_image: "app/assets/images/emoji@2x.png",
         style:        false,
         nocomments:   true,
@@ -155,4 +181,20 @@ namespace :gemojione do
     image.write(image_path) { self.quality = 100 }
     image.destroy!
   end
+
+  EMOJI_IMAGE_PATH_RE = /(.*?)(([0-9a-f]-?)+)\.png$/i
+  def rename_to_named_emoji_image!(emoji_unicode_string_to_name_map, image_path)
+    # Rename file from unicode to emoji name
+    matches = EMOJI_IMAGE_PATH_RE.match(image_path)
+    preceding_path = matches[1]
+    unicode_string = matches[2]
+    name = emoji_unicode_string_to_name_map[unicode_string]
+    if name
+      new_png_path = File.join(preceding_path, "#{name}.png")
+      FileUtils.mv(image_path, new_png_path)
+      new_png_path
+    else
+      puts "Warning: emoji_unicode_string_to_name_map missing entry for #{unicode_string}. Full path: #{image_path}"
+    end
+  end
 end
diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
new file mode 100644
index 00000000000..b27f7475115
--- /dev/null
+++ b/lib/tasks/gettext.rake
@@ -0,0 +1,22 @@
+require "gettext_i18n_rails/tasks"
+
+namespace :gettext do
+  # Customize list of translatable files
+  # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files
+  def files_to_translate
+    folders = %W(app lib config #{locale_path}).join(',')
+    exts = %w(rb erb haml slim rhtml js jsx vue coffee handlebars hbs mustache).join(',')
+
+    Dir.glob(
+      "{#{folders}}/**/*.{#{exts}}"
+    )
+  end
+
+  task :compile do
+    # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/33014#note_31218998
+    FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot'))
+
+    Rake::Task['gettext:pack'].invoke
+    Rake::Task['gettext:po_to_json'].invoke
+  end
+end
diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
new file mode 100644
index 00000000000..003d57adbbd
--- /dev/null
+++ b/lib/tasks/gitlab/assets.rake
@@ -0,0 +1,48 @@
+namespace :gitlab do
+  namespace :assets do
+    desc 'GitLab | Assets | Compile all frontend assets'
+    task compile: [
+      'yarn:check',
+      'rake:assets:precompile',
+      'webpack:compile',
+      'fix_urls'
+    ]
+
+    desc 'GitLab | Assets | Clean up old compiled frontend assets'
+    task clean: ['rake:assets:clean']
+
+    desc 'GitLab | Assets | Remove all compiled frontend assets'
+    task purge: ['rake:assets:clobber']
+
+    desc 'GitLab | Assets | Uninstall frontend dependencies'
+    task purge_modules: ['yarn:clobber']
+
+    desc 'GitLab | Assets | Fix all absolute url references in CSS'
+    task :fix_urls do
+      css_files = Dir['public/assets/*.css']
+      css_files.each do |file|
+        # replace url(/assets/*) with url(./*)
+        puts "Fixing #{file}"
+        system "sed", "-i", "-e", 's/url(\([\"\']\?\)\/assets\//url(\1.\//g', file
+
+        # rewrite the corresponding gzip file (if it exists)
+        gzip = "#{file}.gz"
+        if File.exist?(gzip)
+          puts "Fixing #{gzip}"
+
+          FileUtils.rm(gzip)
+          mtime = File.stat(file).mtime
+
+          File.open(gzip, 'wb+') do |f|
+            gz = Zlib::GzipWriter.new(f, Zlib::BEST_COMPRESSION)
+            gz.mtime = mtime
+            gz.write IO.binread(file)
+            gz.close
+
+            File.utime(mtime, mtime, f.path)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index b43ee5b3383..1650263b98d 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -13,6 +13,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:uploads:create"].invoke
       Rake::Task["gitlab:backup:builds:create"].invoke
       Rake::Task["gitlab:backup:artifacts:create"].invoke
+      Rake::Task["gitlab:backup:pages:create"].invoke
       Rake::Task["gitlab:backup:lfs:create"].invoke
       Rake::Task["gitlab:backup:registry:create"].invoke
 
@@ -51,13 +52,16 @@ namespace :gitlab do
         $progress.puts 'done'.color(:green)
         Rake::Task['gitlab:backup:db:restore'].invoke
       end
+
       Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories')
       Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads')
       Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds')
       Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts')
+      Rake::Task["gitlab:backup:pages:restore"].invoke unless backup.skipped?('pages')
       Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs')
       Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry')
       Rake::Task['gitlab:shell:setup'].invoke
+      Rake::Task['cache:clear'].invoke
 
       backup.cleanup
     end
@@ -157,6 +161,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :pages do
+      task create: :environment do
+        $progress.puts "Dumping pages ... ".color(:blue)
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("pages")
+          $progress.puts "[SKIPPED]".color(:cyan)
+        else
+          Backup::Pages.new.dump
+          $progress.puts "done".color(:green)
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring pages ... ".color(:blue)
+        Backup::Pages.new.restore
+        $progress.puts "done".color(:green)
+      end
+    end
+
     namespace :lfs do
       task create: :environment do
         $progress.puts "Dumping lfs objects ... ".color(:blue)
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 5f4a6bbfa35..858f1cd7b34 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -1,340 +1,48 @@
+# Temporary hack, until we migrate all checks to SystemCheck format
+require 'system_check'
+require 'system_check/helpers'
+
 namespace :gitlab do
-  desc "GitLab | Check the configuration of GitLab and its environment"
+  desc 'GitLab | Check the configuration of GitLab and its environment'
   task check: %w{gitlab:gitlab_shell:check
                  gitlab:sidekiq:check
                  gitlab:incoming_email:check
                  gitlab:ldap:check
                  gitlab:app:check}
 
-
-
   namespace :app do
-    desc "GitLab | Check the configuration of the GitLab Rails app"
+    desc 'GitLab | Check the configuration of the GitLab Rails app'
     task check: :environment  do
       warn_user_is_not_gitlab
-      start_checking "GitLab"
-
-      check_git_config
-      check_database_config_exists
-      check_migrations_are_up
-      check_orphaned_group_members
-      check_gitlab_config_exists
-      check_gitlab_config_not_outdated
-      check_log_writable
-      check_tmp_writable
-      check_uploads
-      check_init_script_exists
-      check_init_script_up_to_date
-      check_projects_have_namespace
-      check_redis_version
-      check_ruby_version
-      check_git_version
-      check_active_users
-
-      finished_checking "GitLab"
-    end
-
-
-    # Checks
-    ########################
-
-    def check_git_config
-      print "Git configured with autocrlf=input? ... "
-
-      options = {
-        "core.autocrlf" => "input"
-      }
-
-      correct_options = options.map do |name, value|
-        run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value
-      end
-
-      if correct_options.all?
-        puts "yes".color(:green)
-      else
-        print "Trying to fix Git error automatically. ..."
-
-        if auto_fix_git_config(options)
-          puts "Success".color(:green)
-        else
-          puts "Failed".color(:red)
-          try_fixing_it(
-            sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{options["core.autocrlf"]}\"")
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab"
-          )
-        end
-      end
-    end
-
-    def check_database_config_exists
-      print "Database config exists? ... "
-
-      database_config_file = Rails.root.join("config", "database.yml")
-
-      if File.exist?(database_config_file)
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Copy config/database.yml.<your db> to config/database.yml",
-          "Check that the information in config/database.yml is correct"
-        )
-        for_more_information(
-          see_database_guide,
-          "http://guides.rubyonrails.org/getting_started.html#configuring-a-database"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_gitlab_config_exists
-      print "GitLab config exists? ... "
-
-      gitlab_config_file = Rails.root.join("config", "gitlab.yml")
-
-      if File.exist?(gitlab_config_file)
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Copy config/gitlab.yml.example to config/gitlab.yml",
-          "Update config/gitlab.yml to match your setup"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_gitlab_config_not_outdated
-      print "GitLab config outdated? ... "
-
-      gitlab_config_file = Rails.root.join("config", "gitlab.yml")
-      unless File.exist?(gitlab_config_file)
-        puts "can't check because of previous errors".color(:magenta)
-      end
 
-      # omniauth or ldap could have been deleted from the file
-      unless Gitlab.config['git_host']
-        puts "no".color(:green)
-      else
-        puts "yes".color(:red)
-        try_fixing_it(
-          "Backup your config/gitlab.yml",
-          "Copy config/gitlab.yml.example to config/gitlab.yml",
-          "Update config/gitlab.yml to match your setup"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_init_script_exists
-      print "Init script exists? ... "
-
-      if omnibus_gitlab?
-        puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta)
-        return
-      end
-
-      script_path = "/etc/init.d/gitlab"
-
-      if File.exist?(script_path)
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Install the init script"
-        )
-        for_more_information(
-          see_installation_guide_section "Install Init Script"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_init_script_up_to_date
-      print "Init script up-to-date? ... "
-
-      if omnibus_gitlab?
-        puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta)
-        return
-      end
-
-      recipe_path = Rails.root.join("lib/support/init.d/", "gitlab")
-      script_path = "/etc/init.d/gitlab"
-
-      unless File.exist?(script_path)
-        puts "can't check because of previous errors".color(:magenta)
-        return
-      end
-
-      recipe_content = File.read(recipe_path)
-      script_content = File.read(script_path)
-
-      if recipe_content == script_content
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Redownload the init script"
-        )
-        for_more_information(
-          see_installation_guide_section "Install Init Script"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_migrations_are_up
-      print "All migrations up? ... "
-
-      migration_status, _ = Gitlab::Popen.popen(%W(bundle exec rake db:migrate:status))
-
-      unless migration_status =~ /down\s+\d{14}/
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          sudo_gitlab("bundle exec rake db:migrate RAILS_ENV=production")
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_orphaned_group_members
-      print "Database contains orphaned GroupMembers? ... "
-      if GroupMember.where("user_id not in (select id from users)").count > 0
-        puts "yes".color(:red)
-        try_fixing_it(
-          "You can delete the orphaned records using something along the lines of:",
-          sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'")
-        )
-      else
-        puts "no".color(:green)
-      end
-    end
-
-    def check_log_writable
-      print "Log directory writable? ... "
-
-      log_path = Rails.root.join("log")
-
-      if File.writable?(log_path)
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "sudo chown -R gitlab #{log_path}",
-          "sudo chmod -R u+rwX #{log_path}"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_tmp_writable
-      print "Tmp directory writable? ... "
-
-      tmp_path = Rails.root.join("tmp")
-
-      if File.writable?(tmp_path)
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "sudo chown -R gitlab #{tmp_path}",
-          "sudo chmod -R u+rwX #{tmp_path}"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_uploads
-      print "Uploads directory setup correctly? ... "
-
-      unless File.directory?(Rails.root.join('public/uploads'))
-        puts "no".color(:red)
-        try_fixing_it(
-          "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-        return
-      end
-
-      upload_path = File.realpath(Rails.root.join('public/uploads'))
-      upload_path_tmp = File.join(upload_path, 'tmp')
-
-      if File.stat(upload_path).mode == 040700
-        unless Dir.exists?(upload_path_tmp)
-          puts 'skipped (no tmp uploads folder yet)'.color(:magenta)
-          return
-        end
-
-        # If tmp upload dir has incorrect permissions, assume others do as well
-        # Verify drwx------ permissions
-        if File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp)
-          puts "yes".color(:green)
-        else
-          puts "no".color(:red)
-          try_fixing_it(
-            "sudo chown -R #{gitlab_user} #{upload_path}",
-            "sudo find #{upload_path} -type f -exec chmod 0644 {} \\;",
-            "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;"
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab"
-          )
-          fix_and_rerun
-        end
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "sudo chmod 700 #{upload_path}"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_redis_version
-      min_redis_version = "2.8.0"
-      print "Redis version >= #{min_redis_version}? ... "
-
-      redis_version = run_command(%W(redis-cli --version))
-      redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/)
-      if redis_version &&
-          (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version))
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Update your redis server to a version >= #{min_redis_version}"
-        )
-        for_more_information(
-          "gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq"
-        )
-        fix_and_rerun
-      end
+      checks = [
+        SystemCheck::App::GitConfigCheck,
+        SystemCheck::App::DatabaseConfigExistsCheck,
+        SystemCheck::App::MigrationsAreUpCheck,
+        SystemCheck::App::OrphanedGroupMembersCheck,
+        SystemCheck::App::GitlabConfigExistsCheck,
+        SystemCheck::App::GitlabConfigUpToDateCheck,
+        SystemCheck::App::LogWritableCheck,
+        SystemCheck::App::TmpWritableCheck,
+        SystemCheck::App::UploadsDirectoryExistsCheck,
+        SystemCheck::App::UploadsPathPermissionCheck,
+        SystemCheck::App::UploadsPathTmpPermissionCheck,
+        SystemCheck::App::InitScriptExistsCheck,
+        SystemCheck::App::InitScriptUpToDateCheck,
+        SystemCheck::App::ProjectsHaveNamespaceCheck,
+        SystemCheck::App::RedisVersionCheck,
+        SystemCheck::App::RubyVersionCheck,
+        SystemCheck::App::GitVersionCheck,
+        SystemCheck::App::ActiveUsersCheck
+      ]
+
+      SystemCheck.run('GitLab', checks)
     end
   end
 
   namespace :gitlab_shell do
+    include SystemCheck::Helpers
+
     desc "GitLab | Check the configuration of GitLab Shell"
     task check: :environment  do
       warn_user_is_not_gitlab
@@ -351,14 +59,14 @@ namespace :gitlab do
       finished_checking "GitLab Shell"
     end
 
-
     # Checks
     ########################
 
     def check_repo_base_exists
       puts "Repo base directory exists?"
 
-      Gitlab.config.repositories.storages.each do |name, repo_base_path|
+      Gitlab.config.repositories.storages.each do |name, repository_storage|
+        repo_base_path = repository_storage['path']
         print "#{name}... "
 
         if File.exist?(repo_base_path)
@@ -382,12 +90,13 @@ namespace :gitlab do
     def check_repo_base_is_not_symlink
       puts "Repo storage directories are symlinks?"
 
-      Gitlab.config.repositories.storages.each do |name, repo_base_path|
+      Gitlab.config.repositories.storages.each do |name, repository_storage|
+        repo_base_path = repository_storage['path']
         print "#{name}... "
 
         unless File.exist?(repo_base_path)
           puts "can't check because of previous errors".color(:magenta)
-          return
+          break
         end
 
         unless File.symlink?(repo_base_path)
@@ -405,12 +114,13 @@ namespace :gitlab do
     def check_repo_base_permissions
       puts "Repo paths access is drwxrws---?"
 
-      Gitlab.config.repositories.storages.each do |name, repo_base_path|
+      Gitlab.config.repositories.storages.each do |name, repository_storage|
+        repo_base_path = repository_storage['path']
         print "#{name}... "
 
         unless File.exist?(repo_base_path)
           puts "can't check because of previous errors".color(:magenta)
-          return
+          break
         end
 
         if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770")
@@ -432,26 +142,27 @@ namespace :gitlab do
 
     def check_repo_base_user_and_group
       gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
-      gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
-      puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?"
+      puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?"
 
-      Gitlab.config.repositories.storages.each do |name, repo_base_path|
+      Gitlab.config.repositories.storages.each do |name, repository_storage|
+        repo_base_path = repository_storage['path']
         print "#{name}... "
 
         unless File.exist?(repo_base_path)
           puts "can't check because of previous errors".color(:magenta)
-          return
+          break
         end
 
-        uid = uid_for(gitlab_shell_ssh_user)
-        gid = gid_for(gitlab_shell_owner_group)
-        if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid
+        user_id = uid_for(gitlab_shell_ssh_user)
+        root_group_id = gid_for('root')
+        group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)]
+        if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid)
           puts "yes".color(:green)
         else
           puts "no".color(:red)
-          puts "  User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue)
+          puts "  User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue)
           try_fixing_it(
-            "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}"
+            "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}"
           )
           for_more_information(
             see_installation_guide_section "GitLab Shell"
@@ -493,7 +204,6 @@ namespace :gitlab do
           )
           fix_and_rerun
         end
-
       end
     end
 
@@ -514,33 +224,6 @@ namespace :gitlab do
       end
     end
 
-    def check_projects_have_namespace
-      print "projects have namespace: ... "
-
-      unless Project.count > 0
-        puts "can't check, you have no projects".color(:magenta)
-        return
-      end
-      puts ""
-
-      Project.find_each(batch_size: 100) do |project|
-        print sanitized_message(project)
-
-        if project.namespace
-          puts "yes".color(:green)
-        else
-          puts "no".color(:red)
-          try_fixing_it(
-            "Migrate global projects"
-          )
-          for_more_information(
-            "doc/update/5.4-to-6.0.md in section \"#global-projects\""
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
     # Helper methods
     ########################
 
@@ -565,9 +248,9 @@ namespace :gitlab do
     end
   end
 
-
-
   namespace :sidekiq do
+    include SystemCheck::Helpers
+
     desc "GitLab | Check the configuration of Sidekiq"
     task check: :environment  do
       warn_user_is_not_gitlab
@@ -579,7 +262,6 @@ namespace :gitlab do
       finished_checking "Sidekiq"
     end
 
-
     # Checks
     ########################
 
@@ -621,13 +303,14 @@ namespace :gitlab do
     end
 
     def sidekiq_process_count
-      ps_ux, _ = Gitlab::Popen.popen(%W(ps ux))
+      ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww))
       ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count
     end
   end
 
-
   namespace :incoming_email do
+    include SystemCheck::Helpers
+
     desc "GitLab | Check the configuration of Reply by email"
     task check: :environment  do
       warn_user_is_not_gitlab
@@ -649,17 +332,13 @@ namespace :gitlab do
       finished_checking "Reply by email"
     end
 
-
     # Checks
     ########################
 
     def check_initd_configured_correctly
-      print "Init.d configured correctly? ... "
+      return if omnibus_gitlab?
 
-      if omnibus_gitlab?
-        puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta)
-        return
-      end
+      print "Init.d configured correctly? ... "
 
       path = "/etc/default/gitlab"
 
@@ -671,7 +350,7 @@ namespace :gitlab do
           "Enable mail_room in the init.d configuration."
         )
         for_more_information(
-          "doc/incoming_email/README.md"
+          "doc/administration/reply_by_email.md"
         )
         fix_and_rerun
       end
@@ -690,13 +369,15 @@ namespace :gitlab do
           "Enable mail_room in your Procfile."
         )
         for_more_information(
-          "doc/incoming_email/README.md"
+          "doc/administration/reply_by_email.md"
         )
         fix_and_rerun
       end
     end
 
     def check_mail_room_running
+      return if omnibus_gitlab?
+
       print "MailRoom running? ... "
 
       path = "/etc/default/gitlab"
@@ -724,8 +405,11 @@ namespace :gitlab do
     def check_imap_authentication
       print "IMAP server credentials are correct? ... "
 
-      config_path = Rails.root.join('config', 'mail_room.yml')
-      config_file = YAML.load(ERB.new(File.read(config_path)).result)
+      config_path = Rails.root.join('config', 'mail_room.yml').to_s
+      erb = ERB.new(File.read(config_path))
+      erb.filename = config_path
+      config_file = YAML.load(erb.result)
+
       config = config_file[:mailboxes].first
 
       if config
@@ -747,20 +431,22 @@ namespace :gitlab do
           "Check that the information in config/gitlab.yml is correct"
         )
         for_more_information(
-          "doc/incoming_email/README.md"
+          "doc/administration/reply_by_email.md"
         )
         fix_and_rerun
       end
     end
 
     def mail_room_running?
-      ps_ux, _ = Gitlab::Popen.popen(%W(ps ux))
+      ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww))
       ps_ux.include?("mail_room")
     end
   end
 
   namespace :ldap do
-    task :check, [:limit] => :environment do |t, args|
+    include SystemCheck::Helpers
+
+    task :check, [:limit] => :environment do |_, args|
       # Only show up to 100 results because LDAP directories can be very big.
       # This setting only affects the `rake gitlab:check` script.
       args.with_defaults(limit: 100)
@@ -768,7 +454,7 @@ namespace :gitlab do
       start_checking "LDAP"
 
       if Gitlab::LDAP::Config.enabled?
-        print_users(args.limit)
+        check_ldap(args.limit)
       else
         puts 'LDAP is disabled in config/gitlab.yml'
       end
@@ -776,28 +462,51 @@ namespace :gitlab do
       finished_checking "LDAP"
     end
 
-    def print_users(limit)
-      puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)"
-
+    def check_ldap(limit)
       servers = Gitlab::LDAP::Config.providers
 
       servers.each do |server|
         puts "Server: #{server}"
-        Gitlab::LDAP::Adapter.open(server) do |adapter|
-          users = adapter.users(adapter.config.uid, '*', limit)
-          users.each do |user|
-            puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}"
+
+        begin
+          Gitlab::LDAP::Adapter.open(server) do |adapter|
+            check_ldap_auth(adapter)
+
+            puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)"
+
+            users = adapter.users(adapter.config.uid, '*', limit)
+            users.each do |user|
+              puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}"
+            end
           end
+        rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e
+          puts "Could not connect to the LDAP server: #{e.message}".color(:red)
         end
       end
     end
+
+    def check_ldap_auth(adapter)
+      auth = adapter.config.has_auth?
+
+      message = if auth && adapter.ldap.bind
+                  'Success'.color(:green)
+                elsif auth
+                  'Failed. Check `bind_dn` and `password` configuration values'.color(:red)
+                else
+                  'Anonymous. No `bind_dn` or `password` configured'.color(:yellow)
+                end
+
+      puts "LDAP authentication... #{message}"
+    end
   end
 
   namespace :repo do
+    include SystemCheck::Helpers
+
     desc "GitLab | Check the integrity of the repositories managed by GitLab"
     task check: :environment do
-      Gitlab.config.repositories.storages.each do |name, path|
-        namespace_dirs = Dir.glob(File.join(path, '*'))
+      Gitlab.config.repositories.storages.each do |name, repository_storage|
+        namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*'))
 
         namespace_dirs.each do |namespace_dir|
           repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
@@ -808,17 +517,19 @@ namespace :gitlab do
   end
 
   namespace :user do
+    include SystemCheck::Helpers
+
     desc "GitLab | Check the integrity of a specific user's repositories"
     task :check_repos, [:username] => :environment do |t, args|
       username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue))
       user = User.find_by(username: username)
       if user
         repo_dirs = user.authorized_projects.map do |p|
-                      File.join(
-                        p.repository_storage_path,
-                        "#{p.path_with_namespace}.git"
-                      )
-                    end
+          File.join(
+            p.repository_storage_path,
+            "#{p.path_with_namespace}.git"
+          )
+        end
 
         repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
       else
@@ -830,55 +541,6 @@ namespace :gitlab do
   # Helper methods
   ##########################
 
-  def fix_and_rerun
-    puts "  Please #{"fix the error above"} and rerun the checks.".color(:red)
-  end
-
-  def for_more_information(*sources)
-    sources = sources.shift if sources.first.is_a?(Array)
-
-    puts "  For more information see:".color(:blue)
-    sources.each do |source|
-      puts "  #{source}"
-    end
-  end
-
-  def finished_checking(component)
-    puts ""
-    puts "Checking #{component.color(:yellow)} ... #{"Finished".color(:green)}"
-    puts ""
-  end
-
-  def see_database_guide
-    "doc/install/databases.md"
-  end
-
-  def see_installation_guide_section(section)
-    "doc/install/installation.md in section \"#{section}\""
-  end
-
-  def sudo_gitlab(command)
-    "sudo -u #{gitlab_user} -H #{command}"
-  end
-
-  def gitlab_user
-    Gitlab.config.gitlab.user
-  end
-
-  def start_checking(component)
-    puts "Checking #{component.color(:yellow)} ..."
-    puts ""
-  end
-
-  def try_fixing_it(*steps)
-    steps = steps.shift if steps.first.is_a?(Array)
-
-    puts "  Try fixing it:".color(:blue)
-    steps.each do |step|
-      puts "  #{step}"
-    end
-  end
-
   def check_gitlab_shell
     required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version)
     current_version = Gitlab::VersionInfo.parse(gitlab_shell_version)
@@ -891,65 +553,6 @@ namespace :gitlab do
     end
   end
 
-  def check_ruby_version
-    required_version = Gitlab::VersionInfo.new(2, 1, 0)
-    current_version = Gitlab::VersionInfo.parse(run_command(%W(ruby --version)))
-
-    print "Ruby version >= #{required_version} ? ... "
-
-    if current_version.valid? && required_version <= current_version
-      puts "yes (#{current_version})".color(:green)
-    else
-      puts "no".color(:red)
-      try_fixing_it(
-        "Update your ruby to a version >= #{required_version} from #{current_version}"
-      )
-      fix_and_rerun
-    end
-  end
-
-  def check_git_version
-    required_version = Gitlab::VersionInfo.new(2, 7, 3)
-    current_version = Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version)))
-
-    puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\""
-    print "Git version >= #{required_version} ? ... "
-
-    if current_version.valid? && required_version <= current_version
-      puts "yes (#{current_version})".color(:green)
-    else
-      puts "no".color(:red)
-      try_fixing_it(
-        "Update your git to a version >= #{required_version} from #{current_version}"
-      )
-      fix_and_rerun
-    end
-  end
-
-  def check_active_users
-    puts "Active users: #{User.active.count}"
-  end
-
-  def omnibus_gitlab?
-    Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails'
-  end
-
-  def sanitized_message(project)
-    if should_sanitize?
-      "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... "
-    else
-      "#{project.name_with_namespace.color(:yellow)} ... "
-    end
-  end
-
-  def should_sanitize?
-    if ENV['SANITIZE'] == "true"
-      true
-    else
-      false
-    end
-  end
-
   def check_repo_integrity(repo_dir)
     puts "\nChecking repo at #{repo_dir.color(:yellow)}"
 
@@ -964,13 +567,13 @@ namespace :gitlab do
   end
 
   def check_config_lock(repo_dir)
-    config_exists = File.exist?(File.join(repo_dir,'config.lock'))
+    config_exists = File.exist?(File.join(repo_dir, 'config.lock'))
     config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green)
     puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}"
   end
 
   def check_ref_locks(repo_dir)
-    lock_files = Dir.glob(File.join(repo_dir,'refs/heads/*.lock'))
+    lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock'))
     if lock_files.present?
       puts "Ref lock files exist:".color(:red)
       lock_files.each do |lock_file|
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index b7cbdc6cd78..f76bef5f4bf 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -6,7 +6,8 @@ namespace :gitlab do
       remove_flag = ENV['REMOVE']
 
       namespaces = Namespace.pluck(:path)
-      Gitlab.config.repositories.storages.each do |name, git_base_path|
+      Gitlab.config.repositories.storages.each do |name, repository_storage|
+        git_base_path = repository_storage['path']
         all_dirs = Dir.glob(git_base_path + '/*')
 
         puts git_base_path.color(:yellow)
@@ -25,7 +26,6 @@ namespace :gitlab do
         end
 
         all_dirs.each do |dir_path|
-
           if remove_flag
             if FileUtils.rm_rf dir_path
               puts "Removed...#{dir_path}".color(:red)
@@ -48,17 +48,18 @@ namespace :gitlab do
       warn_user_is_not_gitlab
 
       move_suffix = "+orphaned+#{Time.now.to_i}"
-      Gitlab.config.repositories.storages.each do |name, repo_root|
+      Gitlab.config.repositories.storages.each do |name, repository_storage|
+        repo_root = repository_storage['path']
         # Look for global repos (legacy, depth 1) and normal repos (depth 2)
         IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
           find.each_line do |path|
             path.chomp!
-            repo_with_namespace = path.
-              sub(repo_root, '').
-              sub(%r{^/*}, '').
-              chomp('.git').
-              chomp('.wiki')
-            next if Project.find_with_namespace(repo_with_namespace)
+            repo_with_namespace = path
+              .sub(repo_root, '')
+              .sub(%r{^/*}, '')
+              .chomp('.git')
+              .chomp('.wiki')
+            next if Project.find_by_full_path(repo_with_namespace)
             new_path = path + move_suffix
             puts path.inspect + ' -> ' + new_path.inspect
             File.rename(path, new_path)
@@ -91,5 +92,28 @@ namespace :gitlab do
         puts "To block these users run this command with BLOCK=true".color(:yellow)
       end
     end
+
+    # This is a rake task which removes faulty refs. These refs where only
+    # created in the 8.13.RC cycle, and fixed in the stable builds which were
+    # released. So likely this should only be run once on gitlab.com
+    # Faulty refs are moved so they are kept around, else some features break.
+    desc 'GitLab | Cleanup | Remove faulty deployment refs'
+    task move_faulty_deployment_refs: :environment do
+      projects = Project.where(id: Deployment.select(:project_id).distinct)
+
+      projects.find_each do |project|
+        rugged = project.repository.rugged
+
+        max_iid = project.deployments.maximum(:iid)
+
+        rugged.references.each('refs/environments/**/*') do |ref|
+          id = ref.name.split('/').last.to_i
+          next unless id > max_iid
+
+          project.deployments.find(id).create_ref
+          rugged.references.delete(ref)
+        end
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 7c96bc864ce..139ab70e125 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -23,7 +23,7 @@ namespace :gitlab do
     end
 
     desc 'Drop all tables'
-    task :drop_tables => :environment do
+    task drop_tables: :environment do
       connection = ActiveRecord::Base.connection
 
       # If MySQL, turn off foreign key checks
@@ -62,9 +62,10 @@ namespace :gitlab do
 
       ref = Shellwords.escape(args[:ref])
 
-      migrations = `git diff #{ref}.. --name-only -- db/migrate`.lines.
-        map { |file| Rails.root.join(file.strip).to_s }.
-        select { |file| File.file?(file) }
+      migrations = `git diff #{ref}.. --diff-filter=A --name-only -- db/migrate`.lines
+        .map { |file| Rails.root.join(file.strip).to_s }
+        .select { |file| File.file?(file) }
+        .select { |file| /\A[0-9]+.*\.rb\z/ =~ File.basename(file) }
 
       Gitlab::DowntimeCheck.new.check_and_print(migrations)
     end
diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake
new file mode 100644
index 00000000000..7ccda04a35f
--- /dev/null
+++ b/lib/tasks/gitlab/dev.rake
@@ -0,0 +1,23 @@
+namespace :gitlab do
+  namespace :dev do
+    desc 'Checks if the branch would apply cleanly to EE'
+    task :ee_compat_check, [:branch] => :environment do |_, args|
+      opts =
+        if ENV['CI']
+          { branch: ENV['CI_COMMIT_REF_NAME'] }
+        else
+          unless args[:branch]
+            puts "Must specify a branch as an argument".color(:red)
+            exit 1
+          end
+          args
+        end
+
+      if Gitlab::EeCompatCheck.new(opts || {}).check
+        exit 0
+      else
+        exit 1
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/generate_docs.rake b/lib/tasks/gitlab/generate_docs.rake
deleted file mode 100644
index f6448c38e10..00000000000
--- a/lib/tasks/gitlab/generate_docs.rake
+++ /dev/null
@@ -1,7 +0,0 @@
-namespace :gitlab do
-  desc "GitLab | Generate sdocs for project"
-  task generate_docs: :environment do
-    system(*%W(bundle exec sdoc -o doc/code app lib))
-  end
-end
-
diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake
index f9834a4dae8..cf82134d97e 100644
--- a/lib/tasks/gitlab/git.rake
+++ b/lib/tasks/gitlab/git.rake
@@ -1,9 +1,8 @@
 namespace :gitlab do
   namespace :git do
-
     desc "GitLab | Git | Repack"
     task repack: :environment do
-      failures = perform_git_cmd(%W(git repack -a --quiet), "Repacking repo")
+      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo")
       if failures.empty?
         puts "Done".color(:green)
       else
@@ -13,17 +12,17 @@ namespace :gitlab do
 
     desc "GitLab | Git | Run garbage collection on all repos"
     task gc: :environment do
-      failures = perform_git_cmd(%W(git gc --auto --quiet), "Garbage Collecting")
+      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting")
       if failures.empty?
         puts "Done".color(:green)
       else
         output_failures(failures)
       end
     end
-    
+
     desc "GitLab | Git | Prune all repos"
     task prune: :environment do
-      failures = perform_git_cmd(%W(git prune), "Git Prune")
+      failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune")
       if failures.empty?
         puts "Done".color(:green)
       else
@@ -50,6 +49,5 @@ namespace :gitlab do
       puts "The following repositories reported errors:".color(:red)
       failures.each { |f| puts "- #{f}" }
     end
-
   end
 end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
new file mode 100644
index 00000000000..a8db5701d0b
--- /dev/null
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -0,0 +1,75 @@
+namespace :gitlab do
+  namespace :gitaly do
+    desc "GitLab | Install or upgrade gitaly"
+    task :install, [:dir, :repo] => :environment do |t, args|
+      require 'toml'
+
+      warn_user_is_not_gitlab
+      unless args.dir.present?
+        abort %(Please specify the directory where you want to install gitaly:\n  rake "gitlab:gitaly:install[/home/git/gitaly]")
+      end
+      args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git')
+
+      version = Gitlab::GitalyClient.expected_server_version
+
+      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
+
+      _, status = Gitlab::Popen.popen(%w[which gmake])
+      command = status.zero? ? 'gmake' : 'make'
+
+      Dir.chdir(args.dir) do
+        create_gitaly_configuration
+        run_command!([command])
+      end
+    end
+
+    desc "GitLab | Print storage configuration in TOML format"
+    task storage_config: :environment do
+      require 'toml'
+
+      puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}"
+      puts "# This is in TOML format suitable for use in Gitaly's config.toml file."
+
+      puts gitaly_configuration_toml
+    end
+
+    private
+
+    # We cannot create config.toml files for all possible Gitaly configuations.
+    # For instance, if Gitaly is running on another machine then it makes no
+    # sense to write a config.toml file on the current machine. This method will
+    # only generate a configuration for the most common and simplest case: when
+    # we have exactly one Gitaly process and we are sure it is running locally
+    # because it uses a Unix socket.
+    def gitaly_configuration_toml
+      storages = []
+      address = nil
+
+      Gitlab.config.repositories.storages.each do |key, val|
+        if address
+          if address != val['gitaly_address']
+            raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address."
+          end
+        elsif URI(val['gitaly_address']).scheme != 'unix'
+          raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses."
+        else
+          address = val['gitaly_address']
+        end
+
+        storages << { name: key, path: val['path'] }
+      end
+      config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
+      config[:auth] = { token: 'secret' } if Rails.env.test?
+      TOML.dump(config)
+    end
+
+    def create_gitaly_configuration
+      File.open("config.toml", "w") do |f|
+        f.puts gitaly_configuration_toml
+      end
+    rescue ArgumentError => e
+      puts "Skipping config.toml generation:"
+      puts e.message
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake
new file mode 100644
index 00000000000..dd2d5861481
--- /dev/null
+++ b/lib/tasks/gitlab/helpers.rake
@@ -0,0 +1,8 @@
+require 'tasks/gitlab/task_helpers'
+
+# Prevent StateMachine warnings from outputting during a cron task
+StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON']
+
+namespace :gitlab do
+  include Gitlab::TaskHelpers
+end
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index dbdd4e977e8..48bd9139ce8 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -11,7 +11,8 @@ namespace :gitlab do
     #
     desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
     task repos: :environment do
-      Gitlab.config.repositories.storages.each do |name, git_base_path|
+      Gitlab.config.repositories.storages.each_value do |repository_storage|
+        git_base_path = repository_storage['path']
         repos_to_import = Dir.glob(git_base_path + '/**/*.git')
 
         repos_to_import.each do |repo_path|
@@ -29,7 +30,7 @@ namespace :gitlab do
             next
           end
 
-          project = Project.find_with_namespace(path)
+          project = Project.find_by_full_path(path)
 
           if project
             puts " * #{project.name} (#{repo_path}) exists"
@@ -46,7 +47,7 @@ namespace :gitlab do
               group = Namespace.find_by(path: group_name)
               # create group namespace
               unless group
-                group = Group.new(:name => group_name)
+                group = Group.new(name: group_name)
                 group.path = group_name
                 group.owner = user
                 if group.save
@@ -63,8 +64,7 @@ namespace :gitlab do
 
             if project.persisted?
               puts " * Created #{project.name} (#{repo_path})".color(:green)
-              project.update_repository_size
-              project.update_commit_count
+              ProjectCacheWorker.perform_async(project.id)
             else
               puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
               puts "   Errors: #{project.errors.messages}".color(:red)
diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake
index c2c6031db67..dd1825c8a9e 100644
--- a/lib/tasks/gitlab/import_export.rake
+++ b/lib/tasks/gitlab/import_export.rake
@@ -7,7 +7,7 @@ namespace :gitlab do
 
     desc "GitLab | Display exported DB structure"
     task data: :environment do
-      puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(:SortKeys => true)
+      puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true)
     end
   end
 end
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index dffea8ed155..e3883278886 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -2,30 +2,37 @@ namespace :gitlab do
   namespace :env do
     desc "GitLab | Show information about GitLab and its environment"
     task info: :environment  do
-
       # check if there is an RVM environment
-      rvm_version = run_and_match(%W(rvm --version), /[\d\.]+/).try(:to_s)
+      rvm_version = run_and_match(%w(rvm --version), /[\d\.]+/).try(:to_s)
       # check Ruby version
-      ruby_version = run_and_match(%W(ruby --version), /[\d\.p]+/).try(:to_s)
+      ruby_version = run_and_match(%w(ruby --version), /[\d\.p]+/).try(:to_s)
       # check Gem version
-      gem_version = run_command(%W(gem --version))
-      # check Bundler version
-      bunder_version = run_and_match(%W(bundle --version), /[\d\.]+/).try(:to_s)
+      gem_version = run_command(%w(gem --version))
       # check Bundler version
-      rake_version = run_and_match(%W(rake --version), /[\d\.]+/).try(:to_s)
+      bunder_version = run_and_match(%w(bundle --version), /[\d\.]+/).try(:to_s)
+      # check Rake version
+      rake_version = run_and_match(%w(rake --version), /[\d\.]+/).try(:to_s)
+      # check redis version
+      redis_version = run_and_match(%w(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a
+      # check Git version
+      git_version = run_and_match([Gitlab.config.git.bin_path, '--version'], /git version ([\d\.]+)/).to_a
+      # check Go version
+      go_version = run_and_match(%w(go version), /go version (.+)/).to_a
 
       puts ""
       puts "System information".color(:yellow)
       puts "System:\t\t#{os_name || "unknown".color(:red)}"
-      puts "Current User:\t#{run_command(%W(whoami))}"
+      puts "Current User:\t#{run_command(%w(whoami))}"
       puts "Using RVM:\t#{rvm_version.present? ? "yes".color(:green) : "no"}"
       puts "RVM Version:\t#{rvm_version}" if rvm_version.present?
       puts "Ruby Version:\t#{ruby_version || "unknown".color(:red)}"
       puts "Gem Version:\t#{gem_version || "unknown".color(:red)}"
       puts "Bundler Version:#{bunder_version || "unknown".color(:red)}"
       puts "Rake Version:\t#{rake_version || "unknown".color(:red)}"
+      puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}"
+      puts "Git Version:\t#{git_version[1] || "unknown".color(:red)}"
       puts "Sidekiq Version:#{Sidekiq::VERSION}"
-
+      puts "Go Version:\t#{go_version[1] || "unknown".color(:red)}"
 
       # check database adapter
       database_adapter = ActiveRecord::Base.connection.adapter_name.downcase
@@ -51,8 +58,6 @@ namespace :gitlab do
       puts "Using Omniauth:\t#{Gitlab.config.omniauth.enabled ? "yes".color(:green) : "no"}"
       puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab.config.omniauth.enabled
 
-
-
       # check Gitolite version
       gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.hooks_path}/../VERSION"
       if File.readable?(gitlab_shell_version_file)
@@ -63,12 +68,11 @@ namespace :gitlab do
       puts "GitLab Shell".color(:yellow)
       puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
       puts "Repository storage paths:"
-      Gitlab.config.repositories.storages.each do |name, path|
-        puts "- #{name}: \t#{path}"
+      Gitlab.config.repositories.storages.each do |name, repository_storage|
+        puts "- #{name}: \t#{repository_storage['path']}"
       end
       puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
       puts "Git:\t\t#{Gitlab.config.git.bin_path}"
-
     end
   end
 end
diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake
new file mode 100644
index 00000000000..c66a2a263dc
--- /dev/null
+++ b/lib/tasks/gitlab/ldap.rake
@@ -0,0 +1,40 @@
+namespace :gitlab do
+  namespace :ldap do
+    desc 'GitLab | LDAP | Rename provider'
+    task :rename_provider, [:old_provider, :new_provider] => :environment  do |_, args|
+      old_provider = args[:old_provider] ||
+        prompt('What is the old provider? Ex. \'ldapmain\': '.color(:blue))
+      new_provider = args[:new_provider] ||
+        prompt('What is the new provider ID? Ex. \'ldapcustom\': '.color(:blue))
+      puts '' # Add some separation in the output
+
+      identities = Identity.where(provider: old_provider)
+      identity_count = identities.count
+
+      if identities.empty?
+        puts "Found no user identities with '#{old_provider}' provider."
+        puts 'Please check the provider name and try again.'
+        exit 1
+      end
+
+      plural_id_count = ActionController::Base.helpers.pluralize(identity_count, 'user')
+
+      unless ENV['force'] == 'yes'
+        puts "#{plural_id_count} with provider '#{old_provider}' will be updated to '#{new_provider}'"
+        puts 'If the new provider is incorrect, users will be unable to sign in'
+        ask_to_continue
+        puts ''
+      end
+
+      updated_count = identities.update_all(provider: new_provider)
+
+      if updated_count == identity_count
+        puts 'User identities were successfully updated'.color(:green)
+      else
+        plural_updated_count = ActionController::Base.helpers.pluralize(updated_count, 'user')
+        puts 'Some user identities could not be updated'.color(:red)
+        puts "Successfully updated #{plural_updated_count} out of #{plural_id_count} total"
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 210899882b4..ee2cdcdea1b 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -1,48 +1,28 @@
 namespace :gitlab do
   namespace :shell do
     desc "GitLab | Install or upgrade gitlab-shell"
-    task :install, [:tag, :repo] => :environment do |t, args|
+    task :install, [:repo] => :environment do |t, args|
       warn_user_is_not_gitlab
 
       default_version = Gitlab::Shell.version_required
-      default_version_tag = 'v' + default_version
-      args.with_defaults(tag: default_version_tag, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git")
+      args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git')
 
-      user = Gitlab.config.gitlab.user
-      home_dir = Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home
       gitlab_url = Gitlab.config.gitlab.url
       # gitlab-shell requires a / at the end of the url
       gitlab_url += '/' unless gitlab_url.end_with?('/')
       target_dir = Gitlab.config.gitlab_shell.path
 
-      # Clone if needed
-      if File.directory?(target_dir)
-        Dir.chdir(target_dir) do
-          system(*%W(Gitlab.config.git.bin_path} fetch --tags --quiet))
-          system(*%W(Gitlab.config.git.bin_path} checkout --quiet #{default_version_tag}))
-        end
-      else
-        system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir}))
-      end
+      checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir)
 
       # Make sure we're on the right tag
       Dir.chdir(target_dir) do
-        # First try to checkout without fetching
-        # to avoid stalling tests if the Internet is down.
-        reseted = reset_to_commit(args)
-
-        unless reseted
-          system(*%W(#{Gitlab.config.git.bin_path} fetch origin))
-          reset_to_commit(args)
-        end
-
         config = {
-          user: user,
+          user: Gitlab.config.gitlab.user,
           gitlab_url: gitlab_url,
-          http_settings: {self_signed_cert: false}.stringify_keys,
-          auth_file: File.join(home_dir, ".ssh", "authorized_keys"),
+          http_settings: { self_signed_cert: false }.stringify_keys,
+          auth_file: File.join(user_home, ".ssh", "authorized_keys"),
           redis: {
-            bin: %x{which redis-cli}.chomp,
+            bin: `which redis-cli`.chomp,
             namespace: "resque:gitlab"
           }.stringify_keys,
           log_level: "INFO",
@@ -61,20 +41,26 @@ namespace :gitlab do
         # Generate config.yml based on existing gitlab settings
         File.open("config.yml", "w+") {|f| f.puts config.to_yaml}
 
-        # Launch installation process
-        system(*%W(bin/install) + repository_storage_paths_args)
-
-        # (Re)create hooks
-        system(*%W(bin/create-hooks) + repository_storage_paths_args)
+        [
+          %w(bin/install) + repository_storage_paths_args,
+          %w(bin/compile)
+        ].each do |cmd|
+          unless Kernel.system(*cmd)
+            raise "command failed: #{cmd.join(' ')}"
+          end
+        end
       end
 
+      # (Re)create hooks
+      Rake::Task['gitlab:shell:create_hooks'].invoke
+
       # Required for debian packaging with PKGR: Setup .ssh/environment with
       # the current PATH, so that the correct ruby version gets loaded
       # Requires to set "PermitUserEnvironment yes" in sshd config (should not
       # be an issue since it is more than likely that there are no "normal"
       # user accounts on a gitlab server). The alternative is for the admin to
       # install a ruby (1.9.3+) in the global path.
-      File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f|
+      File.open(File.join(user_home, ".ssh", "environment"), "w+") do |f|
         f.puts "PATH=#{ENV['PATH']}"
       end
 
@@ -102,6 +88,15 @@ namespace :gitlab do
         end
       end
     end
+
+    desc 'Create or repair repository hooks symlink'
+    task create_hooks: :environment do
+      warn_user_is_not_gitlab
+
+      puts 'Creating/Repairing hooks symlinks for all repositories'
+      system(*%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args)
+      puts 'done'.color(:green)
+    end
   end
 
   def setup
@@ -133,15 +128,4 @@ namespace :gitlab do
     puts "Quitting...".color(:red)
     exit 1
   end
-
-  def reset_to_commit(args)
-    tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag}))
-
-    unless status.zero?
-      tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag}))
-    end
-
-    tag = tag.strip
-    system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag}))
-  end
 end
diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake
index 7e2a6668e59..6cbc83b8973 100644
--- a/lib/tasks/gitlab/sidekiq.rake
+++ b/lib/tasks/gitlab/sidekiq.rake
@@ -1,13 +1,13 @@
 namespace :gitlab do
   namespace :sidekiq do
-    QUEUE = 'queue:post_receive'
+    QUEUE = 'queue:post_receive'.freeze
 
     desc 'Drop all Sidekiq PostReceive jobs for a given project'
-    task :drop_post_receive , [:project] => :environment do |t, args|
+    task :drop_post_receive, [:project] => :environment do |t, args|
       unless args.project.present?
         abort "Please specify the project you want to drop PostReceive jobs for:\n  rake gitlab:sidekiq:drop_post_receive[group/project]"
       end
-      project_path = Project.find_with_namespace(args.project).repository.path_to_repo
+      project_path = Project.find_by_full_path(args.project).repository.path_to_repo
 
       Sidekiq.redis do |redis|
         unless redis.exists(QUEUE)
@@ -21,7 +21,7 @@ namespace :gitlab do
         # new jobs already. We will repopulate it with the old jobs, skipping the
         # ones we want to drop.
         dropped = 0
-        while (job = redis.lpop(temp_queue)) do
+        while (job = redis.lpop(temp_queue))
           if repo_path(job) == project_path
             dropped += 1
           else
diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake
deleted file mode 100644
index 74be413423a..00000000000
--- a/lib/tasks/gitlab/task_helpers.rake
+++ /dev/null
@@ -1,140 +0,0 @@
-module Gitlab
-  class TaskAbortedByUserError < StandardError; end
-end
-
-require 'rainbow/ext/string'
-
-# Prevent StateMachine warnings from outputting during a cron task
-StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON']
-
-namespace :gitlab do
-
-  # Ask if the user wants to continue
-  #
-  # Returns "yes" the user chose to continue
-  # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue
-  def ask_to_continue
-    answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no})
-    raise Gitlab::TaskAbortedByUserError unless answer == "yes"
-  end
-
-  # Check which OS is running
-  #
-  # It will primarily use lsb_relase to determine the OS.
-  # It has fallbacks to Debian, SuSE, OS X and systems running systemd.
-  def os_name
-    os_name = run_command(%W(lsb_release -irs))
-    os_name ||= if File.readable?('/etc/system-release')
-                  File.read('/etc/system-release')
-                end
-    os_name ||= if File.readable?('/etc/debian_version')
-                  debian_version = File.read('/etc/debian_version')
-                  "Debian #{debian_version}"
-                end
-    os_name ||= if File.readable?('/etc/SuSE-release')
-                  File.read('/etc/SuSE-release')
-                end
-    os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion))
-                  "Mac OS X #{os_x_version}"
-                end
-    os_name ||= if File.readable?('/etc/os-release')
-                  File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1]
-                end
-    os_name.try(:squish!)
-  end
-
-  # Prompt the user to input something
-  #
-  # message - the message to display before input
-  # choices - array of strings of acceptable answers or nil for any answer
-  #
-  # Returns the user's answer
-  def prompt(message, choices = nil)
-    begin
-      print(message)
-      answer = STDIN.gets.chomp
-    end while choices.present? && !choices.include?(answer)
-    answer
-  end
-
-  # Runs the given command and matches the output against the given pattern
-  #
-  # Returns nil if nothing matched
-  # Returns the MatchData if the pattern matched
-  #
-  # see also #run_command
-  # see also String#match
-  def run_and_match(command, regexp)
-    run_command(command).try(:match, regexp)
-  end
-
-  # Runs the given command
-  #
-  # Returns nil if the command was not found
-  # Returns the output of the command otherwise
-  #
-  # see also #run_and_match
-  def run_command(command)
-    output, _ = Gitlab::Popen.popen(command)
-    output
-  rescue Errno::ENOENT
-    '' # if the command does not exist, return an empty string
-  end
-
-  def uid_for(user_name)
-    run_command(%W(id -u #{user_name})).chomp.to_i
-  end
-
-  def gid_for(group_name)
-    begin
-      Etc.getgrnam(group_name).gid
-    rescue ArgumentError # no group
-      "group #{group_name} doesn't exist"
-    end
-  end
-
-  def warn_user_is_not_gitlab
-    unless @warned_user_not_gitlab
-      gitlab_user = Gitlab.config.gitlab.user
-      current_user = run_command(%W(whoami)).chomp
-      unless current_user == gitlab_user
-        puts " Warning ".color(:black).background(:yellow)
-        puts "  You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing."
-        puts "  Things may work\/fail for the wrong reasons."
-        puts "  For correct results you should run this as user #{gitlab_user.color(:magenta)}."
-        puts ""
-      end
-      @warned_user_not_gitlab = true
-    end
-  end
-
-  # Tries to configure git itself
-  #
-  # Returns true if all subcommands were successfull (according to their exit code)
-  # Returns false if any or all subcommands failed.
-  def auto_fix_git_config(options)
-    if !@warned_user_not_gitlab
-      command_success = options.map do |name, value|
-        system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value}))
-      end
-
-      command_success.all?
-    else
-      false
-    end
-  end
-
-  def all_repos
-    Gitlab.config.repositories.storages.each do |name, path|
-      IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
-        find.each_line do |path|
-          yield path.chomp
-        end
-      end
-    end
-  end
-
-  def repository_storage_paths_args
-    Gitlab.config.repositories.storages.values
-  end
-end
diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb
new file mode 100644
index 00000000000..964aa0fe1bc
--- /dev/null
+++ b/lib/tasks/gitlab/task_helpers.rb
@@ -0,0 +1,172 @@
+require 'rainbow/ext/string'
+
+module Gitlab
+  TaskFailedError = Class.new(StandardError)
+  TaskAbortedByUserError = Class.new(StandardError)
+
+  module TaskHelpers
+    # Ask if the user wants to continue
+    #
+    # Returns "yes" the user chose to continue
+    # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue
+    def ask_to_continue
+      answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no})
+      raise Gitlab::TaskAbortedByUserError unless answer == "yes"
+    end
+
+    # Check which OS is running
+    #
+    # It will primarily use lsb_relase to determine the OS.
+    # It has fallbacks to Debian, SuSE, OS X and systems running systemd.
+    def os_name
+      os_name = run_command(%w(lsb_release -irs))
+      os_name ||=
+        if File.readable?('/etc/system-release')
+          File.read('/etc/system-release')
+        elsif File.readable?('/etc/debian_version')
+          "Debian #{File.read('/etc/debian_version')}"
+        elsif File.readable?('/etc/SuSE-release')
+          File.read('/etc/SuSE-release')
+        elsif os_x_version = run_command(%w(sw_vers -productVersion))
+          "Mac OS X #{os_x_version}"
+        elsif File.readable?('/etc/os-release')
+          File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1]
+        end
+
+      os_name.try(:squish!)
+    end
+
+    # Prompt the user to input something
+    #
+    # message - the message to display before input
+    # choices - array of strings of acceptable answers or nil for any answer
+    #
+    # Returns the user's answer
+    def prompt(message, choices = nil)
+      begin
+        print(message)
+        answer = STDIN.gets.chomp
+      end while choices.present? && !choices.include?(answer)
+      answer
+    end
+
+    # Runs the given command and matches the output against the given pattern
+    #
+    # Returns nil if nothing matched
+    # Returns the MatchData if the pattern matched
+    #
+    # see also #run_command
+    # see also String#match
+    def run_and_match(command, regexp)
+      run_command(command).try(:match, regexp)
+    end
+
+    # Runs the given command
+    #
+    # Returns '' if the command was not found
+    # Returns the output of the command otherwise
+    #
+    # see also #run_and_match
+    def run_command(command)
+      output, _ = Gitlab::Popen.popen(command)
+      output
+    rescue Errno::ENOENT
+      '' # if the command does not exist, return an empty string
+    end
+
+    # Runs the given command and raises a Gitlab::TaskFailedError exception if
+    # the command does not exit with 0
+    #
+    # Returns the output of the command otherwise
+    def run_command!(command)
+      output, status = Gitlab::Popen.popen(command)
+
+      raise Gitlab::TaskFailedError.new(output) unless status.zero?
+
+      output
+    end
+
+    def uid_for(user_name)
+      run_command(%W(id -u #{user_name})).chomp.to_i
+    end
+
+    def gid_for(group_name)
+      begin
+        Etc.getgrnam(group_name).gid
+      rescue ArgumentError # no group
+        "group #{group_name} doesn't exist"
+      end
+    end
+
+    def gitlab_user
+      Gitlab.config.gitlab.user
+    end
+
+    def is_gitlab_user?
+      return @is_gitlab_user unless @is_gitlab_user.nil?
+
+      current_user = run_command(%w(whoami)).chomp
+      @is_gitlab_user = current_user == gitlab_user
+    end
+
+    def warn_user_is_not_gitlab
+      return if @warned_user_not_gitlab
+
+      unless is_gitlab_user?
+        current_user = run_command(%w(whoami)).chomp
+
+        puts " Warning ".color(:black).background(:yellow)
+        puts "  You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing."
+        puts "  Things may work\/fail for the wrong reasons."
+        puts "  For correct results you should run this as user #{gitlab_user.color(:magenta)}."
+        puts ""
+
+        @warned_user_not_gitlab = true
+      end
+    end
+
+    def all_repos
+      Gitlab.config.repositories.storages.each_value do |repository_storage|
+        IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find|
+          find.each_line do |path|
+            yield path.chomp
+          end
+        end
+      end
+    end
+
+    def repository_storage_paths_args
+      Gitlab.config.repositories.storages.values.map { |rs| rs['path'] }
+    end
+
+    def user_home
+      Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home
+    end
+
+    def checkout_or_clone_version(version:, repo:, target_dir:)
+      version =
+        if version.starts_with?("=")
+          version.sub(/\A=/, '') # tag or branch
+        else
+          "v#{version}" # tag
+        end
+
+      clone_repo(repo, target_dir) unless Dir.exist?(target_dir)
+      checkout_version(version, target_dir)
+      reset_to_version(version, target_dir)
+    end
+
+    def clone_repo(repo, target_dir)
+      run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}])
+    end
+
+    def checkout_version(version, target_dir)
+      run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --quiet])
+      run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{version}])
+    end
+
+    def reset_to_version(version, target_dir)
+      run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{version}])
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/test.rake b/lib/tasks/gitlab/test.rake
index 4d4e746503a..523b0fa055b 100644
--- a/lib/tasks/gitlab/test.rake
+++ b/lib/tasks/gitlab/test.rake
@@ -2,15 +2,15 @@ namespace :gitlab do
   desc "GitLab | Run all tests"
   task :test do
     cmds = [
-      %W(rake brakeman),
-      %W(rake rubocop),
-      %W(rake spinach),
-      %W(rake spec),
-      %W(rake teaspoon)
+      %w(rake brakeman),
+      %w(rake rubocop),
+      %w(rake spinach),
+      %w(rake spec),
+      %w(rake karma)
     ]
 
     cmds.each do |cmd|
-      system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!")
+      system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!")
     end
   end
 end
diff --git a/lib/tasks/gitlab/track_deployment.rake b/lib/tasks/gitlab/track_deployment.rake
index 84aa2e8507a..6f101aea303 100644
--- a/lib/tasks/gitlab/track_deployment.rake
+++ b/lib/tasks/gitlab/track_deployment.rake
@@ -1,8 +1,8 @@
 namespace :gitlab do
   desc 'GitLab | Tracks a deployment in GitLab Performance Monitoring'
   task track_deployment: :environment do
-    metric = Gitlab::Metrics::Metric.
-      new('deployments', version: Gitlab::VERSION)
+    metric = Gitlab::Metrics::Metric
+      .new('deployments', version: Gitlab::VERSION)
 
     Gitlab::Metrics.submit_metrics([metric.to_hash])
   end
diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake
index fc0ccc726ed..7728c485e8d 100644
--- a/lib/tasks/gitlab/two_factor.rake
+++ b/lib/tasks/gitlab/two_factor.rake
@@ -19,5 +19,21 @@ namespace :gitlab do
         puts "There are currently no users with 2FA enabled.".color(:yellow)
       end
     end
+
+    namespace :rotate_key do
+      def rotator
+        @rotator ||= Gitlab::OtpKeyRotator.new(ENV['filename'])
+      end
+
+      desc "Encrypt user OTP secrets with a new encryption key"
+      task apply: :environment do |t, args|
+        rotator.rotate!(old_key: ENV['old_key'], new_key: ENV['new_key'])
+      end
+
+      desc "Rollback to secrets encrypted with the old encryption key"
+      task rollback: :environment do
+        rotator.rollback!
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/update_commit_count.rake b/lib/tasks/gitlab/update_commit_count.rake
deleted file mode 100644
index 3bd10b0208b..00000000000
--- a/lib/tasks/gitlab/update_commit_count.rake
+++ /dev/null
@@ -1,20 +0,0 @@
-namespace :gitlab do
-  desc "GitLab | Update commit count for projects"
-  task update_commit_count: :environment do
-    projects = Project.where(commit_count: 0)
-    puts "#{projects.size} projects need to be updated. This might take a while."
-    ask_to_continue unless ENV['force'] == 'yes'
-
-    projects.find_each(batch_size: 100) do |project|
-      print "#{project.name_with_namespace.color(:yellow)} ... "
-
-      unless project.repo_exists?
-        puts "skipping, because the repo is empty".color(:magenta)
-        next
-      end
-
-      project.update_commit_count
-      puts project.commit_count.to_s.color(:green)
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index 4f76dad7286..59c32bbe7a4 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -5,7 +5,7 @@ namespace :gitlab do
   end
 
   def update(template)
-    sub_dir = template.repo_url.match(/([a-z-]+)\.git\z/)[1]
+    sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1]
     dir = File.join(vendor_directory, sub_dir)
 
     unless clone_repository(template.repo_url, dir)
@@ -44,9 +44,13 @@ namespace :gitlab do
     ),
     Template.new(
       "https://gitlab.com/gitlab-org/gitlab-ci-yml.git",
-      /(\.{1,2}|LICENSE|Pages|\.gitlab-ci.yml)\z/
+      /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/
+    ),
+    Template.new(
+      "https://gitlab.com/gitlab-org/Dockerfile.git",
+      /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/
     )
-  ]
+  ].freeze
 
   def vendor_directory
     Rails.root.join('vendor')
diff --git a/lib/tasks/gitlab/users.rake b/lib/tasks/gitlab/users.rake
new file mode 100644
index 00000000000..3a16ace60bd
--- /dev/null
+++ b/lib/tasks/gitlab/users.rake
@@ -0,0 +1,11 @@
+namespace :gitlab do
+  namespace :users do
+    desc "GitLab | Clear the authentication token for all users"
+    task clear_all_authentication_tokens: :environment  do |t, args|
+      # Do small batched updates because these updates will be slow and locking
+      User.select(:id).find_in_batches(batch_size: 100) do |batch|
+        User.where(id: batch.map(&:id)).update_all(authentication_token: nil)
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake
index 49530e7a372..5a1c8006052 100644
--- a/lib/tasks/gitlab/web_hook.rake
+++ b/lib/tasks/gitlab/web_hook.rake
@@ -1,7 +1,7 @@
 namespace :gitlab do
   namespace :web_hook do
     desc "GitLab | Adds a webhook to the projects"
-    task :add => :environment do
+    task add: :environment do
       web_hook_url = ENV['URL']
       namespace_path = ENV['NAMESPACE']
 
@@ -21,7 +21,7 @@ namespace :gitlab do
     end
 
     desc "GitLab | Remove a webhook from the projects"
-    task :rm => :environment do
+    task rm: :environment do
       web_hook_url = ENV['URL']
       namespace_path = ENV['NAMESPACE']
 
@@ -34,7 +34,7 @@ namespace :gitlab do
     end
 
     desc "GitLab | List webhooks"
-    task :list => :environment do
+    task list: :environment do
       namespace_path = ENV['NAMESPACE']
 
       projects = find_projects(namespace_path)
diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake
new file mode 100644
index 00000000000..e7ac0b5859f
--- /dev/null
+++ b/lib/tasks/gitlab/workhorse.rake
@@ -0,0 +1,23 @@
+namespace :gitlab do
+  namespace :workhorse do
+    desc "GitLab | Install or upgrade gitlab-workhorse"
+    task :install, [:dir, :repo] => :environment do |t, args|
+      warn_user_is_not_gitlab
+      unless args.dir.present?
+        abort %(Please specify the directory where you want to install gitlab-workhorse:\n  rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]")
+      end
+      args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git')
+
+      version = Gitlab::Workhorse.version
+
+      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
+
+      _, status = Gitlab::Popen.popen(%w[which gmake])
+      command = status.zero? ? 'gmake' : 'make'
+
+      Dir.chdir(args.dir) do
+        run_command!([command])
+      end
+    end
+  end
+end
diff --git a/lib/tasks/grape.rake b/lib/tasks/grape.rake
index 9980e0b7984..ea2698da606 100644
--- a/lib/tasks/grape.rake
+++ b/lib/tasks/grape.rake
@@ -2,7 +2,11 @@ namespace :grape do
   desc 'Print compiled grape routes'
   task routes: :environment do
     API::API.routes.each do |route|
-      puts route
+      puts "#{route.options[:method]} #{route.path} - #{route_description(route.options)}"
     end
   end
+
+  def route_description(options)
+    options[:settings][:description][:description] if options[:settings][:description]
+  end
 end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
new file mode 100644
index 00000000000..50b8e331469
--- /dev/null
+++ b/lib/tasks/import.rake
@@ -0,0 +1,142 @@
+require 'benchmark'
+require 'rainbow/ext/string'
+
+class GithubImport
+  def self.run!(*args)
+    new(*args).run!
+  end
+
+  def initialize(token, gitlab_username, project_path, extras)
+    @options = { url: 'https://api.github.com', token: token, verbose: true }
+    @project_path = project_path
+    @current_user = User.find_by_username(gitlab_username)
+    @github_repo = extras.empty? ? nil : extras.first
+  end
+
+  def run!
+    @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one!
+
+    raise 'No repo found!' unless @repo
+
+    show_warning!
+
+    @project = Project.find_by_full_path(@project_path) || new_project
+
+    import!
+  end
+
+  private
+
+  def show_warning!
+    puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}"
+    puts "Permission checks are ignored. Press any key to continue.".color(:red)
+
+    STDIN.getch
+
+    puts 'Starting the import (this could take a while)'.color(:green)
+  end
+
+  def import!
+    @project.force_import_start
+
+    timings = Benchmark.measure do
+      Github::Import.new(@project, @options).execute
+    end
+
+    puts "Import finished. Timings: #{timings}".color(:green)
+
+    @project.import_finish
+  end
+
+  def new_project
+    Project.transaction do
+      namespace_path, _sep, name = @project_path.rpartition('/')
+      namespace = find_or_create_namespace(namespace_path)
+
+      Projects::CreateService.new(
+        @current_user,
+        name: name,
+        path: name,
+        description: @repo['description'],
+        namespace_id: namespace.id,
+        visibility_level: visibility_level,
+        import_type: 'github',
+        import_source: @repo['full_name'],
+        skip_wiki: @repo['has_wiki']
+      ).execute
+    end
+  end
+
+  def find_or_create_namespace(names)
+    return @current_user.namespace if names == @current_user.namespace_path
+    return @current_user.namespace unless @current_user.can_create_group?
+
+    full_path_namespace = Namespace.find_by_full_path(names)
+
+    return full_path_namespace if full_path_namespace
+
+    names.split('/').inject(nil) do |parent, name|
+      begin
+        namespace = Group.create!(name: name,
+                                  path: name,
+                                  owner: @current_user,
+                                  parent: parent)
+        namespace.add_owner(@current_user)
+
+        namespace
+      rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
+        Namespace.where(parent: parent).find_by_path_or_name(name)
+      end
+    end
+  end
+
+  def full_path_namespace(names)
+    @full_path_namespace ||= Namespace.find_by_full_path(names)
+  end
+
+  def visibility_level
+    @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
+  end
+end
+
+class GithubRepos
+  def initialize(options, current_user, github_repo)
+    @options = options
+    @current_user = current_user
+    @github_repo = github_repo
+  end
+
+  def choose_one!
+    return found_github_repo if @github_repo
+
+    repos.each do |repo|
+      print "ID: #{repo['id'].to_s.bright}".color(:green)
+      print "\tName: #{repo['full_name']}\n".color(:green)
+    end
+
+    print 'ID? '.bright
+
+    repos.find { |repo| repo['id'] == repo_id }
+  end
+
+  def found_github_repo
+    repos.find { |repo| repo['full_name'] == @github_repo }
+  end
+
+  def repo_id
+    @repo_id ||= STDIN.gets.chomp.to_i
+  end
+
+  def repos
+    Github::Repositories.new(@options).fetch
+  end
+end
+
+namespace :import do
+  desc 'Import a GitHub project - Example: import:github[ToKeN,root,root/blah,my/github_repo] (optional my/github_repo)'
+  task :github, [:token, :gitlab_username, :project_path] => :environment do |_t, args|
+    abort 'Project path must be: namespace(s)/project_name'.color(:red) unless args.project_path.include?('/')
+
+    GithubImport.run!(args.token, args.gitlab_username, args.project_path, args.extras)
+  end
+end
diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake
new file mode 100644
index 00000000000..62a12174efa
--- /dev/null
+++ b/lib/tasks/karma.rake
@@ -0,0 +1,21 @@
+unless Rails.env.production?
+  namespace :karma do
+    desc 'GitLab | Karma | Generate fixtures for JavaScript tests'
+    RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args|
+      args.with_defaults(pattern: 'spec/javascripts/fixtures/*.rb')
+      ENV['NO_KNAPSACK'] = 'true'
+      t.pattern = args[:pattern]
+      t.rspec_opts = '--format documentation'
+    end
+
+    desc 'GitLab | Karma | Run JavaScript tests'
+    task tests: ['yarn:check'] do
+      sh "yarn run karma" do |ok, res|
+        abort('rake karma:tests failed') unless ok
+      end
+    end
+  end
+
+  desc 'GitLab | Karma | Shortcut for karma:fixtures and karma:tests'
+  task karma: ['karma:fixtures', 'karma:tests']
+end
diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake
new file mode 100644
index 00000000000..7b63e93db0e
--- /dev/null
+++ b/lib/tasks/lint.rake
@@ -0,0 +1,8 @@
+unless Rails.env.production?
+  namespace :lint do
+    desc "GitLab | lint | Lint JavaScript files using ESLint"
+    task :javascript do
+      Rake::Task['eslint'].invoke
+    end
+  end
+end
diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake
index 6ded519aff2..151f42a2222 100644
--- a/lib/tasks/migrate/add_limits_mysql.rake
+++ b/lib/tasks/migrate/add_limits_mysql.rake
@@ -1,7 +1,11 @@
 require Rails.root.join('db/migrate/limits_to_mysql')
+require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql')
+require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql')
 
 desc "GitLab | Add limits to strings in mysql database"
 task add_limits_mysql: :environment do
   puts "Adding limits to schema.rb for mysql"
   LimitsToMysql.new.up
+  MarkdownCacheLimitsToMysql.new.up
+  MergeRequestDiffFileLimitsToMysql.new.up
 end
diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake
index 4f2486157b7..fc2cea8c016 100644
--- a/lib/tasks/migrate/migrate_iids.rake
+++ b/lib/tasks/migrate/migrate_iids.rake
@@ -24,7 +24,7 @@ task migrate_iids: :environment do
       else
         print 'F'
       end
-    rescue => ex
+    rescue
       print 'F'
     end
   end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index 141a0b74ec0..4108cee08b4 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -1,8 +1,16 @@
+require Rails.root.join('lib/gitlab/database')
+require Rails.root.join('lib/gitlab/database/migration_helpers')
 require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes')
 require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes')
+require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes')
+require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like')
+require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like')
 
 desc 'GitLab | Sets up PostgreSQL'
 task setup_postgresql: :environment do
   NamespacesProjectsPathLowerIndexes.new.up
   AddUsersLowerUsernameEmailIndexes.new.up
+  AddLowerPathIndexToRoutes.new.up
+  IndexRoutesPathForLike.new.up
+  IndexRedirectRoutesPathForLike.new.up
 end
diff --git a/lib/tasks/services.rake b/lib/tasks/services.rake
index 39541c0b9c6..56b81106c5f 100644
--- a/lib/tasks/services.rake
+++ b/lib/tasks/services.rake
@@ -76,23 +76,23 @@ namespace :services do
         end
 
         param_hash
-      end.sort_by { |p| p[:required] ? 0 : 1 }
+      end
+      service_hash[:params].sort_by! { |p| p[:required] ? 0 : 1 }
 
-      puts "Collected data for: #{service.title}, #{Time.now-service_start}"
+      puts "Collected data for: #{service.title}, #{Time.now - service_start}"
       service_hash
     end
 
     doc_start = Time.now
     doc_path = File.join(Rails.root, 'doc', 'api', 'services.md')
 
-    result = ERB.new(services_template, 0 , '>')
+    result = ERB.new(services_template, 0, '>')
       .result(OpenStruct.new(services: services).instance_eval { binding })
 
     File.open(doc_path, 'w') do |f|
       f.write result
     end
 
-    puts "write a new service.md to: #{doc_path.to_s}, #{Time.now-doc_start}"
-
+    puts "write a new service.md to: #{doc_path}, #{Time.now - doc_start}"
   end
 end
diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake
index d1f6ed87704..dd9ce86f7ca 100644
--- a/lib/tasks/sidekiq.rake
+++ b/lib/tasks/sidekiq.rake
@@ -1,21 +1,21 @@
 namespace :sidekiq do
   desc "GitLab | Stop sidekiq"
   task :stop do
-    system *%W(bin/background_jobs stop)
+    system(*%w(bin/background_jobs stop))
   end
 
   desc "GitLab | Start sidekiq"
   task :start do
-    system *%W(bin/background_jobs start)
+    system(*%w(bin/background_jobs start))
   end
 
   desc 'GitLab | Restart sidekiq'
   task :restart do
-    system *%W(bin/background_jobs restart)
+    system(*%w(bin/background_jobs restart))
   end
 
   desc "GitLab | Start sidekiq with launchd on Mac OS X"
   task :launchd do
-    system *%W(bin/background_jobs start_no_deamonize)
+    system(*%w(bin/background_jobs start_no_deamonize))
   end
 end
diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake
index 2cf7a25a0fd..2eddcb3c777 100644
--- a/lib/tasks/spec.rake
+++ b/lib/tasks/spec.rake
@@ -4,8 +4,8 @@ namespace :spec do
   desc 'GitLab | Rspec | Run request specs'
   task :api do
     cmds = [
-      %W(rake gitlab:setup),
-      %W(rspec spec --tag @api)
+      %w(rake gitlab:setup),
+      %w(rspec spec --tag @api)
     ]
     run_commands(cmds)
   end
@@ -13,8 +13,8 @@ namespace :spec do
   desc 'GitLab | Rspec | Run feature specs'
   task :feature do
     cmds = [
-      %W(rake gitlab:setup),
-      %W(rspec spec --tag @feature)
+      %w(rake gitlab:setup),
+      %w(rspec spec --tag @feature)
     ]
     run_commands(cmds)
   end
@@ -22,8 +22,8 @@ namespace :spec do
   desc 'GitLab | Rspec | Run model specs'
   task :models do
     cmds = [
-      %W(rake gitlab:setup),
-      %W(rspec spec --tag @models)
+      %w(rake gitlab:setup),
+      %w(rspec spec --tag @models)
     ]
     run_commands(cmds)
   end
@@ -31,8 +31,8 @@ namespace :spec do
   desc 'GitLab | Rspec | Run service specs'
   task :services do
     cmds = [
-      %W(rake gitlab:setup),
-      %W(rspec spec --tag @services)
+      %w(rake gitlab:setup),
+      %w(rspec spec --tag @services)
     ]
     run_commands(cmds)
   end
@@ -40,8 +40,8 @@ namespace :spec do
   desc 'GitLab | Rspec | Run lib specs'
   task :lib do
     cmds = [
-      %W(rake gitlab:setup),
-      %W(rspec spec --tag @lib)
+      %w(rake gitlab:setup),
+      %w(rspec spec --tag @lib)
     ]
     run_commands(cmds)
   end
@@ -49,8 +49,8 @@ namespace :spec do
   desc 'GitLab | Rspec | Run other specs'
   task :other do
     cmds = [
-      %W(rake gitlab:setup),
-      %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services)
+      %w(rake gitlab:setup),
+      %w(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services)
     ]
     run_commands(cmds)
   end
@@ -59,14 +59,14 @@ end
 desc "GitLab | Run specs"
 task :spec do
   cmds = [
-    %W(rake gitlab:setup),
-    %W(rspec spec),
+    %w(rake gitlab:setup),
+    %w(rspec spec)
   ]
   run_commands(cmds)
 end
 
 def run_commands(cmds)
   cmds.each do |cmd|
-    system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!")
+    system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!")
   end
 end
diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake
index 8dbfa7751dc..19ff13f06c0 100644
--- a/lib/tasks/spinach.rake
+++ b/lib/tasks/spinach.rake
@@ -35,7 +35,7 @@ task :spinach do
 end
 
 def run_system_command(cmd)
-  system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd)
+  system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd)
 end
 
 def run_spinach_command(args)
diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake
index d3dcbd2c29b..3e01f91d32c 100644
--- a/lib/tasks/test.rake
+++ b/lib/tasks/test.rake
@@ -7,5 +7,5 @@ end
 
 unless Rails.env.production?
   desc "GitLab | Run all tests on CI with simplecov"
-  task test_ci: [:rubocop, :brakeman, :teaspoon, :spinach, :spec]
+  task test_ci: [:rubocop, :brakeman, :karma, :spinach, :spec]
 end
diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake
new file mode 100644
index 00000000000..ad1818ff1fa
--- /dev/null
+++ b/lib/tasks/tokens.rake
@@ -0,0 +1,48 @@
+require_relative '../../app/models/concerns/token_authenticatable.rb'
+
+namespace :tokens do
+  desc "Reset all GitLab user auth tokens"
+  task reset_all_auth: :environment do
+    reset_all_users_token(:reset_authentication_token!)
+  end
+
+  desc "Reset all GitLab email tokens"
+  task reset_all_email: :environment do
+    reset_all_users_token(:reset_incoming_email_token!)
+  end
+
+  desc "Reset all GitLab RSS tokens"
+  task reset_all_rss: :environment do
+    reset_all_users_token(:reset_rss_token!)
+  end
+
+  def reset_all_users_token(reset_token_method)
+    TmpUser.find_in_batches do |batch|
+      puts "Processing batch starting with user ID: #{batch.first.id}"
+      STDOUT.flush
+
+      batch.each(&reset_token_method)
+    end
+  end
+end
+
+class TmpUser < ActiveRecord::Base
+  include TokenAuthenticatable
+
+  self.table_name = 'users'
+
+  def reset_authentication_token!
+    write_new_token(:authentication_token)
+    save!(validate: false)
+  end
+
+  def reset_incoming_email_token!
+    write_new_token(:incoming_email_token)
+    save!(validate: false)
+  end
+
+  def reset_rss_token!
+    write_new_token(:rss_token)
+    save!(validate: false)
+  end
+end
diff --git a/lib/tasks/yarn.rake b/lib/tasks/yarn.rake
new file mode 100644
index 00000000000..2ac88a039e7
--- /dev/null
+++ b/lib/tasks/yarn.rake
@@ -0,0 +1,40 @@
+
+namespace :yarn do
+  desc 'Ensure Yarn is installed'
+  task :available do
+    unless system('yarn --version', out: File::NULL)
+      warn(
+        'Error: Yarn executable was not detected in the system.'.color(:red),
+        'Download Yarn at https://yarnpkg.com/en/docs/install'.color(:green)
+      )
+      abort
+    end
+  end
+
+  desc 'Ensure Node dependencies are installed'
+  task check: ['yarn:available'] do
+    unless system('yarn check --ignore-engines', out: File::NULL)
+      warn(
+        'Error: You have unmet dependencies. (`yarn check` command failed)'.color(:red),
+        'Run `yarn install` to install missing modules.'.color(:green)
+      )
+      abort
+    end
+  end
+
+  desc 'Install Node dependencies with Yarn'
+  task install: ['yarn:available'] do
+    unless system('yarn install --pure-lockfile --ignore-engines')
+      abort 'Error: Unable to install node modules.'.color(:red)
+    end
+  end
+
+  desc 'Remove Node dependencies'
+  task :clobber do
+    warn 'Purging ./node_modules directory'.color(:red)
+    FileUtils.rm_rf 'node_modules'
+  end
+end
+
+desc 'Install Node dependencies with Yarn'
+task yarn: ['yarn:install']