Merge branch 'master' into feature/gb/persist-pipeline-stages

* master: (524 commits)
  Improve user experience around slash commands in instant comments
  Fix LFS timeouts when trying to save large files
  retryable? is now available for CommitStatus
  Resolve "Documentation of `.gitlab-ci.yml` states incorrect example for variables"
  Fix test failures
  Add slugify project path to CI enviroment variables
  Fixed typo: craeted -> created
  32118 Make New environment empty state btn lowercase
  Expose import_status in Projects API
  32832 Remove overflow from comment form for confidential issues and vertically aligns confidential issue icon
  Fix test failures
  Allow manual bypass of auto_sign_in_with_provider
  Fix keys seed
  Allow users to be hard-deleted from the API
  fixup some classnames and media queries
  Enable the Style/PreferredHashMethods cop
  Lint our factories creation in addition to their build
  Don’t schedule workers from inside transactions
  Allow scheduling from after_commit hooks
  Forbid Sidekiq scheduling in transactions
  ...

Conflicts:
	app/serializers/pipeline_entity.rb
	db/schema.rb
	spec/factories/ci/stages.rb
	spec/lib/gitlab/import_export/safe_model_attributes.yml
	spec/services/ci/create_pipeline_service_spec.rb
	spec/spec_helper.rb

124 files changed, 2764 insertions, 1178 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 52cd7cbe3db..7ae2f3cad40 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -45,9 +45,9 @@ module API
     end
 
     before { allow_access_with_scope :api }
-    before { Gitlab::I18n.set_locale(current_user) }
+    before { Gitlab::I18n.locale = current_user&.preferred_language }
 
-    after { Gitlab::I18n.reset_locale }
+    after { Gitlab::I18n.use_default_locale }
 
     rescue_from Gitlab::Access::AccessDeniedError do
       rack_response({ 'message' => '403 Forbidden' }.to_json, 403)
@@ -94,6 +94,7 @@ module API
     mount ::API::DeployKeys
     mount ::API::Deployments
     mount ::API::Environments
+    mount ::API::Features
     mount ::API::Files
     mount ::API::Groups
     mount ::API::Internal
@@ -110,6 +111,7 @@ module API
     mount ::API::Notes
     mount ::API::NotificationSettings
     mount ::API::Pipelines
+    mount ::API::PipelineSchedules
     mount ::API::ProjectHooks
     mount ::API::Projects
     mount ::API::ProjectSnippets
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index 827a38d33da..10f2d5ef6a3 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -68,7 +68,14 @@ module API
 
         name = params[:name] || params[:context] || 'default'
 
-        pipeline = @project.ensure_pipeline(ref, commit.sha, current_user)
+        pipeline = @project.pipeline_for(ref, commit.sha)
+        unless pipeline
+          pipeline = @project.pipelines.create!(
+            source: :external,
+            sha: commit.sha,
+            ref: ref,
+            user: current_user)
+        end
 
         status = GenericCommitStatus.running_or_pending.find_or_initialize_by(
           project: @project,
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 621b9dcecd9..c6fc17cc391 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -176,7 +176,7 @@ module API
         }
 
         if params[:path]
-          commit.raw_diffs(all_diffs: true).each do |diff|
+          commit.raw_diffs(limits: false).each do |diff|
             next unless diff.new_path == params[:path]
             lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line)
 
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 01cc8e8e1ca..ded5c65e303 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -100,6 +100,8 @@ module API
       expose :creator_id
       expose :namespace, using: 'API::Entities::Namespace'
       expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? }
+      expose :import_status
+      expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] }
       expose :avatar_url do |user, options|
         user.avatar_url(only_path: false)
       end
@@ -152,7 +154,10 @@ module API
       expose :web_url
       expose :request_access_enabled
       expose :full_name, :full_path
-      expose :parent_id
+
+      if ::Group.supports_nested_groups?
+        expose :parent_id
+      end
 
       expose :statistics, if: :statistics do
         with_options format_with: -> (value) { value.to_i } do
@@ -252,7 +257,9 @@ module API
 
     class RepoDiff < Grape::Entity
       expose :old_path, :new_path, :a_mode, :b_mode, :diff
-      expose :new_file, :renamed_file, :deleted_file
+      expose :new_file?, as: :new_file
+      expose :renamed_file?, as: :renamed_file
+      expose :deleted_file?, as: :deleted_file
     end
 
     class Milestone < ProjectEntity
@@ -326,7 +333,7 @@ module API
 
     class MergeRequestChanges < MergeRequest
       expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _|
-        compare.raw_diffs(all_diffs: true).to_a
+        compare.raw_diffs(limits: false).to_a
       end
     end
 
@@ -339,7 +346,7 @@ module API
       expose :commits, using: Entities::RepoCommit
 
       expose :diffs, using: Entities::RepoDiff do |compare, _|
-        compare.raw_diffs(all_diffs: true).to_a
+        compare.raw_diffs(limits: false).to_a
       end
     end
 
@@ -543,7 +550,7 @@ module API
       end
 
       expose :diffs, using: Entities::RepoDiff do |compare, options|
-        compare.diffs(all_diffs: true).to_a
+        compare.diffs(limits: false).to_a
       end
 
       expose :compare_timeout do |compare, options|
@@ -670,6 +677,7 @@ module API
 
     class Variable < Grape::Entity
       expose :key, :value
+      expose :protected?, as: :protected
     end
 
     class Pipeline < PipelineBasic
@@ -681,6 +689,17 @@ module API
       expose :coverage
     end
 
+    class PipelineSchedule < Grape::Entity
+      expose :id
+      expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active
+      expose :created_at, :updated_at
+      expose :owner, using: Entities::UserBasic
+    end
+
+    class PipelineScheduleDetails < PipelineSchedule
+      expose :last_pipeline, using: Entities::PipelineBasic
+    end
+
     class EnvironmentBasic < Grape::Entity
       expose :id, :name, :slug, :external_url
     end
@@ -737,6 +756,28 @@ module API
       expose :impersonation
     end
 
+    class FeatureGate < Grape::Entity
+      expose :key
+      expose :value
+    end
+
+    class Feature < Grape::Entity
+      expose :name
+      expose :state
+      expose :gates, using: FeatureGate do |model|
+        model.gates.map do |gate|
+          value = model.gate_values[gate.key]
+
+          # By default all gate values are populated. Only show relevant ones.
+          if (value.is_a?(Integer) && value.zero?) || (value.is_a?(Set) && value.empty?)
+            next
+          end
+
+          { key: gate.key, value: value }
+        end.compact
+      end
+    end
+
     module JobRequest
       class JobInfo < Grape::Entity
         expose :name, :stage
diff --git a/lib/api/features.rb b/lib/api/features.rb
new file mode 100644
index 00000000000..cff0ba2ddff
--- /dev/null
+++ b/lib/api/features.rb
@@ -0,0 +1,36 @@
+module API
+  class Features < Grape::API
+    before { authenticated_as_admin! }
+
+    resource :features do
+      desc 'Get a list of all features' do
+        success Entities::Feature
+      end
+      get do
+        features = Feature.all
+
+        present features, with: Entities::Feature, current_user: current_user
+      end
+
+      desc 'Set the gate value for the given feature' do
+        success Entities::Feature
+      end
+      params do
+        requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time'
+      end
+      post ':name' do
+        feature = Feature.get(params[:name])
+
+        if %w(0 false).include?(params[:value])
+          feature.disable
+        elsif params[:value] == 'true'
+          feature.enable
+        else
+          feature.enable_percentage_of_time(params[:value].to_i)
+        end
+
+        present feature, with: Entities::Feature, current_user: current_user
+      end
+    end
+  end
+end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 3da7d735da8..e14a988a153 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -70,7 +70,11 @@ module API
       params do
         requires :name, type: String, desc: 'The name of the group'
         requires :path, type: String, desc: 'The path of the group'
-        optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group'
+
+        if ::Group.supports_nested_groups?
+          optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group'
+        end
+
         use :optional_params
       end
       post do
@@ -147,8 +151,8 @@ module API
       end
       get ":id/projects" do
         group = find_group!(params[:id])
-        projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute
-        projects = filter_projects(projects)
+        projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute
+        projects = reorder_projects(projects)
         entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
         present paginate(projects), with: entity, current_user: current_user
       end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 226a7ddd50e..2c73a6fdc4e 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -158,7 +158,7 @@ module API
       params_hash = custom_params || params
       attrs = {}
       keys.each do |key|
-        if params_hash[key].present? || (params_hash.has_key?(key) && params_hash[key] == false)
+        if params_hash[key].present? || (params_hash.key?(key) && params_hash[key] == false)
           attrs[key] = params_hash[key]
         end
       end
@@ -256,31 +256,21 @@ module API
 
     # project helpers
 
-    def filter_projects(projects)
-      if params[:membership]
-        projects = projects.merge(current_user.authorized_projects)
-      end
-
-      if params[:owned]
-        projects = projects.merge(current_user.owned_projects)
-      end
-
-      if params[:starred]
-        projects = projects.merge(current_user.starred_projects)
-      end
-
-      if params[:search].present?
-        projects = projects.search(params[:search])
-      end
-
-      if params[:visibility].present?
-        projects = projects.search_by_visibility(params[:visibility])
-      end
-
-      projects = projects.where(archived: params[:archived])
+    def reorder_projects(projects)
       projects.reorder(params[:order_by] => params[:sort])
     end
 
+    def project_finder_params
+      finder_params = {}
+      finder_params[:owned] = true if params[:owned].present?
+      finder_params[:non_public] = true if params[:membership].present?
+      finder_params[:starred] = true if params[:starred].present?
+      finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility]
+      finder_params[:archived] = params[:archived]
+      finder_params[:search] = params[:search] if params[:search]
+      finder_params
+    end
+
     # file helpers
 
     def uploaded_file(field, uploads_path)
@@ -321,6 +311,16 @@ module API
       end
     end
 
+    def present_artifacts!(artifacts_file)
+      return not_found! unless artifacts_file.exists?
+  
+      if artifacts_file.file_storage?
+        present_file!(artifacts_file.path, artifacts_file.filename)
+      else
+        redirect_to(artifacts_file.url)
+      end
+    end
+
     private
 
     def private_token
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
index 0223957fde1..8a67de10bca 100644
--- a/lib/api/jobs.rb
+++ b/lib/api/jobs.rb
@@ -224,16 +224,6 @@ module API
         find_build(id) || not_found!
       end
 
-      def present_artifacts!(artifacts_file)
-        if !artifacts_file.file_storage?
-          redirect_to(build.artifacts_file.url)
-        elsif artifacts_file.exists?
-          present_file!(artifacts_file.path, artifacts_file.filename)
-        else
-          not_found!
-        end
-      end
-
       def filter_builds(builds, scope)
         return builds if scope.nil? || scope.empty?
 
diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb
new file mode 100644
index 00000000000..93d89209934
--- /dev/null
+++ b/lib/api/pipeline_schedules.rb
@@ -0,0 +1,131 @@
+module API
+  class PipelineSchedules < Grape::API
+    include PaginationParams
+
+    before { authenticate! }
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: { id: %r{[^/]+} } do
+      desc 'Get all pipeline schedules' do
+        success Entities::PipelineSchedule
+      end
+      params do
+        use :pagination
+        optional :scope,    type: String, values: %w[active inactive],
+                            desc: 'The scope of pipeline schedules'
+      end
+      get ':id/pipeline_schedules' do
+        authorize! :read_pipeline_schedule, user_project
+
+        schedules = PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope])
+          .preload([:owner, :last_pipeline])
+        present paginate(schedules), with: Entities::PipelineSchedule
+      end
+
+      desc 'Get a single pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+      end
+      get ':id/pipeline_schedules/:pipeline_schedule_id' do
+        authorize! :read_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        present pipeline_schedule, with: Entities::PipelineScheduleDetails
+      end
+
+      desc 'Create a new pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :description, type: String, desc: 'The description of pipeline schedule'
+        requires :ref, type: String, desc: 'The branch/tag name will be triggered'
+        requires :cron, type: String, desc: 'The cron'
+        optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone'
+        optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule'
+      end
+      post ':id/pipeline_schedules' do
+        authorize! :create_pipeline_schedule, user_project
+
+        pipeline_schedule = Ci::CreatePipelineScheduleService
+          .new(user_project, current_user, declared_params(include_missing: false))
+          .execute
+
+        if pipeline_schedule.persisted?
+          present pipeline_schedule, with: Entities::PipelineScheduleDetails
+        else
+          render_validation_error!(pipeline_schedule)
+        end
+      end
+
+      desc 'Edit a pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+        optional :description, type: String, desc: 'The description of pipeline schedule'
+        optional :ref, type: String, desc: 'The branch/tag name will be triggered'
+        optional :cron, type: String, desc: 'The cron'
+        optional :cron_timezone, type: String, desc: 'The timezone'
+        optional :active, type: Boolean, desc: 'The activation of pipeline schedule'
+      end
+      put ':id/pipeline_schedules/:pipeline_schedule_id' do
+        authorize! :update_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        if pipeline_schedule.update(declared_params(include_missing: false))
+          present pipeline_schedule, with: Entities::PipelineScheduleDetails
+        else
+          render_validation_error!(pipeline_schedule)
+        end
+      end
+
+      desc 'Take ownership of a pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+      end
+      post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do
+        authorize! :update_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        if pipeline_schedule.own!(current_user)
+          present pipeline_schedule, with: Entities::PipelineScheduleDetails
+        else
+          render_validation_error!(pipeline_schedule)
+        end
+      end
+
+      desc 'Delete a pipeline schedule' do
+        success Entities::PipelineScheduleDetails
+      end
+      params do
+        requires :pipeline_schedule_id, type: Integer,  desc: 'The pipeline schedule id'
+      end
+      delete ':id/pipeline_schedules/:pipeline_schedule_id' do
+        authorize! :admin_pipeline_schedule, user_project
+
+        not_found!('PipelineSchedule') unless pipeline_schedule
+
+        status :accepted
+        present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails
+      end
+    end
+
+    helpers do
+      def pipeline_schedule
+        @pipeline_schedule ||=
+          user_project.pipeline_schedules
+                      .preload(:owner, :last_pipeline)
+                      .find_by(id: params.delete(:pipeline_schedule_id))
+      end
+    end
+  end
+end
diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb
index 9117704aa46..e505cae3992 100644
--- a/lib/api/pipelines.rb
+++ b/lib/api/pipelines.rb
@@ -47,7 +47,7 @@ module API
         new_pipeline = Ci::CreatePipelineService.new(user_project,
                                                      current_user,
                                                      declared_params(include_missing: false))
-                           .execute(ignore_skip_ci: true, save_on_errors: false)
+                           .execute(:api, ignore_skip_ci: true, save_on_errors: false)
         if new_pipeline.persisted?
           present new_pipeline, with: Entities::Pipeline
         else
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index ed5004e8d1a..deac3934d57 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -21,6 +21,7 @@ module API
         optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
         optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed'
         optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved'
+        optional :tag_list, type: Array[String], desc: 'The list of tags for a project'
       end
 
       params :optional_params do
@@ -58,6 +59,8 @@ module API
           optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user'
           optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
           optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of'
+          optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
+          optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
         end
 
         params :create_params do
@@ -65,16 +68,19 @@ module API
           optional :import_url, type: String, desc: 'URL from which the project is imported'
         end
 
-        def present_projects(projects, options = {})
+        def present_projects(options = {})
+          projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
+          projects = reorder_projects(projects)
+          projects = projects.with_statistics if params[:statistics]
+          projects = projects.with_issues_enabled if params[:with_issues_enabled]
+          projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
+
           options = options.reverse_merge(
-            with: Entities::Project,
-            current_user: current_user,
-            simple: params[:simple]
+            with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
+            statistics: params[:statistics],
+            current_user: current_user
           )
-
-          projects = filter_projects(projects)
-          projects = projects.with_statistics if options[:statistics]
-          options[:with] = Entities::BasicProjectDetails if options[:simple]
+          options[:with] = Entities::BasicProjectDetails if params[:simple]
 
           present paginate(projects), options
         end
@@ -88,8 +94,7 @@ module API
         use :statistics_params
       end
       get do
-        entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
-        present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics]
+        present_projects
       end
 
       desc 'Create new project' do
@@ -104,7 +109,7 @@ module API
       end
       post do
         attrs = declared_params(include_missing: false)
-        attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled)
+        attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled)
         project = ::Projects::CreateService.new(current_user, attrs).execute
 
         if project.saved?
@@ -124,6 +129,7 @@ module API
       params do
         requires :name, type: String, desc: 'The name of the project'
         requires :user_id, type: Integer, desc: 'The ID of a user'
+        optional :path, type: String, desc: 'The path of the repository'
         optional :default_branch, type: String, desc: 'The default branch of the project'
         use :optional_params
         use :create_params
@@ -225,6 +231,7 @@ module API
             :request_access_enabled,
             :shared_runners_enabled,
             :snippets_enabled,
+            :tag_list,
             :visibility,
             :wiki_enabled
           ]
@@ -241,7 +248,7 @@ module API
         authorize! :rename_project, user_project if attrs[:name].present?
         authorize! :change_visibility_level, user_project if attrs[:visibility].present?
 
-        attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled)
+        attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled)
 
         result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute
 
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 8f16e532ecb..14d2bff9cb5 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -85,7 +85,7 @@ module API
         optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded'
         optional :format, type: String, desc: 'The archive format'
       end
-      get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do
+      get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do
         begin
           send_git_archive user_project.repository, ref: params[:sha], format: params[:format]
         rescue
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index 6fbb02cb3aa..4552115b3e2 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -141,7 +141,7 @@ module API
       patch '/:id/trace' do
         job = authenticate_job!
 
-        error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range')
+        error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
         content_range = request.headers['Content-Range']
         content_range = content_range.split('-')
 
@@ -241,16 +241,7 @@ module API
       get '/:id/artifacts' do
         job = authenticate_job!
 
-        artifacts_file = job.artifacts_file
-        unless artifacts_file.file_storage?
-          return redirect_to job.artifacts_file.url
-        end
-
-        unless artifacts_file.exists?
-          not_found!
-        end
-
-        present_file!(artifacts_file.path, artifacts_file.filename)
+        present_artifacts!(job.artifacts_file)
       end
     end
   end
diff --git a/lib/api/services.rb b/lib/api/services.rb
index cb07df9e249..47bd9940f77 100644
--- a/lib/api/services.rb
+++ b/lib/api/services.rb
@@ -304,7 +304,13 @@ module API
           required: true,
           name: :url,
           type: String,
-          desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com'
+          desc: 'The base URL to the JIRA instance web interface which is being linked to this GitLab project. E.g., https://jira.example.com'
+        },
+        {
+          required: false,
+          name: :api_url,
+          type: String,
+          desc: 'The base URL to the JIRA instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com'
         },
         {
           required: true,
diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb
index 05b4b490e27..df4632346dd 100644
--- a/lib/api/time_tracking_endpoints.rb
+++ b/lib/api/time_tracking_endpoints.rb
@@ -5,7 +5,7 @@ module API
     included do
       helpers do
         def issuable_name
-          declared_params.has_key?(:issue_iid) ? 'issue' : 'merge_request'
+          declared_params.key?(:issue_iid) ? 'issue' : 'merge_request'
         end
 
         def issuable_key
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 3d83720b7b9..2070dbd8bc7 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -286,13 +286,14 @@ module API
       end
       params do
         requires :id, type: Integer, desc: 'The ID of the user'
+        optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions"
       end
       delete ":id" do
         authenticated_as_admin!
         user = User.find_by(id: params[:id])
         not_found!('User') unless user
 
-        DeleteUserWorker.perform_async(current_user.id, user.id)
+        DeleteUserWorker.perform_async(current_user.id, user.id, hard_delete: params[:hard_delete])
       end
 
       desc 'Block a user. Available only for admins.'
diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb
index 21935922414..93ad9eb26b8 100644
--- a/lib/api/v3/builds.rb
+++ b/lib/api/v3/builds.rb
@@ -225,16 +225,6 @@ module API
           find_build(id) || not_found!
         end
 
-        def present_artifacts!(artifacts_file)
-          if !artifacts_file.file_storage?
-            redirect_to(build.artifacts_file.url)
-          elsif artifacts_file.exists?
-            present_file!(artifacts_file.path, artifacts_file.filename)
-          else
-            not_found!
-          end
-        end
-
         def filter_builds(builds, scope)
           return builds if scope.nil? || scope.empty?
 
diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb
index 674de592f0a..5936f4700aa 100644
--- a/lib/api/v3/commits.rb
+++ b/lib/api/v3/commits.rb
@@ -167,7 +167,7 @@ module API
           }
 
           if params[:path]
-            commit.raw_diffs(all_diffs: true).each do |diff|
+            commit.raw_diffs(limits: false).each do |diff|
               next unless diff.new_path == params[:path]
               lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line)
 
diff --git a/lib/api/v3/deploy_keys.rb b/lib/api/v3/deploy_keys.rb
index bbb174b6003..b90e2061da3 100644
--- a/lib/api/v3/deploy_keys.rb
+++ b/lib/api/v3/deploy_keys.rb
@@ -41,6 +41,7 @@ module API
           params do
             requires :key, type: String, desc: 'The new deploy key'
             requires :title, type: String, desc: 'The name of the deploy key'
+            optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository"
           end
           post ":id/#{path}" do
             params[:key].strip!
diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb
index 332f233bf5e..7c5065dee90 100644
--- a/lib/api/v3/entities.rb
+++ b/lib/api/v3/entities.rb
@@ -137,7 +137,10 @@ module API
         expose :web_url
         expose :request_access_enabled
         expose :full_name, :full_path
-        expose :parent_id
+
+        if ::Group.supports_nested_groups?
+          expose :parent_id
+        end
 
         expose :statistics, if: :statistics do
           with_options format_with: -> (value) { value.to_i } do
@@ -223,7 +226,7 @@ module API
 
       class MergeRequestChanges < MergeRequest
         expose :diffs, as: :changes, using: ::API::Entities::RepoDiff do |compare, _|
-          compare.raw_diffs(all_diffs: true).to_a
+          compare.raw_diffs(limits: false).to_a
         end
       end
 
diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb
index 6187445fc8d..2c52d21fa1c 100644
--- a/lib/api/v3/groups.rb
+++ b/lib/api/v3/groups.rb
@@ -74,7 +74,11 @@ module API
         params do
           requires :name, type: String, desc: 'The name of the group'
           requires :path, type: String, desc: 'The path of the group'
-          optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group'
+
+          if ::Group.supports_nested_groups?
+            optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group'
+          end
+
           use :optional_params
         end
         post do
diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb
index 0f234d4cdad..d9e76560d03 100644
--- a/lib/api/v3/helpers.rb
+++ b/lib/api/v3/helpers.rb
@@ -14,6 +14,33 @@ module API
         authorize! access_level, merge_request
         merge_request
       end
+
+      # project helpers
+
+      def filter_projects(projects)
+        if params[:membership]
+          projects = projects.merge(current_user.authorized_projects)
+        end
+
+        if params[:owned]
+          projects = projects.merge(current_user.owned_projects)
+        end
+
+        if params[:starred]
+          projects = projects.merge(current_user.starred_projects)
+        end
+
+        if params[:search].present?
+          projects = projects.search(params[:search])
+        end
+
+        if params[:visibility].present?
+          projects = projects.where(visibility_level: Gitlab::VisibilityLevel.level_value(params[:visibility]))
+        end
+
+        projects = projects.where(archived: params[:archived])
+        projects.reorder(params[:order_by] => params[:sort])
+      end
     end
   end
 end
diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb
index 164612cb8dd..20976b9dd08 100644
--- a/lib/api/v3/projects.rb
+++ b/lib/api/v3/projects.rb
@@ -44,7 +44,7 @@ module API
         end
 
         def set_only_allow_merge_if_pipeline_succeeds!
-          if params.has_key?(:only_allow_merge_if_build_succeeds)
+          if params.key?(:only_allow_merge_if_build_succeeds)
             params[:only_allow_merge_if_pipeline_succeeds] = params.delete(:only_allow_merge_if_build_succeeds)
           end
         end
@@ -147,7 +147,7 @@ module API
         get '/starred' do
           authenticate!
 
-          present_projects current_user.viewable_starred_projects
+          present_projects ProjectsFinder.new(current_user: current_user, params: { starred: true }).execute
         end
 
         desc 'Get all projects for admin user' do
diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb
index e4d14bc8168..0eaa0de2eef 100644
--- a/lib/api/v3/repositories.rb
+++ b/lib/api/v3/repositories.rb
@@ -72,7 +72,7 @@ module API
           optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded'
           optional :format, type: String, desc: 'The archive format'
         end
-        get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do
+        get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do
           begin
             send_git_archive user_project.repository, ref: params[:sha], format: params[:format]
           rescue
diff --git a/lib/api/v3/time_tracking_endpoints.rb b/lib/api/v3/time_tracking_endpoints.rb
index 81ae4e8137d..d5b90e435ba 100644
--- a/lib/api/v3/time_tracking_endpoints.rb
+++ b/lib/api/v3/time_tracking_endpoints.rb
@@ -6,7 +6,7 @@ module API
       included do
         helpers do
           def issuable_name
-            declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request'
+            declared_params.key?(:issue_id) ? 'issue' : 'merge_request'
           end
 
           def issuable_key
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
index 5acde41551b..381c4ef50b0 100644
--- a/lib/api/variables.rb
+++ b/lib/api/variables.rb
@@ -42,6 +42,7 @@ module API
       params do
         requires :key, type: String, desc: 'The key of the variable'
         requires :value, type: String, desc: 'The value of the variable'
+        optional :protected, type: String, desc: 'Whether the variable is protected'
       end
       post ':id/variables' do
         variable = user_project.variables.create(declared(params, include_parent_namespaces: false).to_h)
@@ -59,13 +60,14 @@ module API
       params do
         optional :key, type: String, desc: 'The key of the variable'
         optional :value, type: String, desc: 'The value of the variable'
+        optional :protected, type: String, desc: 'Whether the variable is protected'
       end
       put ':id/variables/:key' do
         variable = user_project.variables.find_by(key: params[:key])
 
         return not_found!('Variable') unless variable
 
-        if variable.update(value: params[:value])
+        if variable.update(declared_params(include_missing: false).except(:key))
           present variable, with: Entities::Variable
         else
           render_validation_error!(variable)
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
index 51fa3867e67..1f4bda6f588 100644
--- a/lib/backup/artifacts.rb
+++ b/lib/backup/artifacts.rb
@@ -3,7 +3,7 @@ require 'backup/files'
 module Backup
   class Artifacts < Files
     def initialize
-      super('artifacts', ArtifactUploader.artifacts_path)
+      super('artifacts', ArtifactUploader.local_artifacts_store)
     end
 
     def create_files_dir
diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb
index c2503fa2adc..d99a3bfa625 100644
--- a/lib/banzai/reference_parser/base_parser.rb
+++ b/lib/banzai/reference_parser/base_parser.rb
@@ -163,14 +163,15 @@ module Banzai
       # been queried the object is returned from the cache.
       def collection_objects_for_ids(collection, ids)
         if RequestStore.active?
+          ids = ids.map(&:to_i)
           cache = collection_cache[collection_cache_key(collection)]
-          to_query = ids.map(&:to_i) - cache.keys
+          to_query = ids - cache.keys
 
           unless to_query.empty?
             collection.where(id: to_query).each { |row| cache[row.id] = row }
           end
 
-          cache.values
+          cache.values_at(*ids)
         else
           collection.where(id: ids)
         end
diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb
index 4f8efe03bae..c52acbc3ddc 100644
--- a/lib/bitbucket/representation/pull_request_comment.rb
+++ b/lib/bitbucket/representation/pull_request_comment.rb
@@ -22,11 +22,11 @@ module Bitbucket
       end
 
       def inline?
-        raw.has_key?('inline')
+        raw.key?('inline')
       end
 
       def has_parent?
-        raw.has_key?('parent')
+        raw.key?('parent')
       end
 
       private
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 67b269b330c..e2e91ce99cd 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -88,7 +88,7 @@ module Ci
         patch ":id/trace.txt" do
           build = authenticate_build!
 
-          error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range')
+          error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
           content_range = request.headers['Content-Range']
           content_range = content_range.split('-')
 
@@ -187,14 +187,14 @@ module Ci
           build = authenticate_build!
           artifacts_file = build.artifacts_file
 
-          unless artifacts_file.file_storage?
-            return redirect_to build.artifacts_file.url
-          end
-
           unless artifacts_file.exists?
             not_found!
           end
 
+          unless artifacts_file.file_storage?
+            return redirect_to build.artifacts_file.url
+          end
+
           present_file!(artifacts_file.path, artifacts_file.filename)
         end
 
diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb
index 0ea2f97352d..6fc1d56d7a0 100644
--- a/lib/constraints/group_url_constrainer.rb
+++ b/lib/constraints/group_url_constrainer.rb
@@ -1,9 +1,9 @@
 class GroupUrlConstrainer
   def matches?(request)
-    id = request.params[:group_id] || request.params[:id]
+    full_path = request.params[:group_id] || request.params[:id]
 
-    return false unless DynamicPathValidator.valid_namespace_path?(id)
+    return false unless DynamicPathValidator.valid_group_path?(full_path)
 
-    Group.find_by_full_path(id, follow_redirects: request.get?).present?
+    Group.find_by_full_path(full_path, follow_redirects: request.get?).present?
   end
 end
diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb
index 4444a1abee3..4c0aee6c48f 100644
--- a/lib/constraints/project_url_constrainer.rb
+++ b/lib/constraints/project_url_constrainer.rb
@@ -2,7 +2,7 @@ class ProjectUrlConstrainer
   def matches?(request)
     namespace_path = request.params[:namespace_id]
     project_path = request.params[:project_id] || request.params[:id]
-    full_path = namespace_path + '/' + project_path
+    full_path = [namespace_path, project_path].join('/')
 
     return false unless DynamicPathValidator.valid_project_path?(full_path)
 
diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb
index 28159dc0dec..d16ae7f3f40 100644
--- a/lib/constraints/user_url_constrainer.rb
+++ b/lib/constraints/user_url_constrainer.rb
@@ -1,5 +1,9 @@
 class UserUrlConstrainer
   def matches?(request)
-    User.find_by_full_path(request.params[:username], follow_redirects: request.get?).present?
+    full_path = request.params[:username]
+
+    return false unless DynamicPathValidator.valid_user_path?(full_path)
+
+    User.find_by_full_path(full_path, follow_redirects: request.get?).present?
   end
 end
diff --git a/lib/feature.rb b/lib/feature.rb
new file mode 100644
index 00000000000..2e2b343f82c
--- /dev/null
+++ b/lib/feature.rb
@@ -0,0 +1,41 @@
+require 'flipper/adapters/active_record'
+
+class Feature
+  # Classes to override flipper table names
+  class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature
+    # Using `self.table_name` won't work. ActiveRecord bug?
+    superclass.table_name = 'features'
+  end
+
+  class FlipperGate < Flipper::Adapters::ActiveRecord::Gate
+    superclass.table_name = 'feature_gates'
+  end
+
+  class << self
+    def all
+      flipper.features.to_a
+    end
+
+    def get(key)
+      flipper.feature(key)
+    end
+
+    def persisted?(feature)
+      # Flipper creates on-memory features when asked for a not-yet-created one.
+      # If we want to check if a feature has been actually set, we look for it
+      # on the persisted features list.
+      all.map(&:name).include?(feature.name)
+    end
+
+    private
+
+    def flipper
+      @flipper ||= begin
+        adapter = Flipper::Adapters::ActiveRecord.new(
+          feature_class: FlipperFeature, gate_class: FlipperGate)
+
+        Flipper.new(adapter)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/chat_commands/presenters/base.rb b/lib/gitlab/chat_commands/presenters/base.rb
index 2700a5a2ad5..05994bee79d 100644
--- a/lib/gitlab/chat_commands/presenters/base.rb
+++ b/lib/gitlab/chat_commands/presenters/base.rb
@@ -45,9 +45,9 @@ module Gitlab
         end
 
         def format_response(response)
-          response[:text] = format(response[:text]) if response.has_key?(:text)
+          response[:text] = format(response[:text]) if response.key?(:text)
 
-          if response.has_key?(:attachments)
+          if response.key?(:attachments)
             response[:attachments].each do |attachment|
               attachment[:pretext] = format(attachment[:pretext]) if attachment[:pretext]
               attachment[:text] = format(attachment[:text]) if attachment[:text]
diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb
index 57b533bad99..439ef0ce015 100644
--- a/lib/gitlab/ci/status/build/cancelable.rb
+++ b/lib/gitlab/ci/status/build/cancelable.rb
@@ -12,7 +12,7 @@ module Gitlab
           end
 
           def action_path
-            cancel_namespace_project_build_path(subject.project.namespace,
+            cancel_namespace_project_job_path(subject.project.namespace,
                                                 subject.project,
                                                 subject)
           end
diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb
index 3fec2c5d4db..b173c23fba4 100644
--- a/lib/gitlab/ci/status/build/common.rb
+++ b/lib/gitlab/ci/status/build/common.rb
@@ -8,7 +8,7 @@ module Gitlab
           end
 
           def details_path
-            namespace_project_build_path(subject.project.namespace,
+            namespace_project_job_path(subject.project.namespace,
                                          subject.project,
                                          subject)
           end
diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb
index c6139f1b716..e80f3263794 100644
--- a/lib/gitlab/ci/status/build/play.rb
+++ b/lib/gitlab/ci/status/build/play.rb
@@ -20,7 +20,7 @@ module Gitlab
           end
 
           def action_path
-            play_namespace_project_build_path(subject.project.namespace,
+            play_namespace_project_job_path(subject.project.namespace,
                                               subject.project,
                                               subject)
           end
diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb
index 505f80848b2..56303e4cb17 100644
--- a/lib/gitlab/ci/status/build/retryable.rb
+++ b/lib/gitlab/ci/status/build/retryable.rb
@@ -16,7 +16,7 @@ module Gitlab
           end
 
           def action_path
-            retry_namespace_project_build_path(subject.project.namespace,
+            retry_namespace_project_job_path(subject.project.namespace,
                                                subject.project,
                                                subject)
           end
diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb
index 0b5199e5483..2778d6f3b52 100644
--- a/lib/gitlab/ci/status/build/stop.rb
+++ b/lib/gitlab/ci/status/build/stop.rb
@@ -20,7 +20,7 @@ module Gitlab
           end
 
           def action_path
-            play_namespace_project_build_path(subject.project.namespace,
+            play_namespace_project_job_path(subject.project.namespace,
                                               subject.project,
                                               subject)
           end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index fa462cbe095..c4c0623df6c 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -73,7 +73,7 @@ module Gitlab
 
           match = ""
 
-          stream.each_line do |line|
+          reverse_line do |line|
             matches = line.scan(regex)
             next unless matches.is_a?(Array)
             next if matches.empty?
@@ -86,34 +86,39 @@ module Gitlab
           nil
         rescue
           # if bad regex or something goes wrong we dont want to interrupt transition
-          # so we just silentrly ignore error for now
+          # so we just silently ignore error for now
         end
 
         private
 
-        def read_last_lines(last_lines)
-          chunks = []
-          pos = lines = 0
-          max = stream.size
-
-          # We want an extra line to make sure fist line has full contents
-          while lines <= last_lines && pos < max
-            pos += BUFFER_SIZE
-
-            buf =
-              if pos <= max
-                stream.seek(-pos, IO::SEEK_END)
-                stream.read(BUFFER_SIZE)
-              else # Reached the head, read only left
-                stream.seek(0)
-                stream.read(BUFFER_SIZE - (pos - max))
-              end
-
-            lines += buf.count("\n")
-            chunks.unshift(buf)
+        def read_last_lines(limit)
+          to_enum(:reverse_line).first(limit).reverse.join
+        end
+
+        def reverse_line
+          stream.seek(0, IO::SEEK_END)
+          debris = ''
+
+          until (buf = read_backward(BUFFER_SIZE)).empty?
+            buf += debris
+            debris, *lines = buf.each_line.to_a
+            lines.reverse_each do |line|
+              yield(line.force_encoding('UTF-8'))
+            end
           end
 
-          chunks.join.lines.last(last_lines).join
+          yield(debris.force_encoding('UTF-8')) unless debris.empty?
+        end
+
+        def read_backward(length)
+          cur_offset = stream.tell
+          start = cur_offset - length
+          start = 0 if start < 0
+
+          stream.seek(start, IO::SEEK_SET)
+          stream.read(cur_offset - start).tap do
+            stream.seek(start, IO::SEEK_SET)
+          end
         end
       end
     end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 82576d197fe..9e14b35b0f8 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -19,7 +19,7 @@ module Gitlab
           settings = ::ApplicationSetting.last
         end
 
-        settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration?
+        settings ||= ::ApplicationSetting.create_from_defaults
       end
 
       settings || in_memory_application_settings
@@ -46,7 +46,8 @@ module Gitlab
       active_db_connection = ActiveRecord::Base.connection.active? rescue false
 
       active_db_connection &&
-        ActiveRecord::Base.connection.table_exists?('application_settings')
+        ActiveRecord::Base.connection.table_exists?('application_settings') &&
+        !ActiveRecord::Migrator.needs_migration?
     rescue ActiveRecord::NoDatabaseError
       false
     end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index e76c9abbe04..a412bb6dbd2 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -42,7 +42,7 @@ module Gitlab
             'in the body of your migration class'
         end
 
-        if Database.postgresql?
+        if supports_drop_index_concurrently?
           options = options.merge({ algorithm: :concurrently })
           disable_statement_timeout
         end
@@ -50,6 +50,39 @@ module Gitlab
         remove_index(table_name, options.merge({ column: column_name }))
       end
 
+      # Removes an existing index, concurrently when supported
+      #
+      # On PostgreSQL this method removes an index concurrently.
+      #
+      # Example:
+      #
+      #     remove_concurrent_index :users, "index_X_by_Y"
+      #
+      # See Rails' `remove_index` for more info on the available arguments.
+      def remove_concurrent_index_by_name(table_name, index_name, options = {})
+        if transaction_open?
+          raise 'remove_concurrent_index_by_name can not be run inside a transaction, ' \
+            'you can disable transactions by calling disable_ddl_transaction! ' \
+            'in the body of your migration class'
+        end
+
+        if supports_drop_index_concurrently?
+          options = options.merge({ algorithm: :concurrently })
+          disable_statement_timeout
+        end
+
+        remove_index(table_name, options.merge({ name: index_name }))
+      end
+
+      # Only available on Postgresql >= 9.2
+      def supports_drop_index_concurrently?
+        return false unless Database.postgresql?
+
+        version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
+
+        version >= 90200
+      end
+
       # Adds a foreign key with only minimal locking on the tables involved.
       #
       # This method only requires minimal locking when using PostgreSQL. When
diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb
index c45ae8feb2c..3192bf6f667 100644
--- a/lib/gitlab/dependency_linker.rb
+++ b/lib/gitlab/dependency_linker.rb
@@ -1,7 +1,16 @@
 module Gitlab
   module DependencyLinker
     LINKERS = [
-      GemfileLinker
+      GemfileLinker,
+      GemspecLinker,
+      PackageJsonLinker,
+      ComposerJsonLinker,
+      PodfileLinker,
+      PodspecLinker,
+      PodspecJsonLinker,
+      CartfileLinker,
+      GodepsJsonLinker,
+      RequirementsTxtLinker
     ].freeze
 
     def self.linker(blob_name)
diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb
index 5f4027e7e81..7bbd154eb03 100644
--- a/lib/gitlab/dependency_linker/base_linker.rb
+++ b/lib/gitlab/dependency_linker/base_linker.rb
@@ -1,6 +1,9 @@
 module Gitlab
   module DependencyLinker
     class BaseLinker
+      URL_REGEX = %r{https?://[^'" ]+}.freeze
+      REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze
+
       class_attribute :file_type
 
       def self.support?(blob_name)
@@ -26,59 +29,20 @@ module Gitlab
 
       private
 
-      def package_url(name)
-        raise NotImplementedError
-      end
-
       def link_dependencies
         raise NotImplementedError
       end
 
-      def package_link(name, url = package_url(name))
-        return name unless url
-
-        %{<a href="#{ERB::Util.html_escape_once(url)}" rel="noopener noreferrer" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}
+      def license_url(name)
+        Licensee::License.find(name)&.url
       end
 
-      # Links package names in a method call or assignment string argument.
-      #
-      # Example:
-      #   link_method_call("gem")
-      #   # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"`
-      #
-      #   link_method_call("gem", "specific_package")
-      #   # Will link `specific_package` in `gem "specific_package"`
-      #
-      #   link_method_call("github", /[^\/]+\/[^\/]+/)
-      #   # Will link `user/repo` in `github "user/repo"`, but not `github "package"`
-      #
-      #   link_method_call(%w[add_dependency add_development_dependency])
-      #   # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"`
-      #
-      #   link_method_call("name")
-      #   # Will link `package` in `self.name = "package"`
-      def link_method_call(method_names, value = nil, &url_proc)
-        value =
-          case value
-          when String
-            Regexp.escape(value)
-          when nil
-            /[^'"]+/
-          else
-            value
-          end
-
-        method_names = Array(method_names).map { |name| Regexp.escape(name) }
-
-        regex = %r{
-          #{Regexp.union(method_names)} # Method name
-          \s*                           # Whitespace
-          [(=]?                         # Opening brace or equals sign
-          \s*                           # Whitespace
-          ['"](?<name>#{value})['"]     # Package name in quotes
-        }x
+      def github_url(name)
+        "https://github.com/#{name}"
+      end
 
-        link_regex(regex, &url_proc)
+      def link_tag(name, url)
+        %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}
       end
 
       # Links package names based on regex.
@@ -86,13 +50,13 @@ module Gitlab
       # Example:
       #   link_regex(/(github:|:github =>)\s*['"](?<name>[^'"]+)['"]/)
       #   # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"`
-      def link_regex(regex)
+      def link_regex(regex, &url_proc)
         highlighted_lines.map!.with_index do |rich_line, i|
           marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe)
 
           marker.mark(regex, group: :name) do |text, left:, right:|
-            url = block_given? ? yield(text) : package_url(text)
-            package_link(text, url)
+            url = yield(text)
+            url ? link_tag(text, url) : text
           end
         end
       end
@@ -104,6 +68,19 @@ module Gitlab
       def highlighted_lines
         @highlighted_lines ||= highlighted_text.lines
       end
+
+      def regexp_for_value(value, default: /[^'" ]+/)
+        case value
+        when Array
+          Regexp.union(value.map { |v| regexp_for_value(v, default: default) })
+        when String
+          Regexp.escape(value)
+        when Regexp
+          value
+        else
+          default
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/dependency_linker/cartfile_linker.rb b/lib/gitlab/dependency_linker/cartfile_linker.rb
new file mode 100644
index 00000000000..4f69f2c4ab2
--- /dev/null
+++ b/lib/gitlab/dependency_linker/cartfile_linker.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module DependencyLinker
+    class CartfileLinker < MethodLinker
+      self.file_type = :cartfile
+
+      private
+
+      def link_dependencies
+        link_method_call('github', REPO_REGEX, &method(:github_url))
+        link_method_call(%w[github git binary], URL_REGEX, &:itself)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/cocoapods.rb b/lib/gitlab/dependency_linker/cocoapods.rb
new file mode 100644
index 00000000000..2fbde7da1b4
--- /dev/null
+++ b/lib/gitlab/dependency_linker/cocoapods.rb
@@ -0,0 +1,10 @@
+module Gitlab
+  module DependencyLinker
+    module Cocoapods
+      def package_url(name)
+        package = name.split("/", 2).first
+        "https://cocoapods.org/pods/#{package}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb
new file mode 100644
index 00000000000..0245bf4077a
--- /dev/null
+++ b/lib/gitlab/dependency_linker/composer_json_linker.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module DependencyLinker
+    class ComposerJsonLinker < PackageJsonLinker
+      self.file_type = :composer_json
+
+      private
+
+      def link_packages
+        link_packages_at_key("require", &method(:package_url))
+        link_packages_at_key("require-dev", &method(:package_url))
+      end
+
+      def package_url(name)
+        "https://packagist.org/packages/#{name}" if name =~ %r{\A#{REPO_REGEX}\z}
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb
index 9b82e126528..d034ea67387 100644
--- a/lib/gitlab/dependency_linker/gemfile_linker.rb
+++ b/lib/gitlab/dependency_linker/gemfile_linker.rb
@@ -1,28 +1,31 @@
 module Gitlab
   module DependencyLinker
-    class GemfileLinker < BaseLinker
+    class GemfileLinker < MethodLinker
       self.file_type = :gemfile
 
       private
 
       def link_dependencies
-        # Link `gem "package_name"` to https://rubygems.org/gems/package_name
-        link_method_call("gem")
+        link_urls
+        link_packages
+      end
 
+      def link_urls
         # Link `github: "user/repo"` to https://github.com/user/repo
-        link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/) do |name|
-          "https://github.com/#{name}"
-        end
+        link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url))
 
         # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo
-        link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>https?://[^'"]+)['"]}) { |url| url }
+        link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself)
 
         # Link `source "https://rubygems.org"` to https://rubygems.org
-        link_method_call("source", %r{https?://[^'"]+}) { |url| url }
+        link_method_call('source', URL_REGEX, &:itself)
       end
 
-      def package_url(name)
-        "https://rubygems.org/gems/#{name}"
+      def link_packages
+        # Link `gem "package_name"` to https://rubygems.org/gems/package_name
+        link_method_call('gem') do |name|
+          "https://rubygems.org/gems/#{name}"
+        end
       end
     end
   end
diff --git a/lib/gitlab/dependency_linker/gemspec_linker.rb b/lib/gitlab/dependency_linker/gemspec_linker.rb
new file mode 100644
index 00000000000..f1783ee2ab4
--- /dev/null
+++ b/lib/gitlab/dependency_linker/gemspec_linker.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module DependencyLinker
+    class GemspecLinker < MethodLinker
+      self.file_type = :gemspec
+
+      private
+
+      def link_dependencies
+        link_method_call('homepage', URL_REGEX, &:itself)
+        link_method_call('license', &method(:license_url))
+
+        link_method_call(%w[name add_dependency add_runtime_dependency add_development_dependency]) do |name|
+          "https://rubygems.org/gems/#{name}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/godeps_json_linker.rb b/lib/gitlab/dependency_linker/godeps_json_linker.rb
new file mode 100644
index 00000000000..fe091baee6d
--- /dev/null
+++ b/lib/gitlab/dependency_linker/godeps_json_linker.rb
@@ -0,0 +1,26 @@
+module Gitlab
+  module DependencyLinker
+    class GodepsJsonLinker < JsonLinker
+      NESTED_REPO_REGEX = %r{([^/]+/)+[^/]+?}.freeze
+
+      self.file_type = :godeps_json
+
+      private
+
+      def link_dependencies
+        link_json('ImportPath') do |path|
+          case path
+          when %r{\A(?<repo>gitlab\.com/#{NESTED_REPO_REGEX})\.git/(?<path>.+)\z},
+              %r{\A(?<repo>git(lab|hub)\.com/#{REPO_REGEX})/(?<path>.+)\z}
+
+            "https://#{$~[:repo]}/tree/master/#{$~[:path]}"
+          when /\Agolang\.org/
+            "https://godoc.org/#{path}"
+          else
+            "https://#{path}"
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/json_linker.rb b/lib/gitlab/dependency_linker/json_linker.rb
new file mode 100644
index 00000000000..a8ef25233d8
--- /dev/null
+++ b/lib/gitlab/dependency_linker/json_linker.rb
@@ -0,0 +1,44 @@
+module Gitlab
+  module DependencyLinker
+    class JsonLinker < BaseLinker
+      def link
+        return highlighted_text unless json
+
+        super
+      end
+
+      private
+
+      # Links package names in a JSON key or values.
+      #
+      # Example:
+      #   link_json('name')
+      #   # Will link `package` in `"name": "package"`
+      #
+      #   link_json('name', 'specific_package')
+      #   # Will link `specific_package` in `"name": "specific_package"`
+      #
+      #   link_json('name', /[^\/]+\/[^\/]+/)
+      #   # Will link `user/repo` in `"name": "user/repo"`, but not `"name": "package"`
+      #
+      #   link_json('specific_package', '1.0.1', link: :key)
+      #   # Will link `specific_package` in `"specific_package": "1.0.1"`
+      def link_json(key, value = nil, link: :value, &url_proc)
+        key = regexp_for_value(key, default: /[^" ]+/)
+        value = regexp_for_value(value, default: /[^" ]+/)
+
+        if link == :value
+          value = /(?<name>#{value})/
+        else
+          key = /(?<name>#{key})/
+        end
+
+        link_regex(/"#{key}":\s*"#{value}"/, &url_proc)
+      end
+
+      def json
+        @json ||= JSON.parse(plain_text) rescue nil
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/method_linker.rb b/lib/gitlab/dependency_linker/method_linker.rb
new file mode 100644
index 00000000000..0ffa2a83c93
--- /dev/null
+++ b/lib/gitlab/dependency_linker/method_linker.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module DependencyLinker
+    class MethodLinker < BaseLinker
+      private
+
+      # Links package names in a method call or assignment string argument.
+      #
+      # Example:
+      #   link_method_call('gem')
+      #   # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"`
+      #
+      #   link_method_call('gem', 'specific_package')
+      #   # Will link `specific_package` in `gem "specific_package"`
+      #
+      #   link_method_call('github', /[^\/"]+\/[^\/"]+/)
+      #   # Will link `user/repo` in `github "user/repo"`, but not `github "package"`
+      #
+      #   link_method_call(%w[add_dependency add_development_dependency])
+      #   # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"`
+      #
+      #   link_method_call('name')
+      #   # Will link `package` in `self.name = "package"`
+      def link_method_call(method_name, value = nil, &url_proc)
+        method_name = regexp_for_value(method_name)
+        value = regexp_for_value(value)
+
+        regex = %r{
+          #{method_name}            # Method name
+          \s*                       # Whitespace
+          [(=]?                     # Opening brace or equals sign
+          \s*                       # Whitespace
+          ['"](?<name>#{value})['"] # Package name in quotes
+        }x
+
+        link_regex(regex, &url_proc)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/package_json_linker.rb b/lib/gitlab/dependency_linker/package_json_linker.rb
new file mode 100644
index 00000000000..330c95f0880
--- /dev/null
+++ b/lib/gitlab/dependency_linker/package_json_linker.rb
@@ -0,0 +1,44 @@
+module Gitlab
+  module DependencyLinker
+    class PackageJsonLinker < JsonLinker
+      self.file_type = :package_json
+
+      private
+
+      def link_dependencies
+        link_json('name', json["name"], &method(:package_url))
+        link_json('license', &method(:license_url))
+        link_json(%w[homepage url], URL_REGEX, &:itself)
+
+        link_packages
+      end
+
+      def link_packages
+        link_packages_at_key("dependencies", &method(:package_url))
+        link_packages_at_key("devDependencies", &method(:package_url))
+      end
+
+      def link_packages_at_key(key, &url_proc)
+        dependencies = json[key]
+        return unless dependencies
+
+        dependencies.each do |name, version|
+          link_json(name, version, link: :key, &url_proc)
+
+          link_json(name) do |value|
+            case value
+            when /\A#{URL_REGEX}\z/
+              value
+            when /\A#{REPO_REGEX}\z/
+              github_url(value)
+            end
+          end
+        end
+      end
+
+      def package_url(name)
+        "https://npmjs.com/package/#{name}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/podfile_linker.rb b/lib/gitlab/dependency_linker/podfile_linker.rb
new file mode 100644
index 00000000000..60ad166ea17
--- /dev/null
+++ b/lib/gitlab/dependency_linker/podfile_linker.rb
@@ -0,0 +1,15 @@
+module Gitlab
+  module DependencyLinker
+    class PodfileLinker < GemfileLinker
+      include Cocoapods
+
+      self.file_type = :podfile
+
+      private
+
+      def link_packages
+        link_method_call('pod', &method(:package_url))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/podspec_json_linker.rb b/lib/gitlab/dependency_linker/podspec_json_linker.rb
new file mode 100644
index 00000000000..d82237ed3f1
--- /dev/null
+++ b/lib/gitlab/dependency_linker/podspec_json_linker.rb
@@ -0,0 +1,32 @@
+module Gitlab
+  module DependencyLinker
+    class PodspecJsonLinker < JsonLinker
+      include Cocoapods
+
+      self.file_type = :podspec_json
+
+      private
+
+      def link_dependencies
+        link_json('name', json["name"], &method(:package_url))
+        link_json('license', &method(:license_url))
+        link_json(%w[homepage git], URL_REGEX, &:itself)
+
+        link_packages_at_key("dependencies", &method(:package_url))
+
+        json["subspecs"]&.each do |subspec|
+          link_packages_at_key("dependencies", subspec, &method(:package_url))
+        end
+      end
+
+      def link_packages_at_key(key, root = json, &url_proc)
+        dependencies = root[key]
+        return unless dependencies
+
+        dependencies.each do |name, _|
+          link_regex(/"(?<name>#{Regexp.escape(name)})":\s*\[/, &url_proc)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb
new file mode 100644
index 00000000000..a52c7a02439
--- /dev/null
+++ b/lib/gitlab/dependency_linker/podspec_linker.rb
@@ -0,0 +1,24 @@
+module Gitlab
+  module DependencyLinker
+    class PodspecLinker < MethodLinker
+      include Cocoapods
+
+      STRING_REGEX = /['"](?<name>[^'"]+)['"]/.freeze
+
+      self.file_type = :podspec
+
+      private
+
+      def link_dependencies
+        link_method_call('homepage', URL_REGEX, &:itself)
+
+        link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself)
+
+        link_method_call('license', &method(:license_url))
+        link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url))
+
+        link_method_call(%w[name dependency], &method(:package_url))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb
new file mode 100644
index 00000000000..2e197e5cd94
--- /dev/null
+++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb
@@ -0,0 +1,17 @@
+module Gitlab
+  module DependencyLinker
+    class RequirementsTxtLinker < BaseLinker
+      self.file_type = :requirements_txt
+
+      private
+
+      def link_dependencies
+        link_regex(/^(?<name>(?![a-z+]+:)[^#.-][^ ><=;\[]+)/) do |name|
+          "https://pypi.python.org/pypi/#{name}"
+        end
+
+        link_regex(%r{^(?<name>https?://[^ ]+)}, &:itself)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index c6bf25b5874..2aef7fdaa35 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -1,16 +1,17 @@
 module Gitlab
   module Diff
     class File
-      attr_reader :diff, :repository, :diff_refs
+      attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs
 
-      delegate :new_file, :deleted_file, :renamed_file,
-        :old_path, :new_path, :a_mode, :b_mode,
+      delegate :new_file?, :deleted_file?, :renamed_file?,
+        :old_path, :new_path, :a_mode, :b_mode, :mode_changed?,
         :submodule?, :too_large?, :collapsed?, to: :diff, prefix: false
 
-      def initialize(diff, repository:, diff_refs: nil)
+      def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil)
         @diff = diff
         @repository = repository
         @diff_refs = diff_refs
+        @fallback_diff_refs = fallback_diff_refs
       end
 
       def position(line)
@@ -49,24 +50,60 @@ module Gitlab
         line_code(line) if line
       end
 
+      def old_sha
+        diff_refs&.base_sha
+      end
+
+      def new_sha
+        diff_refs&.head_sha
+      end
+
+      def content_sha
+        return old_content_sha if deleted_file?
+        return @content_sha if defined?(@content_sha)
+
+        refs = diff_refs || fallback_diff_refs
+        @content_sha = refs&.head_sha
+      end
+
       def content_commit
-        return unless diff_refs
+        return @content_commit if defined?(@content_commit)
+
+        sha = content_sha
+        @content_commit = repository.commit(sha) if sha
+      end
+
+      def old_content_sha
+        return if new_file?
+        return @old_content_sha if defined?(@old_content_sha)
 
-        repository.commit(deleted_file ? old_ref : new_ref)
+        refs = diff_refs || fallback_diff_refs
+        @old_content_sha = refs&.base_sha
       end
 
       def old_content_commit
-        return unless diff_refs
+        return @old_content_commit if defined?(@old_content_commit)
 
-        repository.commit(old_ref)
+        sha = old_content_sha
+        @old_content_commit = repository.commit(sha) if sha
       end
 
-      def old_ref
-        diff_refs.try(:base_sha)
+      def blob
+        return @blob if defined?(@blob)
+
+        sha = content_sha
+        return @blob = nil unless sha
+
+        repository.blob_at(sha, file_path)
       end
 
-      def new_ref
-        diff_refs.try(:head_sha)
+      def old_blob
+        return @old_blob if defined?(@old_blob)
+
+        sha = old_content_sha
+        return @old_blob = nil unless sha
+
+        @old_blob = repository.blob_at(sha, old_path)
       end
 
       attr_writer :highlighted_diff_lines
@@ -85,10 +122,6 @@ module Gitlab
         @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize
       end
 
-      def mode_changed?
-        a_mode && b_mode && a_mode != b_mode
-      end
-
       def raw_diff
         diff.diff.to_s
       end
@@ -117,20 +150,8 @@ module Gitlab
         diff_lines.count(&:removed?)
       end
 
-      def old_blob(commit = old_content_commit)
-        return unless commit
-
-        repository.blob_at(commit.id, old_path)
-      end
-
-      def blob(commit = content_commit)
-        return unless commit
-
-        repository.blob_at(commit.id, file_path)
-      end
-
       def file_identifier
-        "#{file_path}-#{new_file}-#{deleted_file}-#{renamed_file}"
+        "#{file_path}-#{new_file?}-#{deleted_file?}-#{renamed_file?}"
       end
     end
   end
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
index 7c32adc6ce7..a6007ebf531 100644
--- a/lib/gitlab/diff/file_collection/base.rb
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -2,22 +2,23 @@ module Gitlab
   module Diff
     module FileCollection
       class Base
-        attr_reader :project, :diff_options, :diff_view, :diff_refs
+        attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs
 
         delegate :count, :size, :real_size, to: :diff_files
 
         def self.default_options
-          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false)
+          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false)
         end
 
-        def initialize(diffable, project:, diff_options: nil, diff_refs: nil)
+        def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil)
           diff_options = self.class.default_options.merge(diff_options || {})
 
-          @diffable     = diffable
-          @diffs        = diffable.raw_diffs(diff_options)
-          @project      = project
+          @diffable = diffable
+          @diffs = diffable.raw_diffs(diff_options)
+          @project = project
           @diff_options = diff_options
-          @diff_refs    = diff_refs
+          @diff_refs = diff_refs
+          @fallback_diff_refs = fallback_diff_refs
         end
 
         def diff_files
@@ -35,7 +36,7 @@ module Gitlab
         private
 
         def decorate_diff!(diff)
-          Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs)
+          Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs)
         end
       end
     end
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb
index 0bd226ef050..9a58b500a2c 100644
--- a/lib/gitlab/diff/file_collection/merge_request_diff.rb
+++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb
@@ -8,7 +8,8 @@ module Gitlab
           super(merge_request_diff,
             project: merge_request_diff.project,
             diff_options: diff_options,
-            diff_refs: merge_request_diff.diff_refs)
+            diff_refs: merge_request_diff.diff_refs,
+            fallback_diff_refs: merge_request_diff.fallback_diff_refs)
         end
 
         def diff_files
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index 7db896522a9..ed2f541977a 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -3,7 +3,7 @@ module Gitlab
     class Highlight
       attr_reader :diff_file, :diff_lines, :raw_lines, :repository
 
-      delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff
+      delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff
 
       def initialize(diff_lines, repository: nil)
         @repository = repository
@@ -61,12 +61,12 @@ module Gitlab
 
       def old_lines
         return unless diff_file
-        @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_ref, diff_old_path)
+        @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_sha, diff_old_path)
       end
 
       def new_lines
         return unless diff_file
-        @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_ref, diff_new_path)
+        @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_sha, diff_new_path)
       end
     end
   end
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 0a15c6d9358..bd52ae47e9f 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -59,6 +59,10 @@ module Gitlab
         type == 'match'
       end
 
+      def discussable?
+        !['match', 'new-nonewline', 'old-nonewline'].include?(type)
+      end
+
       def as_json(opts = nil)
         {
           type: type,
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index 6c69cd9e6a9..ea035e33eff 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -42,7 +42,7 @@ module Gitlab
           return unless compare
 
           # This diff is more moderated in number of files and lines
-          @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, no_collapse: true).diff_files
+          @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, expanded: true).diff_files
         end
 
         def diffs_count
diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
new file mode 100644
index 00000000000..dbe28e6bb93
--- /dev/null
+++ b/lib/gitlab/encoding_helper.rb
@@ -0,0 +1,62 @@
+module Gitlab
+  module EncodingHelper
+    extend self
+
+    # This threshold is carefully tweaked to prevent usage of encodings detected
+    # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
+    # we're better off sticking with utf8 encoding.
+    # Reason: git diff can return strings with invalid utf8 byte sequences if it
+    # truncates a diff in the middle of a multibyte character. In this case
+    # CharlockHolmes will try to guess the encoding and will likely suggest an
+    # obscure encoding with low confidence.
+    # There is a lot more info with this merge request:
+    # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
+    ENCODING_CONFIDENCE_THRESHOLD = 40
+
+    def encode!(message)
+      return nil unless message.respond_to? :force_encoding
+
+      # if message is utf-8 encoding, just return it
+      message.force_encoding("UTF-8")
+      return message if message.valid_encoding?
+
+      # return message if message type is binary
+      detect = CharlockHolmes::EncodingDetector.detect(message)
+      return message.force_encoding("BINARY") if detect && detect[:type] == :binary
+
+      # force detected encoding if we have sufficient confidence.
+      if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
+        message.force_encoding(detect[:encoding])
+      end
+
+      # encode and clean the bad chars
+      message.replace clean(message)
+    rescue
+      encoding = detect ? detect[:encoding] : "unknown"
+      "--broken encoding: #{encoding}"
+    end
+
+    def encode_utf8(message)
+      detect = CharlockHolmes::EncodingDetector.detect(message)
+      if detect
+        begin
+          CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
+        rescue ArgumentError => e
+          Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}")
+
+          ''
+        end
+      else
+        clean(message)
+      end
+    end
+
+    private
+
+    def clean(message)
+      message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
+        .encode("UTF-8")
+        .gsub("\0".encode("UTF-8"), "")
+    end
+  end
+end
diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb
index 2b0e19b338b..ca49eda51fb 100644
--- a/lib/gitlab/etag_caching/router.rb
+++ b/lib/gitlab/etag_caching/router.rb
@@ -9,9 +9,11 @@ module Gitlab
       #   - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route
       #   - Ending in `issues/id`/realtime_changes` for the `issue_title` route
       USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes
-                          commit pipelines merge_requests new].freeze
-      RESERVED_WORDS = Gitlab::Regex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES
-      RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS)
+                          commit pipelines merge_requests builds
+                          new environments].freeze
+      RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES
+      RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape)))
+
       ROUTES = [
         Gitlab::EtagCaching::Router::Route.new(
           %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z),
@@ -40,6 +42,14 @@ module Gitlab
         Gitlab::EtagCaching::Router::Route.new(
           %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines/\d+\.json\z),
           'project_pipeline'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/builds/\d+\.json\z),
+          'project_build'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z),
+          'environments'
         )
       ].freeze
 
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 58193391926..66829a03c2e 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Git
     class Blame
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       attr_reader :lines, :blames
 
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index c1b31618e0d..d60e607b02b 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Git
     class Blob
       include Linguist::BlobHelper
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       # This number is the maximum amount of data that we want to display to
       # the user. We load as much as we can for encoding detection
@@ -88,6 +88,7 @@ module Gitlab
           new(
             id: blob_entry[:oid],
             name: blob_entry[:name],
+            size: 0,
             data: '',
             path: path,
             commit_id: sha
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 297531db4cc..bb04731f08c 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -2,7 +2,7 @@
 module Gitlab
   module Git
     class Commit
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       attr_accessor :raw_commit, :head, :refs
 
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index 31d1b66b4f7..7e21994a084 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -3,7 +3,7 @@ module Gitlab
   module Git
     class Diff
       TimeoutError = Class.new(StandardError)
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       # Diff properties
       attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff
@@ -11,13 +11,20 @@ module Gitlab
       # Stats properties
       attr_accessor :new_file, :renamed_file, :deleted_file
 
+      alias_method :new_file?, :new_file
+      alias_method :deleted_file?, :deleted_file
+      alias_method :renamed_file?, :renamed_file
+
+      attr_accessor :expanded
+
+      # We need this accessor because of `to_hash` and `init_from_hash`
       attr_accessor :too_large
 
       # The maximum size of a diff to display.
-      DIFF_SIZE_LIMIT = 102400 # 100 KB
+      SIZE_LIMIT = 100.kilobytes
 
       # The maximum size before a diff is collapsed.
-      DIFF_COLLAPSE_LIMIT = 10240 # 10 KB
+      COLLAPSE_LIMIT = 10.kilobytes
 
       class << self
         def between(repo, head, base, options = {}, *paths)
@@ -148,7 +155,7 @@ module Gitlab
                              :include_untracked_content, :skip_binary_check,
                              :include_typechange, :include_typechange_trees,
                              :ignore_filemode, :recurse_ignored_dirs, :paths,
-                             :max_files, :max_lines, :all_diffs, :no_collapse]
+                             :max_files, :max_lines, :limits, :expanded]
 
           if default_options
             actual_defaults = default_options.dup
@@ -173,16 +180,18 @@ module Gitlab
         end
       end
 
-      def initialize(raw_diff, collapse: false)
+      def initialize(raw_diff, expanded: true)
+        @expanded = expanded
+
         case raw_diff
         when Hash
           init_from_hash(raw_diff)
-          prune_diff_if_eligible(collapse)
+          prune_diff_if_eligible
         when Rugged::Patch, Rugged::Diff::Delta
-          init_from_rugged(raw_diff, collapse: collapse)
-        when Gitaly::CommitDiffResponse
+          init_from_rugged(raw_diff)
+        when Gitlab::GitalyClient::Diff
           init_from_gitaly(raw_diff)
-          prune_diff_if_eligible(collapse)
+          prune_diff_if_eligible
         when Gitaly::CommitDelta
           init_from_gitaly(raw_diff)
         when nil
@@ -208,6 +217,10 @@ module Gitlab
         hash
       end
 
+      def mode_changed?
+        a_mode && b_mode && a_mode != b_mode
+      end
+
       def submodule?
         a_mode == '160000' || b_mode == '160000'
       end
@@ -218,17 +231,13 @@ module Gitlab
 
       def too_large?
         if @too_large.nil?
-          @too_large = @diff.bytesize >= DIFF_SIZE_LIMIT
+          @too_large = @diff.bytesize >= SIZE_LIMIT
         else
           @too_large
         end
       end
 
-      def collapsible?
-        @diff.bytesize >= DIFF_COLLAPSE_LIMIT
-      end
-
-      def prune_large_diff!
+      def too_large!
         @diff = ''
         @line_count = 0
         @too_large = true
@@ -236,10 +245,11 @@ module Gitlab
 
       def collapsed?
         return @collapsed if defined?(@collapsed)
-        false
+
+        @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT
       end
 
-      def prune_collapsed_diff!
+      def collapse!
         @diff = ''
         @line_count = 0
         @collapsed = true
@@ -247,9 +257,9 @@ module Gitlab
 
       private
 
-      def init_from_rugged(rugged, collapse: false)
+      def init_from_rugged(rugged)
         if rugged.is_a?(Rugged::Patch)
-          init_from_rugged_patch(rugged, collapse: collapse)
+          init_from_rugged_patch(rugged)
           d = rugged.delta
         else
           d = rugged
@@ -264,10 +274,10 @@ module Gitlab
         @deleted_file = d.deleted?
       end
 
-      def init_from_rugged_patch(patch, collapse: false)
+      def init_from_rugged_patch(patch)
         # Don't bother initializing diffs that are too large. If a diff is
         # binary we're not going to display anything so we skip the size check.
-        return if !patch.delta.binary? && prune_large_patch(patch, collapse)
+        return if !patch.delta.binary? && prune_large_patch(patch)
 
         @diff = encode!(strip_diff_headers(patch.to_s))
       end
@@ -280,40 +290,43 @@ module Gitlab
         end
       end
 
-      def init_from_gitaly(msg)
-        @diff = msg.raw_chunks.join if msg.respond_to?(:raw_chunks)
-        @new_path = encode!(msg.to_path.dup)
-        @old_path = encode!(msg.from_path.dup)
-        @a_mode = msg.old_mode.to_s(8)
-        @b_mode = msg.new_mode.to_s(8)
-        @new_file = msg.from_id == BLANK_SHA
-        @renamed_file = msg.from_path != msg.to_path
-        @deleted_file = msg.to_id == BLANK_SHA
+      def init_from_gitaly(diff)
+        @diff = diff.patch if diff.respond_to?(:patch)
+        @new_path = encode!(diff.to_path.dup)
+        @old_path = encode!(diff.from_path.dup)
+        @a_mode = diff.old_mode.to_s(8)
+        @b_mode = diff.new_mode.to_s(8)
+        @new_file = diff.from_id == BLANK_SHA
+        @renamed_file = diff.from_path != diff.to_path
+        @deleted_file = diff.to_id == BLANK_SHA
       end
 
-      def prune_diff_if_eligible(collapse = false)
-        prune_large_diff! if too_large?
-        prune_collapsed_diff! if collapse && collapsible?
+      def prune_diff_if_eligible
+        if too_large?
+          too_large!
+        elsif collapsed?
+          collapse!
+        end
       end
 
       # If the patch surpasses any of the diff limits it calls the appropiate
       # prune method and returns true. Otherwise returns false.
-      def prune_large_patch(patch, collapse)
+      def prune_large_patch(patch)
         size = 0
 
         patch.each_hunk do |hunk|
           hunk.each_line do |line|
             size += line.content.bytesize
 
-            if size >= DIFF_SIZE_LIMIT
-              prune_large_diff!
+            if size >= SIZE_LIMIT
+              too_large!
               return true
             end
           end
         end
 
-        if collapse && size >= DIFF_COLLAPSE_LIMIT
-          prune_collapsed_diff!
+        if !expanded && size >= COLLAPSE_LIMIT
+          collapse!
           return true
         end
 
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index bcbad8ec829..334e06a6eca 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -9,32 +9,29 @@ module Gitlab
         @iterator = iterator
         @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files])
         @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines])
-        @max_bytes = @max_files * 5120 # Average 5 KB per file
+        @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file
         @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min
         @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min
-        @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file
-        @all_diffs = !!options.fetch(:all_diffs, false)
-        @no_collapse = !!options.fetch(:no_collapse, true)
+        @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file
+        @enforce_limits = !!options.fetch(:limits, true)
+        @expanded = !!options.fetch(:expanded, true)
 
         @line_count = 0
         @byte_count = 0
         @overflow = false
+        @empty = true
         @array = Array.new
       end
 
       def each(&block)
-        if @populated
-          # @iterator.each is slower than just iterating the array in place
-          @array.each(&block)
-        else
-          Gitlab::GitalyClient.migrate(:commit_raw_diffs) do
-            each_patch(&block)
-          end
+        Gitlab::GitalyClient.migrate(:commit_raw_diffs) do
+          each_patch(&block)
         end
       end
 
       def empty?
-        !@iterator.any?
+        any? # Make sure the iterator has been exercised
+        @empty
       end
 
       def overflow?
@@ -60,17 +57,17 @@ module Gitlab
         collection = each_with_index do |element, i|
           @array[i] = yield(element)
         end
-        @populated = true
         collection
       end
 
+      alias_method :to_ary, :to_a
+
       private
 
       def populate!
         return if @populated
 
         each { nil } # force a loop through all diffs
-        @populated = true
         nil
       end
 
@@ -79,33 +76,35 @@ module Gitlab
       end
 
       def each_patch
-        @iterator.each_with_index do |raw, i|
-          # First yield cached Diff instances from @array
-          if @array[i]
-            yield @array[i]
-            next
-          end
+        i = 0
+        @array.each do |diff|
+          yield diff
+          i += 1
+        end
+
+        return if @overflow
+        return if @iterator.nil?
 
-          # We have exhausted @array, time to create new Diff instances or stop.
-          break if @overflow
+        @iterator.each do |raw|
+          @empty = false
 
-          if !@all_diffs && i >= @max_files
+          if @enforce_limits && i >= @max_files
             @overflow = true
             break
           end
 
-          collapse = !@all_diffs && !@no_collapse
+          expanded = !@enforce_limits || @expanded
 
-          diff = Gitlab::Git::Diff.new(raw, collapse: collapse)
+          diff = Gitlab::Git::Diff.new(raw, expanded: expanded)
 
-          if collapse && over_safe_limits?(i)
-            diff.prune_collapsed_diff!
+          if !expanded && over_safe_limits?(i)
+            diff.collapse!
           end
 
           @line_count += diff.line_count
           @byte_count += diff.diff.bytesize
 
-          if !@all_diffs && (@line_count >= @max_lines || @byte_count >= @max_bytes)
+          if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes)
             # This last Diff instance pushes us over the lines limit. We stop and
             # discard it.
             @overflow = true
@@ -113,7 +112,13 @@ module Gitlab
           end
 
           yield @array[i] = diff
+          i += 1
         end
+
+        @populated = true
+
+        # Allow iterator to be garbage-collected. It cannot be reused anyway.
+        @iterator = nil
       end
     end
   end
diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb
deleted file mode 100644
index f918074cb14..00000000000
--- a/lib/gitlab/git/encoding_helper.rb
+++ /dev/null
@@ -1,64 +0,0 @@
-module Gitlab
-  module Git
-    module EncodingHelper
-      extend self
-
-      # This threshold is carefully tweaked to prevent usage of encodings detected
-      # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
-      # we're better off sticking with utf8 encoding.
-      # Reason: git diff can return strings with invalid utf8 byte sequences if it
-      # truncates a diff in the middle of a multibyte character. In this case
-      # CharlockHolmes will try to guess the encoding and will likely suggest an
-      # obscure encoding with low confidence.
-      # There is a lot more info with this merge request:
-      # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
-      ENCODING_CONFIDENCE_THRESHOLD = 40
-
-      def encode!(message)
-        return nil unless message.respond_to? :force_encoding
-
-        # if message is utf-8 encoding, just return it
-        message.force_encoding("UTF-8")
-        return message if message.valid_encoding?
-
-        # return message if message type is binary
-        detect = CharlockHolmes::EncodingDetector.detect(message)
-        return message.force_encoding("BINARY") if detect && detect[:type] == :binary
-
-        # force detected encoding if we have sufficient confidence.
-        if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
-          message.force_encoding(detect[:encoding])
-        end
-
-        # encode and clean the bad chars
-        message.replace clean(message)
-      rescue
-        encoding = detect ? detect[:encoding] : "unknown"
-        "--broken encoding: #{encoding}"
-      end
-
-      def encode_utf8(message)
-        detect = CharlockHolmes::EncodingDetector.detect(message)
-        if detect
-          begin
-            CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
-          rescue ArgumentError => e
-            Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}")
-
-            ''
-          end
-        else
-          clean(message)
-        end
-      end
-
-      private
-
-      def clean(message)
-        message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
-          .encode("UTF-8")
-          .gsub("\0".encode("UTF-8"), "")
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb
index 37ef6836742..ebf7393dc61 100644
--- a/lib/gitlab/git/ref.rb
+++ b/lib/gitlab/git/ref.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Git
     class Ref
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       # Branch or tag name
       # without "refs/tags|heads" prefix
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index b9f1ac144b6..9d6adbdb4ac 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -1006,31 +1006,39 @@ module Gitlab
       # Parses the contents of a .gitmodules file and returns a hash of
       # submodule information.
       def parse_gitmodules(commit, content)
-        results = {}
+        modules = {}
 
-        current = ""
-        content.split("\n").each do |txt|
-          if txt =~ /^\s*\[/
-            current = txt.match(/(?<=").*(?=")/)[0]
-            results[current] = {}
-          else
-            next unless results[current]
-            match_data = txt.match(/(\w+)\s*=\s*(.*)/)
-            next unless match_data
-            target = match_data[2].chomp
-            results[current][match_data[1]] = target
+        name = nil
+        content.each_line do |line|
+          case line.strip
+          when /\A\[submodule "(?<name>[^"]+)"\]\z/ # Submodule header
+            name = $~[:name]
+            modules[name] = {}
+          when /\A(?<key>\w+)\s*=\s*(?<value>.*)\z/ # Key/value pair
+            key = $~[:key]
+            value = $~[:value].chomp
+
+            next unless name && modules[name]
+
+            modules[name][key] = value
 
-            if match_data[1] == "path"
+            if key == 'path'
               begin
-                results[current]["id"] = blob_content(commit, target)
+                modules[name]['id'] = blob_content(commit, value)
               rescue InvalidBlobName
-                results.delete(current)
+                # The current entry is invalid
+                modules.delete(name)
+                name = nil
               end
             end
+          when /\A#/ # Comment
+            next
+          else # Invalid line
+            name = nil
           end
         end
 
-        results
+        modules
       end
 
       # Returns true if +commit+ introduced changes to +path+, using commit
@@ -1086,7 +1094,12 @@ module Gitlab
           elsif tmp_entry.nil?
             return nil
           else
-            tmp_entry = rugged.lookup(tmp_entry[:oid])
+            begin
+              tmp_entry = rugged.lookup(tmp_entry[:oid])
+            rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError
+              return nil
+            end
+
             return nil unless tmp_entry.type == :tree
             tmp_entry = tmp_entry[dir]
           end
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index d41256d9a84..b9afa05c819 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Git
     class Tree
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       attr_accessor :id, :root_id, :name, :path, :type,
         :mode, :commit_id, :submodule_url
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 72466700c05..2343446bf22 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -2,6 +2,12 @@ require 'gitaly'
 
 module Gitlab
   module GitalyClient
+    module MigrationStatus
+      DISABLED = 1
+      OPT_IN = 2
+      OPT_OUT = 3
+    end
+
     SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze
 
     MUTEX = Mutex.new
@@ -46,8 +52,20 @@ module Gitlab
       Gitlab.config.gitaly.enabled
     end
 
-    def self.feature_enabled?(feature)
-      enabled? && ENV["GITALY_#{feature.upcase}"] == '1'
+    def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN)
+      return false if !enabled? || status == MigrationStatus::DISABLED
+
+      feature = Feature.get("gitaly_#{feature}")
+
+      # If the feature hasn't been set, turn it on if it's opt-out
+      return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature)
+
+      if feature.percentage_of_time_value > 0
+        # Probabilistically enable this feature
+        return Random.rand() * 100 < feature.percentage_of_time_value
+      end
+
+      feature.enabled?
     end
 
     def self.migrate(feature)
diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb
index 4491903d788..ba3da781dad 100644
--- a/lib/gitlab/gitaly_client/commit.rb
+++ b/lib/gitlab/gitaly_client/commit.rb
@@ -26,7 +26,7 @@ module Gitlab
         request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false)
 
         response = diff_service_stub.commit_diff(Gitaly::CommitDiffRequest.new(request_params))
-        Gitlab::Git::DiffCollection.new(response, options)
+        Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options)
       end
 
       def commit_deltas(commit)
diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb
new file mode 100644
index 00000000000..1e117b7e74a
--- /dev/null
+++ b/lib/gitlab/gitaly_client/diff.rb
@@ -0,0 +1,21 @@
+module Gitlab
+  module GitalyClient
+    class Diff
+      FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch).freeze
+
+      attr_accessor(*FIELDS)
+
+      def initialize(params)
+        params.each do |key, val|
+          public_send(:"#{key}=", val)
+        end
+      end
+
+      def ==(other)
+        FIELDS.all? do |field|
+          public_send(field) == other.public_send(field)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb
new file mode 100644
index 00000000000..d84e8d752dc
--- /dev/null
+++ b/lib/gitlab/gitaly_client/diff_stitcher.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module GitalyClient
+    class DiffStitcher
+      include Enumerable
+
+      def initialize(rpc_response)
+        @rpc_response = rpc_response
+      end
+
+      def each
+        current_diff = nil
+
+        @rpc_response.each do |diff_msg|
+          if current_diff.nil?
+            diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS)
+            diff_params[:patch] = diff_msg.raw_patch_data
+
+            current_diff = GitalyClient::Diff.new(diff_params)
+          else
+            current_diff.patch += diff_msg.raw_patch_data
+          end
+
+          if diff_msg.end_of_patch
+            yield current_diff
+            current_diff = nil
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 21f2e6b6970..319633656ff 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -1,3 +1,5 @@
+# rubocop:disable Metrics/AbcSize
+
 module Gitlab
   module GonHelper
     def add_gon_variables
@@ -13,6 +15,7 @@ module Gitlab
       gon.sentry_dsn             = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled
       gon.gitlab_url             = Gitlab.config.gitlab.url
       gon.revision               = Gitlab::REVISION
+      gon.gitlab_logo            = ActionController::Base.helpers.asset_path('gitlab_logo.png')
 
       if current_user
         gon.current_user_id = current_user.id
diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb
index 890bd9a3554..b1dbf554e41 100644
--- a/lib/gitlab/google_code_import/client.rb
+++ b/lib/gitlab/google_code_import/client.rb
@@ -14,7 +14,7 @@ module Gitlab
       end
 
       def valid?
-        raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects")
+        raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.key?("projects")
       end
 
       def repos
diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb
index 1b43440673c..ab38c0c3e34 100644
--- a/lib/gitlab/google_code_import/importer.rb
+++ b/lib/gitlab/google_code_import/importer.rb
@@ -95,7 +95,7 @@ module Gitlab
           labels = import_issue_labels(raw_issue)
 
           assignee_id = nil
-          if raw_issue.has_key?("owner")
+          if raw_issue.key?("owner")
             username = user_map[raw_issue["owner"]["name"]]
 
             if username.start_with?("@")
@@ -144,7 +144,7 @@ module Gitlab
       def import_issue_comments(issue, comments)
         Note.transaction do
           while raw_comment = comments.shift
-            next if raw_comment.has_key?("deletedBy")
+            next if raw_comment.key?("deletedBy")
 
             content     = format_content(raw_comment["content"])
             updates     = format_updates(raw_comment["updates"])
@@ -235,15 +235,15 @@ module Gitlab
       def format_updates(raw_updates)
         updates = []
 
-        if raw_updates.has_key?("status")
+        if raw_updates.key?("status")
           updates << "*Status: #{raw_updates["status"]}*"
         end
 
-        if raw_updates.has_key?("owner")
+        if raw_updates.key?("owner")
           updates << "*Owner: #{user_map[raw_updates["owner"]]}*"
         end
 
-        if raw_updates.has_key?("cc")
+        if raw_updates.key?("cc")
           cc = raw_updates["cc"].map do |l|
             deleted = l.start_with?("-")
             l = l[1..-1] if deleted
@@ -255,7 +255,7 @@ module Gitlab
           updates << "*Cc: #{cc.join(", ")}*"
         end
 
-        if raw_updates.has_key?("labels")
+        if raw_updates.key?("labels")
           labels = raw_updates["labels"].map do |l|
             deleted = l.start_with?("-")
             l = l[1..-1] if deleted
@@ -267,11 +267,11 @@ module Gitlab
           updates << "*Labels: #{labels.join(", ")}*"
         end
 
-        if raw_updates.has_key?("mergedInto")
+        if raw_updates.key?("mergedInto")
           updates << "*Merged into: ##{raw_updates["mergedInto"]}*"
         end
 
-        if raw_updates.has_key?("blockedOn")
+        if raw_updates.key?("blockedOn")
           blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on|
             format_blocking_updates(raw_blocked_on)
           end
@@ -279,7 +279,7 @@ module Gitlab
           updates << "*Blocked on: #{blocked_ons.join(", ")}*"
         end
 
-        if raw_updates.has_key?("blocking")
+        if raw_updates.key?("blocking")
           blockings = raw_updates["blocking"].map do |raw_blocked_on|
             format_blocking_updates(raw_blocked_on)
           end
diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb
new file mode 100644
index 00000000000..e9d5d52cabb
--- /dev/null
+++ b/lib/gitlab/group_hierarchy.rb
@@ -0,0 +1,104 @@
+module Gitlab
+  # Retrieving of parent or child groups based on a base ActiveRecord relation.
+  #
+  # This class uses recursive CTEs and as a result will only work on PostgreSQL.
+  class GroupHierarchy
+    attr_reader :base, :model
+
+    # base - An instance of ActiveRecord::Relation for which to get parent or
+    #        child groups.
+    def initialize(base)
+      @base = base
+      @model = base.model
+    end
+
+    # Returns a relation that includes the base set of groups and all their
+    # ancestors (recursively).
+    def base_and_ancestors
+      return model.none unless Group.supports_nested_groups?
+
+      base_and_ancestors_cte.apply_to(model.all)
+    end
+
+    # Returns a relation that includes the base set of groups and all their
+    # descendants (recursively).
+    def base_and_descendants
+      return model.none unless Group.supports_nested_groups?
+
+      base_and_descendants_cte.apply_to(model.all)
+    end
+
+    # Returns a relation that includes the base groups, their ancestors, and the
+    # descendants of the base groups.
+    #
+    # The resulting query will roughly look like the following:
+    #
+    #     WITH RECURSIVE ancestors AS ( ... ),
+    #       descendants AS ( ... )
+    #     SELECT *
+    #     FROM (
+    #       SELECT *
+    #       FROM ancestors namespaces
+    #
+    #       UNION
+    #
+    #       SELECT *
+    #       FROM descendants namespaces
+    #     ) groups;
+    #
+    # Using this approach allows us to further add criteria to the relation with
+    # Rails thinking it's selecting data the usual way.
+    def all_groups
+      return base unless Group.supports_nested_groups?
+
+      ancestors = base_and_ancestors_cte
+      descendants = base_and_descendants_cte
+
+      ancestors_table = ancestors.alias_to(groups_table)
+      descendants_table = descendants.alias_to(groups_table)
+
+      union = SQL::Union.new([model.unscoped.from(ancestors_table),
+                              model.unscoped.from(descendants_table)])
+
+      model.
+        unscoped.
+        with.
+        recursive(ancestors.to_arel, descendants.to_arel).
+        from("(#{union.to_sql}) #{model.table_name}")
+    end
+
+    private
+
+    def base_and_ancestors_cte
+      cte = SQL::RecursiveCTE.new(:base_and_ancestors)
+
+      cte << base.except(:order)
+
+      # Recursively get all the ancestors of the base set.
+      cte << model.
+        from([groups_table, cte.table]).
+        where(groups_table[:id].eq(cte.table[:parent_id])).
+        except(:order)
+
+      cte
+    end
+
+    def base_and_descendants_cte
+      cte = SQL::RecursiveCTE.new(:base_and_descendants)
+
+      cte << base.except(:order)
+
+      # Recursively get all the descendants of the base set.
+      cte << model.
+        from([groups_table, cte.table]).
+        where(groups_table[:parent_id].eq(cte.table[:id])).
+        except(:order)
+
+      cte
+    end
+
+    def groups_table
+      model.arel_table
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb
index df962d203b7..e78b7f22e03 100644
--- a/lib/gitlab/health_checks/fs_shards_check.rb
+++ b/lib/gitlab/health_checks/fs_shards_check.rb
@@ -2,6 +2,9 @@ module Gitlab
   module HealthChecks
     class FsShardsCheck
       extend BaseAbstractCheck
+      RANDOM_STRING = SecureRandom.hex(1000).freeze
+      COMMAND_TIMEOUT = '1'.freeze
+      TIMEOUT_EXECUTABLE = 'timeout'.freeze
 
       class << self
         def readiness
@@ -41,8 +44,6 @@ module Gitlab
 
         private
 
-        RANDOM_STRING = SecureRandom.hex(1000).freeze
-
         def operation_metrics(ok_metric, latency_metric, operation, **labels)
           with_timing operation do |result, elapsed|
             [
@@ -63,8 +64,8 @@ module Gitlab
           @storage_paths ||= Gitlab.config.repositories.storages
         end
 
-        def with_timeout(args)
-          %w{timeout 1}.concat(args)
+        def exec_with_timeout(cmd_args, *args, &block)
+          Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block)
         end
 
         def tmp_file_path(storage_name)
@@ -78,7 +79,7 @@ module Gitlab
         def storage_stat_test(storage_name)
           stat_path = File.join(path(storage_name), '.')
           begin
-            _, status = Gitlab::Popen.popen(with_timeout(%W{ stat #{stat_path} }))
+            _, status = exec_with_timeout(%W{ stat #{stat_path} })
             status == 0
           rescue Errno::ENOENT
             File.exist?(stat_path) && File::Stat.new(stat_path).readable?
@@ -86,7 +87,7 @@ module Gitlab
         end
 
         def storage_write_test(tmp_path)
-          _, status = Gitlab::Popen.popen(with_timeout(%W{ tee #{tmp_path} })) do |stdin|
+          _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin|
             stdin.write(RANDOM_STRING)
           end
           status == 0
@@ -96,7 +97,7 @@ module Gitlab
         end
 
         def storage_read_test(tmp_path)
-          _, status = Gitlab::Popen.popen(with_timeout(%W{ diff #{tmp_path} - })) do |stdin|
+          _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin|
             stdin.write(RANDOM_STRING)
           end
           status == 0
@@ -106,7 +107,7 @@ module Gitlab
         end
 
         def delete_test_file(tmp_path)
-          _, status = Gitlab::Popen.popen(with_timeout(%W{ rm -f #{tmp_path} }))
+          _, status = exec_with_timeout(%W{ rm -f #{tmp_path} })
           status == 0
         rescue Errno::ENOENT
           File.delete(tmp_path) rescue Errno::ENOENT
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index 3411516319f..5ab3eeb3aff 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -12,15 +12,36 @@ module Gitlab
       AVAILABLE_LANGUAGES.keys
     end
 
-    def set_locale(current_user)
-      requested_locale = current_user&.preferred_language || ::I18n.default_locale
-      locale = FastGettext.set_locale(requested_locale)
-      ::I18n.locale = locale
+    def locale
+      FastGettext.locale
     end
 
-    def reset_locale
+    def locale=(locale_string)
+      requested_locale = locale_string || ::I18n.default_locale
+      new_locale = FastGettext.set_locale(requested_locale)
+      ::I18n.locale = new_locale
+    end
+
+    def use_default_locale
       FastGettext.set_locale(::I18n.default_locale)
       ::I18n.locale = ::I18n.default_locale
     end
+
+    def with_locale(locale_string)
+      original_locale = locale
+
+      self.locale = locale_string
+      yield
+    ensure
+      self.locale = original_locale
+    end
+
+    def with_user_locale(user, &block)
+      with_locale(user&.preferred_language, &block)
+    end
+
+    def with_default_locale(&block)
+      with_locale(::I18n.default_locale, &block)
+    end
   end
 end
diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb
index 9ad7a38d505..ac9d66c836d 100644
--- a/lib/gitlab/o_auth/provider.rb
+++ b/lib/gitlab/o_auth/provider.rb
@@ -22,7 +22,11 @@ module Gitlab
       def self.config_for(name)
         name = name.to_s
         if ldap_provider?(name)
-          Gitlab::LDAP::Config.new(name).options
+          if Gitlab::LDAP::Config.valid_provider?(name)
+            Gitlab::LDAP::Config.new(name).options
+          else
+            nil
+          end
         else
           Gitlab.config.omniauth.providers.find { |provider| provider.name == name }
         end
diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb
new file mode 100644
index 00000000000..9ff6829cd49
--- /dev/null
+++ b/lib/gitlab/path_regex.rb
@@ -0,0 +1,265 @@
+module Gitlab
+  module PathRegex
+    extend self
+
+    # All routes that appear on the top level must be listed here.
+    # This will make sure that groups cannot be created with these names
+    # as these routes would be masked by the paths already in place.
+    #
+    # Example:
+    #   /api/api-project
+    #
+    #  the path `api` shouldn't be allowed because it would be masked by `api/*`
+    #
+    TOP_LEVEL_ROUTES = %w[
+      -
+      .well-known
+      abuse_reports
+      admin
+      all
+      api
+      assets
+      autocomplete
+      ci
+      dashboard
+      explore
+      files
+      groups
+      health_check
+      help
+      hooks
+      import
+      invites
+      issues
+      jwt
+      koding
+      member
+      merge_requests
+      new
+      notes
+      notification_settings
+      oauth
+      profile
+      projects
+      public
+      repository
+      robots.txt
+      s
+      search
+      sent_notifications
+      services
+      snippets
+      teams
+      u
+      unicorn_test
+      unsubscribes
+      uploads
+      users
+    ].freeze
+
+    # This list should contain all words following `/*namespace_id/:project_id` in
+    # routes that contain a second wildcard.
+    #
+    # Example:
+    #   /*namespace_id/:project_id/badges/*ref/build
+    #
+    # If `badges` was allowed as a project/group name, we would not be able to access the
+    # `badges` route for those projects:
+    #
+    # Consider a namespace with path `foo/bar` and a project called `badges`.
+    # The route to the build badge would then be `/foo/bar/badges/badges/master/build.svg`
+    #
+    # When accessing this path the route would be matched to the `badges` path
+    # with the following params:
+    #   - namespace_id: `foo`
+    #   - project_id: `bar`
+    #   - ref: `badges/master`
+    #
+    # Failing to find the project, this would result in a 404.
+    #
+    # By rejecting `badges` the router can _count_ on the fact that `badges` will
+    # be preceded by the `namespace/project`.
+    PROJECT_WILDCARD_ROUTES = %w[
+      -
+      badges
+      blame
+      blob
+      builds
+      commits
+      create
+      create_dir
+      edit
+      environments/folders
+      files
+      find_file
+      gitlab-lfs/objects
+      info/lfs/objects
+      new
+      preview
+      raw
+      refs
+      tree
+      update
+      wikis
+    ].freeze
+
+    # These are all the paths that follow `/groups/*id/ or `/groups/*group_id`
+    # We need to reject these because we have a `/groups/*id` page that is the same
+    # as the `/*id`.
+    #
+    # If we would allow a subgroup to be created with the name `activity` then
+    # this group would not be accessible through `/groups/parent/activity` since
+    # this would map to the activity-page of its parent.
+    GROUP_ROUTES = %w[
+      activity
+      analytics
+      audit_events
+      avatar
+      edit
+      group_members
+      hooks
+      issues
+      labels
+      ldap
+      ldap_group_links
+      merge_requests
+      milestones
+      notification_setting
+      pipeline_quota
+      projects
+      subgroups
+    ].freeze
+
+    ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES
+    ILLEGAL_GROUP_PATH_WORDS = (PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze
+
+    # The namespace regex is used in JavaScript to validate usernames in the "Register" form. However, Javascript
+    # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`.
+    # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to
+    # allow non-regex validations, etc), `NAMESPACE_FORMAT_REGEX_JS` serves as a Javascript-compatible version of
+    # `NAMESPACE_FORMAT_REGEX`, with the negative lookbehind assertion removed. This means that the client-side validation
+    # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation.
+    PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze
+    NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze
+
+    NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze
+    NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/.freeze
+    PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/.freeze
+    FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/)*#{NAMESPACE_FORMAT_REGEX}}.freeze
+
+    def root_namespace_route_regex
+      @root_namespace_route_regex ||= begin
+        illegal_words = Regexp.new(Regexp.union(TOP_LEVEL_ROUTES).source, Regexp::IGNORECASE)
+
+        single_line_regexp %r{
+          (?!(#{illegal_words})/)
+          #{NAMESPACE_FORMAT_REGEX}
+        }x
+      end
+    end
+
+    def full_namespace_route_regex
+      @full_namespace_route_regex ||= begin
+        illegal_words = Regexp.new(Regexp.union(ILLEGAL_GROUP_PATH_WORDS).source, Regexp::IGNORECASE)
+
+        single_line_regexp %r{
+          #{root_namespace_route_regex}
+          (?:
+            /
+            (?!#{illegal_words}/)
+            #{NAMESPACE_FORMAT_REGEX}
+          )*
+        }x
+      end
+    end
+
+    def project_route_regex
+      @project_route_regex ||= begin
+        illegal_words = Regexp.new(Regexp.union(ILLEGAL_PROJECT_PATH_WORDS).source, Regexp::IGNORECASE)
+
+        single_line_regexp %r{
+          (?!(#{illegal_words})/)
+          #{PROJECT_PATH_FORMAT_REGEX}
+        }x
+      end
+    end
+
+    def project_git_route_regex
+      @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze
+    end
+
+    def root_namespace_path_regex
+      @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z}
+    end
+
+    def full_namespace_path_regex
+      @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z}
+    end
+
+    def project_path_regex
+      @project_path_regex ||= %r{\A#{project_route_regex}/\z}
+    end
+
+    def full_project_path_regex
+      @full_project_path_regex ||= %r{\A#{full_namespace_route_regex}/#{project_route_regex}/\z}
+    end
+
+    def full_namespace_format_regex
+      @namespace_format_regex ||= /A#{FULL_NAMESPACE_FORMAT_REGEX}\z/.freeze
+    end
+
+    def namespace_format_regex
+      @namespace_format_regex ||= /\A#{NAMESPACE_FORMAT_REGEX}\z/.freeze
+    end
+
+    def namespace_format_message
+      "can contain only letters, digits, '_', '-' and '.'. " \
+      "Cannot start with '-' or end in '.', '.git' or '.atom'." \
+    end
+
+    def project_path_format_regex
+      @project_path_format_regex ||= /\A#{PROJECT_PATH_FORMAT_REGEX}\z/.freeze
+    end
+
+    def project_path_format_message
+      "can contain only letters, digits, '_', '-' and '.'. " \
+      "Cannot start with '-', end in '.git' or end in '.atom'" \
+    end
+
+    def archive_formats_regex
+      #                           |zip|tar|    tar.gz    |         tar.bz2         |
+      @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze
+    end
+
+    def git_reference_regex
+      # Valid git ref regex, see:
+      # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html
+
+      @git_reference_regex ||= single_line_regexp %r{
+        (?!
+           (?# doesn't begins with)
+           \/|                    (?# rule #6)
+           (?# doesn't contain)
+           .*(?:
+              [\/.]\.|            (?# rule #1,3)
+              \/\/|               (?# rule #6)
+              @\{|                (?# rule #8)
+              \\                  (?# rule #9)
+           )
+        )
+        [^\000-\040\177~^:?*\[]+  (?# rule #4-5)
+        (?# doesn't end with)
+        (?<!\.lock)               (?# rule #1)
+        (?<![\/.])                (?# rule #6-7)
+      }x
+    end
+
+    private
+
+    def single_line_regexp(regex)
+      # Turns a multiline extended regexp into a single line one,
+      # beacuse `rake routes` breaks on multiline regexes.
+      Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze
+    end
+  end
+end
diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb
new file mode 100644
index 00000000000..bb0df1e3dad
--- /dev/null
+++ b/lib/gitlab/project_authorizations/with_nested_groups.rb
@@ -0,0 +1,125 @@
+module Gitlab
+  module ProjectAuthorizations
+    # Calculating new project authorizations when supporting nested groups.
+    #
+    # This class relies on Common Table Expressions to efficiently get all data,
+    # including data for nested groups. As a result this class can only be used
+    # on PostgreSQL.
+    class WithNestedGroups
+      attr_reader :user
+
+      # user - The User object for which to calculate the authorizations.
+      def initialize(user)
+        @user = user
+      end
+
+      def calculate
+        cte = recursive_cte
+        cte_alias = cte.table.alias(Group.table_name)
+        projects = Project.arel_table
+        links = ProjectGroupLink.arel_table
+
+        relations = [
+          # The project a user has direct access to.
+          user.projects.select_for_project_authorization,
+
+          # The personal projects of the user.
+          user.personal_projects.select_as_master_for_project_authorization,
+
+          # Projects that belong directly to any of the groups the user has
+          # access to.
+          Namespace.
+            unscoped.
+            select([alias_as_column(projects[:id], 'project_id'),
+                    cte_alias[:access_level]]).
+            from(cte_alias).
+            joins(:projects),
+
+          # Projects shared with any of the namespaces the user has access to.
+          Namespace.
+            unscoped.
+            select([links[:project_id],
+                    least(cte_alias[:access_level],
+                          links[:group_access],
+                          'access_level')]).
+            from(cte_alias).
+            joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id').
+            joins('INNER JOIN projects ON projects.id = project_group_links.project_id').
+            joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id').
+            where('p_ns.share_with_group_lock IS FALSE')
+        ]
+
+        union = Gitlab::SQL::Union.new(relations)
+
+        ProjectAuthorization.
+          unscoped.
+          with.
+          recursive(cte.to_arel).
+          select_from_union(union)
+      end
+
+      private
+
+      # Builds a recursive CTE that gets all the groups the current user has
+      # access to, including any nested groups.
+      def recursive_cte
+        cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte)
+        members = Member.arel_table
+        namespaces = Namespace.arel_table
+
+        # Namespaces the user is a member of.
+        cte << user.groups.
+          select([namespaces[:id], members[:access_level]]).
+          except(:order)
+
+        # Sub groups of any groups the user is a member of.
+        cte << Group.select([namespaces[:id],
+                             greatest(members[:access_level],
+                                      cte.table[:access_level], 'access_level')]).
+          joins(join_cte(cte)).
+          joins(join_members).
+          except(:order)
+
+        cte
+      end
+
+      # Builds a LEFT JOIN to join optional memberships onto the CTE.
+      def join_members
+        members = Member.arel_table
+        namespaces = Namespace.arel_table
+
+        cond = members[:source_id].
+          eq(namespaces[:id]).
+          and(members[:source_type].eq('Namespace')).
+          and(members[:requested_at].eq(nil)).
+          and(members[:user_id].eq(user.id))
+
+        Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond))
+      end
+
+      # Builds an INNER JOIN to join namespaces onto the CTE.
+      def join_cte(cte)
+        namespaces = Namespace.arel_table
+        cond = cte.table[:id].eq(namespaces[:parent_id])
+
+        Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond))
+      end
+
+      def greatest(left, right, column_alias)
+        sql_function('GREATEST', [left, right], column_alias)
+      end
+
+      def least(left, right, column_alias)
+        sql_function('LEAST', [left, right], column_alias)
+      end
+
+      def sql_function(name, args, column_alias)
+        alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias)
+      end
+
+      def alias_as_column(value, alias_to)
+        Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb
new file mode 100644
index 00000000000..627e8c5fba2
--- /dev/null
+++ b/lib/gitlab/project_authorizations/without_nested_groups.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module ProjectAuthorizations
+    # Calculating new project authorizations when not supporting nested groups.
+    class WithoutNestedGroups
+      attr_reader :user
+
+      # user - The User object for which to calculate the authorizations.
+      def initialize(user)
+        @user = user
+      end
+
+      def calculate
+        relations = [
+          # Projects the user is a direct member of
+          user.projects.select_for_project_authorization,
+
+          # Personal projects
+          user.personal_projects.select_as_master_for_project_authorization,
+
+          # Projects of groups the user is a member of
+          user.groups_projects.select_for_project_authorization,
+
+          # Projects shared with groups the user is a member of
+          user.groups.joins(:shared_projects).select_for_project_authorization
+        ]
+
+        union = Gitlab::SQL::Union.new(relations)
+
+        ProjectAuthorization.
+          unscoped.
+          select_from_union(union)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index f609850f8fa..e4d2a992470 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -2,203 +2,6 @@ module Gitlab
   module Regex
     extend self
 
-    # All routes that appear on the top level must be listed here.
-    # This will make sure that groups cannot be created with these names
-    # as these routes would be masked by the paths already in place.
-    #
-    # Example:
-    #   /api/api-project
-    #
-    #  the path `api` shouldn't be allowed because it would be masked by `api/*`
-    #
-    TOP_LEVEL_ROUTES = %w[
-      -
-      .well-known
-      abuse_reports
-      admin
-      all
-      api
-      assets
-      autocomplete
-      ci
-      dashboard
-      explore
-      files
-      groups
-      health_check
-      help
-      hooks
-      import
-      invites
-      issues
-      jwt
-      koding
-      member
-      merge_requests
-      new
-      notes
-      notification_settings
-      oauth
-      profile
-      projects
-      public
-      repository
-      robots.txt
-      s
-      search
-      sent_notifications
-      services
-      snippets
-      teams
-      u
-      unicorn_test
-      unsubscribes
-      uploads
-      users
-    ].freeze
-
-    # This list should contain all words following `/*namespace_id/:project_id` in
-    # routes that contain a second wildcard.
-    #
-    # Example:
-    #   /*namespace_id/:project_id/badges/*ref/build
-    #
-    # If `badges` was allowed as a project/group name, we would not be able to access the
-    # `badges` route for those projects:
-    #
-    # Consider a namespace with path `foo/bar` and a project called `badges`.
-    # The route to the build badge would then be `/foo/bar/badges/badges/master/build.svg`
-    #
-    # When accessing this path the route would be matched to the `badges` path
-    # with the following params:
-    #   - namespace_id: `foo`
-    #   - project_id: `bar`
-    #   - ref: `badges/master`
-    #
-    # Failing to find the project, this would result in a 404.
-    #
-    # By rejecting `badges` the router can _count_ on the fact that `badges` will
-    # be preceded by the `namespace/project`.
-    PROJECT_WILDCARD_ROUTES = %w[
-      badges
-      blame
-      blob
-      builds
-      commits
-      create
-      create_dir
-      edit
-      environments/folders
-      files
-      find_file
-      gitlab-lfs/objects
-      info/lfs/objects
-      new
-      preview
-      raw
-      refs
-      tree
-      update
-      wikis
-    ].freeze
-
-    # These are all the paths that follow `/groups/*id/ or `/groups/*group_id`
-    # We need to reject these because we have a `/groups/*id` page that is the same
-    # as the `/*id`.
-    #
-    # If we would allow a subgroup to be created with the name `activity` then
-    # this group would not be accessible through `/groups/parent/activity` since
-    # this would map to the activity-page of its parent.
-    GROUP_ROUTES = %w[
-      activity
-      analytics
-      audit_events
-      avatar
-      edit
-      group_members
-      hooks
-      issues
-      labels
-      ldap
-      ldap_group_links
-      merge_requests
-      milestones
-      notification_setting
-      pipeline_quota
-      projects
-      subgroups
-    ].freeze
-
-    ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES
-    ILLEGAL_GROUP_PATH_WORDS = (PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze
-
-    # The namespace regex is used in Javascript to validate usernames in the "Register" form. However, Javascript
-    # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`.
-    # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to
-    # allow non-regex validatiions, etc), `NAMESPACE_REGEX_STR_JS` serves as a Javascript-compatible version of
-    # `NAMESPACE_REGEX_STR`, with the negative lookbehind assertion removed. This means that the client-side validation
-    # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation.
-    PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze
-    NAMESPACE_REGEX_STR_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze
-    NO_SUFFIX_REGEX_STR = '(?<!\.git|\.atom)'.freeze
-    NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR_JS})#{NO_SUFFIX_REGEX_STR}".freeze
-    PROJECT_REGEX_STR = "(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX_STR}".freeze
-
-    # Same as NAMESPACE_REGEX_STR but allows `/` in the path.
-    # So `group/subgroup` will match this regex but not NAMESPACE_REGEX_STR
-    FULL_NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR}/)*#{NAMESPACE_REGEX_STR}".freeze
-
-    def root_namespace_route_regex
-      @root_namespace_route_regex ||= begin
-        illegal_words = Regexp.new(Regexp.union(TOP_LEVEL_ROUTES).source, Regexp::IGNORECASE)
-
-        single_line_regexp %r{
-          (?!(#{illegal_words})/)
-          #{NAMESPACE_REGEX_STR}
-        }x
-      end
-    end
-
-    def root_namespace_path_regex
-      @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z}
-    end
-
-    def full_namespace_path_regex
-      @full_namespace_path_regex ||= %r{\A#{namespace_route_regex}/\z}
-    end
-
-    def full_project_path_regex
-      @full_project_path_regex ||= %r{\A#{namespace_route_regex}/#{project_route_regex}/\z}
-    end
-
-    def namespace_regex
-      @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze
-    end
-
-    def full_namespace_regex
-      @full_namespace_regex ||= %r{\A#{FULL_NAMESPACE_REGEX_STR}\z}
-    end
-
-    def namespace_route_regex
-      @namespace_route_regex ||= begin
-        illegal_words = Regexp.new(Regexp.union(ILLEGAL_GROUP_PATH_WORDS).source, Regexp::IGNORECASE)
-
-        single_line_regexp %r{
-          #{root_namespace_route_regex}
-          (?:
-            /
-            (?!#{illegal_words}/)
-            #{NAMESPACE_REGEX_STR}
-          )*
-        }x
-      end
-    end
-
-    def namespace_regex_message
-      "can contain only letters, digits, '_', '-' and '.'. " \
-      "Cannot start with '-' or end in '.', '.git' or '.atom'." \
-    end
-
     def namespace_name_regex
       @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze
     end
@@ -216,34 +19,6 @@ module Gitlab
       "It must start with letter, digit, emoji or '_'."
     end
 
-    def project_path_regex
-      @project_path_regex ||= %r{\A#{project_route_regex}/\z}
-    end
-
-    def project_route_regex
-      @project_route_regex ||= begin
-        illegal_words = Regexp.new(Regexp.union(ILLEGAL_PROJECT_PATH_WORDS).source, Regexp::IGNORECASE)
-
-        single_line_regexp %r{
-          (?!(#{illegal_words})/)
-          #{PROJECT_REGEX_STR}
-        }x
-      end
-    end
-
-    def project_git_route_regex
-      @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze
-    end
-
-    def project_path_format_regex
-      @project_path_format_regex ||= /\A#{PROJECT_REGEX_STR}\z/.freeze
-    end
-
-    def project_path_regex_message
-      "can contain only letters, digits, '_', '-' and '.'. " \
-      "Cannot start with '-', end in '.git' or end in '.atom'" \
-    end
-
     def file_name_regex
       @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze
     end
@@ -252,36 +27,8 @@ module Gitlab
       "can contain only letters, digits, '_', '-', '@', '+' and '.'."
     end
 
-    def archive_formats_regex
-      #                           |zip|tar|    tar.gz    |         tar.bz2         |
-      @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze
-    end
-
-    def git_reference_regex
-      # Valid git ref regex, see:
-      # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html
-
-      @git_reference_regex ||= single_line_regexp %r{
-        (?!
-           (?# doesn't begins with)
-           \/|                    (?# rule #6)
-           (?# doesn't contain)
-           .*(?:
-              [\/.]\.|            (?# rule #1,3)
-              \/\/|               (?# rule #6)
-              @\{|                (?# rule #8)
-              \\                  (?# rule #9)
-           )
-        )
-        [^\000-\040\177~^:?*\[]+  (?# rule #4-5)
-        (?# doesn't end with)
-        (?<!\.lock)               (?# rule #1)
-        (?<![\/.])                (?# rule #6-7)
-      }x
-    end
-
     def container_registry_reference_regex
-      git_reference_regex
+      Gitlab::PathRegex.git_reference_regex
     end
 
     ##
@@ -315,13 +62,5 @@ module Gitlab
       "can contain only lowercase letters, digits, and '-'. " \
       "Must start with a letter, and cannot end with '-'"
     end
-
-    private
-
-    def single_line_regexp(regex)
-      # Turns a multiline extended regexp into a single line one,
-      # beacuse `rake routes` breaks on multiline regexes.
-      Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze
-    end
   end
 end
diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb
index 36791fae60f..877aa6e6a28 100644
--- a/lib/gitlab/route_map.rb
+++ b/lib/gitlab/route_map.rb
@@ -25,8 +25,8 @@ module Gitlab
 
     def parse_entry(entry)
       raise FormatError, 'Route map entry is not a hash' unless entry.is_a?(Hash)
-      raise FormatError, 'Route map entry does not have a source key' unless entry.has_key?('source')
-      raise FormatError, 'Route map entry does not have a public key' unless entry.has_key?('public')
+      raise FormatError, 'Route map entry does not have a source key' unless entry.key?('source')
+      raise FormatError, 'Route map entry does not have a public key' unless entry.key?('public')
 
       source_pattern = entry['source']
       public_path = entry['public']
diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb
new file mode 100644
index 00000000000..36d1a8a6f64
--- /dev/null
+++ b/lib/gitlab/routes/legacy_builds.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Routes
+    class LegacyBuilds
+      def initialize(map)
+        @map = map
+      end
+
+      def draw
+        @map.instance_eval do
+          resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do
+            collection do
+              resources :artifacts, only: [], controller: 'build_artifacts' do
+                collection do
+                  get :latest_succeeded,
+                    path: '*ref_name_and_path',
+                    format: false
+                end
+              end
+            end
+
+            member do
+              get :raw
+            end
+
+            resource :artifacts, only: [], controller: 'build_artifacts' do
+              get :download
+              get :browse, path: 'browse(/*path)', format: false
+              get :file, path: 'file/*path', format: false
+              get :raw, path: 'raw/*path', format: false
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb
new file mode 100644
index 00000000000..5b1b03820a3
--- /dev/null
+++ b/lib/gitlab/sql/recursive_cte.rb
@@ -0,0 +1,62 @@
+module Gitlab
+  module SQL
+    # Class for easily building recursive CTE statements.
+    #
+    # Example:
+    #
+    #     cte = RecursiveCTE.new(:my_cte_name)
+    #     ns = Arel::Table.new(:namespaces)
+    #
+    #     cte << Namespace.
+    #       where(ns[:parent_id].eq(some_namespace_id))
+    #
+    #     cte << Namespace.
+    #       from([ns, cte.table]).
+    #       where(ns[:parent_id].eq(cte.table[:id]))
+    #
+    #     Namespace.with.
+    #       recursive(cte.to_arel).
+    #       from(cte.alias_to(ns))
+    class RecursiveCTE
+      attr_reader :table
+
+      # name - The name of the CTE as a String or Symbol.
+      def initialize(name)
+        @table = Arel::Table.new(name)
+        @queries = []
+      end
+
+      # Adds a query to the body of the CTE.
+      #
+      # relation - The relation object to add to the body of the CTE.
+      def <<(relation)
+        @queries << relation
+      end
+
+      # Returns the Arel relation for this CTE.
+      def to_arel
+        sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries).to_sql)
+
+        Arel::Nodes::As.new(table, Arel::Nodes::Grouping.new(sql))
+      end
+
+      # Returns an "AS" statement that aliases the CTE name as the given table
+      # name. This allows one to trick ActiveRecord into thinking it's selecting
+      # from an actual table, when in reality it's selecting from a CTE.
+      #
+      # alias_table - The Arel table to use as the alias.
+      def alias_to(alias_table)
+        Arel::Nodes::As.new(table, alias_table)
+      end
+
+      # Applies the CTE to the given relation, returning a new one that will
+      # query from it.
+      def apply_to(relation)
+        relation.except(:where).
+          with.
+          recursive(to_arel).
+          from(alias_to(relation.model.arel_table))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb
index 9ce13feb79a..c81dc7e30d0 100644
--- a/lib/gitlab/url_sanitizer.rb
+++ b/lib/gitlab/url_sanitizer.rb
@@ -18,12 +18,6 @@ module Gitlab
       false
     end
 
-    def self.http_credentials_for_user(user)
-      return {} unless user.respond_to?(:username)
-
-      { user: user.username }
-    end
-
     def initialize(url, credentials: nil)
       @url = Addressable::URI.parse(url.strip)
       @credentials = credentials
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index 4c395b4266e..fa182c4deda 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -21,5 +21,13 @@ module Gitlab
 
       nil
     end
+
+    def boolean_to_yes_no(bool)
+      if bool
+        'Yes'
+      else
+        'No'
+      end
+    end
   end
 end
diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb
index 2e31f4462f9..85da4c8660b 100644
--- a/lib/gitlab/visibility_level.rb
+++ b/lib/gitlab/visibility_level.rb
@@ -83,7 +83,7 @@ module Gitlab
       end
 
       def valid_level?(level)
-        options.has_value?(level)
+        options.value?(level)
       end
 
       def level_name(level)
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index fe37e4da94f..7f27317775c 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -31,8 +31,7 @@ module Gitlab
 
           feature_enabled = case action.to_s
                             when 'git_receive_pack'
-                              # Disabled for now, see https://gitlab.com/gitlab-org/gitaly/issues/172
-                              false
+                              Gitlab::GitalyClient.feature_enabled?(:post_receive_pack)
                             when 'git_upload_pack'
                               Gitlab::GitalyClient.feature_enabled?(:post_upload_pack)
                             when 'info_refs'
@@ -130,7 +129,7 @@ module Gitlab
             'MaxSessionTime' => terminal[:max_session_time]
           }
         }
-        details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem)
+        details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.key?(:ca_pem)
 
         details
       end
diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab
index 6e351365de0..c5f93336346 100755
--- a/lib/support/init.d/gitlab
+++ b/lib/support/init.d/gitlab
@@ -48,7 +48,7 @@ gitlab_pages_pid_path="$pid_path/gitlab-pages.pid"
 gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090"
 gitlab_pages_log="$app_root/log/gitlab-pages.log"
 shell_path="/bin/bash"
-gitaly_enabled=false
+gitaly_enabled=true
 gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd)
 gitaly_pid_path="$pid_path/gitaly.pid"
 gitaly_log="$app_root/log/gitaly.log"
diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example
index 9472c3c992f..295c79fccfc 100644
--- a/lib/support/init.d/gitlab.default.example
+++ b/lib/support/init.d/gitlab.default.example
@@ -86,5 +86,7 @@ mail_room_pid_path="$pid_path/mail_room.pid"
 shell_path="/bin/bash"
 
 # This variable controls whether the init script starts/stops Gitaly
-gitaly_enabled=false
+gitaly_enabled=true
+gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd)
+gitaly_pid_path="$pid_path/gitaly.pid"
 gitaly_log="$app_root/log/gitaly.log"
diff --git a/lib/system_check.rb b/lib/system_check.rb
new file mode 100644
index 00000000000..466c39904fa
--- /dev/null
+++ b/lib/system_check.rb
@@ -0,0 +1,21 @@
+# Library to perform System Checks
+#
+# Every Check is implemented as its own class inherited from SystemCheck::BaseCheck
+# Execution coordination and boilerplate output is done by the SystemCheck::SimpleExecutor
+#
+# This structure decouples checks from Rake tasks and facilitates unit-testing
+module SystemCheck
+  # Executes a bunch of checks for specified component
+  #
+  # @param [String] component name of the component relative to the checks being executed
+  # @param [Array<BaseCheck>] checks classes of corresponding checks to be executed in the same order
+  def self.run(component, checks = [])
+    executor = SimpleExecutor.new(component)
+
+    checks.each do |check|
+      executor << check
+    end
+
+    executor.execute
+  end
+end
diff --git a/lib/system_check/app/active_users_check.rb b/lib/system_check/app/active_users_check.rb
new file mode 100644
index 00000000000..1d72c8d6903
--- /dev/null
+++ b/lib/system_check/app/active_users_check.rb
@@ -0,0 +1,17 @@
+module SystemCheck
+  module App
+    class ActiveUsersCheck < SystemCheck::BaseCheck
+      set_name 'Active users:'
+
+      def multi_check
+        active_users = User.active.count
+
+        if active_users > 0
+          $stdout.puts active_users.to_s.color(:green)
+        else
+          $stdout.puts active_users.to_s.color(:red)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/database_config_exists_check.rb b/lib/system_check/app/database_config_exists_check.rb
new file mode 100644
index 00000000000..d1fae192350
--- /dev/null
+++ b/lib/system_check/app/database_config_exists_check.rb
@@ -0,0 +1,25 @@
+module SystemCheck
+  module App
+    class DatabaseConfigExistsCheck < SystemCheck::BaseCheck
+      set_name 'Database config exists?'
+
+      def check?
+        database_config_file = Rails.root.join('config', 'database.yml')
+
+        File.exist?(database_config_file)
+      end
+
+      def show_error
+        try_fixing_it(
+          'Copy config/database.yml.<your db> to config/database.yml',
+          'Check that the information in config/database.yml is correct'
+        )
+        for_more_information(
+          'doc/install/databases.md',
+          'http://guides.rubyonrails.org/getting_started.html#configuring-a-database'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb
new file mode 100644
index 00000000000..198867f7ac6
--- /dev/null
+++ b/lib/system_check/app/git_config_check.rb
@@ -0,0 +1,42 @@
+module SystemCheck
+  module App
+    class GitConfigCheck < SystemCheck::BaseCheck
+      OPTIONS = {
+        'core.autocrlf' => 'input'
+      }.freeze
+
+      set_name 'Git configured correctly?'
+
+      def check?
+        correct_options = OPTIONS.map do |name, value|
+          run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value
+        end
+
+        correct_options.all?
+      end
+
+      # Tries to configure git itself
+      #
+      # Returns true if all subcommands were successful (according to their exit code)
+      # Returns false if any or all subcommands failed.
+      def repair!
+        return false unless is_gitlab_user?
+
+        command_success = OPTIONS.map do |name, value|
+          system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value}))
+        end
+
+        command_success.all?
+      end
+
+      def show_error
+        try_fixing_it(
+          sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{OPTIONS['core.autocrlf']}\"")
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb
new file mode 100644
index 00000000000..c388682dfb4
--- /dev/null
+++ b/lib/system_check/app/git_version_check.rb
@@ -0,0 +1,29 @@
+module SystemCheck
+  module App
+    class GitVersionCheck < SystemCheck::BaseCheck
+      set_name -> { "Git version >= #{self.required_version} ?" }
+      set_check_pass -> { "yes (#{self.current_version})" }
+
+      def self.required_version
+        @required_version ||= Gitlab::VersionInfo.new(2, 7, 3)
+      end
+
+      def self.current_version
+        @current_version ||= Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version)))
+      end
+
+      def check?
+        self.class.current_version.valid? && self.class.required_version <= self.class.current_version
+      end
+
+      def show_error
+        $stdout.puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\""
+
+        try_fixing_it(
+          "Update your git to a version >= #{self.class.required_version} from #{self.class.current_version}"
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/gitlab_config_exists_check.rb b/lib/system_check/app/gitlab_config_exists_check.rb
new file mode 100644
index 00000000000..247aa0994e4
--- /dev/null
+++ b/lib/system_check/app/gitlab_config_exists_check.rb
@@ -0,0 +1,24 @@
+module SystemCheck
+  module App
+    class GitlabConfigExistsCheck < SystemCheck::BaseCheck
+      set_name 'GitLab config exists?'
+
+      def check?
+        gitlab_config_file = Rails.root.join('config', 'gitlab.yml')
+
+        File.exist?(gitlab_config_file)
+      end
+
+      def show_error
+        try_fixing_it(
+          'Copy config/gitlab.yml.example to config/gitlab.yml',
+          'Update config/gitlab.yml to match your setup'
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/gitlab_config_up_to_date_check.rb b/lib/system_check/app/gitlab_config_up_to_date_check.rb
new file mode 100644
index 00000000000..c609e48e133
--- /dev/null
+++ b/lib/system_check/app/gitlab_config_up_to_date_check.rb
@@ -0,0 +1,30 @@
+module SystemCheck
+  module App
+    class GitlabConfigUpToDateCheck < SystemCheck::BaseCheck
+      set_name 'GitLab config up to date?'
+      set_skip_reason "can't check because of previous errors"
+
+      def skip?
+        gitlab_config_file = Rails.root.join('config', 'gitlab.yml')
+        !File.exist?(gitlab_config_file)
+      end
+
+      def check?
+        # omniauth or ldap could have been deleted from the file
+        !Gitlab.config['git_host']
+      end
+
+      def show_error
+        try_fixing_it(
+          'Back-up your config/gitlab.yml',
+          'Copy config/gitlab.yml.example to config/gitlab.yml',
+          'Update config/gitlab.yml to match your setup'
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/init_script_exists_check.rb b/lib/system_check/app/init_script_exists_check.rb
new file mode 100644
index 00000000000..d246e058e86
--- /dev/null
+++ b/lib/system_check/app/init_script_exists_check.rb
@@ -0,0 +1,27 @@
+module SystemCheck
+  module App
+    class InitScriptExistsCheck < SystemCheck::BaseCheck
+      set_name 'Init script exists?'
+      set_skip_reason 'skipped (omnibus-gitlab has no init script)'
+
+      def skip?
+        omnibus_gitlab?
+      end
+
+      def check?
+        script_path = '/etc/init.d/gitlab'
+        File.exist?(script_path)
+      end
+
+      def show_error
+        try_fixing_it(
+          'Install the init script'
+        )
+        for_more_information(
+          see_installation_guide_section 'Install Init Script'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb
new file mode 100644
index 00000000000..015c7ed1731
--- /dev/null
+++ b/lib/system_check/app/init_script_up_to_date_check.rb
@@ -0,0 +1,43 @@
+module SystemCheck
+  module App
+    class InitScriptUpToDateCheck < SystemCheck::BaseCheck
+      SCRIPT_PATH = '/etc/init.d/gitlab'.freeze
+
+      set_name 'Init script up-to-date?'
+      set_skip_reason 'skipped (omnibus-gitlab has no init script)'
+
+      def skip?
+        omnibus_gitlab?
+      end
+
+      def multi_check
+        recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab')
+
+        unless File.exist?(SCRIPT_PATH)
+          $stdout.puts "can't check because of previous errors".color(:magenta)
+          return
+        end
+
+        recipe_content = File.read(recipe_path)
+        script_content = File.read(SCRIPT_PATH)
+
+        if recipe_content == script_content
+          $stdout.puts 'yes'.color(:green)
+        else
+          $stdout.puts 'no'.color(:red)
+          show_error
+        end
+      end
+
+      def show_error
+        try_fixing_it(
+          'Re-download the init script'
+        )
+        for_more_information(
+          see_installation_guide_section 'Install Init Script'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/log_writable_check.rb b/lib/system_check/app/log_writable_check.rb
new file mode 100644
index 00000000000..3e0c436d6ee
--- /dev/null
+++ b/lib/system_check/app/log_writable_check.rb
@@ -0,0 +1,28 @@
+module SystemCheck
+  module App
+    class LogWritableCheck < SystemCheck::BaseCheck
+      set_name 'Log directory writable?'
+
+      def check?
+        File.writable?(log_path)
+      end
+
+      def show_error
+        try_fixing_it(
+          "sudo chown -R gitlab #{log_path}",
+          "sudo chmod -R u+rwX #{log_path}"
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+
+      private
+
+      def log_path
+        Rails.root.join('log')
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/migrations_are_up_check.rb b/lib/system_check/app/migrations_are_up_check.rb
new file mode 100644
index 00000000000..5eedbacce77
--- /dev/null
+++ b/lib/system_check/app/migrations_are_up_check.rb
@@ -0,0 +1,20 @@
+module SystemCheck
+  module App
+    class MigrationsAreUpCheck < SystemCheck::BaseCheck
+      set_name 'All migrations up?'
+
+      def check?
+        migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status))
+
+        migration_status !~ /down\s+\d{14}/
+      end
+
+      def show_error
+        try_fixing_it(
+          sudo_gitlab('bundle exec rake db:migrate RAILS_ENV=production')
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/orphaned_group_members_check.rb b/lib/system_check/app/orphaned_group_members_check.rb
new file mode 100644
index 00000000000..2b46d36fe51
--- /dev/null
+++ b/lib/system_check/app/orphaned_group_members_check.rb
@@ -0,0 +1,20 @@
+module SystemCheck
+  module App
+    class OrphanedGroupMembersCheck < SystemCheck::BaseCheck
+      set_name 'Database contains orphaned GroupMembers?'
+      set_check_pass 'no'
+      set_check_fail 'yes'
+
+      def check?
+        !GroupMember.where('user_id not in (select id from users)').exists?
+      end
+
+      def show_error
+        try_fixing_it(
+          'You can delete the orphaned records using something along the lines of:',
+          sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'")
+        )
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/projects_have_namespace_check.rb b/lib/system_check/app/projects_have_namespace_check.rb
new file mode 100644
index 00000000000..a6ec9f7665c
--- /dev/null
+++ b/lib/system_check/app/projects_have_namespace_check.rb
@@ -0,0 +1,37 @@
+module SystemCheck
+  module App
+    class ProjectsHaveNamespaceCheck < SystemCheck::BaseCheck
+      set_name 'Projects have namespace:'
+      set_skip_reason "can't check, you have no projects"
+
+      def skip?
+        !Project.exists?
+      end
+
+      def multi_check
+        $stdout.puts ''
+
+        Project.find_each(batch_size: 100) do |project|
+          $stdout.print sanitized_message(project)
+
+          if project.namespace
+            $stdout.puts 'yes'.color(:green)
+          else
+            $stdout.puts 'no'.color(:red)
+            show_error
+          end
+        end
+      end
+
+      def show_error
+        try_fixing_it(
+          "Migrate global projects"
+        )
+        for_more_information(
+          "doc/update/5.4-to-6.0.md in section \"#global-projects\""
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb
new file mode 100644
index 00000000000..a0610e73576
--- /dev/null
+++ b/lib/system_check/app/redis_version_check.rb
@@ -0,0 +1,25 @@
+module SystemCheck
+  module App
+    class RedisVersionCheck < SystemCheck::BaseCheck
+      MIN_REDIS_VERSION = '2.8.0'.freeze
+      set_name "Redis version >= #{MIN_REDIS_VERSION}?"
+
+      def check?
+        redis_version = run_command(%w(redis-cli --version))
+        redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/)
+
+        redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(MIN_REDIS_VERSION))
+      end
+
+      def show_error
+        try_fixing_it(
+          "Update your redis server to a version >= #{MIN_REDIS_VERSION}"
+        )
+        for_more_information(
+          'gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb
new file mode 100644
index 00000000000..fd82f5f8a4a
--- /dev/null
+++ b/lib/system_check/app/ruby_version_check.rb
@@ -0,0 +1,27 @@
+module SystemCheck
+  module App
+    class RubyVersionCheck < SystemCheck::BaseCheck
+      set_name -> { "Ruby version >= #{self.required_version} ?" }
+      set_check_pass -> { "yes (#{self.current_version})" }
+
+      def self.required_version
+        @required_version ||= Gitlab::VersionInfo.new(2, 3, 3)
+      end
+
+      def self.current_version
+        @current_version ||= Gitlab::VersionInfo.parse(run_command(%w(ruby --version)))
+      end
+
+      def check?
+        self.class.current_version.valid? && self.class.required_version <= self.class.current_version
+      end
+
+      def show_error
+        try_fixing_it(
+          "Update your ruby to a version >= #{self.class.required_version} from #{self.class.current_version}"
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/tmp_writable_check.rb b/lib/system_check/app/tmp_writable_check.rb
new file mode 100644
index 00000000000..99a75e57abf
--- /dev/null
+++ b/lib/system_check/app/tmp_writable_check.rb
@@ -0,0 +1,28 @@
+module SystemCheck
+  module App
+    class TmpWritableCheck < SystemCheck::BaseCheck
+      set_name 'Tmp directory writable?'
+
+      def check?
+        File.writable?(tmp_path)
+      end
+
+      def show_error
+        try_fixing_it(
+          "sudo chown -R gitlab #{tmp_path}",
+          "sudo chmod -R u+rwX #{tmp_path}"
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+
+      private
+
+      def tmp_path
+        Rails.root.join('tmp')
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/uploads_directory_exists_check.rb b/lib/system_check/app/uploads_directory_exists_check.rb
new file mode 100644
index 00000000000..7026d0ba075
--- /dev/null
+++ b/lib/system_check/app/uploads_directory_exists_check.rb
@@ -0,0 +1,21 @@
+module SystemCheck
+  module App
+    class UploadsDirectoryExistsCheck < SystemCheck::BaseCheck
+      set_name 'Uploads directory exists?'
+
+      def check?
+        File.directory?(Rails.root.join('public/uploads'))
+      end
+
+      def show_error
+        try_fixing_it(
+          "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads"
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/uploads_path_permission_check.rb b/lib/system_check/app/uploads_path_permission_check.rb
new file mode 100644
index 00000000000..7df6c060254
--- /dev/null
+++ b/lib/system_check/app/uploads_path_permission_check.rb
@@ -0,0 +1,36 @@
+module SystemCheck
+  module App
+    class UploadsPathPermissionCheck < SystemCheck::BaseCheck
+      set_name 'Uploads directory has correct permissions?'
+      set_skip_reason 'skipped (no uploads folder found)'
+
+      def skip?
+        !File.directory?(rails_uploads_path)
+      end
+
+      def check?
+        File.stat(uploads_fullpath).mode == 040700
+      end
+
+      def show_error
+        try_fixing_it(
+          "sudo chmod 700 #{uploads_fullpath}"
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+
+      private
+
+      def rails_uploads_path
+        Rails.root.join('public/uploads')
+      end
+
+      def uploads_fullpath
+        File.realpath(rails_uploads_path)
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/uploads_path_tmp_permission_check.rb b/lib/system_check/app/uploads_path_tmp_permission_check.rb
new file mode 100644
index 00000000000..b276a81eac1
--- /dev/null
+++ b/lib/system_check/app/uploads_path_tmp_permission_check.rb
@@ -0,0 +1,40 @@
+module SystemCheck
+  module App
+    class UploadsPathTmpPermissionCheck < SystemCheck::BaseCheck
+      set_name 'Uploads directory tmp has correct permissions?'
+      set_skip_reason 'skipped (no tmp uploads folder yet)'
+
+      def skip?
+        !File.directory?(uploads_fullpath) || !Dir.exist?(upload_path_tmp)
+      end
+
+      def check?
+        # If tmp upload dir has incorrect permissions, assume others do as well
+        # Verify drwx------ permissions
+        File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp)
+      end
+
+      def show_error
+        try_fixing_it(
+          "sudo chown -R #{gitlab_user} #{uploads_fullpath}",
+          "sudo find #{uploads_fullpath} -type f -exec chmod 0644 {} \\;",
+          "sudo find #{uploads_fullpath} -type d -not -path #{uploads_fullpath} -exec chmod 0700 {} \\;"
+        )
+        for_more_information(
+          see_installation_guide_section 'GitLab'
+        )
+        fix_and_rerun
+      end
+
+      private
+
+      def upload_path_tmp
+        File.join(uploads_fullpath, 'tmp')
+      end
+
+      def uploads_fullpath
+        File.realpath(Rails.root.join('public/uploads'))
+      end
+    end
+  end
+end
diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb
new file mode 100644
index 00000000000..5dcb3f0886b
--- /dev/null
+++ b/lib/system_check/base_check.rb
@@ -0,0 +1,129 @@
+module SystemCheck
+  # Base class for Checks. You must inherit from here
+  # and implement the methods below when necessary
+  class BaseCheck
+    include ::SystemCheck::Helpers
+
+    # Define a custom term for when check passed
+    #
+    # @param [String] term used when check passed (default: 'yes')
+    def self.set_check_pass(term)
+      @check_pass = term
+    end
+
+    # Define a custom term for when check failed
+    #
+    # @param [String] term used when check failed (default: 'no')
+    def self.set_check_fail(term)
+      @check_fail = term
+    end
+
+    # Define the name of the SystemCheck that will be displayed during execution
+    #
+    # @param [String] name of the check
+    def self.set_name(name)
+      @name = name
+    end
+
+    # Define the reason why we skipped the SystemCheck
+    #
+    # This is only used if subclass implements `#skip?`
+    #
+    # @param [String] reason to be displayed
+    def self.set_skip_reason(reason)
+      @skip_reason = reason
+    end
+
+    # Term to be displayed when check passed
+    #
+    # @return [String] term when check passed ('yes' if not re-defined in a subclass)
+    def self.check_pass
+      call_or_return(@check_pass) || 'yes'
+    end
+
+    ## Term to be displayed when check failed
+    #
+    # @return [String] term when check failed ('no' if not re-defined in a subclass)
+    def self.check_fail
+      call_or_return(@check_fail) || 'no'
+    end
+
+    # Name of the SystemCheck defined by the subclass
+    #
+    # @return [String] the name
+    def self.display_name
+      call_or_return(@name) || self.name
+    end
+
+    # Skip reason defined by the subclass
+    #
+    # @return [String] the reason
+    def self.skip_reason
+      call_or_return(@skip_reason) || 'skipped'
+    end
+
+    # Does the check support automatically repair routine?
+    #
+    # @return [Boolean] whether check implemented `#repair!` method or not
+    def can_repair?
+      self.class.instance_methods(false).include?(:repair!)
+    end
+
+    def can_skip?
+      self.class.instance_methods(false).include?(:skip?)
+    end
+
+    def is_multi_check?
+      self.class.instance_methods(false).include?(:multi_check)
+    end
+
+    # Execute the check routine
+    #
+    # This is where you should implement the main logic that will return
+    # a boolean at the end
+    #
+    # You should not print any output to STDOUT here, use the specific methods instead
+    #
+    # @return [Boolean] whether check passed or failed
+    def check?
+      raise NotImplementedError
+    end
+
+    # Execute a custom check that cover multiple unities
+    #
+    # When using multi_check you have to provide the output yourself
+    def multi_check
+      raise NotImplementedError
+    end
+
+    # Prints troubleshooting instructions
+    #
+    # This is where you should print detailed information for any error found during #check?
+    #
+    # You may use helper methods to help format the output:
+    #
+    # @see #try_fixing_it
+    # @see #fix_and_rerun
+    # @see #for_more_infromation
+    def show_error
+      raise NotImplementedError
+    end
+
+    # When implemented by a subclass, will attempt to fix the issue automatically
+    def repair!
+      raise NotImplementedError
+    end
+
+    # When implemented by a subclass, will evaluate whether check should be skipped or not
+    #
+    # @return [Boolean] whether or not this check should be skipped
+    def skip?
+      raise NotImplementedError
+    end
+
+    def self.call_or_return(input)
+      input.respond_to?(:call) ? input.call : input
+    end
+    private_class_method :call_or_return
+  end
+end
diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb
new file mode 100644
index 00000000000..c42ae4fe4c4
--- /dev/null
+++ b/lib/system_check/helpers.rb
@@ -0,0 +1,75 @@
+require 'tasks/gitlab/task_helpers'
+
+module SystemCheck
+  module Helpers
+    include ::Gitlab::TaskHelpers
+
+    # Display a message telling to fix and rerun the checks
+    def fix_and_rerun
+      $stdout.puts '  Please fix the error above and rerun the checks.'.color(:red)
+    end
+
+    # Display a formatted list of references (documentation or links) where to find more information
+    #
+    # @param [Array<String>] sources one or more references (documentation or links)
+    def for_more_information(*sources)
+      $stdout.puts '  For more information see:'.color(:blue)
+      sources.each do |source|
+        $stdout.puts "  #{source}"
+      end
+    end
+
+    def see_installation_guide_section(section)
+      "doc/install/installation.md in section \"#{section}\""
+    end
+
+    # @deprecated This will no longer be used when all checks were executed using SystemCheck
+    def finished_checking(component)
+      $stdout.puts ''
+      $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}"
+      $stdout.puts ''
+    end
+
+    # @deprecated This will no longer be used when all checks were executed using SystemCheck
+    def start_checking(component)
+      $stdout.puts "Checking #{component.color(:yellow)} ..."
+      $stdout.puts ''
+    end
+
+    # Display a formatted list of instructions on how to fix the issue identified by the #check?
+    #
+    # @param [Array<String>] steps one or short sentences with help how to fix the issue
+    def try_fixing_it(*steps)
+      steps = steps.shift if steps.first.is_a?(Array)
+
+      $stdout.puts '  Try fixing it:'.color(:blue)
+      steps.each do |step|
+        $stdout.puts "  #{step}"
+      end
+    end
+
+    def sanitized_message(project)
+      if should_sanitize?
+        "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... "
+      else
+        "#{project.name_with_namespace.color(:yellow)} ... "
+      end
+    end
+
+    def should_sanitize?
+      if ENV['SANITIZE'] == 'true'
+        true
+      else
+        false
+      end
+    end
+
+    def omnibus_gitlab?
+      Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails'
+    end
+
+    def sudo_gitlab(command)
+      "sudo -u #{gitlab_user} -H #{command}"
+    end
+  end
+end
diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb
new file mode 100644
index 00000000000..dc2d4643a01
--- /dev/null
+++ b/lib/system_check/simple_executor.rb
@@ -0,0 +1,99 @@
+module SystemCheck
+  # Simple Executor is current default executor for GitLab
+  # It is a simple port from display logic in the old check.rake
+  #
+  # There is no concurrency level and the output is progressively
+  # printed into the STDOUT
+  #
+  # @attr_reader [Array<BaseCheck>] checks classes of corresponding checks to be executed in the same order
+  # @attr_reader [String] component name of the component relative to the checks being executed
+  class SimpleExecutor
+    attr_reader :checks
+    attr_reader :component
+
+    # @param [String] component name of the component relative to the checks being executed
+    def initialize(component)
+      raise ArgumentError unless component.is_a? String
+
+      @component = component
+      @checks = Set.new
+    end
+
+    # Add a check to be executed
+    #
+    # @param [BaseCheck] check class
+    def <<(check)
+      raise ArgumentError unless check < BaseCheck
+      @checks << check
+    end
+
+    # Executes defined checks in the specified order and outputs confirmation or error information
+    def execute
+      start_checking(component)
+
+      @checks.each do |check|
+        run_check(check)
+      end
+
+      finished_checking(component)
+    end
+
+    # Executes a single check
+    #
+    # @param [SystemCheck::BaseCheck] check_klass
+    def run_check(check_klass)
+      $stdout.print "#{check_klass.display_name} ... "
+
+      check = check_klass.new
+
+      # When implements skip method, we run it first, and if true, skip the check
+      if check.can_skip? && check.skip?
+        $stdout.puts check_klass.skip_reason.color(:magenta)
+        return
+      end
+
+      # When implements a multi check, we don't control the output
+      if check.is_multi_check?
+        check.multi_check
+        return
+      end
+
+      if check.check?
+        $stdout.puts check_klass.check_pass.color(:green)
+      else
+        $stdout.puts check_klass.check_fail.color(:red)
+
+        if check.can_repair?
+          $stdout.print 'Trying to fix error automatically. ...'
+          if check.repair!
+            $stdout.puts 'Success'.color(:green)
+            return
+          else
+            $stdout.puts 'Failed'.color(:red)
+          end
+        end
+
+        check.show_error
+      end
+    end
+
+    private
+
+    # Prints header content for the series of checks to be executed for this component
+    #
+    # @param [String] component name of the component relative to the checks being executed
+    def start_checking(component)
+      $stdout.puts "Checking #{component.color(:yellow)} ..."
+      $stdout.puts ''
+    end
+
+    # Prints footer content for the series of checks executed for this component
+    #
+    # @param [String] component name of the component relative to the checks being executed
+    def finished_checking(component)
+      $stdout.puts ''
+      $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}"
+      $stdout.puts ''
+    end
+  end
+end
diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
index 0aa21a4bd13..b27f7475115 100644
--- a/lib/tasks/gettext.rake
+++ b/lib/tasks/gettext.rake
@@ -11,4 +11,12 @@ namespace :gettext do
       "{#{folders}}/**/*.{#{exts}}"
     )
   end
+
+  task :compile do
+    # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/33014#note_31218998
+    FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot'))
+
+    Rake::Task['gettext:pack'].invoke
+    Rake::Task['gettext:po_to_json'].invoke
+  end
 end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index f41c73154f5..63c5e9b9c83 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -1,5 +1,9 @@
+# Temporary hack, until we migrate all checks to SystemCheck format
+require 'system_check'
+require 'system_check/helpers'
+
 namespace :gitlab do
-  desc "GitLab | Check the configuration of GitLab and its environment"
+  desc 'GitLab | Check the configuration of GitLab and its environment'
   task check: %w{gitlab:gitlab_shell:check
                  gitlab:sidekiq:check
                  gitlab:incoming_email:check
@@ -7,331 +11,38 @@ namespace :gitlab do
                  gitlab:app:check}
 
   namespace :app do
-    desc "GitLab | Check the configuration of the GitLab Rails app"
+    desc 'GitLab | Check the configuration of the GitLab Rails app'
     task check: :environment  do
       warn_user_is_not_gitlab
-      start_checking "GitLab"
-
-      check_git_config
-      check_database_config_exists
-      check_migrations_are_up
-      check_orphaned_group_members
-      check_gitlab_config_exists
-      check_gitlab_config_not_outdated
-      check_log_writable
-      check_tmp_writable
-      check_uploads
-      check_init_script_exists
-      check_init_script_up_to_date
-      check_projects_have_namespace
-      check_redis_version
-      check_ruby_version
-      check_git_version
-      check_active_users
-
-      finished_checking "GitLab"
-    end
-
-    # Checks
-    ########################
-
-    def check_git_config
-      print "Git configured with autocrlf=input? ... "
-
-      options = {
-        "core.autocrlf" => "input"
-      }
-
-      correct_options = options.map do |name, value|
-        run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value
-      end
-
-      if correct_options.all?
-        puts "yes".color(:green)
-      else
-        print "Trying to fix Git error automatically. ..."
-
-        if auto_fix_git_config(options)
-          puts "Success".color(:green)
-        else
-          puts "Failed".color(:red)
-          try_fixing_it(
-            sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{options["core.autocrlf"]}\"")
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab"
-          )
-        end
-      end
-    end
-
-    def check_database_config_exists
-      print "Database config exists? ... "
-
-      database_config_file = Rails.root.join("config", "database.yml")
-
-      if File.exist?(database_config_file)
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Copy config/database.yml.<your db> to config/database.yml",
-          "Check that the information in config/database.yml is correct"
-        )
-        for_more_information(
-          see_database_guide,
-          "http://guides.rubyonrails.org/getting_started.html#configuring-a-database"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_gitlab_config_exists
-      print "GitLab config exists? ... "
-
-      gitlab_config_file = Rails.root.join("config", "gitlab.yml")
-
-      if File.exist?(gitlab_config_file)
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Copy config/gitlab.yml.example to config/gitlab.yml",
-          "Update config/gitlab.yml to match your setup"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_gitlab_config_not_outdated
-      print "GitLab config outdated? ... "
-
-      gitlab_config_file = Rails.root.join("config", "gitlab.yml")
-      unless File.exist?(gitlab_config_file)
-        puts "can't check because of previous errors".color(:magenta)
-      end
-
-      # omniauth or ldap could have been deleted from the file
-      unless Gitlab.config['git_host']
-        puts "no".color(:green)
-      else
-        puts "yes".color(:red)
-        try_fixing_it(
-          "Backup your config/gitlab.yml",
-          "Copy config/gitlab.yml.example to config/gitlab.yml",
-          "Update config/gitlab.yml to match your setup"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_init_script_exists
-      print "Init script exists? ... "
-
-      if omnibus_gitlab?
-        puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta)
-        return
-      end
-
-      script_path = "/etc/init.d/gitlab"
-
-      if File.exist?(script_path)
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Install the init script"
-        )
-        for_more_information(
-          see_installation_guide_section "Install Init Script"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_init_script_up_to_date
-      print "Init script up-to-date? ... "
-
-      if omnibus_gitlab?
-        puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta)
-        return
-      end
-
-      recipe_path = Rails.root.join("lib/support/init.d/", "gitlab")
-      script_path = "/etc/init.d/gitlab"
-
-      unless File.exist?(script_path)
-        puts "can't check because of previous errors".color(:magenta)
-        return
-      end
-
-      recipe_content = File.read(recipe_path)
-      script_content = File.read(script_path)
-
-      if recipe_content == script_content
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Redownload the init script"
-        )
-        for_more_information(
-          see_installation_guide_section "Install Init Script"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_migrations_are_up
-      print "All migrations up? ... "
-
-      migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status))
-
-      unless migration_status =~ /down\s+\d{14}/
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          sudo_gitlab("bundle exec rake db:migrate RAILS_ENV=production")
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_orphaned_group_members
-      print "Database contains orphaned GroupMembers? ... "
-      if GroupMember.where("user_id not in (select id from users)").count > 0
-        puts "yes".color(:red)
-        try_fixing_it(
-          "You can delete the orphaned records using something along the lines of:",
-          sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'")
-        )
-      else
-        puts "no".color(:green)
-      end
-    end
-
-    def check_log_writable
-      print "Log directory writable? ... "
-
-      log_path = Rails.root.join("log")
-
-      if File.writable?(log_path)
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "sudo chown -R gitlab #{log_path}",
-          "sudo chmod -R u+rwX #{log_path}"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-      end
-    end
 
-    def check_tmp_writable
-      print "Tmp directory writable? ... "
-
-      tmp_path = Rails.root.join("tmp")
-
-      if File.writable?(tmp_path)
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "sudo chown -R gitlab #{tmp_path}",
-          "sudo chmod -R u+rwX #{tmp_path}"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_uploads
-      print "Uploads directory setup correctly? ... "
-
-      unless File.directory?(Rails.root.join('public/uploads'))
-        puts "no".color(:red)
-        try_fixing_it(
-          "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-        return
-      end
-
-      upload_path = File.realpath(Rails.root.join('public/uploads'))
-      upload_path_tmp = File.join(upload_path, 'tmp')
-
-      if File.stat(upload_path).mode == 040700
-        unless Dir.exist?(upload_path_tmp)
-          puts 'skipped (no tmp uploads folder yet)'.color(:magenta)
-          return
-        end
-
-        # If tmp upload dir has incorrect permissions, assume others do as well
-        # Verify drwx------ permissions
-        if File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp)
-          puts "yes".color(:green)
-        else
-          puts "no".color(:red)
-          try_fixing_it(
-            "sudo chown -R #{gitlab_user} #{upload_path}",
-            "sudo find #{upload_path} -type f -exec chmod 0644 {} \\;",
-            "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;"
-          )
-          for_more_information(
-            see_installation_guide_section "GitLab"
-          )
-          fix_and_rerun
-        end
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "sudo chmod 700 #{upload_path}"
-        )
-        for_more_information(
-          see_installation_guide_section "GitLab"
-        )
-        fix_and_rerun
-      end
-    end
-
-    def check_redis_version
-      min_redis_version = "2.8.0"
-      print "Redis version >= #{min_redis_version}? ... "
-
-      redis_version = run_command(%w(redis-cli --version))
-      redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/)
-      if redis_version &&
-          (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version))
-        puts "yes".color(:green)
-      else
-        puts "no".color(:red)
-        try_fixing_it(
-          "Update your redis server to a version >= #{min_redis_version}"
-        )
-        for_more_information(
-          "gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq"
-        )
-        fix_and_rerun
-      end
+      checks = [
+        SystemCheck::App::GitConfigCheck,
+        SystemCheck::App::DatabaseConfigExistsCheck,
+        SystemCheck::App::MigrationsAreUpCheck,
+        SystemCheck::App::OrphanedGroupMembersCheck,
+        SystemCheck::App::GitlabConfigExistsCheck,
+        SystemCheck::App::GitlabConfigUpToDateCheck,
+        SystemCheck::App::LogWritableCheck,
+        SystemCheck::App::TmpWritableCheck,
+        SystemCheck::App::UploadsDirectoryExistsCheck,
+        SystemCheck::App::UploadsPathPermissionCheck,
+        SystemCheck::App::UploadsPathTmpPermissionCheck,
+        SystemCheck::App::InitScriptExistsCheck,
+        SystemCheck::App::InitScriptUpToDateCheck,
+        SystemCheck::App::ProjectsHaveNamespaceCheck,
+        SystemCheck::App::RedisVersionCheck,
+        SystemCheck::App::RubyVersionCheck,
+        SystemCheck::App::GitVersionCheck,
+        SystemCheck::App::ActiveUsersCheck
+      ]
+
+      SystemCheck.run('GitLab', checks)
     end
   end
 
   namespace :gitlab_shell do
+    include SystemCheck::Helpers
+
     desc "GitLab | Check the configuration of GitLab Shell"
     task check: :environment  do
       warn_user_is_not_gitlab
@@ -513,33 +224,6 @@ namespace :gitlab do
       end
     end
 
-    def check_projects_have_namespace
-      print "projects have namespace: ... "
-
-      unless Project.count > 0
-        puts "can't check, you have no projects".color(:magenta)
-        return
-      end
-      puts ""
-
-      Project.find_each(batch_size: 100) do |project|
-        print sanitized_message(project)
-
-        if project.namespace
-          puts "yes".color(:green)
-        else
-          puts "no".color(:red)
-          try_fixing_it(
-            "Migrate global projects"
-          )
-          for_more_information(
-            "doc/update/5.4-to-6.0.md in section \"#global-projects\""
-          )
-          fix_and_rerun
-        end
-      end
-    end
-
     # Helper methods
     ########################
 
@@ -565,6 +249,8 @@ namespace :gitlab do
   end
 
   namespace :sidekiq do
+    include SystemCheck::Helpers
+
     desc "GitLab | Check the configuration of Sidekiq"
     task check: :environment  do
       warn_user_is_not_gitlab
@@ -623,6 +309,8 @@ namespace :gitlab do
   end
 
   namespace :incoming_email do
+    include SystemCheck::Helpers
+
     desc "GitLab | Check the configuration of Reply by email"
     task check: :environment  do
       warn_user_is_not_gitlab
@@ -757,6 +445,8 @@ namespace :gitlab do
   end
 
   namespace :ldap do
+    include SystemCheck::Helpers
+
     task :check, [:limit] => :environment do |_, args|
       # Only show up to 100 results because LDAP directories can be very big.
       # This setting only affects the `rake gitlab:check` script.
@@ -812,6 +502,8 @@ namespace :gitlab do
   end
 
   namespace :repo do
+    include SystemCheck::Helpers
+
     desc "GitLab | Check the integrity of the repositories managed by GitLab"
     task check: :environment do
       Gitlab.config.repositories.storages.each do |name, repository_storage|
@@ -826,6 +518,8 @@ namespace :gitlab do
   end
 
   namespace :user do
+    include SystemCheck::Helpers
+
     desc "GitLab | Check the integrity of a specific user's repositories"
     task :check_repos, [:username] => :environment do |t, args|
       username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue))
@@ -848,55 +542,6 @@ namespace :gitlab do
   # Helper methods
   ##########################
 
-  def fix_and_rerun
-    puts "  Please fix the error above and rerun the checks.".color(:red)
-  end
-
-  def for_more_information(*sources)
-    sources = sources.shift if sources.first.is_a?(Array)
-
-    puts "  For more information see:".color(:blue)
-    sources.each do |source|
-      puts "  #{source}"
-    end
-  end
-
-  def finished_checking(component)
-    puts ""
-    puts "Checking #{component.color(:yellow)} ... #{"Finished".color(:green)}"
-    puts ""
-  end
-
-  def see_database_guide
-    "doc/install/databases.md"
-  end
-
-  def see_installation_guide_section(section)
-    "doc/install/installation.md in section \"#{section}\""
-  end
-
-  def sudo_gitlab(command)
-    "sudo -u #{gitlab_user} -H #{command}"
-  end
-
-  def gitlab_user
-    Gitlab.config.gitlab.user
-  end
-
-  def start_checking(component)
-    puts "Checking #{component.color(:yellow)} ..."
-    puts ""
-  end
-
-  def try_fixing_it(*steps)
-    steps = steps.shift if steps.first.is_a?(Array)
-
-    puts "  Try fixing it:".color(:blue)
-    steps.each do |step|
-      puts "  #{step}"
-    end
-  end
-
   def check_gitlab_shell
     required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version)
     current_version = Gitlab::VersionInfo.parse(gitlab_shell_version)
@@ -909,65 +554,6 @@ namespace :gitlab do
     end
   end
 
-  def check_ruby_version
-    required_version = Gitlab::VersionInfo.new(2, 1, 0)
-    current_version = Gitlab::VersionInfo.parse(run_command(%w(ruby --version)))
-
-    print "Ruby version >= #{required_version} ? ... "
-
-    if current_version.valid? && required_version <= current_version
-      puts "yes (#{current_version})".color(:green)
-    else
-      puts "no".color(:red)
-      try_fixing_it(
-        "Update your ruby to a version >= #{required_version} from #{current_version}"
-      )
-      fix_and_rerun
-    end
-  end
-
-  def check_git_version
-    required_version = Gitlab::VersionInfo.new(2, 7, 3)
-    current_version = Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version)))
-
-    puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\""
-    print "Git version >= #{required_version} ? ... "
-
-    if current_version.valid? && required_version <= current_version
-      puts "yes (#{current_version})".color(:green)
-    else
-      puts "no".color(:red)
-      try_fixing_it(
-        "Update your git to a version >= #{required_version} from #{current_version}"
-      )
-      fix_and_rerun
-    end
-  end
-
-  def check_active_users
-    puts "Active users: #{User.active.count}"
-  end
-
-  def omnibus_gitlab?
-    Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails'
-  end
-
-  def sanitized_message(project)
-    if should_sanitize?
-      "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... "
-    else
-      "#{project.name_with_namespace.color(:yellow)} ... "
-    end
-  end
-
-  def should_sanitize?
-    if ENV['SANITIZE'] == "true"
-      true
-    else
-      false
-    end
-  end
-
   def check_repo_integrity(repo_dir)
     puts "\nChecking repo at #{repo_dir.color(:yellow)}"
 
diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb
index e3c9d3b491c..964aa0fe1bc 100644
--- a/lib/tasks/gitlab/task_helpers.rb
+++ b/lib/tasks/gitlab/task_helpers.rb
@@ -98,34 +98,30 @@ module Gitlab
       end
     end
 
+    def gitlab_user
+      Gitlab.config.gitlab.user
+    end
+
+    def is_gitlab_user?
+      return @is_gitlab_user unless @is_gitlab_user.nil?
+
+      current_user = run_command(%w(whoami)).chomp
+      @is_gitlab_user = current_user == gitlab_user
+    end
+
     def warn_user_is_not_gitlab
-      unless @warned_user_not_gitlab
-        gitlab_user = Gitlab.config.gitlab.user
+      return if @warned_user_not_gitlab
+
+      unless is_gitlab_user?
         current_user = run_command(%w(whoami)).chomp
-        unless current_user == gitlab_user
-          puts " Warning ".color(:black).background(:yellow)
-          puts "  You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing."
-          puts "  Things may work\/fail for the wrong reasons."
-          puts "  For correct results you should run this as user #{gitlab_user.color(:magenta)}."
-          puts ""
-        end
-        @warned_user_not_gitlab = true
-      end
-    end
 
-    # Tries to configure git itself
-    #
-    # Returns true if all subcommands were successfull (according to their exit code)
-    # Returns false if any or all subcommands failed.
-    def auto_fix_git_config(options)
-      if !@warned_user_not_gitlab
-        command_success = options.map do |name, value|
-          system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value}))
-        end
+        puts " Warning ".color(:black).background(:yellow)
+        puts "  You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing."
+        puts "  Things may work\/fail for the wrong reasons."
+        puts "  For correct results you should run this as user #{gitlab_user.color(:magenta)}."
+        puts ""
 
-        command_success.all?
-      else
-        false
+        @warned_user_not_gitlab = true
       end
     end