21 files changed, 154 insertions, 170 deletions

diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index 88cb7e7b5a4..9895db9e451 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -53,6 +53,8 @@ module Backup
           FileUtils.mv(files, timestamped_files_path)
         rescue Errno::EACCES
           access_denied_error(app_files_dir)
+        rescue Errno::EBUSY
+          resource_busy_error(app_files_dir)
         end
       end
     end
diff --git a/lib/backup/helper.rb b/lib/backup/helper.rb
index a1ee0faefe9..54b9ce10b4d 100644
--- a/lib/backup/helper.rb
+++ b/lib/backup/helper.rb
@@ -13,5 +13,19 @@ module Backup
       EOS
       raise message
     end
+
+    def resource_busy_error(path)
+      message = <<~EOS
+
+      ### NOTICE ###
+      As part of restore, the task tried to rename `#{path}` before restoring.
+      This could not be completed, perhaps `#{path}` is a mountpoint?
+
+      To complete the restore, please move the contents of `#{path}` to a
+      different location and run the restore task again.
+
+      EOS
+      raise message
+    end
   end
 end
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 89e3f1d9076..65e06fd78c0 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -81,6 +81,8 @@ module Backup
           FileUtils.mv(files, bk_repos_path)
         rescue Errno::EACCES
           access_denied_error(path)
+        rescue Errno::EBUSY
+          resource_busy_error(path)
         end
       end
     end
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index 0a167104bf4..c5498d0da1a 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -1,4 +1,3 @@
-require_dependency 'settings'
 require_dependency 'gitlab/popen'
 
 module Gitlab
@@ -30,6 +29,6 @@ module Gitlab
   end
 
   def self.dev_env_or_com?
-    Rails.env.test? || Rails.env.development? || org? || com?
+    Rails.env.development? || org? || com?
   end
 end
diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb
index 068212d9a21..922d0567d99 100644
--- a/lib/gitlab/auth/ldap/user.rb
+++ b/lib/gitlab/auth/ldap/user.rb
@@ -8,6 +8,8 @@ module Gitlab
   module Auth
     module LDAP
       class User < Gitlab::Auth::OAuth::User
+        extend ::Gitlab::Utils::Override
+
         class << self
           def find_by_uid_and_provider(uid, provider)
             identity = ::Identity.with_extern_uid(provider, uid).take
@@ -29,7 +31,8 @@ module Gitlab
           self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider)
         end
 
-        def changed?
+        override :should_save?
+        def should_save?
           gl_user.changed? || gl_user.identities.any?(&:changed?)
         end
 
@@ -41,6 +44,10 @@ module Gitlab
           Gitlab::Auth::LDAP::Access.allowed?(gl_user)
         end
 
+        def valid_sign_in?
+          allowed? && super
+        end
+
         def ldap_config
           Gitlab::Auth::LDAP::Config.new(auth_hash.provider)
         end
diff --git a/lib/gitlab/auth/o_auth/identity_linker.rb b/lib/gitlab/auth/o_auth/identity_linker.rb
new file mode 100644
index 00000000000..de92d7a214d
--- /dev/null
+++ b/lib/gitlab/auth/o_auth/identity_linker.rb
@@ -0,0 +1,8 @@
+module Gitlab
+  module Auth
+    module OAuth
+      class IdentityLinker < OmniauthIdentityLinkerBase
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb
index d0c6b0386ba..6c5d0788a0a 100644
--- a/lib/gitlab/auth/o_auth/user.rb
+++ b/lib/gitlab/auth/o_auth/user.rb
@@ -30,6 +30,10 @@ module Gitlab
           gl_user.try(:valid?)
         end
 
+        def valid_sign_in?
+          valid? && persisted?
+        end
+
         def save(provider = 'OAuth')
           raise SigninDisabledForProviderError if oauth_provider_disabled?
           raise SignupDisabledError unless gl_user
@@ -64,8 +68,18 @@ module Gitlab
           user
         end
 
+        def find_and_update!
+          save if should_save?
+
+          gl_user
+        end
+
         protected
 
+        def should_save?
+          true
+        end
+
         def add_or_update_user_identities
           return unless gl_user
 
diff --git a/lib/gitlab/auth/omniauth_identity_linker_base.rb b/lib/gitlab/auth/omniauth_identity_linker_base.rb
new file mode 100644
index 00000000000..ae365fcdfaa
--- /dev/null
+++ b/lib/gitlab/auth/omniauth_identity_linker_base.rb
@@ -0,0 +1,47 @@
+module Gitlab
+  module Auth
+    class OmniauthIdentityLinkerBase
+      attr_reader :current_user, :oauth
+
+      def initialize(current_user, oauth)
+        @current_user = current_user
+        @oauth = oauth
+        @changed = false
+      end
+
+      def link
+        save if identity.new_record?
+      end
+
+      def changed?
+        @changed
+      end
+
+      def error_message
+        identity.validate
+
+        identity.errors.full_messages.join(', ')
+      end
+
+      private
+
+      def save
+        @changed = identity.save
+      end
+
+      def identity
+        @identity ||= current_user.identities
+                                  .with_extern_uid(provider, uid)
+                                  .first_or_initialize(extern_uid: uid)
+      end
+
+      def provider
+        oauth['provider']
+      end
+
+      def uid
+        oauth['uid']
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/auth/saml/identity_linker.rb b/lib/gitlab/auth/saml/identity_linker.rb
new file mode 100644
index 00000000000..7e4b191d512
--- /dev/null
+++ b/lib/gitlab/auth/saml/identity_linker.rb
@@ -0,0 +1,8 @@
+module Gitlab
+  module Auth
+    module Saml
+      class IdentityLinker < OmniauthIdentityLinkerBase
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb
index d4024e9ec39..cb01cd8004c 100644
--- a/lib/gitlab/auth/saml/user.rb
+++ b/lib/gitlab/auth/saml/user.rb
@@ -7,6 +7,8 @@ module Gitlab
   module Auth
     module Saml
       class User < Gitlab::Auth::OAuth::User
+        extend ::Gitlab::Utils::Override
+
         def save
           super('SAML')
         end
@@ -21,13 +23,14 @@ module Gitlab
           if external_users_enabled? && user
             # Check if there is overlap between the user's groups and the external groups
             # setting then set user as external or internal.
-            user.external = !(auth_hash.groups & Gitlab::Auth::Saml::Config.external_groups).empty?
+            user.external = !(auth_hash.groups & saml_config.external_groups).empty?
           end
 
           user
         end
 
-        def changed?
+        override :should_save?
+        def should_save?
           return true unless gl_user
 
           gl_user.changed? || gl_user.identities.any?(&:changed?)
@@ -35,12 +38,16 @@ module Gitlab
 
         protected
 
+        def saml_config
+          Gitlab::Auth::Saml::Config
+        end
+
         def auto_link_saml_user?
           Gitlab.config.omniauth.auto_link_saml_user
         end
 
         def external_users_enabled?
-          !Gitlab::Auth::Saml::Config.external_groups.nil?
+          !saml_config.external_groups.nil?
         end
 
         def auth_hash=(auth_hash)
diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb
index 1a25138e7d6..4ca5a78e068 100644
--- a/lib/gitlab/bare_repository_import/importer.rb
+++ b/lib/gitlab/bare_repository_import/importer.rb
@@ -75,10 +75,11 @@ module Gitlab
       end
 
       def mv_repo(project)
-        FileUtils.mv(repo_path, File.join(project.repository_storage_path, project.disk_path + '.git'))
+        storage_path = storage_path_for_shard(project.repository_storage)
+        FileUtils.mv(repo_path, project.repository.path_to_repo)
 
         if bare_repo.wiki_exists?
-          FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git'))
+          FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git'))
         end
 
         true
@@ -88,6 +89,10 @@ module Gitlab
         false
       end
 
+      def storage_path_for_shard(shard)
+        Gitlab.config.repositories.storages[shard].legacy_disk_path
+      end
+
       def find_or_create_groups
         return nil unless group_path.present?
 
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
index 05b86f32ce2..73971af6a74 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
@@ -62,21 +62,20 @@ module Gitlab
           end
 
           def move_repositories(namespace, old_full_path, new_full_path)
-            repo_paths_for_namespace(namespace).each do |repository_storage_path|
+            repo_shards_for_namespace(namespace).each do |repository_storage|
               # Ensure old directory exists before moving it
-              gitlab_shell.add_namespace(repository_storage_path, old_full_path)
+              gitlab_shell.add_namespace(repository_storage, old_full_path)
 
-              unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
-                message = "Exception moving path #{repository_storage_path} \
-                           from #{old_full_path} to #{new_full_path}"
+              unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path)
+                message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}"
                 Rails.logger.error message
               end
             end
           end
 
-          def repo_paths_for_namespace(namespace)
+          def repo_shards_for_namespace(namespace)
             projects_for_namespace(namespace).distinct.select(:repository_storage)
-              .map(&:repository_storage_path)
+              .map(&:repository_storage)
           end
 
           def projects_for_namespace(namespace)
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
index 979225dd216..827aeb12a02 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
@@ -51,7 +51,7 @@ module Gitlab
           end
 
           def move_repository(project, old_path, new_path)
-            unless gitlab_shell.mv_repository(project.repository_storage_path,
+            unless gitlab_shell.mv_repository(project.repository_storage,
                                               old_path,
                                               new_path)
               Rails.logger.error "Error moving #{old_path} to #{new_path}"
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index af203ff711d..b713fa7e1cd 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -3,7 +3,7 @@ module Gitlab
     extend self
 
     # For every version update, the version history in import_export.md has to be kept up to date.
-    VERSION = '0.2.2'.freeze
+    VERSION = '0.2.3'.freeze
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index ec91c02dbe7..0d1c4f73c6e 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -64,6 +64,7 @@ project_tree:
   - :project_feature
   - :custom_attributes
   - :project_badges
+  - :ci_cd_settings
 
 # Only include the following attributes for the models specified.
 included_attributes:
@@ -73,6 +74,8 @@ included_attributes:
     - :username
   author:
     - :name
+  ci_cd_settings:
+    - :group_runners_enabled
 
 # Do not include the following attributes for the models specified.
 excluded_attributes:
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 598832fb2df..e3e9f156fb4 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -17,7 +17,8 @@ module Gitlab
                     auto_devops: :project_auto_devops,
                     label: :project_label,
                     custom_attributes: 'ProjectCustomAttribute',
-                    project_badges: 'Badge' }.freeze
+                    project_badges: 'Badge',
+                    ci_cd_settings: 'ProjectCiCdSetting' }.freeze
 
       USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze
 
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index ac4ac537a8a..156115f8a8f 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -65,11 +65,11 @@ module Gitlab
 
     # Init new repository
     #
-    # storage - project's storage name
+    # storage - the shard key
     # name - project disk path
     #
     # Ex.
-    #   create_repository("/path/to/storage", "gitlab/gitlab-ci")
+    #   create_repository("default", "gitlab/gitlab-ci")
     #
     def create_repository(storage, name)
       relative_path = name.dup
@@ -291,13 +291,13 @@ module Gitlab
     # Add empty directory for storing repositories
     #
     # Ex.
-    #   add_namespace("/path/to/storage", "gitlab")
+    #   add_namespace("default", "gitlab")
     #
     def add_namespace(storage, name)
       Gitlab::GitalyClient.migrate(:add_namespace,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).add(name)
+          Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
         else
           path = full_path(storage, name)
           FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
@@ -313,13 +313,13 @@ module Gitlab
     # Every repository inside this directory will be removed too
     #
     # Ex.
-    #   rm_namespace("/path/to/storage", "gitlab")
+    #   rm_namespace("default", "gitlab")
     #
     def rm_namespace(storage, name)
       Gitlab::GitalyClient.migrate(:remove_namespace,
                                status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).remove(name)
+          Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
         else
           FileUtils.rm_r(full_path(storage, name), force: true)
         end
@@ -338,7 +338,8 @@ module Gitlab
       Gitlab::GitalyClient.migrate(:rename_namespace,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).rename(old_name, new_name)
+          Gitlab::GitalyClient::NamespaceService.new(storage)
+            .rename(old_name, new_name)
         else
           break false if exists?(storage, new_name) || !exists?(storage, old_name)
 
@@ -374,7 +375,8 @@ module Gitlab
       Gitlab::GitalyClient.migrate(:namespace_exists,
                                    status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
         if enabled
-          gitaly_namespace_client(storage).exists?(dir_name)
+          Gitlab::GitalyClient::NamespaceService.new(storage)
+            .exists?(dir_name)
         else
           File.exist?(full_path(storage, dir_name))
         end
@@ -398,7 +400,7 @@ module Gitlab
     def full_path(storage, dir_name)
       raise ArgumentError.new("Directory name can't be blank") if dir_name.blank?
 
-      File.join(storage, dir_name)
+      File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, dir_name)
     end
 
     def gitlab_shell_projects_path
@@ -475,14 +477,6 @@ module Gitlab
       Bundler.with_original_env { Popen.popen(cmd, nil, vars) }
     end
 
-    def gitaly_namespace_client(storage_path)
-      storage, _value = Gitlab.config.repositories.storages.find do |storage, value|
-        value.legacy_disk_path == storage_path
-      end
-
-      Gitlab::GitalyClient::NamespaceService.new(storage)
-    end
-
     def git_timeout
       Gitlab.config.gitlab_shell.git_timeout
     end
diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb
index 69952cbb47c..8cf5d636743 100644
--- a/lib/gitlab/user_access.rb
+++ b/lib/gitlab/user_access.rb
@@ -63,10 +63,12 @@ module Gitlab
 
     request_cache def can_push_to_branch?(ref)
       return false unless can_access_git?
-      return false unless user.can?(:push_code, project) || project.branch_allows_maintainer_push?(user, ref)
+      return false unless project
+
+      return false if !user.can?(:push_code, project) && !project.branch_allows_maintainer_push?(user, ref)
 
       if protected?(ProtectedBranch, project, ref)
-        project.user_can_push_to_empty_repo?(user) || protected_branch_accessible_to?(ref, action: :push)
+        protected_branch_accessible_to?(ref, action: :push)
       else
         true
       end
@@ -101,6 +103,7 @@ module Gitlab
     def protected_branch_accessible_to?(ref, action:)
       ProtectedBranch.protected_ref_accessible_to?(
         ref, user,
+        project: project,
         action: action,
         protected_refs: project.protected_branches)
     end
@@ -108,6 +111,7 @@ module Gitlab
     def protected_tag_accessible_to?(ref, action:)
       ProtectedTag.protected_ref_accessible_to?(
         ref, user,
+        project: project,
         action: action,
         protected_refs: project.protected_tags)
     end
diff --git a/lib/settings.rb b/lib/settings.rb
deleted file mode 100644
index 69d637761ea..00000000000
--- a/lib/settings.rb
+++ /dev/null
@@ -1,126 +0,0 @@
-require 'settingslogic'
-
-class Settings < Settingslogic
-  source ENV.fetch('GITLAB_CONFIG') { Pathname.new(File.expand_path('..', __dir__)).join('config/gitlab.yml') }
-  namespace ENV.fetch('GITLAB_ENV') { Rails.env }
-
-  class << self
-    def gitlab_on_standard_port?
-      on_standard_port?(gitlab)
-    end
-
-    def host_without_www(url)
-      host(url).sub('www.', '')
-    end
-
-    def build_gitlab_ci_url
-      custom_port =
-        if on_standard_port?(gitlab)
-          nil
-        else
-          ":#{gitlab.port}"
-        end
-
-      [
-        gitlab.protocol,
-        "://",
-        gitlab.host,
-        custom_port,
-        gitlab.relative_url_root
-      ].join('')
-    end
-
-    def build_pages_url
-      base_url(pages).join('')
-    end
-
-    def build_gitlab_shell_ssh_path_prefix
-      user_host = "#{gitlab_shell.ssh_user}@#{gitlab_shell.ssh_host}"
-
-      if gitlab_shell.ssh_port != 22
-        "ssh://#{user_host}:#{gitlab_shell.ssh_port}/"
-      else
-        if gitlab_shell.ssh_host.include? ':'
-          "[#{user_host}]:"
-        else
-          "#{user_host}:"
-        end
-      end
-    end
-
-    def build_base_gitlab_url
-      base_url(gitlab).join('')
-    end
-
-    def build_gitlab_url
-      (base_url(gitlab) + [gitlab.relative_url_root]).join('')
-    end
-
-    # check that values in `current` (string or integer) is a contant in `modul`.
-    def verify_constant_array(modul, current, default)
-      values = default || []
-      unless current.nil?
-        values = []
-        current.each do |constant|
-          values.push(verify_constant(modul, constant, nil))
-        end
-        values.delete_if { |value| value.nil? }
-      end
-
-      values
-    end
-
-    # check that `current` (string or integer) is a contant in `modul`.
-    def verify_constant(modul, current, default)
-      constant = modul.constants.find { |name| modul.const_get(name) == current }
-      value = constant.nil? ? default : modul.const_get(constant)
-      if current.is_a? String
-        value = modul.const_get(current.upcase) rescue default
-      end
-
-      value
-    end
-
-    def absolute(path)
-      File.expand_path(path, Rails.root)
-    end
-
-    private
-
-    def base_url(config)
-      custom_port = on_standard_port?(config) ? nil : ":#{config.port}"
-
-      [
-        config.protocol,
-        "://",
-        config.host,
-        custom_port
-      ]
-    end
-
-    def on_standard_port?(config)
-      config.port.to_i == (config.https ? 443 : 80)
-    end
-
-    # Extract the host part of the given +url+.
-    def host(url)
-      url = url.downcase
-      url = "http://#{url}" unless url.start_with?('http')
-
-      # Get rid of the path so that we don't even have to encode it
-      url_without_path = url.sub(%r{(https?://[^/]+)/?.*}, '\1')
-
-      URI.parse(url_without_path).host
-    end
-
-    # Runs every minute in a random ten-minute period on Sundays, to balance the
-    # load on the server receiving these pings. The usage ping is safe to run
-    # multiple times because of a 24 hour exclusive lock.
-    def cron_for_usage_ping
-      hour = rand(24)
-      minute = rand(6)
-
-      "#{minute}0-#{minute}9 #{hour} * * 0"
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index abef8cd2bcc..c04dae7446f 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -427,10 +427,7 @@ namespace :gitlab do
       user = User.find_by(username: username)
       if user
         repo_dirs = user.authorized_projects.map do |p|
-          File.join(
-            p.repository_storage_path,
-            "#{p.disk_path}.git"
-          )
+          p.repository.path_to_repo
         end
 
         repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) }
diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake
index d7f28691098..b854c34a8e5 100644
--- a/lib/tasks/gitlab/list_repos.rake
+++ b/lib/tasks/gitlab/list_repos.rake
@@ -10,9 +10,8 @@ namespace :gitlab do
     end
 
     scope.find_each do |project|
-      base = File.join(project.repository_storage_path, project.disk_path)
-      puts base + '.git'
-      puts base + '.wiki.git'
+      puts project.repository.path_to_repo
+      puts project.wiki.repository.path_to_repo
     end
   end
 end