diff options
author | Yorick Peterse <yorickpeterse@gmail.com> | 2016-06-30 11:02:43 +0000 |
---|---|---|
committer | Yorick Peterse <yorickpeterse@gmail.com> | 2016-06-30 11:02:43 +0000 |
commit | 1a68a0a6f55049b4ad42222b0dc667fe625a7604 (patch) | |
tree | 541a0d45943fcfc765192252ab761235e5978ef1 | |
parent | 2efee5f6410f4e8e40e9b0339393de80f4affdde (diff) | |
parent | 1735fdd447e9d589388f7d2ec2e21f9fb3615854 (diff) | |
download | gitlab-ce-1a68a0a6f55049b4ad42222b0dc667fe625a7604.tar.gz |
Merge branch 'shards' into 'master'
Implement multiple repository mount points
See merge request !4578
57 files changed, 779 insertions, 413 deletions
diff --git a/CHANGELOG b/CHANGELOG index ca7c3f29c72..19bd1f0ba31 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,6 +3,8 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.10.0 (unreleased) - Fix commit builds API, return all builds for all pipelines for given commit. !4849 - Replace Haml with Hamlit to make view rendering faster. !3666 + - Refactor repository paths handling to allow multiple git mount points + - Add Application Setting to configure default Repository Path for new projects - Wrap code blocks on Activies and Todos page. !4783 (winniehell) - Align flash messages with left side of page content !4959 (winniehell) - Display last commit of deleted branch in push events !4699 (winniehell) diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index 4a36342fcab..fd2a01863fd 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -3.0.0 +3.1.0 diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index f4eda864aac..5f65dd3aff0 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -109,6 +109,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController :metrics_packet_size, :send_user_confirmation_email, :container_registry_token_expire_delay, + :repository_storage, restricted_visibility_levels: [], import_sources: [], disabled_oauth_sign_in_sources: [] diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 55313fd8357..6e580c62ccd 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -78,4 +78,12 @@ module ApplicationSettingsHelper end end end + + def repository_storage_options_for_select + options = Gitlab.config.repositories.storages.map do |name, path| + ["#{name} - #{path}", name] + end + + options_for_select(options, @application_setting.repository_storage) + end end diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 26796b08fac..f312a7ccca3 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -327,9 +327,9 @@ module ProjectsHelper end end - def sanitize_repo_path(message) + def sanitize_repo_path(project, message) return '' unless message.present? - message.strip.gsub(Gitlab.config.gitlab_shell.repos_path.chomp('/'), "[REPOS PATH]") + message.strip.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]") end end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index d914b0b26eb..5fa6eacd234 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -55,6 +55,10 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { only_integer: true, greater_than: 0 } + validates :repository_storage, + presence: true, + inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } + validates_each :restricted_visibility_levels do |record, attr, value| unless value.nil? value.each do |level| @@ -134,6 +138,7 @@ class ApplicationSetting < ActiveRecord::Base disabled_oauth_sign_in_sources: [], send_user_confirmation_email: false, container_registry_token_expire_delay: 5, + repository_storage: 'default', ) end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index da19462f265..8b52cc824cd 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -21,8 +21,10 @@ class Namespace < ActiveRecord::Base delegate :name, to: :owner, allow_nil: true, prefix: true - after_create :ensure_dir_exist after_update :move_dir, if: :path_changed? + + # Save the storage paths before the projects are destroyed to use them on after destroy + before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths } after_destroy :rm_dir scope :root, -> { where('type IS NULL') } @@ -87,51 +89,35 @@ class Namespace < ActiveRecord::Base owner_name end - def ensure_dir_exist - gitlab_shell.add_namespace(path) - end - - def rm_dir - # Move namespace directory into trash. - # We will remove it later async - new_path = "#{path}+#{id}+deleted" - - if gitlab_shell.mv_namespace(path, new_path) - message = "Namespace directory \"#{path}\" moved to \"#{new_path}\"" - Gitlab::AppLogger.info message - - # Remove namespace directroy async with delay so - # GitLab has time to remove all projects first - GitlabShellWorker.perform_in(5.minutes, :rm_namespace, new_path) - end - end - def move_dir - # Ensure old directory exists before moving it - gitlab_shell.add_namespace(path_was) - if any_project_has_container_registry_tags? raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry') end - if gitlab_shell.mv_namespace(path_was, path) - Gitlab::UploadsTransfer.new.rename_namespace(path_was, path) - - # If repositories moved successfully we need to - # send update instructions to users. - # However we cannot allow rollback since we moved namespace dir - # So we basically we mute exceptions in next actions - begin - send_update_instructions - rescue - # Returning false does not rollback after_* transaction but gives - # us information about failing some of tasks - false + # Move the namespace directory in all storages paths used by member projects + repository_storage_paths.each do |repository_storage_path| + # Ensure old directory exists before moving it + gitlab_shell.add_namespace(repository_storage_path, path_was) + + unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path) + # if we cannot move namespace directory we should rollback + # db changes in order to prevent out of sync between db and fs + raise Exception.new('namespace directory cannot be moved') end - else - # if we cannot move namespace directory we should rollback - # db changes in order to prevent out of sync between db and fs - raise Exception.new('namespace directory cannot be moved') + end + + Gitlab::UploadsTransfer.new.rename_namespace(path_was, path) + + # If repositories moved successfully we need to + # send update instructions to users. + # However we cannot allow rollback since we moved namespace dir + # So we basically we mute exceptions in next actions + begin + send_update_instructions + rescue + # Returning false does not rollback after_* transaction but gives + # us information about failing some of tasks + false end end @@ -152,4 +138,33 @@ class Namespace < ActiveRecord::Base def find_fork_of(project) projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id) end + + private + + def repository_storage_paths + # We need to get the storage paths for all the projects, even the ones that are + # pending delete. Unscoping also get rids of the default order, which causes + # problems with SELECT DISTINCT. + Project.unscoped do + projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path) + end + end + + def rm_dir + # Remove the namespace directory in all storages paths used by member projects + @old_repository_storage_paths.each do |repository_storage_path| + # Move namespace directory into trash. + # We will remove it later async + new_path = "#{path}+#{id}+deleted" + + if gitlab_shell.mv_namespace(repository_storage_path, path, new_path) + message = "Namespace directory \"#{path}\" moved to \"#{new_path}\"" + Gitlab::AppLogger.info message + + # Remove namespace directroy async with delay so + # GitLab has time to remove all projects first + GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path) + end + end + end end diff --git a/app/models/project.rb b/app/models/project.rb index 73ded09c162..6a950ee830d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -24,8 +24,12 @@ class Project < ActiveRecord::Base default_value_for :wiki_enabled, gitlab_config_features.wiki default_value_for :snippets_enabled, gitlab_config_features.snippets default_value_for :container_registry_enabled, gitlab_config_features.container_registry + default_value_for(:repository_storage) { current_application_settings.repository_storage } default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled } + after_create :ensure_dir_exist + after_save :ensure_dir_exist, if: :namespace_id_changed? + # set last_activity_at to the same as created_at after_create :set_last_activity_at def set_last_activity_at @@ -165,6 +169,9 @@ class Project < ActiveRecord::Base validate :visibility_level_allowed_by_group validate :visibility_level_allowed_as_fork validate :check_wiki_path_conflict + validates :repository_storage, + presence: true, + inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } add_authentication_token_field :runners_token before_save :ensure_runners_token @@ -376,6 +383,10 @@ class Project < ActiveRecord::Base end end + def repository_storage_path + Gitlab.config.repositories.storages[repository_storage] + end + def team @team ||= ProjectTeam.new(self) end @@ -842,12 +853,12 @@ class Project < ActiveRecord::Base raise Exception.new('Project cannot be renamed, because tags are present in its container registry') end - if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace) + if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) # If repository moved successfully we need to send update instructions to users. # However we cannot allow rollback since we moved repository # So we basically we mute exceptions in next actions begin - gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") + gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") send_move_instructions(old_path_with_namespace) reset_events_cache @@ -988,7 +999,7 @@ class Project < ActiveRecord::Base def create_repository # Forked import is handled asynchronously unless forked? - if gitlab_shell.add_repository(path_with_namespace) + if gitlab_shell.add_repository(repository_storage_path, path_with_namespace) repository.after_create true else @@ -1140,4 +1151,8 @@ class Project < ActiveRecord::Base _, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete)) status.zero? end + + def ensure_dir_exist + gitlab_shell.add_namespace(repository_storage_path, namespace.path) + end end diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 25d82929c0b..a255710f577 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -159,7 +159,7 @@ class ProjectWiki private def init_repo(path_with_namespace) - gitlab_shell.add_repository(path_with_namespace) + gitlab_shell.add_repository(project.repository_storage_path, path_with_namespace) end def commit_details(action, message = nil, title = nil) @@ -173,7 +173,7 @@ class ProjectWiki end def path_to_repo - @path_to_repo ||= File.join(Gitlab.config.gitlab_shell.repos_path, "#{path_with_namespace}.git") + @path_to_repo ||= File.join(project.repository_storage_path, "#{path_with_namespace}.git") end def update_project_activity diff --git a/app/models/repository.rb b/app/models/repository.rb index 2a6a3b086c2..f45c3d06abd 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -39,7 +39,7 @@ class Repository # Return absolute path to repository def path_to_repo @path_to_repo ||= File.expand_path( - File.join(Gitlab.config.gitlab_shell.repos_path, path_with_namespace + ".git") + File.join(@project.repository_storage_path, path_with_namespace + ".git") ) end diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index f09072975c3..882606e38d0 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -51,13 +51,13 @@ module Projects return true if params[:skip_repo] == true # There is a possibility project does not have repository or wiki - return true unless gitlab_shell.exists?(path + '.git') + return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git') new_path = removal_path(path) - if gitlab_shell.mv_repository(path, new_path) + if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path) log_info("Repository \"#{path}\" moved to \"#{new_path}\"") - GitlabShellWorker.perform_in(5.minutes, :remove_repository, new_path) + GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path) else false end diff --git a/app/services/projects/housekeeping_service.rb b/app/services/projects/housekeeping_service.rb index 43db29315a1..a47df22f1ba 100644 --- a/app/services/projects/housekeeping_service.rb +++ b/app/services/projects/housekeeping_service.rb @@ -24,7 +24,7 @@ module Projects def execute raise LeaseTaken unless try_obtain_lease - GitlabShellOneShotWorker.perform_async(:gc, @project.path_with_namespace) + GitlabShellOneShotWorker.perform_async(:gc, @project.repository_storage_path, @project.path_with_namespace) ensure Gitlab::Metrics.measure(:reset_pushes_since_gc) do @project.update_column(:pushes_since_gc, 0) diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb index 9159ec08959..163ebf26c84 100644 --- a/app/services/projects/import_service.rb +++ b/app/services/projects/import_service.rb @@ -42,7 +42,7 @@ module Projects def import_repository begin - gitlab_shell.import_repository(project.path_with_namespace, project.import_url) + gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url) rescue Gitlab::Shell::Error => e raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}" end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 03b57dea51e..bc7f8bf433b 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -50,12 +50,12 @@ module Projects project.send_move_instructions(old_path) # Move main repository - unless gitlab_shell.mv_repository(old_path, new_path) + unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, new_path) raise TransferError.new('Cannot move project') end # Move wiki repo also if present - gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki") + gitlab_shell.mv_repository(project.repository_storage_path, "#{old_path}.wiki", "#{new_path}.wiki") # clear project cached events project.reset_events_cache diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index 30ab0717164..c1f70bc1866 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -311,6 +311,15 @@ = f.text_field :sentry_dsn, class: 'form-control' %fieldset + %legend Repository Storage + .form-group + = f.label :repository_storage, 'Storage path for new projects', class: 'control-label col-sm-2' + .col-sm-10 + = f.select :repository_storage, repository_storage_options_for_select, {}, class: 'form-control' + .help-block + You can manage the repository storage paths in your gitlab.yml configuration file + + %fieldset %legend Repository Checks .form-group .col-sm-offset-2.col-sm-10 diff --git a/app/views/projects/imports/new.html.haml b/app/views/projects/imports/new.html.haml index a8a8caf7280..2cd8d03e30e 100644 --- a/app/views/projects/imports/new.html.haml +++ b/app/views/projects/imports/new.html.haml @@ -10,7 +10,7 @@ .panel-body %pre :preserve - #{sanitize_repo_path(@project.import_error)} + #{sanitize_repo_path(@project, @project.import_error)} = form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f| = render "shared/import_form", f: f diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index f3327ca9e61..09035a7cf2d 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -4,10 +4,10 @@ class PostReceive sidekiq_options queue: :post_receive def perform(repo_path, identifier, changes) - if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s) - repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "") + if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) } + repo_path.gsub!(path[1].to_s, "") else - log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"") + log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"") end post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes) diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index d947f105516..f7604e48f83 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -12,7 +12,7 @@ class RepositoryForkWorker return end - result = gitlab_shell.fork_repository(source_path, target_path) + result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path) unless result logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}") project.mark_import_as_failed('The project could not be forked.') diff --git a/config/gitlab.teatro.yml b/config/gitlab.teatro.yml index 01c8dc5ff98..75b79b837e0 100644 --- a/config/gitlab.teatro.yml +++ b/config/gitlab.teatro.yml @@ -47,11 +47,13 @@ production: &base backup: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) + repositories: + storages: # REPO PATHS MUST NOT BE A SYMLINK!!! + default: /apps/repositories/ + gitlab_shell: path: /apps/gitlab-shell/ - # REPOS_PATH MUST NOT BE A SYMLINK!!! - repos_path: /apps/repositories/ hooks_path: /apps/gitlab-shell/hooks/ upload_pack: true diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 75e1a3c1093..325eca72862 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -428,6 +428,13 @@ production: &base satellites: path: /home/git/gitlab-satellites/ + ## Repositories settings + repositories: + # Paths where repositories can be stored. Give the canonicalized absolute pathname. + # NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!! + storages: # You must have at least a `default` storage path. + default: /home/git/repositories/ + ## Backup settings backup: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) @@ -452,9 +459,6 @@ production: &base ## GitLab Shell settings gitlab_shell: path: /home/git/gitlab-shell/ - - # REPOS_PATH MUST NOT BE A SYMLINK!!! - repos_path: /home/git/repositories/ hooks_path: /home/git/gitlab-shell/hooks/ # File that contains the secret key for verifying access for gitlab-shell. @@ -528,11 +532,13 @@ test: # user: YOUR_USERNAME satellites: path: tmp/tests/gitlab-satellites/ + repositories: + storages: + default: tmp/tests/repositories/ backup: path: tmp/tests/backups gitlab_shell: path: tmp/tests/gitlab-shell/ - repos_path: tmp/tests/repositories/ hooks_path: tmp/tests/gitlab-shell/hooks/ issues_tracker: redmine: diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index c6dc1e4ab38..a93996cec72 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -304,7 +304,6 @@ Settings.gitlab_shell['hooks_path'] ||= Settings.gitlab['user_home'] + '/gitla Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret') Settings.gitlab_shell['receive_pack'] = true if Settings.gitlab_shell['receive_pack'].nil? Settings.gitlab_shell['upload_pack'] = true if Settings.gitlab_shell['upload_pack'].nil? -Settings.gitlab_shell['repos_path'] ||= Settings.gitlab['user_home'] + '/repositories/' Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host Settings.gitlab_shell['ssh_port'] ||= 22 Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user @@ -312,6 +311,14 @@ Settings.gitlab_shell['owner_group'] ||= Settings.gitlab.user Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix) # +# Repositories +# +Settings['repositories'] ||= Settingslogic.new({}) +Settings.repositories['storages'] ||= {} +# Setting gitlab_shell.repos_path is DEPRECATED and WILL BE REMOVED in version 9.0 +Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'] || Settings.gitlab['user_home'] + '/repositories/' + +# # Backup # Settings['backup'] ||= Settingslogic.new({}) diff --git a/config/initializers/6_validations.rb b/config/initializers/6_validations.rb new file mode 100644 index 00000000000..3ba9e36c567 --- /dev/null +++ b/config/initializers/6_validations.rb @@ -0,0 +1,24 @@ +def storage_name_valid?(name) + !!(name =~ /\A[a-zA-Z0-9\-_]+\z/) +end + +def find_parent_path(name, path) + Gitlab.config.repositories.storages.detect do |n, p| + name != n && path.chomp('/').start_with?(p.chomp('/')) + end +end + +def error(message) + raise "#{message}. Please fix this in your gitlab.yml before starting GitLab." +end + +error('No repository storage path defined') if Gitlab.config.repositories.storages.empty? + +Gitlab.config.repositories.storages.each do |name, path| + error("\"#{name}\" is not a valid storage name") unless storage_name_valid?(name) + + parent_name, _parent_path = find_parent_path(name, path) + if parent_name + error("#{name} is a nested path of #{parent_name}. Nested paths are not supported for repository storages") + end +end diff --git a/config/initializers/gitlab_shell_secret_token.rb b/config/initializers/gitlab_shell_secret_token.rb index 751fccead07..7454c33c9dd 100644 --- a/config/initializers/gitlab_shell_secret_token.rb +++ b/config/initializers/gitlab_shell_secret_token.rb @@ -1,19 +1 @@ -# Be sure to restart your server when you modify this file. - -require 'securerandom' - -# Your secret key for verifying the gitlab_shell. - - -secret_file = Gitlab.config.gitlab_shell.secret_file - -unless File.exist? secret_file - # Generate a new token of 16 random hexadecimal characters and store it in secret_file. - token = SecureRandom.hex(16) - File.write(secret_file, token) -end - -link_path = File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret') -if File.exist?(Gitlab.config.gitlab_shell.path) && !File.exist?(link_path) - FileUtils.symlink(secret_file, link_path) -end +Gitlab::Shell.new.generate_and_link_secret_token diff --git a/db/migrate/20160608195742_add_repository_storage_to_projects.rb b/db/migrate/20160608195742_add_repository_storage_to_projects.rb new file mode 100644 index 00000000000..c700d2b569d --- /dev/null +++ b/db/migrate/20160608195742_add_repository_storage_to_projects.rb @@ -0,0 +1,12 @@ +class AddRepositoryStorageToProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + + def up + add_column_with_default(:projects, :repository_storage, :string, default: 'default') + end + + def down + remove_column(:projects, :repository_storage) + end +end diff --git a/db/migrate/20160614182521_add_repository_storage_to_application_settings.rb b/db/migrate/20160614182521_add_repository_storage_to_application_settings.rb new file mode 100644 index 00000000000..6dae91b700b --- /dev/null +++ b/db/migrate/20160614182521_add_repository_storage_to_application_settings.rb @@ -0,0 +1,5 @@ +class AddRepositoryStorageToApplicationSettings < ActiveRecord::Migration + def change + add_column :application_settings, :repository_storage, :string, default: 'default' + end +end diff --git a/db/schema.rb b/db/schema.rb index 7a8377f687c..beb723c3bc5 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -85,6 +85,7 @@ ActiveRecord::Schema.define(version: 20160620115026) do t.boolean "send_user_confirmation_email", default: false t.integer "container_registry_token_expire_delay", default: 5 t.text "after_sign_up_text" + t.string "repository_storage", default: "default" end create_table "audit_events", force: :cascade do |t| @@ -796,38 +797,39 @@ ActiveRecord::Schema.define(version: 20160620115026) do t.datetime "created_at" t.datetime "updated_at" t.integer "creator_id" - t.boolean "issues_enabled", default: true, null: false - t.boolean "merge_requests_enabled", default: true, null: false - t.boolean "wiki_enabled", default: true, null: false + t.boolean "issues_enabled", default: true, null: false + t.boolean "merge_requests_enabled", default: true, null: false + t.boolean "wiki_enabled", default: true, null: false t.integer "namespace_id" - t.boolean "snippets_enabled", default: true, null: false + t.boolean "snippets_enabled", default: true, null: false t.datetime "last_activity_at" t.string "import_url" - t.integer "visibility_level", default: 0, null: false - t.boolean "archived", default: false, null: false + t.integer "visibility_level", default: 0, null: false + t.boolean "archived", default: false, null: false t.string "avatar" t.string "import_status" t.float "repository_size", default: 0.0 - t.integer "star_count", default: 0, null: false + t.integer "star_count", default: 0, null: false t.string "import_type" t.string "import_source" t.integer "commit_count", default: 0 t.text "import_error" t.integer "ci_id" - t.boolean "builds_enabled", default: true, null: false - t.boolean "shared_runners_enabled", default: true, null: false + t.boolean "builds_enabled", default: true, null: false + t.boolean "shared_runners_enabled", default: true, null: false t.string "runners_token" t.string "build_coverage_regex" - t.boolean "build_allow_git_fetch", default: true, null: false - t.integer "build_timeout", default: 3600, null: false + t.boolean "build_allow_git_fetch", default: true, null: false + t.integer "build_timeout", default: 3600, null: false t.boolean "pending_delete", default: false - t.boolean "public_builds", default: true, null: false + t.boolean "public_builds", default: true, null: false t.integer "pushes_since_gc", default: 0 t.boolean "last_repository_check_failed" t.datetime "last_repository_check_at" t.boolean "container_registry_enabled" - t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false + t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false t.boolean "has_external_issue_tracker" + t.string "repository_storage", default: "default", null: false end add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree diff --git a/doc/README.md b/doc/README.md index be0d17084c7..b98d6812a81 100644 --- a/doc/README.md +++ b/doc/README.md @@ -34,6 +34,7 @@ - [Operations](operations/README.md) Keeping GitLab up and running. - [Raketasks](raketasks/README.md) Backups, maintenance, automatic webhook setup and the importing of projects. - [Repository checks](administration/repository_checks.md) Periodic Git repository checks. +- [Repository storages](administration/repository_storages.md) Manage the paths used to store repositories. - [Security](security/README.md) Learn what you can do to further secure your GitLab instance. - [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed. - [Update](update/README.md) Update guides to upgrade your installation. diff --git a/doc/administration/repository_storages.md b/doc/administration/repository_storages.md new file mode 100644 index 00000000000..81bfe173151 --- /dev/null +++ b/doc/administration/repository_storages.md @@ -0,0 +1,18 @@ +# Repository storages + +GitLab allows you to define repository storage paths to enable distribution of +storage load between several mount points. + +## For installations from source + +Add your repository storage paths in your `gitlab.yml` under repositories -> storages, using key -> value pairs. + +>**Notes:** +- You must have at least one storage path called `default`. +- In order for backups to work correctly the storage path must **not** be a +mount point and the GitLab user should have correct permissions for the parent +directory of the path. + +## For omnibus installations + +Follow the instructions at https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/configuration.md#storing-git-data-in-an-alternative-directory diff --git a/doc/api/settings.md b/doc/api/settings.md index b5152311f28..741c5a29581 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -38,7 +38,8 @@ Example response: "default_project_visibility" : 0, "gravatar_enabled" : true, "sign_in_text" : null, - "container_registry_token_expire_delay": 5 + "container_registry_token_expire_delay": 5, + "repository_storage": "default" } ``` @@ -66,6 +67,7 @@ PUT /application/settings | `user_oauth_applications` | boolean | no | Allow users to register any application to use GitLab as an OAuth provider | | `after_sign_out_path` | string | no | Where to redirect users after logout | | `container_registry_token_expire_delay` | integer | no | Container Registry token duration in minutes | +| `repository_storage` | string | no | Storage path for new projects. The value should be the name of one of the repository storage paths defined in your gitlab.yml | ```bash curl -X PUT -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/application/settings?signup_enabled=false&default_project_visibility=1 @@ -93,6 +95,7 @@ Example response: "restricted_signup_domains": [], "user_oauth_applications": true, "after_sign_out_path": "", - "container_registry_token_expire_delay": 5 + "container_registry_token_expire_delay": 5, + "repository_storage": "default" } ``` diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index 8a38937062e..2b305cb5c99 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -14,7 +14,8 @@ - For omnibus-gitlab, it is located at: `/var/opt/gitlab/git-data/repositories` by default, unless you changed it in the `/etc/gitlab/gitlab.rb` file. - For installations from source, it is usually located at: `/home/git/repositories` or you can see where -your repositories are located by looking at `config/gitlab.yml` under the `gitlab_shell => repos_path` entry. +your repositories are located by looking at `config/gitlab.yml` under the `repositories => storages` entries +(you'll usually use the `default` storage path to start). New folder needs to have git user ownership and read/write/execute access for git user and its group: diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 5a23a18fe9c..4e2a43e45e2 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -376,6 +376,7 @@ module API expose :user_oauth_applications expose :after_sign_out_path expose :container_registry_token_expire_delay + expose :repository_storage end class Release < Grape::Entity diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 1d361569d59..b32503e8516 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -20,6 +20,20 @@ module API @wiki ||= params[:project].end_with?('.wiki') && !Project.find_with_namespace(params[:project]) end + + def project + @project ||= begin + project_path = params[:project] + + # Check for *.wiki repositories. + # Strip out the .wiki from the pathname before finding the + # project. This applies the correct project permissions to + # the wiki repository as well. + project_path.chomp!('.wiki') if wiki? + + Project.find_with_namespace(project_path) + end + end end post "/allowed" do @@ -32,16 +46,6 @@ module API User.find_by(id: params[:user_id]) end - project_path = params[:project] - - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - project = Project.find_with_namespace(project_path) - access = if wiki? Gitlab::GitAccessWiki.new(actor, project) @@ -49,7 +53,17 @@ module API Gitlab::GitAccess.new(actor, project) end - access.check(params[:action], params[:changes]) + access_status = access.check(params[:action], params[:changes]) + + response = { status: access_status.status, message: access_status.message } + + if access_status.status + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + response[:repository_path] = project.repository.path_to_repo + end + + response end # diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 7b91215d50b..b9773f98d75 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,8 +2,6 @@ require 'yaml' module Backup class Repository - attr_reader :repos_path - def dump prepare @@ -50,10 +48,12 @@ module Backup end def restore - if File.exists?(repos_path) + Gitlab.config.repositories.storages.each do |name, path| + next unless File.exists?(path) + # Move repos dir to 'repositories.old' dir - bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s) - FileUtils.mv(repos_path, bk_repos_path) + bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) + FileUtils.mv(path, bk_repos_path) end FileUtils.mkdir_p(repos_path) @@ -61,7 +61,7 @@ module Backup Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " - project.namespace.ensure_dir_exist if project.namespace + project.ensure_dir_exist if File.exists?(path_to_bundle(project)) FileUtils.mkdir_p(path_to_repo(project)) @@ -100,8 +100,8 @@ module Backup end $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) - cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks" - if system(cmd) + cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args + if system(*cmd) $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) @@ -120,10 +120,6 @@ module Backup File.join(backup_repos_path, project.path_with_namespace + ".bundle") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - def backup_repos_path File.join(Gitlab.config.backup.path, "repositories") end @@ -139,5 +135,11 @@ module Backup def silent {err: '/dev/null', out: '/dev/null'} end + + private + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 3e3986d6382..34e0143a82e 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -1,3 +1,5 @@ +require 'securerandom' + module Gitlab class Shell class Error < StandardError; end @@ -18,77 +20,82 @@ module Gitlab # Init new repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # add_repository("gitlab/gitlab-ci") + # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - def add_repository(name) + def add_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'add-project', "#{name}.git"]) + 'add-project', storage, "#{name}.git"]) end # Import repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") + # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # - def import_repository(name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900']) + def import_repository(storage, name, url) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, '900']) raise Error, output unless status.zero? true end # Move repository - # + # storage - project's storage path # path - project path with namespace # new_path - new project path with namespace # # Ex. - # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new") + # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - def mv_repository(path, new_path) + def mv_repository(storage, path, new_path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', - "#{path}.git", "#{new_path}.git"]) + storage, "#{path}.git", "#{new_path}.git"]) end # Fork repository to new namespace - # + # storage - project's storage path # path - project path with namespace # fork_namespace - namespace for forked project # # Ex. - # fork_repository("gitlab/gitlab-ci", "randx") + # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx") # - def fork_repository(path, fork_namespace) + def fork_repository(storage, path, fork_namespace) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - "#{path}.git", fork_namespace]) + storage, "#{path}.git", fork_namespace]) end # Remove repository from file system # + # storage - project's storage path # name - project path with namespace # # Ex. - # remove_repository("gitlab/gitlab-ci") + # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - def remove_repository(name) + def remove_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'rm-project', "#{name}.git"]) + 'rm-project', storage, "#{name}.git"]) end # Gc repository # + # storage - project storage path # path - project path with namespace # # Ex. - # gc("gitlab/gitlab-ci") + # gc("/path/to/storage", "gitlab/gitlab-ci") # - def gc(path) + def gc(storage, path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', - "#{path}.git"]) + storage, "#{path}.git"]) end # Add new key to gitlab-shell @@ -133,31 +140,31 @@ module Gitlab # Add empty directory for storing repositories # # Ex. - # add_namespace("gitlab") + # add_namespace("/path/to/storage", "gitlab") # - def add_namespace(name) - FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name) + def add_namespace(storage, name) + FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name) end # Remove directory from repositories storage # Every repository inside this directory will be removed too # # Ex. - # rm_namespace("gitlab") + # rm_namespace("/path/to/storage", "gitlab") # - def rm_namespace(name) - FileUtils.rm_r(full_path(name), force: true) + def rm_namespace(storage, name) + FileUtils.rm_r(full_path(storage, name), force: true) end # Move namespace directory inside repositories storage # # Ex. - # mv_namespace("gitlab", "gitlabhq") + # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - def mv_namespace(old_name, new_name) - return false if exists?(new_name) || !exists?(old_name) + def mv_namespace(storage, old_name, new_name) + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(old_name), full_path(new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) end def url_to_repo(path) @@ -176,11 +183,26 @@ module Gitlab # Check if such directory exists in repositories. # # Usage: - # exists?('gitlab') - # exists?('gitlab/cookies.git') + # exists?(storage, 'gitlab') + # exists?(storage, 'gitlab/cookies.git') # - def exists?(dir_name) - File.exist?(full_path(dir_name)) + def exists?(storage, dir_name) + File.exist?(full_path(storage, dir_name)) + end + + # Create (if necessary) and link the secret token file + def generate_and_link_secret_token + secret_file = Gitlab.config.gitlab_shell.secret_file + unless File.exist? secret_file + # Generate a new token of 16 random hexadecimal characters and store it in secret_file. + token = SecureRandom.hex(16) + File.write(secret_file, token) + end + + link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret') + if File.exist?(gitlab_shell_path) && !File.exist?(link_path) + FileUtils.symlink(secret_file, link_path) + end end protected @@ -193,14 +215,10 @@ module Gitlab File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - - def full_path(dir_name) + def full_path(storage, dir_name) raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? - File.join(repos_path, dir_name) + File.join(storage, dir_name) end def gitlab_shell_projects_path diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2286ac8829c..730978d502b 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -167,7 +167,7 @@ module Gitlab def import_wiki unless project.wiki_enabled? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 12d6ac45fb6..e9a4e37ec48 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -356,97 +356,108 @@ namespace :gitlab do ######################## def check_repo_base_exists - print "Repo base directory exists? ... " + puts "Repo base directory exists?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.exists?(repo_base_path) - puts "yes".color(:green) - else - puts "no".color(:red) - puts "#{repo_base_path} is missing".color(:red) - try_fixing_it( - "This should have been created when setting up GitLab Shell.", - "Make sure it's set correctly in config/gitlab.yml", - "Make sure GitLab Shell is installed correctly." - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + if File.exists?(repo_base_path) + puts "yes".color(:green) + else + puts "no".color(:red) + puts "#{repo_base_path} is missing".color(:red) + try_fixing_it( + "This should have been created when setting up GitLab Shell.", + "Make sure it's set correctly in config/gitlab.yml", + "Make sure GitLab Shell is installed correctly." + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_is_not_symlink - print "Repo base directory is a symlink? ... " + puts "Repo storage directories are symlinks?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - unless File.symlink?(repo_base_path) - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Make sure it's set to the real directory in config/gitlab.yml" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + unless File.symlink?(repo_base_path) + puts "no".color(:green) + else + puts "yes".color(:red) + try_fixing_it( + "Make sure it's set to the real directory in config/gitlab.yml" + ) + fix_and_rerun + end end end def check_repo_base_permissions - print "Repo base access is drwxrws---? ... " + puts "Repo paths access is drwxrws---?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", - "sudo chmod -R ug-s #{repo_base_path}", - "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") + puts "yes".color(:green) + else + puts "no".color(:red) + try_fixing_it( + "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", + "sudo chmod -R ug-s #{repo_base_path}", + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group - print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... " + puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - uid = uid_for(gitlab_shell_ssh_user) - gid = gid_for(gitlab_shell_owner_group) - if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid - puts "yes".color(:green) - else - puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) - try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + uid = uid_for(gitlab_shell_ssh_user) + gid = gid_for(gitlab_shell_owner_group) + if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid + puts "yes".color(:green) + else + puts "no".color(:red) + puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) + try_fixing_it( + "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end @@ -473,7 +484,7 @@ namespace :gitlab do else puts "wrong or missing hooks".color(:red) try_fixing_it( - sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"), + sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"), 'Check the hooks_path in config/gitlab.yml', 'Check your gitlab-shell installation' ) @@ -785,13 +796,13 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - namespace_dirs = Dir.glob( - File.join(Gitlab.config.gitlab_shell.repos_path, '*') - ) + Gitlab.config.repositories.storages.each do |name, path| + namespace_dirs = Dir.glob(File.join(path, '*')) - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + namespace_dirs.each do |namespace_dir| + repo_dirs = Dir.glob(File.join(namespace_dir, '*')) + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + end end end end @@ -799,12 +810,12 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| - username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue)) + username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| File.join( - Gitlab.config.gitlab_shell.repos_path, + p.repository_storage_path, "#{p.path_with_namespace}.git" ) end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index ab0028d6603..b7cbdc6cd78 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -5,36 +5,36 @@ namespace :gitlab do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) - git_base_path = Gitlab.config.gitlab_shell.repos_path - all_dirs = Dir.glob(git_base_path + '/*') + Gitlab.config.repositories.storages.each do |name, git_base_path| + all_dirs = Dir.glob(git_base_path + '/*') - puts git_base_path.color(:yellow) - puts "Looking for directories to remove... " + puts git_base_path.color(:yellow) + puts "Looking for directories to remove... " - all_dirs.reject! do |dir| - # skip if git repo - dir =~ /.git$/ - end + all_dirs.reject! do |dir| + # skip if git repo + dir =~ /.git$/ + end - all_dirs.reject! do |dir| - dir_name = File.basename dir + all_dirs.reject! do |dir| + dir_name = File.basename dir - # skip if namespace present - namespaces.include?(dir_name) - end + # skip if namespace present + namespaces.include?(dir_name) + end - all_dirs.each do |dir_path| + all_dirs.each do |dir_path| - if remove_flag - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".color(:red) + if remove_flag + if FileUtils.rm_rf dir_path + puts "Removed...#{dir_path}".color(:red) + else + puts "Cannot remove #{dir_path}".color(:red) + end else - puts "Cannot remove #{dir_path}".color(:red) + puts "Can be removed: #{dir_path}".color(:red) end - else - puts "Can be removed: #{dir_path}".color(:red) end end @@ -48,20 +48,21 @@ namespace :gitlab do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" - repo_root = Gitlab.config.gitlab_shell.repos_path - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| - find.each_line do |path| - path.chomp! - repo_with_namespace = path. - sub(repo_root, ''). - sub(%r{^/*}, ''). - chomp('.git'). - chomp('.wiki') - next if Project.find_with_namespace(repo_with_namespace) - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - File.rename(path, new_path) + Gitlab.config.repositories.storages.each do |name, repo_root| + # Look for global repos (legacy, depth 1) and normal repos (depth 2) + IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| + find.each_line do |path| + path.chomp! + repo_with_namespace = path. + sub(repo_root, ''). + sub(%r{^/*}, ''). + chomp('.git'). + chomp('.wiki') + next if Project.find_with_namespace(repo_with_namespace) + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + File.rename(path, new_path) + end end end end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 4753f00c26a..dbdd4e977e8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -2,73 +2,73 @@ namespace :gitlab do namespace :import do # How to use: # - # 1. copy the bare repos under the repos_path (commonly /home/git/repositories) + # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories) # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production # # Notes: # * The project owner will set to the first administator of the system # * Existing projects will be skipped # - desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance" + desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do + Gitlab.config.repositories.storages.each do |name, git_base_path| + repos_to_import = Dir.glob(git_base_path + '/**/*.git') - git_base_path = Gitlab.config.gitlab_shell.repos_path - repos_to_import = Dir.glob(git_base_path + '/**/*.git') + repos_to_import.each do |repo_path| + # strip repo base path + repo_path[0..git_base_path.length] = '' - repos_to_import.each do |repo_path| - # strip repo base path - repo_path[0..git_base_path.length] = '' + path = repo_path.sub(/\.git$/, '') + group_name, name = File.split(path) + group_name = nil if group_name == '.' - path = repo_path.sub(/\.git$/, '') - group_name, name = File.split(path) - group_name = nil if group_name == '.' + puts "Processing #{repo_path}".color(:yellow) - puts "Processing #{repo_path}".color(:yellow) - - if path.end_with?('.wiki') - puts " * Skipping wiki repo" - next - end + if path.end_with?('.wiki') + puts " * Skipping wiki repo" + next + end - project = Project.find_with_namespace(path) + project = Project.find_with_namespace(path) - if project - puts " * #{project.name} (#{repo_path}) exists" - else - user = User.admins.reorder("id").first + if project + puts " * #{project.name} (#{repo_path}) exists" + else + user = User.admins.reorder("id").first - project_params = { - name: name, - path: name - } + project_params = { + name: name, + path: name + } - # find group namespace - if group_name - group = Namespace.find_by(path: group_name) - # create group namespace - unless group - group = Group.new(:name => group_name) - group.path = group_name - group.owner = user - if group.save - puts " * Created Group #{group.name} (#{group.id})".color(:green) - else - puts " * Failed trying to create group #{group.name}".color(:red) + # find group namespace + if group_name + group = Namespace.find_by(path: group_name) + # create group namespace + unless group + group = Group.new(:name => group_name) + group.path = group_name + group.owner = user + if group.save + puts " * Created Group #{group.name} (#{group.id})".color(:green) + else + puts " * Failed trying to create group #{group.name}".color(:red) + end end + # set project group + project_params[:namespace_id] = group.id end - # set project group - project_params[:namespace_id] = group.id - end - project = Projects::CreateService.new(user, project_params).execute + project = Projects::CreateService.new(user, project_params).execute - if project.persisted? - puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count - else - puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) - puts " Errors: #{project.errors.messages}".color(:red) + if project.persisted? + puts " * Created #{project.name} (#{repo_path})".color(:green) + project.update_repository_size + project.update_commit_count + else + puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) + puts " Errors: #{project.errors.messages}".color(:red) + end end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 352b566df24..fe43d40e6d2 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -62,7 +62,10 @@ namespace :gitlab do puts "" puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" - puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}" + puts "Repository storage paths:" + Gitlab.config.repositories.storages.each do |name, path| + puts "- #{name}: \t#{path}" + end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index c7596e7abcb..ffcc76e5498 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -9,7 +9,7 @@ namespace :gitlab do scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end scope.find_each do |project| - base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + base = File.join(project.repository_storage_path, project.path_with_namespace) puts base + '.git' puts base + '.wiki.git' end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index b1648a4602a..c85ebdf8619 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -12,7 +12,6 @@ namespace :gitlab do gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') - repos_path = Gitlab.config.gitlab_shell.repos_path target_dir = Gitlab.config.gitlab_shell.path # Clone if needed @@ -35,7 +34,6 @@ namespace :gitlab do user: user, gitlab_url: gitlab_url, http_settings: {self_signed_cert: false}.stringify_keys, - repos_path: repos_path, auth_file: File.join(home_dir, ".ssh", "authorized_keys"), redis: { bin: %x{which redis-cli}.chomp, @@ -58,10 +56,10 @@ namespace :gitlab do File.open("config.yml", "w+") {|f| f.puts config.to_yaml} # Launch installation process - system(*%W(bin/install)) + system(*%W(bin/install) + repository_storage_paths_args) # (Re)create hooks - system(*%W(bin/create-hooks)) + system(*%W(bin/create-hooks) + repository_storage_paths_args) end # Required for debian packaging with PKGR: Setup .ssh/environment with @@ -73,6 +71,8 @@ namespace :gitlab do File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f| f.puts "PATH=#{ENV['PATH']}" end + + Gitlab::Shell.new.generate_and_link_secret_token end desc "GitLab | Setup gitlab-shell" @@ -87,7 +87,8 @@ namespace :gitlab do if File.exists?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.path_with_namespace) + if Gitlab::Shell.new.add_repository(project.repository_storage_path, + project.path_with_namespace) print '.' else print 'F' @@ -138,4 +139,3 @@ namespace :gitlab do system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) end end - diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index d0c019044b7..ab96b1d3593 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -125,10 +125,16 @@ namespace :gitlab do end def all_repos - IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end end end end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index 09e0bbfd00b..604204cca0a 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -123,11 +123,17 @@ describe ProjectsHelper do end describe '#sanitized_import_error' do + let(:project) { create(:project) } + + before do + allow(project).to receive(:repository_storage_path).and_return('/base/repo/path') + end + it 'removes the repo path' do - repo = File.join(Gitlab.config.gitlab_shell.repos_path, '/namespace/test.git') + repo = '/base/repo/path/namespace/test.git' import_error = "Could not clone #{repo}\n" - expect(sanitize_repo_path(import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git') + expect(sanitize_repo_path(project, import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git') end end end diff --git a/spec/initializers/6_validations_spec.rb b/spec/initializers/6_validations_spec.rb new file mode 100644 index 00000000000..5178bd130f4 --- /dev/null +++ b/spec/initializers/6_validations_spec.rb @@ -0,0 +1,41 @@ +require 'spec_helper' + +describe '6_validations', lib: true do + context 'with correct settings' do + before do + mock_storages('foo' => '/a/b/c', 'bar' => 'a/b/d') + end + + it 'passes through' do + expect { load_validations }.not_to raise_error + end + end + + context 'with invalid storage names' do + before do + mock_storages('name with spaces' => '/a/b/c') + end + + it 'throws an error' do + expect { load_validations }.to raise_error('"name with spaces" is not a valid storage name. Please fix this in your gitlab.yml before starting GitLab.') + end + end + + context 'with nested storage paths' do + before do + mock_storages('foo' => '/a/b/c', 'bar' => '/a/b/c/d') + end + + it 'throws an error' do + expect { load_validations }.to raise_error('bar is a nested path of foo. Nested paths are not supported for repository storages. Please fix this in your gitlab.yml before starting GitLab.') + end + end + + def mock_storages(storages) + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + end + + def load_validations + load File.join(__dir__, '../../config/initializers/6_validations.rb') + end +end diff --git a/spec/lib/gitlab/backend/shell_spec.rb b/spec/lib/gitlab/backend/shell_spec.rb index fd869f48b5c..6e5ba211382 100644 --- a/spec/lib/gitlab/backend/shell_spec.rb +++ b/spec/lib/gitlab/backend/shell_spec.rb @@ -13,9 +13,37 @@ describe Gitlab::Shell, lib: true do it { is_expected.to respond_to :add_repository } it { is_expected.to respond_to :remove_repository } it { is_expected.to respond_to :fork_repository } + it { is_expected.to respond_to :gc } + it { is_expected.to respond_to :add_namespace } + it { is_expected.to respond_to :rm_namespace } + it { is_expected.to respond_to :mv_namespace } + it { is_expected.to respond_to :exists? } it { expect(gitlab_shell.url_to_repo('diaspora')).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + "diaspora.git") } + describe 'generate_and_link_secret_token' do + let(:secret_file) { 'tmp/tests/.secret_shell_test' } + let(:link_file) { 'tmp/tests/shell-secret-test/.gitlab_shell_secret' } + + before do + allow(Gitlab.config.gitlab_shell).to receive(:path).and_return('tmp/tests/shell-secret-test') + allow(Gitlab.config.gitlab_shell).to receive(:secret_file).and_return(secret_file) + FileUtils.mkdir('tmp/tests/shell-secret-test') + gitlab_shell.generate_and_link_secret_token + end + + after do + FileUtils.rm_rf('tmp/tests/shell-secret-test') + FileUtils.rm_rf(secret_file) + end + + it 'creates and links the secret token file' do + expect(File.exist?(secret_file)).to be(true) + expect(File.symlink?(link_file)).to be(true) + expect(File.readlink(link_file)).to eq(secret_file) + end + end + describe Gitlab::Shell::KeyAdder, lib: true do describe '#add_key' do it 'normalizes space characters in the key' do diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index d84f3e998f5..2ea1320267c 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -40,6 +40,16 @@ describe ApplicationSetting, models: true do it_behaves_like 'an object with email-formated attributes', :admin_notification_email do subject { setting } end + + context 'repository storages inclussion' do + before do + storages = { 'custom' => 'tmp/tests/custom_repositories' } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + end + + it { is_expected.to allow_value('custom').for(:repository_storage) } + it { is_expected.not_to allow_value('alternative').for(:repository_storage) } + end end context 'restricted signup domains' do diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 4e68ac5e63a..cbea407f9bf 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -57,6 +57,7 @@ describe Namespace, models: true do describe :move_dir do before do @namespace = create :namespace + @project = create :project, namespace: @namespace allow(@namespace).to receive(:path_changed?).and_return(true) end @@ -87,8 +88,13 @@ describe Namespace, models: true do end describe :rm_dir do - it "should remove dir" do - expect(namespace.rm_dir).to be_truthy + let!(:project) { create(:project, namespace: namespace) } + let!(:path) { File.join(Gitlab.config.repositories.storages.default, namespace.path) } + + before { namespace.destroy } + + it "should remove its dirs when deleted" do + expect(File.exist?(path)).to be(false) end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index d305cd9ff1e..42308035d8c 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -56,6 +56,7 @@ describe Project, models: true do it { is_expected.to validate_length_of(:description).is_within(0..2000) } it { is_expected.to validate_presence_of(:creator) } it { is_expected.to validate_presence_of(:namespace) } + it { is_expected.to validate_presence_of(:repository_storage) } it 'should not allow new projects beyond user limits' do project2 = build(:project) @@ -84,6 +85,20 @@ describe Project, models: true do end end end + + context 'repository storages inclussion' do + let(:project2) { build(:project, repository_storage: 'missing') } + + before do + storages = { 'custom' => 'tmp/tests/custom_repositories' } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + end + + it "should not allow repository storages that don't match a label in the configuration" do + expect(project2).not_to be_valid + expect(project2.errors[:repository_storage].first).to match(/is not included in the list/) + end + end end describe 'default_scope' do @@ -131,6 +146,24 @@ describe Project, models: true do end end + describe '#repository_storage_path' do + let(:project) { create(:project, repository_storage: 'custom') } + + before do + FileUtils.mkdir('tmp/tests/custom_repositories') + storages = { 'custom' => 'tmp/tests/custom_repositories' } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + end + + after do + FileUtils.rm_rf('tmp/tests/custom_repositories') + end + + it 'returns the repository storage path' do + expect(project.repository_storage_path).to eq('tmp/tests/custom_repositories') + end + end + it 'should return valid url to repo' do project = Project.new(path: 'somewhere') expect(project.url_to_repo).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + 'somewhere.git') @@ -574,6 +607,21 @@ describe Project, models: true do end end + context 'repository storage by default' do + let(:project) { create(:empty_project) } + + subject { project.repository_storage } + + before do + storages = { 'alternative_storage' => '/some/path' } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + stub_application_setting(repository_storage: 'alternative_storage') + allow_any_instance_of(Project).to receive(:ensure_dir_exist).and_return(true) + end + + it { is_expected.to eq('alternative_storage') } + end + context 'shared runners by default' do let(:project) { create(:empty_project) } @@ -729,12 +777,12 @@ describe Project, models: true do expect(gitlab_shell).to receive(:mv_repository). ordered. - with("#{ns}/foo", "#{ns}/#{project.path}"). + with(project.repository_storage_path, "#{ns}/foo", "#{ns}/#{project.path}"). and_return(true) expect(gitlab_shell).to receive(:mv_repository). ordered. - with("#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki"). + with(project.repository_storage_path, "#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki"). and_return(true) expect_any_instance_of(SystemHooksService). @@ -826,7 +874,7 @@ describe Project, models: true do context 'using a regular repository' do it 'creates the repository' do expect(shell).to receive(:add_repository). - with(project.path_with_namespace). + with(project.repository_storage_path, project.path_with_namespace). and_return(true) expect(project.repository).to receive(:after_create) @@ -836,7 +884,7 @@ describe Project, models: true do it 'adds an error if the repository could not be created' do expect(shell).to receive(:add_repository). - with(project.path_with_namespace). + with(project.repository_storage_path, project.path_with_namespace). and_return(false) expect(project.repository).not_to receive(:after_create) diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb index 437c89c3577..fcea45f19ba 100644 --- a/spec/requests/api/internal_spec.rb +++ b/spec/requests/api/internal_spec.rb @@ -72,6 +72,7 @@ describe API::API, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_truthy + expect(json_response["repository_path"]).to eq(project.repository.path_to_repo) end end @@ -81,6 +82,7 @@ describe API::API, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_truthy + expect(json_response["repository_path"]).to eq(project.repository.path_to_repo) end end end diff --git a/spec/requests/api/settings_spec.rb b/spec/requests/api/settings_spec.rb index f756101c514..6629a5a65e2 100644 --- a/spec/requests/api/settings_spec.rb +++ b/spec/requests/api/settings_spec.rb @@ -14,16 +14,23 @@ describe API::API, 'Settings', api: true do expect(json_response).to be_an Hash expect(json_response['default_projects_limit']).to eq(42) expect(json_response['signin_enabled']).to be_truthy + expect(json_response['repository_storage']).to eq('default') end end describe "PUT /application/settings" do + before do + storages = { 'custom' => 'tmp/tests/custom_repositories' } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + end + it "should update application settings" do put api("/application/settings", admin), - default_projects_limit: 3, signin_enabled: false + default_projects_limit: 3, signin_enabled: false, repository_storage: 'custom' expect(response).to have_http_status(200) expect(json_response['default_projects_limit']).to eq(3) expect(json_response['signin_enabled']).to be_falsey + expect(json_response['repository_storage']).to eq('custom') end end end diff --git a/spec/services/destroy_group_service_spec.rb b/spec/services/destroy_group_service_spec.rb index afa89b84175..eca8ddd8ea4 100644 --- a/spec/services/destroy_group_service_spec.rb +++ b/spec/services/destroy_group_service_spec.rb @@ -23,8 +23,8 @@ describe DestroyGroupService, services: true do Sidekiq::Testing.inline! { destroy_group(group, user) } end - it { expect(gitlab_shell.exists?(group.path)).to be_falsey } - it { expect(gitlab_shell.exists?(remove_path)).to be_falsey } + it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey } + it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_falsey } end context 'Sidekiq fake' do @@ -33,8 +33,8 @@ describe DestroyGroupService, services: true do Sidekiq::Testing.fake! { destroy_group(group, user) } end - it { expect(gitlab_shell.exists?(group.path)).to be_falsey } - it { expect(gitlab_shell.exists?(remove_path)).to be_truthy } + it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey } + it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_truthy } end end diff --git a/spec/services/projects/housekeeping_service_spec.rb b/spec/services/projects/housekeeping_service_spec.rb index 4c5ced7e746..bd4dc6a0f79 100644 --- a/spec/services/projects/housekeeping_service_spec.rb +++ b/spec/services/projects/housekeeping_service_spec.rb @@ -12,7 +12,7 @@ describe Projects::HousekeepingService do it 'enqueues a sidekiq job' do expect(subject).to receive(:try_obtain_lease).and_return(true) - expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.path_with_namespace) + expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.repository_storage_path, project.path_with_namespace) subject.execute expect(project.pushes_since_gc).to eq(0) diff --git a/spec/services/projects/import_service_spec.rb b/spec/services/projects/import_service_spec.rb index 068c9a1219c..d5d4d7c56ef 100644 --- a/spec/services/projects/import_service_spec.rb +++ b/spec/services/projects/import_service_spec.rb @@ -36,7 +36,7 @@ describe Projects::ImportService, services: true do end it 'succeeds if repository import is successfully' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) result = subject.execute @@ -44,7 +44,7 @@ describe Projects::ImportService, services: true do end it 'fails if repository import fails' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository')) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository')) result = subject.execute @@ -64,7 +64,7 @@ describe Projects::ImportService, services: true do end it 'succeeds if importer succeeds' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(true) result = subject.execute @@ -74,7 +74,7 @@ describe Projects::ImportService, services: true do it 'flushes various caches' do expect_any_instance_of(Gitlab::Shell).to receive(:import_repository). - with(project.path_with_namespace, project.import_url). + with(project.repository_storage_path, project.path_with_namespace, project.import_url). and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute). @@ -90,7 +90,7 @@ describe Projects::ImportService, services: true do end it 'fails if importer fails' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(false) result = subject.execute @@ -100,7 +100,7 @@ describe Projects::ImportService, services: true do end it 'fails if importer raise an error' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_raise(Projects::ImportService::Error.new('Github: failed to connect API')) result = subject.execute diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb index 426bf53f198..be5331e4770 100644 --- a/spec/support/test_env.rb +++ b/spec/support/test_env.rb @@ -80,8 +80,9 @@ module TestEnv end def setup_gitlab_shell - unless File.directory?(Rails.root.join(*%w(tmp tests gitlab-shell))) - `rake gitlab:shell:install` + unless File.directory?(Gitlab.config.gitlab_shell.path) + # TODO: Remove `[shards]` when gitlab-shell v3.1.0 is published + `rake gitlab:shell:install[shards]` end end @@ -127,14 +128,14 @@ module TestEnv def copy_repo(project) base_repo_path = File.expand_path(factory_repo_path_bare) - target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git") + target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git") FileUtils.mkdir_p(target_repo_path) FileUtils.cp_r("#{base_repo_path}/.", target_repo_path) FileUtils.chmod_R 0755, target_repo_path end def repos_path - Gitlab.config.gitlab_shell.repos_path + Gitlab.config.repositories.storages.default end def backup_path @@ -143,7 +144,7 @@ module TestEnv def copy_forked_repo_with_submodules(project) base_repo_path = File.expand_path(forked_repo_path_bare) - target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git") + target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git") FileUtils.mkdir_p(target_repo_path) FileUtils.cp_r("#{base_repo_path}/.", target_repo_path) FileUtils.chmod_R 0755, target_repo_path diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 25da0917134..02308530d13 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -98,67 +98,107 @@ describe 'gitlab:app namespace rake task' do @backup_tar = tars_glob.first end - before do - create_backup - end - - after do - FileUtils.rm(@backup_tar) - end + context 'tar creation' do + before do + create_backup + end - context 'archive file permissions' do - it 'should set correct permissions on the tar file' do - expect(File.exist?(@backup_tar)).to be_truthy - expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100600') + after do + FileUtils.rm(@backup_tar) end - context 'with custom archive_permissions' do - before do - allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0651) - # We created a backup in a before(:all) so it got the default permissions. - # We now need to do some work to create a _new_ backup file using our stub. - FileUtils.rm(@backup_tar) - create_backup + context 'archive file permissions' do + it 'should set correct permissions on the tar file' do + expect(File.exist?(@backup_tar)).to be_truthy + expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100600') end - it 'uses the custom permissions' do - expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100651') + context 'with custom archive_permissions' do + before do + allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0651) + # We created a backup in a before(:all) so it got the default permissions. + # We now need to do some work to create a _new_ backup file using our stub. + FileUtils.rm(@backup_tar) + create_backup + end + + it 'uses the custom permissions' do + expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100651') + end end end - end - it 'should set correct permissions on the tar contents' do - tar_contents, exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz} - ) - expect(exit_status).to eq(0) - expect(tar_contents).to match('db/') - expect(tar_contents).to match('uploads.tar.gz') - expect(tar_contents).to match('repositories/') - expect(tar_contents).to match('builds.tar.gz') - expect(tar_contents).to match('artifacts.tar.gz') - expect(tar_contents).to match('lfs.tar.gz') - expect(tar_contents).to match('registry.tar.gz') - expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz|registry.tar.gz)\/$/) - end + it 'should set correct permissions on the tar contents' do + tar_contents, exit_status = Gitlab::Popen.popen( + %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz} + ) + expect(exit_status).to eq(0) + expect(tar_contents).to match('db/') + expect(tar_contents).to match('uploads.tar.gz') + expect(tar_contents).to match('repositories/') + expect(tar_contents).to match('builds.tar.gz') + expect(tar_contents).to match('artifacts.tar.gz') + expect(tar_contents).to match('lfs.tar.gz') + expect(tar_contents).to match('registry.tar.gz') + expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz|registry.tar.gz)\/$/) + end - it 'should delete temp directories' do - temp_dirs = Dir.glob( - File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs,registry}') - ) + it 'should delete temp directories' do + temp_dirs = Dir.glob( + File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs,registry}') + ) + + expect(temp_dirs).to be_empty + end - expect(temp_dirs).to be_empty + context 'registry disabled' do + let(:enable_registry) { false } + + it 'should not create registry.tar.gz' do + tar_contents, exit_status = Gitlab::Popen.popen( + %W{tar -tvf #{@backup_tar}} + ) + expect(exit_status).to eq(0) + expect(tar_contents).not_to match('registry.tar.gz') + end + end end - context 'registry disabled' do - let(:enable_registry) { false } + context 'multiple repository storages' do + let(:project_a) { create(:project, repository_storage: 'default') } + let(:project_b) { create(:project, repository_storage: 'custom') } + + before do + FileUtils.mkdir('tmp/tests/default_storage') + FileUtils.mkdir('tmp/tests/custom_storage') + storages = { + 'default' => 'tmp/tests/default_storage', + 'custom' => 'tmp/tests/custom_storage' + } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + + # Create the projects now, after mocking the settings but before doing the backup + project_a + project_b + + # We only need a backup of the repositories for this test + ENV["SKIP"] = "db,uploads,builds,artifacts,lfs,registry" + create_backup + end + + after do + FileUtils.rm_rf('tmp/tests/default_storage') + FileUtils.rm_rf('tmp/tests/custom_storage') + FileUtils.rm(@backup_tar) + end - it 'should not create registry.tar.gz' do + it 'should include repositories in all repository storages' do tar_contents, exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar}} + %W{tar -tvf #{@backup_tar} repositories} ) expect(exit_status).to eq(0) - expect(tar_contents).not_to match('registry.tar.gz') + expect(tar_contents).to match("repositories/#{project_a.path_with_namespace}.bundle") + expect(tar_contents).to match("repositories/#{project_b.path_with_namespace}.bundle") end end end # backup_create task diff --git a/spec/workers/post_receive_spec.rb b/spec/workers/post_receive_spec.rb index b8e73682c91..20b1a343c27 100644 --- a/spec/workers/post_receive_spec.rb +++ b/spec/workers/post_receive_spec.rb @@ -91,6 +91,6 @@ describe PostReceive do end def pwd(project) - File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + File.join(Gitlab.config.repositories.storages.default, project.path_with_namespace) end end diff --git a/spec/workers/repository_fork_worker_spec.rb b/spec/workers/repository_fork_worker_spec.rb index 4ef05eb29d2..5f762282b5e 100644 --- a/spec/workers/repository_fork_worker_spec.rb +++ b/spec/workers/repository_fork_worker_spec.rb @@ -14,6 +14,7 @@ describe RepositoryForkWorker do describe "#perform" do it "creates a new repository from a fork" do expect(shell).to receive(:fork_repository).with( + project.repository_storage_path, project.path_with_namespace, fork_project.namespace.path ).and_return(true) @@ -25,9 +26,11 @@ describe RepositoryForkWorker do end it 'flushes various caches' do - expect(shell).to receive(:fork_repository). - with(project.path_with_namespace, fork_project.namespace.path). - and_return(true) + expect(shell).to receive(:fork_repository).with( + project.repository_storage_path, + project.path_with_namespace, + fork_project.namespace.path + ).and_return(true) expect_any_instance_of(Repository).to receive(:expire_emptiness_caches). and_call_original |